Backups: backup to home server storage

files/backups/backup-all.py

@@ -11,6 +11,7 @@ import sys
 import subprocess
 import logging
 import pwd
+import time
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -53,7 +54,28 @@ class Application:
     backup_targets: List[Path]
 
 
+@dataclass
+class StorageRunResult:
+    name: str
+    success: bool
+    duration: float
+
+
+def format_duration(seconds: float) -> str:
+    if seconds < 60:
+        return f"{seconds:.1f}s"
+    minutes = int(seconds // 60)
+    secs = int(seconds % 60)
+    if minutes < 60:
+        return f"{minutes}m{secs:02d}s"
+    hours = minutes // 60
+    minutes = minutes % 60
+    return f"{hours}h{minutes:02d}m{secs:02d}s"
+
+
 class Storage(ABC):
+    name: str
+
     def backup(self, backup_dirs: List[str]) -> bool:
         """Backup directories"""
         raise NotImplementedError()
@@ -66,19 +88,15 @@ class ResticStorage(Storage):
         self.name = name
         self.restic_repository = str(params.get("restic_repository", ""))
         self.restic_password = str(params.get("restic_password", ""))
-        self.aws_access_key_id = str(params.get("aws_access_key_id", ""))
-        self.aws_secret_access_key = str(params.get("aws_secret_access_key", ""))
-        self.aws_default_region = str(params.get("aws_default_region", ""))
 
-        if not all(
-            [
-                self.restic_repository,
-                self.restic_password,
-                self.aws_access_key_id,
-                self.aws_secret_access_key,
-                self.aws_default_region,
-            ]
-        ):
+        env_raw = params.get("env") or {}
+        if not isinstance(env_raw, dict):
+            raise ValueError(
+                f"'env' must be a table for storage backend ResticStorage: '{self.name}'"
+            )
+        self.env: Dict[str, str] = {str(k): str(v) for k, v in env_raw.items()}
+
+        if not self.restic_repository or not self.restic_password:
             raise ValueError(
                 f"Missing storage configuration values for backend ResticStorage: '{self.name}'"
             )
@@ -94,19 +112,13 @@ class ResticStorage(Storage):
             return False
 
     def __backup_internal(self, backup_dirs: List[str]) -> bool:
-        logger.info("Starting restic backup")
+        logger.info("Starting restic backup for storage '%s'", self.name)
         logger.info("Destination: %s", self.restic_repository)
 
         env = os.environ.copy()
-        env.update(
-            {
-                "RESTIC_REPOSITORY": self.restic_repository,
-                "RESTIC_PASSWORD": self.restic_password,
-                "AWS_ACCESS_KEY_ID": self.aws_access_key_id,
-                "AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key,
-                "AWS_DEFAULT_REGION": self.aws_default_region,
-            }
-        )
+        env["RESTIC_REPOSITORY"] = self.restic_repository
+        env["RESTIC_PASSWORD"] = self.restic_password
+        env.update(self.env)
 
         backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs
         result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True)
@@ -295,12 +307,15 @@ class BackupManager:
         self.config = config
         self.storages = storages
         self.notifiers = notifiers
+        self.archive_duration: float = 0.0
+        self.storage_results: List[StorageRunResult] = []
 
     def run_backup_process(self, applications: List[Application]) -> bool:
         """Main backup process"""
         logger.info("Starting backup process")
         logger.info(f"Found {len(applications)} application directories")
 
+        archive_start = time.monotonic()
         # Process each user's backup
         for app in applications:
             app_dir = str(app.path)
@@ -316,6 +331,10 @@ class BackupManager:
                 continue
 
             self._run_app_backup(str(app.backup_script), app_dir, username)
+        self.archive_duration = time.monotonic() - archive_start
+        logger.info(
+            "Archive phase finished in %s", format_duration(self.archive_duration)
+        )
 
         # Collect backup directories from applications
         backup_dirs: List[str] = []
@@ -328,10 +347,33 @@ class BackupManager:
 
         overall_success = True
 
+        # Each storage is processed independently: a failure in one storage
+        # must not prevent the others from being attempted.
         for storage in self.storages:
-            backup_result = storage.backup(backup_dirs)
+            storage_start = time.monotonic()
+            try:
+                backup_result = storage.backup(backup_dirs)
+            except Exception as exc:  # noqa: BLE001
+                logger.error(
+                    "Storage '%s' raised an unexpected error: %s", storage.name, exc
+                )
+                backup_result = False
+            storage_duration = time.monotonic() - storage_start
+            self.storage_results.append(
+                StorageRunResult(
+                    name=storage.name,
+                    success=backup_result,
+                    duration=storage_duration,
+                )
+            )
+            logger.info(
+                "Storage '%s' finished in %s (success=%s)",
+                storage.name,
+                format_duration(storage_duration),
+                backup_result,
+            )
             if not backup_result:
-                self.errors.append("Restic backup failed")
+                self.errors.append(f"Storage '{storage.name}' backup failed")
 
             # Determine overall success
             overall_success = overall_success and backup_result
@@ -417,6 +459,14 @@ class BackupManager:
                 items = "".join(f"<li>{e}</li>" for e in self.errors)
                 message += f"<p>❌ Ошибки:</p><ul>{items}</ul>"
 
+        message += f"<p>⏱ Время архивации: {format_duration(self.archive_duration)}</p>"
+        if self.storage_results:
+            items = "".join(
+                f"<li>{'✅' if r.success else '❌'} {r.name}: {format_duration(r.duration)}</li>"
+                for r in self.storage_results
+            )
+            message += f"<p>⏱ Время записи в хранилища:</p><ul>{items}</ul>"
+
         for notificator in self.notifiers:
             try:
                 notificator.send(title, message)

files/backups/config.template.toml

@@ -8,9 +8,19 @@ roots = [
 type = "restic"
 restic_repository = "{{ restic_repository }}"
 restic_password = "{{ restic_password }}"
-aws_access_key_id = "{{ restic_s3_access_key }}"
-aws_secret_access_key = "{{ restic_s3_access_secret }}"
-aws_default_region = "{{ restic_s3_region }}"
+
+[storage.yandex_cloud_s3.env]
+AWS_ACCESS_KEY_ID = "{{ restic_s3_access_key }}"
+AWS_SECRET_ACCESS_KEY = "{{ restic_s3_access_secret }}"
+AWS_DEFAULT_REGION = "{{ restic_s3_region }}"
+
+[storage.pr86keedav]
+type = "restic"
+restic_repository = "{{ restic_pr86keedav_repository }}"
+restic_password = "{{ restic_pr86keedav_password }}"
+
+[storage.pr86keedav.env]
+RCLONE_CONFIG = "{{ rclone_config_file }}"
 
 [notifier.apprise]
 type = "apprise"

files/backups/rclone.template.conf

@@ -0,0 +1,6 @@
+[pr86keedav]
+type = webdav
+url = {{ rclone_pr86keedav_url }}
+vendor = other
+user = {{ rclone_pr86keedav_user }}
+pass = {{ rclone_pr86keedav_password }}