Backup: remove old config

With support of multiple config values

AGENTS.md

@@ -59,6 +59,7 @@ Ansible-based server automation for personal services. Playbooks provision Docke
 - Ansible lint: `ansible-lint .` (CI default).
 - Authelia config validation: `task authelia-validate-config` (renders with secrets then validates via docker).
 - Black formatting for Python helpers: `task format-py-files`.
+- Python types validation with mypy: `mypy <file.py>`.
 
 ## Operational Notes
 - Deployments rely on `production.yml` inventory and per-app playbooks; run with `--diff` for visibility.

files/backups/backup-all.py

@@ -4,18 +4,28 @@ Backup script for all applications
 Automatically discovers and runs backup scripts for all users,
 then creates restic backups and sends notifications.
 """
-
+import itertools
 import os
 import sys
 import subprocess
 import logging
 import pwd
+from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional
+from typing import Dict, List, Optional, Any
 import requests
-import configparser
+import tomllib
 
+# Default config path
+CONFIG_PATH = Path("/etc/backup/config.toml")
+
+# File name to store directories and files to back up
+BACKUP_TARGETS_FILE = "backup-targets"
+
+# Default directory fo backups (relative to app dir)
+# Used when backup-targets file not exists
+BACKUP_DEFAULT_DIR = "backups"
 
 # Configure logging
 logging.basicConfig(
@@ -28,24 +38,11 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-config = configparser.ConfigParser()
-config.read("/etc/backup/config.ini")
 
-RESTIC_REPOSITORY = config.get("restic", "RESTIC_REPOSITORY")
-RESTIC_PASSWORD = config.get("restic", "RESTIC_PASSWORD")
-AWS_ACCESS_KEY_ID = config.get("restic", "AWS_ACCESS_KEY_ID")
-AWS_SECRET_ACCESS_KEY = config.get("restic", "AWS_SECRET_ACCESS_KEY")
-AWS_DEFAULT_REGION = config.get("restic", "AWS_DEFAULT_REGION")
-TELEGRAM_BOT_TOKEN = config.get("telegram", "TELEGRAM_BOT_TOKEN")
-TELEGRAM_CHAT_ID = config.get("telegram", "TELEGRAM_CHAT_ID")
-NOTIFICATIONS_NAME = config.get("telegram", "NOTIFICATIONS_NAME")
-
-# File name to store directories and files to back up
-BACKUP_TARGETS_FILE = "backup-targets"
-
-# Default directory fo backups (relative to app dir)
-# Used when backup-targets file not exists
-BACKUP_DEFAULT_DIR = "backups"
+@dataclass
+class Config:
+    host_name: str
+    roots: List[Path]
 
 
 @dataclass
@@ -54,17 +51,167 @@ class Application:
     owner: str
 
 
+class Storage(ABC):
+    def backup(self, backup_dirs: List[str]) -> bool:
+        """Backup directories"""
+        raise NotImplementedError()
+
+
+class ResticStorage(Storage):
+    TYPE_NAME = "restic"
+
+    def __init__(self, name: str, params: Dict[str, Any]):
+        self.name = name
+        self.restic_repository = str(params.get("restic_repository", ""))
+        self.restic_password = str(params.get("restic_password", ""))
+        self.aws_access_key_id = str(params.get("aws_access_key_id", ""))
+        self.aws_secret_access_key = str(params.get("aws_secret_access_key", ""))
+        self.aws_default_region = str(params.get("aws_default_region", ""))
+
+        if not all(
+            [
+                self.restic_repository,
+                self.restic_password,
+                self.aws_access_key_id,
+                self.aws_secret_access_key,
+                self.aws_default_region,
+            ]
+        ):
+            raise ValueError(
+                f"Missing storage configuration values for backend ResticStorage: '{self.name}'"
+            )
+
+    def backup(self, backup_dirs: List[str]) -> bool:
+        if not backup_dirs:
+            logger.warning("No backup directories found")
+            return True
+        try:
+            return self.__backup_internal(backup_dirs)
+        except Exception as exc:  # noqa: BLE001
+            logger.error("Restic backup process failed: %s", exc)
+            return False
+
+    def __backup_internal(self, backup_dirs: List[str]) -> bool:
+        logger.info("Starting restic backup")
+        logger.info("Destination: %s", self.restic_repository)
+
+        env = os.environ.copy()
+        env.update(
+            {
+                "RESTIC_REPOSITORY": self.restic_repository,
+                "RESTIC_PASSWORD": self.restic_password,
+                "AWS_ACCESS_KEY_ID": self.aws_access_key_id,
+                "AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key,
+                "AWS_DEFAULT_REGION": self.aws_default_region,
+            }
+        )
+
+        backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs
+        result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            logger.error("Restic backup failed: %s", result.stderr)
+            return False
+
+        logger.info("Restic backup completed successfully")
+
+        check_cmd = ["restic", "check"]
+        result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            logger.error("Restic check failed: %s", result.stderr)
+            return False
+
+        logger.info("Restic check completed successfully")
+
+        forget_cmd = [
+            "restic",
+            "forget",
+            "--compact",
+            "--prune",
+            "--keep-daily",
+            "90",
+            "--keep-monthly",
+            "36",
+        ]
+        result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            logger.error("Restic forget/prune failed: %s", result.stderr)
+            return False
+
+        logger.info("Restic forget/prune completed successfully")
+
+        result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            logger.error("Final restic check failed: %s", result.stderr)
+            return False
+
+        logger.info("Final restic check completed successfully")
+        return True
+
+
+class Notifier(ABC):
+    def send(self, html_message: str):
+        raise NotImplementedError()
+
+
+class TelegramNotifier(Notifier):
+    TYPE_NAME = "telegram"
+
+    def __init__(self, name: str, params: Dict[str, Any]):
+        self.name = name
+        self.telegram_bot_token = str(params.get("telegram_bot_token", ""))
+        self.telegram_chat_id = str(params.get("telegram_chat_id", ""))
+        if not all(
+            [
+                self.telegram_bot_token,
+                self.telegram_chat_id,
+            ]
+        ):
+            raise ValueError(
+                f"Missing notification configuration values for backend {name}"
+            )
+
+    def send(self, html_message: str):
+        url = f"https://api.telegram.org/bot{self.telegram_bot_token}/sendMessage"
+        data = {
+            "chat_id": self.telegram_chat_id,
+            "parse_mode": "HTML",
+            "text": html_message,
+        }
+
+        response = requests.post(url, data=data, timeout=30)
+
+        if response.status_code == 200:
+            logger.info("Telegram notification sent successfully")
+        else:
+            logger.error(
+                f"Failed to send Telegram notification: {response.status_code} - {response.text}"
+            )
+
+
 class BackupManager:
-    def __init__(self):
+    def __init__(
+        self,
+        config: Config,
+        roots: List[Path],
+        storages: List[Storage],
+        notifiers: List[Notifier],
+    ):
         self.errors: List[str] = []
         self.warnings: List[str] = []
         self.successful_backups: List[str] = []
+        self.config = config
+        self.roots: List[Path] = roots
+        self.storages = storages
+        self.notifiers = notifiers
 
     def find_applications(self) -> List[Application]:
         """Get all application directories and their owners."""
         applications: List[Application] = []
-        applications_path = Path("/mnt/applications")
-        source_dirs = applications_path.iterdir()
+        source_dirs = itertools.chain(*(root.iterdir() for root in self.roots))
 
         for app_dir in source_dirs:
             if "lost+found" in str(app_dir):
@@ -195,128 +342,32 @@ class BackupManager:
 
         return backup_dirs
 
-    def run_restic_backup(self, backup_dirs: List[str]) -> bool:
-        """Run restic backup for all backup directories"""
-        if not backup_dirs:
-            logger.warning("No backup directories found")
-            return True
+    def send_notification(self, success: bool) -> None:
+        """Send notification to Notifiers"""
 
-        try:
-            logger.info("Starting restic backup")
-            logger.info("Destination: %s", RESTIC_REPOSITORY)
+        if success and not self.errors:
+            message = f"<b>{self.config.host_name}</b>: бекап успешно завершен!"
+            if self.successful_backups:
+                message += f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}"
+        else:
+            message = f"<b>{self.config.host_name}</b>: бекап завершен с ошибками!"
 
-            # Set environment variables for restic
-            env = os.environ.copy()
-            env.update(
-                {
-                    "RESTIC_REPOSITORY": RESTIC_REPOSITORY,
-                    "RESTIC_PASSWORD": RESTIC_PASSWORD,
-                    "AWS_ACCESS_KEY_ID": AWS_ACCESS_KEY_ID,
-                    "AWS_SECRET_ACCESS_KEY": AWS_SECRET_ACCESS_KEY,
-                    "AWS_DEFAULT_REGION": AWS_DEFAULT_REGION,
-                }
-            )
-
-            # Run backup
-            backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs
-            result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True)
-
-            if result.returncode != 0:
-                error_msg = f"Restic backup failed: {result.stderr}"
-                logger.error(error_msg)
-                self.errors.append(f"Restic backup: {error_msg}")
-                return False
-
-            logger.info("Restic backup completed successfully")
-
-            # Run check
-            check_cmd = ["restic", "check"]
-            result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
-
-            if result.returncode != 0:
-                error_msg = f"Restic check failed: {result.stderr}"
-                logger.error(error_msg)
-                self.errors.append(f"Restic check: {error_msg}")
-                return False
-
-            logger.info("Restic check completed successfully")
-
-            # Run forget and prune
-            forget_cmd = [
-                "restic",
-                "forget",
-                "--compact",
-                "--prune",
-                "--keep-daily",
-                "90",
-                "--keep-monthly",
-                "36",
-            ]
-            result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True)
-
-            if result.returncode != 0:
-                error_msg = f"Restic forget/prune failed: {result.stderr}"
-                logger.error(error_msg)
-                self.errors.append(f"Restic forget/prune: {error_msg}")
-                return False
-
-            logger.info("Restic forget/prune completed successfully")
-
-            # Final check
-            result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
-
-            if result.returncode != 0:
-                error_msg = f"Final restic check failed: {result.stderr}"
-                logger.error(error_msg)
-                self.errors.append(f"Final restic check: {error_msg}")
-                return False
-
-            logger.info("Final restic check completed successfully")
-            return True
-
-        except Exception as e:
-            error_msg = f"Restic backup process failed: {str(e)}"
-            logger.error(error_msg)
-            self.errors.append(f"Restic: {error_msg}")
-            return False
-
-    def send_telegram_notification(self, success: bool) -> None:
-        """Send notification to Telegram"""
-        try:
-            if success and not self.errors:
-                message = f"<b>{NOTIFICATIONS_NAME}</b>: бекап успешно завершен!"
-                if self.successful_backups:
-                    message += (
-                        f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}"
-                    )
-            else:
-                message = f"<b>{NOTIFICATIONS_NAME}</b>: бекап завершен с ошибками!"
-
-                if self.successful_backups:
-                    message += (
-                        f"\n\n✅ Успешные бекапы: {', '.join(self.successful_backups)}"
-                    )
-
-                if self.warnings:
-                    message += f"\n\n⚠️ Предупреждения:\n" + "\n".join(self.warnings)
-
-                if self.errors:
-                    message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors)
-
-            url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
-            data = {"chat_id": TELEGRAM_CHAT_ID, "parse_mode": "HTML", "text": message}
-
-            response = requests.post(url, data=data, timeout=30)
-
-            if response.status_code == 200:
-                logger.info("Telegram notification sent successfully")
-            else:
-                logger.error(
-                    f"Failed to send Telegram notification: {response.status_code} - {response.text}"
+            if self.successful_backups:
+                message += (
+                    f"\n\n✅ Успешные бекапы: {', '.join(self.successful_backups)}"
                 )
 
-        except Exception as e:
-            logger.error(f"Failed to send Telegram notification: {str(e)}")
+            if self.warnings:
+                message += f"\n\n⚠️ Предупреждения:\n" + "\n".join(self.warnings)
+
+            if self.errors:
+                message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors)
+
+        for notificator in self.notifiers:
+            try:
+                notificator.send(message)
+            except Exception as e:
+                logger.error(f"Failed to send notification: {str(e)}")
 
     def run_backup_process(self) -> bool:
         """Main backup process"""
@@ -349,14 +400,18 @@ class BackupManager:
         backup_dirs = self.get_backup_directories()
         logger.info(f"Found backup directories: {backup_dirs}")
 
-        # Run restic backup
-        restic_success = self.run_restic_backup(backup_dirs)
+        overall_success = True
 
-        # Determine overall success
-        overall_success = restic_success and len(self.errors) == 0
+        for storage in self.storages:
+            backup_result = storage.backup(backup_dirs)
+            if not backup_result:
+                self.errors.append("Restic backup failed")
+
+            # Determine overall success
+            overall_success = overall_success and backup_result
 
         # Send notification
-        self.send_telegram_notification(overall_success)
+        self.send_notification(overall_success)
 
         logger.info("Backup process completed")
 
@@ -371,9 +426,53 @@ class BackupManager:
             return True
 
 
+def initialize(config_path: Path) -> BackupManager:
+    try:
+        with config_path.open("rb") as config_file:
+            raw_config = tomllib.load(config_file)
+    except OSError as e:
+        logger.error(f"Failed to read config file {config_path}: {e}")
+        raise
+
+    host_name = str(raw_config.get("host_name", "unknown"))
+
+    roots_raw = raw_config.get("roots") or []
+    if not isinstance(roots_raw, list) or not roots_raw:
+        raise ValueError("roots must be a non-empty list of paths in config.toml")
+    roots = [Path(root) for root in roots_raw]
+
+    storage_raw = raw_config.get("storage") or {}
+    storages: List[Storage] = []
+    for name, params in storage_raw.items():
+        if not isinstance(params, dict):
+            raise ValueError(f"Storage config for {name} must be a table")
+        storage_type = params.get("type", "")
+        if storage_type == ResticStorage.TYPE_NAME:
+            storages.append(ResticStorage(name, params))
+    if not storages:
+        raise ValueError("At least one storage backend must be configured")
+
+    notifications_raw = raw_config.get("notifier") or {}
+    notifiers: List[Notifier] = []
+    for name, params in notifications_raw.items():
+        if not isinstance(params, dict):
+            raise ValueError(f"Notificator config for {name} must be a table")
+        notifier_type = params.get("type", "")
+        if notifier_type == TelegramNotifier.TYPE_NAME:
+            notifiers.append(TelegramNotifier(name, params))
+    if not notifiers:
+        raise ValueError("At least one notification backend must be configured")
+
+    config = Config(host_name=host_name, roots=roots)
+
+    return BackupManager(
+        config=config, roots=roots, storages=storages, notifiers=notifiers
+    )
+
+
 def main():
     try:
-        backup_manager = BackupManager()
+        backup_manager = initialize(CONFIG_PATH)
         success = backup_manager.run_backup_process()
         if not success:
             sys.exit(1)

files/backups/config.template.ini

@@ -1,11 +0,0 @@
-[restic]
-RESTIC_REPOSITORY={{ restic_repository }}
-RESTIC_PASSWORD={{ restic_password }}
-AWS_ACCESS_KEY_ID={{ restic_s3_access_key }}
-AWS_SECRET_ACCESS_KEY={{ restic_s3_access_secret }}
-AWS_DEFAULT_REGION={{ restic_s3_region }}
-
-[telegram]
-TELEGRAM_BOT_TOKEN={{ notifications_tg_bot_token }}
-TELEGRAM_CHAT_ID={{ notifications_tg_chat_id }}
-NOTIFICATIONS_NAME={{ notifications_name }}

files/backups/config.template.toml

@@ -0,0 +1,18 @@
+host_name = "{{ notifications_name }}"
+
+roots = [
+    "{{ application_dir }}"
+]
+
+[storage.yandex_cloud_s3]
+type = "restic"
+restic_repository = "{{ restic_repository }}"
+restic_password = "{{ restic_password }}"
+aws_access_key_id = "{{ restic_s3_access_key }}"
+aws_secret_access_key = "{{ restic_s3_access_secret }}"
+aws_default_region = "{{ restic_s3_region }}"
+
+[notifier.server_notifications_channel]
+type = "telegram"
+telegram_bot_token = "{{ notifications_tg_bot_token }}"
+telegram_chat_id = "{{ notifications_tg_chat_id }}"

lefthook.yml

@@ -1,6 +1,8 @@
 #   Refer for explanation to following link:
 #   https://lefthook.dev/configuration/
 
+glob_matcher: doublestar
+
 templates:
   av-hooks-dir: "/home/av/projects/private/git-hooks"
 
@@ -12,3 +14,12 @@ pre-commit:
 
     - name: "check secret files"
       run: "python3 {av-hooks-dir}/pre-commit/check-secrets-encrypted-with-ansible-vault.py"
+
+    - name: "format python"
+      glob: "**/*.py"
+      run: "black --quiet {staged_files}"
+      stage_fixed: true
+
+    - name: "mypy"
+      glob: "**/*.py"
+      run: "mypy {staged_files}"

playbook-backups.yml

@@ -7,7 +7,7 @@
 
   vars:
     backup_config_dir: "/etc/backup"
-    backup_config_file: "{{ (backup_config_dir, 'config.ini') | path_join }}"
+    backup_config_file: "{{ (backup_config_dir, 'config.toml') | path_join }}"
 
     restic_shell_script: "{{ (bin_prefix, 'restic-shell.sh') | path_join }}"
     backup_all_script: "{{ (bin_prefix, 'backup-all.py') | path_join }}"
@@ -23,7 +23,7 @@
 
     - name: "Create backup config file"
       ansible.builtin.template:
-        src: "files/backups/config.template.ini"
+        src: "files/backups/config.template.toml"
         dest: "{{ backup_config_file }}"
         owner: root
         group: root