From a217c79e7d916c9542e3c16ed3a5c8e2ba558848 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 21:03:32 +0300 Subject: [PATCH 1/7] Backup: extract restic storage into separate class --- files/backups/backup-all.py | 334 ++++++++++++++++++------------------ 1 file changed, 170 insertions(+), 164 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index 0ad787f..18880bf 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -15,7 +15,6 @@ from pathlib import Path from typing import Dict, List, Optional import requests import tomllib -from collections.abc import Iterable # Configure logging @@ -55,78 +54,80 @@ class Config: notifications: Dict[str, TelegramConfig] -def read_config(config_path: Path) -> Config: - try: - with config_path.open("rb") as config_file: - raw_config = tomllib.load(config_file) - except OSError as e: - logger.error(f"Failed to read config file {config_path}: {e}") - raise +class ResticStorage: + def __init__(self, cfg: StorageConfig): + if cfg.type != "restic": + raise ValueError(f"Unsupported storage type for ResticStorage: {cfg.type}") + self.cfg = cfg - roots_raw = raw_config.get("roots") or [] - if not isinstance(roots_raw, list) or not roots_raw: - raise ValueError("roots must be a non-empty list of paths in config.toml") - roots = [Path(root) for root in roots_raw] + def backup(self, backup_dirs: List[str]) -> bool: + if not backup_dirs: + logger.warning("No backup directories found") + return True - storage_raw = raw_config.get("storage") or {} - storage: Dict[str, StorageConfig] = {} - for name, cfg in storage_raw.items(): - if not isinstance(cfg, dict): - raise ValueError(f"Storage config for {name} must be a table") - storage[name] = StorageConfig( - type=cfg.get("type", ""), - restic_repository=cfg.get("restic_repository", ""), - restic_password=cfg.get("restic_password", ""), - aws_access_key_id=cfg.get("aws_access_key_id", ""), - aws_secret_access_key=cfg.get("aws_secret_access_key", ""), - aws_default_region=cfg.get("aws_default_region", ""), - ) + try: + logger.info("Starting restic backup") + logger.info("Destination: %s", self.cfg.restic_repository) - if not storage: - raise ValueError("At least one storage backend must be configured") - - notifications_raw = raw_config.get("notifications") or {} - notifications: Dict[str, TelegramConfig] = {} - for name, cfg in notifications_raw.items(): - if not isinstance(cfg, dict): - raise ValueError(f"Notification config for {name} must be a table") - notifications[name] = TelegramConfig( - type=cfg.get("type", ""), - telegram_bot_token=cfg.get("telegram_bot_token", ""), - telegram_chat_id=cfg.get("telegram_chat_id", ""), - notifications_name=cfg.get("notifications_name", ""), - ) - - if not notifications: - raise ValueError("At least one notification backend must be configured") - - for name, cfg in storage.items(): - if not all( - [ - cfg.type, - cfg.restic_repository, - cfg.restic_password, - cfg.aws_access_key_id, - cfg.aws_secret_access_key, - cfg.aws_default_region, - ] - ): - raise ValueError(f"Missing storage configuration values for backend {name}") - - for name, cfg in notifications.items(): - if not all( - [ - cfg.type, - cfg.telegram_bot_token, - cfg.telegram_chat_id, - cfg.notifications_name, - ] - ): - raise ValueError( - f"Missing notification configuration values for backend {name}" + env = os.environ.copy() + env.update( + { + "RESTIC_REPOSITORY": self.cfg.restic_repository, + "RESTIC_PASSWORD": self.cfg.restic_password, + "AWS_ACCESS_KEY_ID": self.cfg.aws_access_key_id, + "AWS_SECRET_ACCESS_KEY": self.cfg.aws_secret_access_key, + "AWS_DEFAULT_REGION": self.cfg.aws_default_region, + } ) - return Config(roots=roots, storage=storage, notifications=notifications) + backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs + result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Restic backup failed: %s", result.stderr) + return False + + logger.info("Restic backup completed successfully") + + check_cmd = ["restic", "check"] + result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Restic check failed: %s", result.stderr) + return False + + logger.info("Restic check completed successfully") + + forget_cmd = [ + "restic", + "forget", + "--compact", + "--prune", + "--keep-daily", + "90", + "--keep-monthly", + "36", + ] + result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Restic forget/prune failed: %s", result.stderr) + return False + + logger.info("Restic forget/prune completed successfully") + + result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Final restic check failed: %s", result.stderr) + return False + + logger.info("Final restic check completed successfully") + return True + + except Exception as exc: # noqa: BLE001 + logger.error("Restic backup process failed: %s", exc) + return False CONFIG_PATH = Path("/etc/backup/config.toml") @@ -146,16 +147,12 @@ class Application: class BackupManager: - def __init__(self): + def __init__(self, config: Config, storage: ResticStorage): self.errors: List[str] = [] self.warnings: List[str] = [] self.successful_backups: List[str] = [] - self.config = read_config(CONFIG_PATH) - - def _select_storage(self) -> StorageConfig: - if "yandex" in self.config.storage: - return self.config.storage["yandex"] - return next(iter(self.config.storage.values())) + self.config = config + self.storage = storage def _select_telegram(self) -> Optional[TelegramConfig]: if "telegram" in self.config.notifications: @@ -296,93 +293,6 @@ class BackupManager: return backup_dirs - def run_restic_backup(self, backup_dirs: List[str]) -> bool: - """Run restic backup for all backup directories""" - if not backup_dirs: - logger.warning("No backup directories found") - return True - - storage_cfg = self._select_storage() - - try: - logger.info("Starting restic backup") - logger.info("Destination: %s", storage_cfg.restic_repository) - - # Set environment variables for restic - env = os.environ.copy() - env.update( - { - "RESTIC_REPOSITORY": storage_cfg.restic_repository, - "RESTIC_PASSWORD": storage_cfg.restic_password, - "AWS_ACCESS_KEY_ID": storage_cfg.aws_access_key_id, - "AWS_SECRET_ACCESS_KEY": storage_cfg.aws_secret_access_key, - "AWS_DEFAULT_REGION": storage_cfg.aws_default_region, - } - ) - - # Run backup - backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs - result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - error_msg = f"Restic backup failed: {result.stderr}" - logger.error(error_msg) - self.errors.append(f"Restic backup: {error_msg}") - return False - - logger.info("Restic backup completed successfully") - - # Run check - check_cmd = ["restic", "check"] - result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - error_msg = f"Restic check failed: {result.stderr}" - logger.error(error_msg) - self.errors.append(f"Restic check: {error_msg}") - return False - - logger.info("Restic check completed successfully") - - # Run forget and prune - forget_cmd = [ - "restic", - "forget", - "--compact", - "--prune", - "--keep-daily", - "90", - "--keep-monthly", - "36", - ] - result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - error_msg = f"Restic forget/prune failed: {result.stderr}" - logger.error(error_msg) - self.errors.append(f"Restic forget/prune: {error_msg}") - return False - - logger.info("Restic forget/prune completed successfully") - - # Final check - result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - error_msg = f"Final restic check failed: {result.stderr}" - logger.error(error_msg) - self.errors.append(f"Final restic check: {error_msg}") - return False - - logger.info("Final restic check completed successfully") - return True - - except Exception as e: - error_msg = f"Restic backup process failed: {str(e)}" - logger.error(error_msg) - self.errors.append(f"Restic: {error_msg}") - return False - def send_telegram_notification(self, success: bool) -> None: """Send notification to Telegram""" telegram_cfg = self._select_telegram() @@ -464,7 +374,10 @@ class BackupManager: logger.info(f"Found backup directories: {backup_dirs}") # Run restic backup - restic_success = self.run_restic_backup(backup_dirs) + restic_success = self.storage.backup(backup_dirs) + + if not restic_success: + self.errors.append("Restic backup failed") # Determine overall success overall_success = restic_success and len(self.errors) == 0 @@ -485,9 +398,102 @@ class BackupManager: return True +def initialize(config_path: Path) -> BackupManager: + try: + with config_path.open("rb") as config_file: + raw_config = tomllib.load(config_file) + except OSError as e: + logger.error(f"Failed to read config file {config_path}: {e}") + raise + + roots_raw = raw_config.get("roots") or [] + if not isinstance(roots_raw, list) or not roots_raw: + raise ValueError("roots must be a non-empty list of paths in config.toml") + roots = [Path(root) for root in roots_raw] + + storage_raw = raw_config.get("storage") or {} + storage: Dict[str, StorageConfig] = {} + for name, cfg in storage_raw.items(): + if not isinstance(cfg, dict): + raise ValueError(f"Storage config for {name} must be a table") + storage[name] = StorageConfig( + type=cfg.get("type", ""), + restic_repository=cfg.get("restic_repository", ""), + restic_password=cfg.get("restic_password", ""), + aws_access_key_id=cfg.get("aws_access_key_id", ""), + aws_secret_access_key=cfg.get("aws_secret_access_key", ""), + aws_default_region=cfg.get("aws_default_region", ""), + ) + + if not storage: + raise ValueError("At least one storage backend must be configured") + + notifications_raw = raw_config.get("notifications") or {} + notifications: Dict[str, TelegramConfig] = {} + for name, cfg in notifications_raw.items(): + if not isinstance(cfg, dict): + raise ValueError(f"Notification config for {name} must be a table") + notifications[name] = TelegramConfig( + type=cfg.get("type", ""), + telegram_bot_token=cfg.get("telegram_bot_token", ""), + telegram_chat_id=cfg.get("telegram_chat_id", ""), + notifications_name=cfg.get("notifications_name", ""), + ) + + if not notifications: + raise ValueError("At least one notification backend must be configured") + + for name, cfg in storage.items(): + if not all( + [ + cfg.type, + cfg.restic_repository, + cfg.restic_password, + cfg.aws_access_key_id, + cfg.aws_secret_access_key, + cfg.aws_default_region, + ] + ): + raise ValueError(f"Missing storage configuration values for backend {name}") + + for name, cfg in notifications.items(): + if not all( + [ + cfg.type, + cfg.telegram_bot_token, + cfg.telegram_chat_id, + cfg.notifications_name, + ] + ): + raise ValueError( + f"Missing notification configuration values for backend {name}" + ) + + config = Config(roots=roots, storage=storage, notifications=notifications) + storage_backend = _create_storage(config) + + return BackupManager(config=config, storage=storage_backend) + + +def _create_storage(config: Config) -> ResticStorage: + # Prefer explicit yandex key if present + if "yandex" in config.storage: + candidate = config.storage["yandex"] + if candidate.type != "restic": + raise ValueError("Storage 'yandex' must be of type 'restic'") + return ResticStorage(candidate) + + # Otherwise take the first restic storage + for name, cfg in config.storage.items(): + if cfg.type == "restic": + return ResticStorage(cfg) + + raise ValueError("No restic storage backend configured") + + def main(): try: - backup_manager = BackupManager() + backup_manager = initialize(CONFIG_PATH) success = backup_manager.run_backup_process() if not success: sys.exit(1) -- 2.49.1 From 0e96b5030db55f39ed1aa3f80f4fbac417c6f61e Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 21:04:54 +0300 Subject: [PATCH 2/7] Backup: refactoring --- files/backups/backup-all.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index 18880bf..b1d2df2 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -16,6 +16,14 @@ from typing import Dict, List, Optional import requests import tomllib +CONFIG_PATH = Path("/etc/backup/config.toml") + +# File name to store directories and files to back up +BACKUP_TARGETS_FILE = "backup-targets" + +# Default directory fo backups (relative to app dir) +# Used when backup-targets file not exists +BACKUP_DEFAULT_DIR = "backups" # Configure logging logging.basicConfig( @@ -54,6 +62,12 @@ class Config: notifications: Dict[str, TelegramConfig] +@dataclass +class Application: + path: Path + owner: str + + class ResticStorage: def __init__(self, cfg: StorageConfig): if cfg.type != "restic": @@ -130,22 +144,6 @@ class ResticStorage: return False -CONFIG_PATH = Path("/etc/backup/config.toml") - -# File name to store directories and files to back up -BACKUP_TARGETS_FILE = "backup-targets" - -# Default directory fo backups (relative to app dir) -# Used when backup-targets file not exists -BACKUP_DEFAULT_DIR = "backups" - - -@dataclass -class Application: - path: Path - owner: str - - class BackupManager: def __init__(self, config: Config, storage: ResticStorage): self.errors: List[str] = [] -- 2.49.1 From 265586981418fbdd6f768b5e209ef5be401a1a7e Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 21:19:06 +0300 Subject: [PATCH 3/7] Backup: support for multiple storages --- files/backups/backup-all.py | 132 +++++++++++++++--------------------- 1 file changed, 56 insertions(+), 76 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index b1d2df2..c08154a 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -10,12 +10,14 @@ import sys import subprocess import logging import pwd +from abc import ABC from dataclasses import dataclass from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Any import requests import tomllib +# Default config path CONFIG_PATH = Path("/etc/backup/config.toml") # File name to store directories and files to back up @@ -37,16 +39,6 @@ logging.basicConfig( logger = logging.getLogger(__name__) -@dataclass -class StorageConfig: - type: str - restic_repository: str - restic_password: str - aws_access_key_id: str - aws_secret_access_key: str - aws_default_region: str - - @dataclass class TelegramConfig: type: str @@ -58,7 +50,6 @@ class TelegramConfig: @dataclass class Config: roots: List[Path] - storage: Dict[str, StorageConfig] notifications: Dict[str, TelegramConfig] @@ -68,11 +59,35 @@ class Application: owner: str -class ResticStorage: - def __init__(self, cfg: StorageConfig): - if cfg.type != "restic": - raise ValueError(f"Unsupported storage type for ResticStorage: {cfg.type}") - self.cfg = cfg +class Storage(ABC): + def backup(self, backup_dirs: List[str]) -> bool: + """Backup directories""" + raise NotImplementedError() + + +class ResticStorage(Storage): + TYPE_NAME = "restic" + + def __init__(self, name: str, params: Dict[str, Any]): + self.name = name + self.restic_repository = str(params.get("restic_repository", "")) + self.restic_password = str(params.get("restic_password", "")) + self.aws_access_key_id = str(params.get("aws_access_key_id", "")) + self.aws_secret_access_key = str(params.get("aws_secret_access_key", "")) + self.aws_default_region = str(params.get("aws_default_region", "")) + + if not all( + [ + self.restic_repository, + self.restic_password, + self.aws_access_key_id, + self.aws_secret_access_key, + self.aws_default_region, + ] + ): + raise ValueError( + f"Missing storage configuration values for backend ResticStorage: '{self.name}'" + ) def backup(self, backup_dirs: List[str]) -> bool: if not backup_dirs: @@ -81,16 +96,16 @@ class ResticStorage: try: logger.info("Starting restic backup") - logger.info("Destination: %s", self.cfg.restic_repository) + logger.info("Destination: %s", self.restic_repository) env = os.environ.copy() env.update( { - "RESTIC_REPOSITORY": self.cfg.restic_repository, - "RESTIC_PASSWORD": self.cfg.restic_password, - "AWS_ACCESS_KEY_ID": self.cfg.aws_access_key_id, - "AWS_SECRET_ACCESS_KEY": self.cfg.aws_secret_access_key, - "AWS_DEFAULT_REGION": self.cfg.aws_default_region, + "RESTIC_REPOSITORY": self.restic_repository, + "RESTIC_PASSWORD": self.restic_password, + "AWS_ACCESS_KEY_ID": self.aws_access_key_id, + "AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key, + "AWS_DEFAULT_REGION": self.aws_default_region, } ) @@ -145,12 +160,12 @@ class ResticStorage: class BackupManager: - def __init__(self, config: Config, storage: ResticStorage): + def __init__(self, config: Config, storages: List[Storage]): self.errors: List[str] = [] self.warnings: List[str] = [] self.successful_backups: List[str] = [] self.config = config - self.storage = storage + self.storages = storages def _select_telegram(self) -> Optional[TelegramConfig]: if "telegram" in self.config.notifications: @@ -371,14 +386,15 @@ class BackupManager: backup_dirs = self.get_backup_directories() logger.info(f"Found backup directories: {backup_dirs}") - # Run restic backup - restic_success = self.storage.backup(backup_dirs) + overall_success = True - if not restic_success: - self.errors.append("Restic backup failed") + for storage in self.storages: + backup_result = storage.backup(backup_dirs) + if not backup_result: + self.errors.append("Restic backup failed") - # Determine overall success - overall_success = restic_success and len(self.errors) == 0 + # Determine overall success + overall_success = overall_success and backup_result # Send notification self.send_telegram_notification(overall_success) @@ -410,20 +426,14 @@ def initialize(config_path: Path) -> BackupManager: roots = [Path(root) for root in roots_raw] storage_raw = raw_config.get("storage") or {} - storage: Dict[str, StorageConfig] = {} - for name, cfg in storage_raw.items(): - if not isinstance(cfg, dict): + storages: List[Storage] = [] + for name, params in storage_raw.items(): + if not isinstance(params, dict): raise ValueError(f"Storage config for {name} must be a table") - storage[name] = StorageConfig( - type=cfg.get("type", ""), - restic_repository=cfg.get("restic_repository", ""), - restic_password=cfg.get("restic_password", ""), - aws_access_key_id=cfg.get("aws_access_key_id", ""), - aws_secret_access_key=cfg.get("aws_secret_access_key", ""), - aws_default_region=cfg.get("aws_default_region", ""), - ) - - if not storage: + storage_type = params.get("type", "") + if storage_type == ResticStorage.TYPE_NAME: + storages.append(ResticStorage(name, params)) + if not storages: raise ValueError("At least one storage backend must be configured") notifications_raw = raw_config.get("notifications") or {} @@ -441,19 +451,6 @@ def initialize(config_path: Path) -> BackupManager: if not notifications: raise ValueError("At least one notification backend must be configured") - for name, cfg in storage.items(): - if not all( - [ - cfg.type, - cfg.restic_repository, - cfg.restic_password, - cfg.aws_access_key_id, - cfg.aws_secret_access_key, - cfg.aws_default_region, - ] - ): - raise ValueError(f"Missing storage configuration values for backend {name}") - for name, cfg in notifications.items(): if not all( [ @@ -467,26 +464,9 @@ def initialize(config_path: Path) -> BackupManager: f"Missing notification configuration values for backend {name}" ) - config = Config(roots=roots, storage=storage, notifications=notifications) - storage_backend = _create_storage(config) + config = Config(roots=roots, notifications=notifications) - return BackupManager(config=config, storage=storage_backend) - - -def _create_storage(config: Config) -> ResticStorage: - # Prefer explicit yandex key if present - if "yandex" in config.storage: - candidate = config.storage["yandex"] - if candidate.type != "restic": - raise ValueError("Storage 'yandex' must be of type 'restic'") - return ResticStorage(candidate) - - # Otherwise take the first restic storage - for name, cfg in config.storage.items(): - if cfg.type == "restic": - return ResticStorage(cfg) - - raise ValueError("No restic storage backend configured") + return BackupManager(config=config, storages=storages) def main(): -- 2.49.1 From 037e0cab9bed49e407daf25aa0733c9782c0eb6b Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 21:31:39 +0300 Subject: [PATCH 4/7] Backup: restic backup refactoring --- files/backups/backup-all.py | 121 ++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index c08154a..a327127 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -93,71 +93,72 @@ class ResticStorage(Storage): if not backup_dirs: logger.warning("No backup directories found") return True - try: - logger.info("Starting restic backup") - logger.info("Destination: %s", self.restic_repository) - - env = os.environ.copy() - env.update( - { - "RESTIC_REPOSITORY": self.restic_repository, - "RESTIC_PASSWORD": self.restic_password, - "AWS_ACCESS_KEY_ID": self.aws_access_key_id, - "AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key, - "AWS_DEFAULT_REGION": self.aws_default_region, - } - ) - - backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs - result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - logger.error("Restic backup failed: %s", result.stderr) - return False - - logger.info("Restic backup completed successfully") - - check_cmd = ["restic", "check"] - result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - logger.error("Restic check failed: %s", result.stderr) - return False - - logger.info("Restic check completed successfully") - - forget_cmd = [ - "restic", - "forget", - "--compact", - "--prune", - "--keep-daily", - "90", - "--keep-monthly", - "36", - ] - result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - logger.error("Restic forget/prune failed: %s", result.stderr) - return False - - logger.info("Restic forget/prune completed successfully") - - result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) - - if result.returncode != 0: - logger.error("Final restic check failed: %s", result.stderr) - return False - - logger.info("Final restic check completed successfully") - return True - + return self.__backup_internal(backup_dirs) except Exception as exc: # noqa: BLE001 logger.error("Restic backup process failed: %s", exc) return False + def __backup_internal(self, backup_dirs: List[str]) -> bool: + logger.info("Starting restic backup") + logger.info("Destination: %s", self.restic_repository) + + env = os.environ.copy() + env.update( + { + "RESTIC_REPOSITORY": self.restic_repository, + "RESTIC_PASSWORD": self.restic_password, + "AWS_ACCESS_KEY_ID": self.aws_access_key_id, + "AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key, + "AWS_DEFAULT_REGION": self.aws_default_region, + } + ) + + backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs + result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Restic backup failed: %s", result.stderr) + return False + + logger.info("Restic backup completed successfully") + + check_cmd = ["restic", "check"] + result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Restic check failed: %s", result.stderr) + return False + + logger.info("Restic check completed successfully") + + forget_cmd = [ + "restic", + "forget", + "--compact", + "--prune", + "--keep-daily", + "90", + "--keep-monthly", + "36", + ] + result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Restic forget/prune failed: %s", result.stderr) + return False + + logger.info("Restic forget/prune completed successfully") + + result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + logger.error("Final restic check failed: %s", result.stderr) + return False + + logger.info("Final restic check completed successfully") + return True + class BackupManager: def __init__(self, config: Config, storages: List[Storage]): -- 2.49.1 From e1379bc480b71a9566240a8d9ece859493192423 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 21:33:21 +0300 Subject: [PATCH 5/7] Backup: roots parameter --- files/backups/backup-all.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index a327127..b585778 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -161,11 +161,12 @@ class ResticStorage(Storage): class BackupManager: - def __init__(self, config: Config, storages: List[Storage]): + def __init__(self, config: Config, roots: List[Path], storages: List[Storage]): self.errors: List[str] = [] self.warnings: List[str] = [] self.successful_backups: List[str] = [] self.config = config + self.roots: List[Path] = roots self.storages = storages def _select_telegram(self) -> Optional[TelegramConfig]: @@ -176,7 +177,7 @@ class BackupManager: def find_applications(self) -> List[Application]: """Get all application directories and their owners.""" applications: List[Application] = [] - source_dirs = itertools.chain(*(root.iterdir() for root in self.config.roots)) + source_dirs = itertools.chain(*(root.iterdir() for root in self.roots)) for app_dir in source_dirs: if "lost+found" in str(app_dir): @@ -467,7 +468,7 @@ def initialize(config_path: Path) -> BackupManager: config = Config(roots=roots, notifications=notifications) - return BackupManager(config=config, storages=storages) + return BackupManager(config=config, roots=roots, storages=storages) def main(): -- 2.49.1 From 54a951b96ae71662231124568ba63d297d5c5708 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sun, 21 Dec 2025 10:10:12 +0300 Subject: [PATCH 6/7] Backup: refactor notifications --- files/backups/backup-all.py | 175 ++++++++++++++--------------- files/backups/config.template.toml | 7 +- 2 files changed, 91 insertions(+), 91 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index b585778..42235ab 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -39,18 +39,10 @@ logging.basicConfig( logger = logging.getLogger(__name__) -@dataclass -class TelegramConfig: - type: str - telegram_bot_token: str - telegram_chat_id: str - notifications_name: str - - @dataclass class Config: + host_name: str roots: List[Path] - notifications: Dict[str, TelegramConfig] @dataclass @@ -160,19 +152,61 @@ class ResticStorage(Storage): return True +class Notifier(ABC): + def send(self, html_message: str): + raise NotImplementedError() + + +class TelegramNotifier(Notifier): + TYPE_NAME = "telegram" + + def __init__(self, name: str, params: Dict[str, Any]): + self.name = name + self.telegram_bot_token = str(params.get("telegram_bot_token", "")) + self.telegram_chat_id = str(params.get("telegram_chat_id", "")) + if not all( + [ + self.telegram_bot_token, + self.telegram_chat_id, + ] + ): + raise ValueError( + f"Missing notification configuration values for backend {name}" + ) + + def send(self, html_message: str): + url = f"https://api.telegram.org/bot{self.telegram_bot_token}/sendMessage" + data = { + "chat_id": self.telegram_chat_id, + "parse_mode": "HTML", + "text": html_message, + } + + response = requests.post(url, data=data, timeout=30) + + if response.status_code == 200: + logger.info("Telegram notification sent successfully") + else: + logger.error( + f"Failed to send Telegram notification: {response.status_code} - {response.text}" + ) + + class BackupManager: - def __init__(self, config: Config, roots: List[Path], storages: List[Storage]): + def __init__( + self, + config: Config, + roots: List[Path], + storages: List[Storage], + notifiers: List[Notifier], + ): self.errors: List[str] = [] self.warnings: List[str] = [] self.successful_backups: List[str] = [] self.config = config self.roots: List[Path] = roots self.storages = storages - - def _select_telegram(self) -> Optional[TelegramConfig]: - if "telegram" in self.config.notifications: - return self.config.notifications["telegram"] - return next(iter(self.config.notifications.values()), None) + self.notifiers = notifiers def find_applications(self) -> List[Application]: """Get all application directories and their owners.""" @@ -308,54 +342,32 @@ class BackupManager: return backup_dirs - def send_telegram_notification(self, success: bool) -> None: - """Send notification to Telegram""" - telegram_cfg = self._select_telegram() - if telegram_cfg is None: - logger.warning("No telegram notification backend configured") - return + def send_notification(self, success: bool) -> None: + """Send notification to Notifiers""" - try: - if success and not self.errors: - message = ( - f"{telegram_cfg.notifications_name}: бекап успешно завершен!" - ) - if self.successful_backups: - message += ( - f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}" - ) - else: - message = f"{telegram_cfg.notifications_name}: бекап завершен с ошибками!" + if success and not self.errors: + message = f"{self.config.host_name}: бекап успешно завершен!" + if self.successful_backups: + message += f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}" + else: + message = f"{self.config.host_name}: бекап завершен с ошибками!" - if self.successful_backups: - message += ( - f"\n\n✅ Успешные бекапы: {', '.join(self.successful_backups)}" - ) - - if self.warnings: - message += f"\n\n⚠️ Предупреждения:\n" + "\n".join(self.warnings) - - if self.errors: - message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors) - - url = f"https://api.telegram.org/bot{telegram_cfg.telegram_bot_token}/sendMessage" - data = { - "chat_id": telegram_cfg.telegram_chat_id, - "parse_mode": "HTML", - "text": message, - } - - response = requests.post(url, data=data, timeout=30) - - if response.status_code == 200: - logger.info("Telegram notification sent successfully") - else: - logger.error( - f"Failed to send Telegram notification: {response.status_code} - {response.text}" + if self.successful_backups: + message += ( + f"\n\n✅ Успешные бекапы: {', '.join(self.successful_backups)}" ) - except Exception as e: - logger.error(f"Failed to send Telegram notification: {str(e)}") + if self.warnings: + message += f"\n\n⚠️ Предупреждения:\n" + "\n".join(self.warnings) + + if self.errors: + message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors) + + for notificator in self.notifiers: + try: + notificator.send(message) + except Exception as e: + logger.error(f"Failed to send notification: {str(e)}") def run_backup_process(self) -> bool: """Main backup process""" @@ -399,7 +411,7 @@ class BackupManager: overall_success = overall_success and backup_result # Send notification - self.send_telegram_notification(overall_success) + self.send_notification(overall_success) logger.info("Backup process completed") @@ -422,6 +434,8 @@ def initialize(config_path: Path) -> BackupManager: logger.error(f"Failed to read config file {config_path}: {e}") raise + host_name = str(raw_config.get("host_name", "unknown")) + roots_raw = raw_config.get("roots") or [] if not isinstance(roots_raw, list) or not roots_raw: raise ValueError("roots must be a non-empty list of paths in config.toml") @@ -438,37 +452,22 @@ def initialize(config_path: Path) -> BackupManager: if not storages: raise ValueError("At least one storage backend must be configured") - notifications_raw = raw_config.get("notifications") or {} - notifications: Dict[str, TelegramConfig] = {} - for name, cfg in notifications_raw.items(): - if not isinstance(cfg, dict): - raise ValueError(f"Notification config for {name} must be a table") - notifications[name] = TelegramConfig( - type=cfg.get("type", ""), - telegram_bot_token=cfg.get("telegram_bot_token", ""), - telegram_chat_id=cfg.get("telegram_chat_id", ""), - notifications_name=cfg.get("notifications_name", ""), - ) - - if not notifications: + notifications_raw = raw_config.get("notifier") or {} + notifiers: List[Notifier] = [] + for name, params in notifications_raw.items(): + if not isinstance(params, dict): + raise ValueError(f"Notificator config for {name} must be a table") + notifier_type = params.get("type", "") + if notifier_type == TelegramNotifier.TYPE_NAME: + notifiers.append(TelegramNotifier(name, params)) + if not notifiers: raise ValueError("At least one notification backend must be configured") - for name, cfg in notifications.items(): - if not all( - [ - cfg.type, - cfg.telegram_bot_token, - cfg.telegram_chat_id, - cfg.notifications_name, - ] - ): - raise ValueError( - f"Missing notification configuration values for backend {name}" - ) + config = Config(host_name=host_name, roots=roots) - config = Config(roots=roots, notifications=notifications) - - return BackupManager(config=config, roots=roots, storages=storages) + return BackupManager( + config=config, roots=roots, storages=storages, notifiers=notifiers + ) def main(): diff --git a/files/backups/config.template.toml b/files/backups/config.template.toml index b8ba4d5..c4e4ead 100644 --- a/files/backups/config.template.toml +++ b/files/backups/config.template.toml @@ -1,8 +1,10 @@ +host_name = "{{ notifications_name }}" + roots = [ "{{ application_dir }}" ] -[storage.yandex] +[storage.yandex_cloud_s3] type = "restic" restic_repository = "{{ restic_repository }}" restic_password = "{{ restic_password }}" @@ -10,8 +12,7 @@ aws_access_key_id = "{{ restic_s3_access_key }}" aws_secret_access_key = "{{ restic_s3_access_secret }}" aws_default_region = "{{ restic_s3_region }}" -[notifications.telegram] +[notifier.server_notifications_channel] type = "telegram" telegram_bot_token = "{{ notifications_tg_bot_token }}" telegram_chat_id = "{{ notifications_tg_chat_id }}" -notifications_name = "{{ notifications_name }}" -- 2.49.1 From a31a07bd16a59b98e54f76db99d11e10f6eb7da6 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sun, 21 Dec 2025 10:13:05 +0300 Subject: [PATCH 7/7] Backup: remove old config --- files/backups/config.template.ini | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 files/backups/config.template.ini diff --git a/files/backups/config.template.ini b/files/backups/config.template.ini deleted file mode 100644 index da958da..0000000 --- a/files/backups/config.template.ini +++ /dev/null @@ -1,11 +0,0 @@ -[restic] -RESTIC_REPOSITORY={{ restic_repository }} -RESTIC_PASSWORD={{ restic_password }} -AWS_ACCESS_KEY_ID={{ restic_s3_access_key }} -AWS_SECRET_ACCESS_KEY={{ restic_s3_access_secret }} -AWS_DEFAULT_REGION={{ restic_s3_region }} - -[telegram] -TELEGRAM_BOT_TOKEN={{ notifications_tg_bot_token }} -TELEGRAM_CHAT_ID={{ notifications_tg_chat_id }} -NOTIFICATIONS_NAME={{ notifications_name }} -- 2.49.1