From bcd8e6269181d72731ea82793025f1b035d6a661 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Thu, 7 Aug 2025 12:06:07 +0300 Subject: [PATCH] Backups: rewrite backup script To avoid specifying individual applications --- files/backups/backup-all.sh.j2 | 37 --- files/backups/backup-all.template.py | 326 +++++++++++++++++++++++++++ playbook-backups.yml | 4 +- 3 files changed, 328 insertions(+), 39 deletions(-) delete mode 100644 files/backups/backup-all.sh.j2 create mode 100644 files/backups/backup-all.template.py diff --git a/files/backups/backup-all.sh.j2 b/files/backups/backup-all.sh.j2 deleted file mode 100644 index cadcbf0..0000000 --- a/files/backups/backup-all.sh.j2 +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - -echo "Backup: perform gitea backup" -su --login gitea --command '/home/gitea/backup.sh' - -echo "Backup: perform outline backup" -su --login outline --command '/home/outline/backup.sh' - -echo "Backup: perform gramps backup" -su --login gramps --command '/home/gramps/backup.sh' - -echo "Backup: perform miniflux backup" -su --login miniflux --command '/home/miniflux/backup.sh' - -echo "Backup: perform wakapi backup" -su --login wakapi --command '/home/wakapi/backup.sh' - -echo "Backup: send backups to remote storage with retic" - -restic-shell.sh backup --verbose /home/gitea/backups /home/outline/backups /home/gramps/backups /home/miniflux/backups /home/wakapi/backups \ - && restic-shell.sh check \ - && restic-shell.sh forget --compact --prune --keep-daily 90 --keep-monthly 36 \ - && restic-shell.sh check - - -echo "Backup: send notification" - -curl -s -X POST 'https://api.telegram.org/bot{{ notifications_tg_bot_token }}/sendMessage' \ - -d 'chat_id={{ notifications_tg_chat_id }}' \ - -d 'parse_mode=HTML' \ - -d 'text={{ notifications_name }}: бекап успешно завершен!' - - -echo -e "\nBackup: done" diff --git a/files/backups/backup-all.template.py b/files/backups/backup-all.template.py new file mode 100644 index 0000000..1e7b2f9 --- /dev/null +++ b/files/backups/backup-all.template.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +""" +Backup script for all applications +Automatically discovers and runs backup scripts for all users, +then creates restic backups and sends notifications. +""" + +import os +import sys +import subprocess +import logging +import pwd +from pathlib import Path +from typing import List, Tuple, Optional +import requests + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler("/var/log/backup-all.log"), + ], +) +logger = logging.getLogger(__name__) + +# Configuration from Ansible template variables +RESTIC_REPOSITORY = "{{ restic_repository }}" +RESTIC_PASSWORD = "{{ restic_password }}" +AWS_ACCESS_KEY_ID = "{{ restic_s3_access_key }}" +AWS_SECRET_ACCESS_KEY = "{{ restic_s3_access_secret }}" +AWS_DEFAULT_REGION = "{{ restic_s3_region }}" +TELEGRAM_BOT_TOKEN = "{{ notifications_tg_bot_token }}" +TELEGRAM_CHAT_ID = "{{ notifications_tg_chat_id }}" +NOTIFICATIONS_NAME = "{{ notifications_name }}" + + +class BackupManager: + def __init__(self): + self.errors = [] + self.warnings = [] + self.successful_backups = [] + + def get_home_directories(self) -> List[Tuple[str, str]]: + """Get all home directories and their owners""" + home_dirs = [] + home_path = Path("/home") + + if not home_path.exists(): + logger.error("/home directory does not exist") + return home_dirs + + for user_dir in home_path.iterdir(): + if user_dir.is_dir(): + try: + # Get the owner of the directory + stat_info = user_dir.stat() + owner = pwd.getpwuid(stat_info.st_uid).pw_name + home_dirs.append((str(user_dir), owner)) + except (KeyError, OSError) as e: + logger.warning(f"Could not get owner for {user_dir}: {e}") + + return home_dirs + + def find_backup_script(self, home_dir: str) -> Optional[str]: + """Find backup script in user's home directory""" + possible_scripts = [ + os.path.join(home_dir, "backup.sh"), + os.path.join(home_dir, "backup"), + ] + + for script_path in possible_scripts: + if os.path.exists(script_path): + # Check if file is executable + if os.access(script_path, os.X_OK): + return script_path + else: + logger.warning( + f"Backup script {script_path} exists but is not executable" + ) + + return None + + def run_user_backup(self, script_path: str, username: str) -> bool: + """Run backup script as the specified user""" + try: + logger.info(f"Running backup script {script_path} as user {username}") + + # Use su to run the script as the user + cmd = ["su", "--login", username, "--command", script_path] + + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=3600 # 1 hour timeout + ) + + if result.returncode == 0: + logger.info(f"Backup script for {username} completed successfully") + self.successful_backups.append(username) + return True + else: + error_msg = f"Backup script for {username} failed with return code {result.returncode}" + if result.stderr: + error_msg += f": {result.stderr}" + logger.error(error_msg) + self.errors.append(f"User {username}: {error_msg}") + return False + + except subprocess.TimeoutExpired: + error_msg = f"Backup script for {username} timed out" + logger.error(error_msg) + self.errors.append(f"User {username}: {error_msg}") + return False + except Exception as e: + error_msg = f"Failed to run backup script for {username}: {str(e)}" + logger.error(error_msg) + self.errors.append(f"User {username}: {error_msg}") + return False + + def get_backup_directories(self) -> List[str]: + """Get all backup directories that exist""" + backup_dirs = [] + home_dirs = self.get_home_directories() + + for home_dir, _ in home_dirs: + backup_path = os.path.join(home_dir, "backups") + if os.path.exists(backup_path) and os.path.isdir(backup_path): + backup_dirs.append(backup_path) + + return backup_dirs + + def run_restic_backup(self, backup_dirs: List[str]) -> bool: + """Run restic backup for all backup directories""" + if not backup_dirs: + logger.warning("No backup directories found") + return True + + try: + logger.info("Starting restic backup") + + # Set environment variables for restic + env = os.environ.copy() + env.update( + { + "RESTIC_REPOSITORY": RESTIC_REPOSITORY, + "RESTIC_PASSWORD": RESTIC_PASSWORD, + "AWS_ACCESS_KEY_ID": AWS_ACCESS_KEY_ID, + "AWS_SECRET_ACCESS_KEY": AWS_SECRET_ACCESS_KEY, + "AWS_DEFAULT_REGION": AWS_DEFAULT_REGION, + } + ) + + # Run backup + backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs + result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + error_msg = f"Restic backup failed: {result.stderr}" + logger.error(error_msg) + self.errors.append(f"Restic backup: {error_msg}") + return False + + logger.info("Restic backup completed successfully") + + # Run check + check_cmd = ["restic", "check"] + result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + error_msg = f"Restic check failed: {result.stderr}" + logger.error(error_msg) + self.errors.append(f"Restic check: {error_msg}") + return False + + logger.info("Restic check completed successfully") + + # Run forget and prune + forget_cmd = [ + "restic", + "forget", + "--compact", + "--prune", + "--keep-daily", + "90", + "--keep-monthly", + "36", + ] + result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + error_msg = f"Restic forget/prune failed: {result.stderr}" + logger.error(error_msg) + self.errors.append(f"Restic forget/prune: {error_msg}") + return False + + logger.info("Restic forget/prune completed successfully") + + # Final check + result = subprocess.run(check_cmd, env=env, capture_output=True, text=True) + + if result.returncode != 0: + error_msg = f"Final restic check failed: {result.stderr}" + logger.error(error_msg) + self.errors.append(f"Final restic check: {error_msg}") + return False + + logger.info("Final restic check completed successfully") + return True + + except Exception as e: + error_msg = f"Restic backup process failed: {str(e)}" + logger.error(error_msg) + self.errors.append(f"Restic: {error_msg}") + return False + + def send_telegram_notification(self, success: bool) -> None: + """Send notification to Telegram""" + try: + if success and not self.errors: + message = f"{NOTIFICATIONS_NAME}: бекап успешно завершен!" + if self.successful_backups: + message += ( + f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}" + ) + else: + message = f"{NOTIFICATIONS_NAME}: бекап завершен с ошибками!" + + if self.successful_backups: + message += ( + f"\n\n✅ Успешные бекапы: {', '.join(self.successful_backups)}" + ) + + if self.warnings: + message += f"\n\n⚠️ Предупреждения:\n" + "\n".join(self.warnings) + + if self.errors: + message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors) + + url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage" + data = {"chat_id": TELEGRAM_CHAT_ID, "parse_mode": "HTML", "text": message} + + response = requests.post(url, data=data, timeout=30) + + if response.status_code == 200: + logger.info("Telegram notification sent successfully") + else: + logger.error( + f"Failed to send Telegram notification: {response.status_code} - {response.text}" + ) + + except Exception as e: + logger.error(f"Failed to send Telegram notification: {str(e)}") + + def run_backup_process(self) -> bool: + """Main backup process""" + logger.info("Starting backup process") + + # Get all home directories + home_dirs = self.get_home_directories() + logger.info(f"Found {len(home_dirs)} home directories") + + # Process each user's backup + for home_dir, username in home_dirs: + logger.info(f"Processing backup for user: {username} ({home_dir})") + + # Find backup script + backup_script = self.find_backup_script(home_dir) + + if backup_script is None: + warning_msg = ( + f"No backup script found for user {username} in {home_dir}" + ) + logger.warning(warning_msg) + self.warnings.append(warning_msg) + continue + + # Run backup script + self.run_user_backup(backup_script, username) + + # Get backup directories + backup_dirs = self.get_backup_directories() + logger.info(f"Found backup directories: {backup_dirs}") + + # Run restic backup + restic_success = self.run_restic_backup(backup_dirs) + + # Determine overall success + overall_success = restic_success and len(self.errors) == 0 + + # Send notification + self.send_telegram_notification(overall_success) + + logger.info("Backup process completed") + + if self.errors: + logger.error(f"Backup completed with {len(self.errors)} errors") + return False + elif self.warnings: + logger.warning(f"Backup completed with {len(self.warnings)} warnings") + return True + else: + logger.info("Backup completed successfully") + return True + + +def main(): + """Main entry point""" + try: + backup_manager = BackupManager() + success = backup_manager.run_backup_process() + + if success: + sys.exit(0) + else: + sys.exit(1) + + except KeyboardInterrupt: + logger.info("Backup process interrupted by user") + sys.exit(130) + except Exception as e: + logger.error(f"Unexpected error in backup process: {str(e)}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/playbook-backups.yml b/playbook-backups.yml index 8c8e5a9..7710cf0 100644 --- a/playbook-backups.yml +++ b/playbook-backups.yml @@ -8,7 +8,7 @@ vars: restic_shell_script: "{{ (bin_prefix, 'restic-shell.sh') | path_join }}" - backup_all_script: "{{ (bin_prefix, 'backup-all.sh') | path_join }}" + backup_all_script: "{{ (bin_prefix, 'backup-all.py') | path_join }}" tasks: - name: "Copy restic shell script" @@ -21,7 +21,7 @@ - name: "Copy backup all script" ansible.builtin.template: - src: "files/backups/backup-all.sh.j2" + src: "files/backups/backup-all.template.py" dest: "{{ backup_all_script }}" owner: root group: root