From 11e5b5752e9082f2d614e1c18308468acf36df81 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 10:32:00 +0300 Subject: [PATCH 1/8] Backups: add backup-targets file support --- files/backups/backup-all.py | 59 ++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index c6cd829..279e305 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -14,7 +14,6 @@ from pathlib import Path from typing import List, Tuple, Optional import requests import configparser -import itertools # Configure logging @@ -126,14 +125,60 @@ class BackupManager: return False def get_backup_directories(self) -> List[str]: - """Get all backup directories that exist""" - backup_dirs = [] + """Collect backup targets according to backup-targets rules""" + backup_dirs: List[str] = [] app_dirs = self.get_application_directories() - for app_dir, _ in app_dirs: - backup_path = os.path.join(app_dir, "backups") - if os.path.exists(backup_path) and os.path.isdir(backup_path): - backup_dirs.append(backup_path) + def parse_targets_file(targets_file: Path) -> List[str]: + """Parse backup-targets file, skipping comments and empty lines.""" + targets: List[str] = [] + try: + for raw_line in targets_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + targets.append(line) + except OSError as e: + warning_msg = f"Could not read backup targets file {targets_file}: {e}" + logger.warning(warning_msg) + self.warnings.append(warning_msg) + return targets + + for app_dir_str, _ in app_dirs: + app_dir = Path(app_dir_str) + targets_file = app_dir / "backup-targets" + resolved_targets: List[Path] = [] + + if targets_file.exists(): + # Read custom targets defined by the application. + for target_line in parse_targets_file(targets_file): + target_path = Path(target_line) + if not target_path.is_absolute(): + target_path = (app_dir / target_path).resolve() + else: + target_path = target_path.resolve() + if target_path.exists(): + resolved_targets.append(target_path) + else: + warning_msg = ( + f"Backup target does not exist for {app_dir}: {target_path}" + ) + logger.warning(warning_msg) + self.warnings.append(warning_msg) + else: + # Fallback to default backups directory when no list is provided. + default_target = (app_dir / "backups").resolve() + if default_target.exists(): + resolved_targets.append(default_target) + else: + warning_msg = f"Default backup path does not exist for {app_dir}: {default_target}" + logger.warning(warning_msg) + self.warnings.append(warning_msg) + + for target in resolved_targets: + target_str = str(target) + if target_str not in backup_dirs: + backup_dirs.append(target_str) return backup_dirs -- 2.49.1 From 479e256b1e099eb2f1ca90eb78562d46c563826f Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 10:36:19 +0300 Subject: [PATCH 2/8] Backups: use constants for file names --- files/backups/backup-all.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index 279e305..2d80286 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -39,6 +39,13 @@ TELEGRAM_BOT_TOKEN = config.get("telegram", "TELEGRAM_BOT_TOKEN") TELEGRAM_CHAT_ID = config.get("telegram", "TELEGRAM_CHAT_ID") NOTIFICATIONS_NAME = config.get("telegram", "NOTIFICATIONS_NAME") +# File name to store directories and files to back up +BACKUP_TARGETS_FILE = "backup-targets" + +# Default directory fo backups (relative to app dir) +# Used when backup-targets file not exists +BACKUP_DEFAULT_DIR = "backups" + class BackupManager: def __init__(self): @@ -146,7 +153,7 @@ class BackupManager: for app_dir_str, _ in app_dirs: app_dir = Path(app_dir_str) - targets_file = app_dir / "backup-targets" + targets_file = app_dir / BACKUP_TARGETS_FILE resolved_targets: List[Path] = [] if targets_file.exists(): @@ -167,7 +174,7 @@ class BackupManager: self.warnings.append(warning_msg) else: # Fallback to default backups directory when no list is provided. - default_target = (app_dir / "backups").resolve() + default_target = (app_dir / BACKUP_DEFAULT_DIR).resolve() if default_target.exists(): resolved_targets.append(default_target) else: -- 2.49.1 From ca7f089fe6aaaa12fae48b180f320f99905c3597 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 10:48:40 +0300 Subject: [PATCH 3/8] Backups: use dataclass Application for app info --- files/backups/backup-all.py | 40 ++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index 2d80286..6f7e416 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -10,8 +10,9 @@ import sys import subprocess import logging import pwd +from dataclasses import dataclass from pathlib import Path -from typing import List, Tuple, Optional +from typing import List, Optional import requests import configparser @@ -47,15 +48,21 @@ BACKUP_TARGETS_FILE = "backup-targets" BACKUP_DEFAULT_DIR = "backups" +@dataclass +class Application: + path: Path + owner: str + + class BackupManager: def __init__(self): - self.errors = [] - self.warnings = [] - self.successful_backups = [] + self.errors: List[str] = [] + self.warnings: List[str] = [] + self.successful_backups: List[str] = [] - def get_application_directories(self) -> List[Tuple[str, str]]: - """Get all home directories and their owners""" - app_dirs = [] + def find_applications(self) -> List[Application]: + """Get all application directories and their owners.""" + applications: List[Application] = [] applications_path = Path("/mnt/applications") source_dirs = applications_path.iterdir() @@ -64,14 +71,13 @@ class BackupManager: continue if app_dir.is_dir(): try: - # Get the owner of the directory stat_info = app_dir.stat() owner = pwd.getpwuid(stat_info.st_uid).pw_name - app_dirs.append((str(app_dir), owner)) + applications.append(Application(path=app_dir, owner=owner)) except (KeyError, OSError) as e: logger.warning(f"Could not get owner for {app_dir}: {e}") - return app_dirs + return applications def find_backup_script(self, app_dir: str) -> Optional[str]: """Find backup script in user's home directory""" @@ -134,7 +140,7 @@ class BackupManager: def get_backup_directories(self) -> List[str]: """Collect backup targets according to backup-targets rules""" backup_dirs: List[str] = [] - app_dirs = self.get_application_directories() + applications = self.find_applications() def parse_targets_file(targets_file: Path) -> List[str]: """Parse backup-targets file, skipping comments and empty lines.""" @@ -151,8 +157,8 @@ class BackupManager: self.warnings.append(warning_msg) return targets - for app_dir_str, _ in app_dirs: - app_dir = Path(app_dir_str) + for app in applications: + app_dir = app.path targets_file = app_dir / BACKUP_TARGETS_FILE resolved_targets: List[Path] = [] @@ -317,11 +323,13 @@ class BackupManager: logger.info("Starting backup process") # Get all home directories - app_dirs = self.get_application_directories() - logger.info(f"Found {len(app_dirs)} application directories") + applications = self.find_applications() + logger.info(f"Found {len(applications)} application directories") # Process each user's backup - for app_dir, username in app_dirs: + for app in applications: + app_dir = str(app.path) + username = app.owner logger.info(f"Processing backup for app: {app_dir} (user {username})") # Find backup script -- 2.49.1 From 91c5eab23615bef807cdffc6e6c0fe260f265c36 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 11:04:50 +0300 Subject: [PATCH 4/8] Gramps: exclude media files from gobackup Backup media files with backup-targets --- files/gramps/gobackup.template.yml | 4 ---- playbook-gramps.yml | 10 ++++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/files/gramps/gobackup.template.yml b/files/gramps/gobackup.template.yml index 77c2fb7..4f6e53e 100644 --- a/files/gramps/gobackup.template.yml +++ b/files/gramps/gobackup.template.yml @@ -23,7 +23,3 @@ models: undo: type: sqlite path: "{{ (data_dir, 'gramps_db/59a0f3d6-1c3d-4410-8c1d-1c9c6689659f/undo.db') | path_join }}" - archive: - includes: - - "{{ data_dir }}" - - "{{ media_dir }}" diff --git a/playbook-gramps.yml b/playbook-gramps.yml index 4243009..3ab62fd 100644 --- a/playbook-gramps.yml +++ b/playbook-gramps.yml @@ -57,6 +57,16 @@ group: "{{ app_user }}" mode: "0750" + - name: "Create backup targets file" + ansible.builtin.lineinfile: + path: "{{ base_dir }}/backup-targets" + line: "{{ item }}" + create: true + loop: + - "{{ data_dir }}" + - "{{ media_dir }}" + - "{{ backups_dir }}" + - name: "Copy rename script" ansible.builtin.copy: src: "files/{{ app_name }}/gramps_rename.py" -- 2.49.1 From e3d847939710aee614e99ff66c1885d3cbfd31c2 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 11:06:56 +0300 Subject: [PATCH 5/8] Memos: exclude media files from gobackup Backup media files with backup-targets --- files/memos/gobackup.yml.j2 | 7 +------ playbook-memos.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/files/memos/gobackup.yml.j2 b/files/memos/gobackup.yml.j2 index 5493672..9dd5725 100644 --- a/files/memos/gobackup.yml.j2 +++ b/files/memos/gobackup.yml.j2 @@ -2,7 +2,7 @@ models: - gramps: + memos: compress_with: type: 'tgz' storages: @@ -14,8 +14,3 @@ models: users: type: sqlite path: "{{ (data_dir, 'memos_prod.db') | path_join }}" - archive: - includes: - - "{{ data_dir }}" - excludes: - - "{{ (data_dir, '.thumbnail_cache') | path_join }}" diff --git a/playbook-memos.yml b/playbook-memos.yml index cb8a7b8..84ffd25 100644 --- a/playbook-memos.yml +++ b/playbook-memos.yml @@ -53,6 +53,15 @@ group: "{{ app_user }}" mode: "0750" + - name: "Create backup targets file" + ansible.builtin.lineinfile: + path: "{{ base_dir }}/backup-targets" + line: "{{ item }}" + create: true + loop: + - "{{ data_dir }}" + - "{{ backups_dir }}" + - name: "Copy docker compose file" ansible.builtin.template: src: "./files/{{ app_name }}/docker-compose.template.yml" -- 2.49.1 From 2eac1362b5f424cf905828faa3e50d26a3df449e Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 11:18:11 +0300 Subject: [PATCH 6/8] Wanderer: backup all data with restic --- playbook-wanderer.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/playbook-wanderer.yml b/playbook-wanderer.yml index cb83b95..d8e3c33 100644 --- a/playbook-wanderer.yml +++ b/playbook-wanderer.yml @@ -51,13 +51,29 @@ group: "{{ app_user }}" mode: "0640" - - name: "Copy backup script" - ansible.builtin.template: - src: "files/{{ app_name }}/backup.template.sh" +# - name: "Copy backup script" +# ansible.builtin.template: +# src: "files/{{ app_name }}/backup.template.sh" +# dest: "{{ base_dir }}/backup.sh" +# owner: "{{ app_user }}" +# group: "{{ app_user }}" +# mode: "0750" + + - name: "Disable backup script" + ansible.builtin.file: dest: "{{ base_dir }}/backup.sh" + state: absent + + - name: "Create backup targets file" + ansible.builtin.lineinfile: + path: "{{ base_dir }}/backup-targets" + line: "{{ item }}" + create: true owner: "{{ app_user }}" group: "{{ app_user }}" mode: "0750" + loop: + - "{{ data_dir }}" - name: "Copy docker compose file" ansible.builtin.template: -- 2.49.1 From dcc4970b2087d72e726e6af77c6264d10da6fef0 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 11:18:37 +0300 Subject: [PATCH 7/8] Add owner and group to backup-targets files --- playbook-gramps.yml | 3 +++ playbook-memos.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/playbook-gramps.yml b/playbook-gramps.yml index 3ab62fd..625dbad 100644 --- a/playbook-gramps.yml +++ b/playbook-gramps.yml @@ -62,6 +62,9 @@ path: "{{ base_dir }}/backup-targets" line: "{{ item }}" create: true + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: "0750" loop: - "{{ data_dir }}" - "{{ media_dir }}" diff --git a/playbook-memos.yml b/playbook-memos.yml index 84ffd25..37ff13b 100644 --- a/playbook-memos.yml +++ b/playbook-memos.yml @@ -58,6 +58,9 @@ path: "{{ base_dir }}/backup-targets" line: "{{ item }}" create: true + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: "0750" loop: - "{{ data_dir }}" - "{{ backups_dir }}" -- 2.49.1 From 4fbe9bd5dec91669912179e8a985324cfbf367cd Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Sat, 20 Dec 2025 11:22:24 +0300 Subject: [PATCH 8/8] Backups: skip system dir lost+found --- files/backups/backup-all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/backups/backup-all.py b/files/backups/backup-all.py index 6f7e416..0b600d8 100644 --- a/files/backups/backup-all.py +++ b/files/backups/backup-all.py @@ -67,7 +67,7 @@ class BackupManager: source_dirs = applications_path.iterdir() for app_dir in source_dirs: - if app_dir == "lost+found": + if "lost+found" in str(app_dir): continue if app_dir.is_dir(): try: -- 2.49.1