Backups: skip system dir lost+found

Backup media files with backup-targets

files/backups/backup-all.py

@@ -10,11 +10,11 @@ import sys
 import subprocess
 import logging
 import pwd
+from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Tuple, Optional
+from typing import List, Optional
 import requests
 import configparser
-import itertools
 
 
 # Configure logging
@@ -40,32 +40,44 @@ TELEGRAM_BOT_TOKEN = config.get("telegram", "TELEGRAM_BOT_TOKEN")
 TELEGRAM_CHAT_ID = config.get("telegram", "TELEGRAM_CHAT_ID")
 NOTIFICATIONS_NAME = config.get("telegram", "NOTIFICATIONS_NAME")
 
+# File name to store directories and files to back up
+BACKUP_TARGETS_FILE = "backup-targets"
+
+# Default directory fo backups (relative to app dir)
+# Used when backup-targets file not exists
+BACKUP_DEFAULT_DIR = "backups"
+
+
+@dataclass
+class Application:
+    path: Path
+    owner: str
+
 
 class BackupManager:
     def __init__(self):
-        self.errors = []
-        self.warnings = []
-        self.successful_backups = []
+        self.errors: List[str] = []
+        self.warnings: List[str] = []
+        self.successful_backups: List[str] = []
 
-    def get_application_directories(self) -> List[Tuple[str, str]]:
-        """Get all home directories and their owners"""
-        app_dirs = []
+    def find_applications(self) -> List[Application]:
+        """Get all application directories and their owners."""
+        applications: List[Application] = []
         applications_path = Path("/mnt/applications")
         source_dirs = applications_path.iterdir()
 
         for app_dir in source_dirs:
-            if app_dir == "lost+found":
+            if "lost+found" in str(app_dir):
                 continue
             if app_dir.is_dir():
                 try:
-                    # Get the owner of the directory
                     stat_info = app_dir.stat()
                     owner = pwd.getpwuid(stat_info.st_uid).pw_name
-                    app_dirs.append((str(app_dir), owner))
+                    applications.append(Application(path=app_dir, owner=owner))
                 except (KeyError, OSError) as e:
                     logger.warning(f"Could not get owner for {app_dir}: {e}")
 
-        return app_dirs
+        return applications
 
     def find_backup_script(self, app_dir: str) -> Optional[str]:
         """Find backup script in user's home directory"""
@@ -126,14 +138,60 @@ class BackupManager:
             return False
 
     def get_backup_directories(self) -> List[str]:
-        """Get all backup directories that exist"""
-        backup_dirs = []
-        app_dirs = self.get_application_directories()
+        """Collect backup targets according to backup-targets rules"""
+        backup_dirs: List[str] = []
+        applications = self.find_applications()
 
-        for app_dir, _ in app_dirs:
-            backup_path = os.path.join(app_dir, "backups")
-            if os.path.exists(backup_path) and os.path.isdir(backup_path):
-                backup_dirs.append(backup_path)
+        def parse_targets_file(targets_file: Path) -> List[str]:
+            """Parse backup-targets file, skipping comments and empty lines."""
+            targets: List[str] = []
+            try:
+                for raw_line in targets_file.read_text(encoding="utf-8").splitlines():
+                    line = raw_line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    targets.append(line)
+            except OSError as e:
+                warning_msg = f"Could not read backup targets file {targets_file}: {e}"
+                logger.warning(warning_msg)
+                self.warnings.append(warning_msg)
+            return targets
+
+        for app in applications:
+            app_dir = app.path
+            targets_file = app_dir / BACKUP_TARGETS_FILE
+            resolved_targets: List[Path] = []
+
+            if targets_file.exists():
+                # Read custom targets defined by the application.
+                for target_line in parse_targets_file(targets_file):
+                    target_path = Path(target_line)
+                    if not target_path.is_absolute():
+                        target_path = (app_dir / target_path).resolve()
+                    else:
+                        target_path = target_path.resolve()
+                    if target_path.exists():
+                        resolved_targets.append(target_path)
+                    else:
+                        warning_msg = (
+                            f"Backup target does not exist for {app_dir}: {target_path}"
+                        )
+                        logger.warning(warning_msg)
+                        self.warnings.append(warning_msg)
+            else:
+                # Fallback to default backups directory when no list is provided.
+                default_target = (app_dir / BACKUP_DEFAULT_DIR).resolve()
+                if default_target.exists():
+                    resolved_targets.append(default_target)
+                else:
+                    warning_msg = f"Default backup path does not exist for {app_dir}: {default_target}"
+                    logger.warning(warning_msg)
+                    self.warnings.append(warning_msg)
+
+            for target in resolved_targets:
+                target_str = str(target)
+                if target_str not in backup_dirs:
+                    backup_dirs.append(target_str)
 
         return backup_dirs
 
@@ -265,11 +323,13 @@ class BackupManager:
         logger.info("Starting backup process")
 
         # Get all home directories
-        app_dirs = self.get_application_directories()
-        logger.info(f"Found {len(app_dirs)} application directories")
+        applications = self.find_applications()
+        logger.info(f"Found {len(applications)} application directories")
 
         # Process each user's backup
-        for app_dir, username in app_dirs:
+        for app in applications:
+            app_dir = str(app.path)
+            username = app.owner
             logger.info(f"Processing backup for app: {app_dir} (user {username})")
 
             # Find backup script

files/gramps/gobackup.template.yml

@@ -23,7 +23,3 @@ models:
       undo:
         type: sqlite
         path: "{{ (data_dir, 'gramps_db/59a0f3d6-1c3d-4410-8c1d-1c9c6689659f/undo.db') | path_join }}"
-    archive:
-      includes:
-        - "{{ data_dir }}"
-        - "{{ media_dir }}"

files/memos/gobackup.yml.j2

@@ -2,7 +2,7 @@
 
 models:
 
-  gramps:
+  memos:
     compress_with:
       type: 'tgz'
     storages:
@@ -14,8 +14,3 @@ models:
       users:
         type: sqlite
         path: "{{ (data_dir, 'memos_prod.db') | path_join }}"
-    archive:
-      includes:
-        - "{{ data_dir }}"
-      excludes:
-        - "{{ (data_dir, '.thumbnail_cache') | path_join }}"

playbook-gramps.yml

@@ -57,6 +57,19 @@
         group: "{{ app_user }}"
         mode: "0750"
 
+    - name: "Create backup targets file"
+      ansible.builtin.lineinfile:
+        path: "{{ base_dir }}/backup-targets"
+        line: "{{ item }}"
+        create: true
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0750"
+      loop:
+        - "{{ data_dir }}"
+        - "{{ media_dir }}"
+        - "{{ backups_dir }}"
+
     - name: "Copy rename script"
       ansible.builtin.copy:
         src: "files/{{ app_name }}/gramps_rename.py"

playbook-memos.yml

@@ -53,6 +53,18 @@
         group: "{{ app_user }}"
         mode: "0750"
 
+    - name: "Create backup targets file"
+      ansible.builtin.lineinfile:
+        path: "{{ base_dir }}/backup-targets"
+        line: "{{ item }}"
+        create: true
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0750"
+      loop:
+        - "{{ data_dir }}"
+        - "{{ backups_dir }}"
+
     - name: "Copy docker compose file"
       ansible.builtin.template:
         src: "./files/{{ app_name }}/docker-compose.template.yml"

playbook-wanderer.yml

@@ -51,13 +51,29 @@
         group: "{{ app_user }}"
         mode: "0640"
 
-    - name: "Copy backup script"
-      ansible.builtin.template:
-        src: "files/{{ app_name }}/backup.template.sh"
+#    - name: "Copy backup script"
+#      ansible.builtin.template:
+#        src: "files/{{ app_name }}/backup.template.sh"
+#        dest: "{{ base_dir }}/backup.sh"
+#        owner: "{{ app_user }}"
+#        group: "{{ app_user }}"
+#        mode: "0750"
+
+    - name: "Disable backup script"
+      ansible.builtin.file:
         dest: "{{ base_dir }}/backup.sh"
+        state: absent
+
+    - name: "Create backup targets file"
+      ansible.builtin.lineinfile:
+        path: "{{ base_dir }}/backup-targets"
+        line: "{{ item }}"
+        create: true
         owner: "{{ app_user }}"
         group: "{{ app_user }}"
         mode: "0750"
+      loop:
+        - "{{ data_dir }}"
 
     - name: "Copy docker compose file"
       ansible.builtin.template: