Compare commits
27 Commits
d09a26b73a
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
e5c1e19e5e
|
|||
|
8439ab3693
|
|||
|
dcb09e349b
|
|||
|
a6781b4f64
|
|||
|
a31a07bd16
|
|||
|
54a951b96a
|
|||
|
e1379bc480
|
|||
|
037e0cab9b
|
|||
|
2655869814
|
|||
|
0e96b5030d
|
|||
|
a217c79e7d
|
|||
|
6a16ebf084
|
|||
|
2617aa2bd2
|
|||
|
b686e4da4d
|
|||
|
439c239ac8
|
|||
|
acf599f905
|
|||
|
eae4f5e27b
|
|||
|
4fbe9bd5de
|
|||
|
dcc4970b20
|
|||
|
2eac1362b5
|
|||
|
e3d8479397
|
|||
|
91c5eab236
|
|||
|
ca7f089fe6
|
|||
|
479e256b1e
|
|||
|
11e5b5752e
|
|||
|
392938d0fb
|
|||
|
2cc059104e
|
2
.gitignore
vendored
2
.gitignore
vendored
@@ -7,3 +7,5 @@
|
|||||||
/ansible-vault-password-file
|
/ansible-vault-password-file
|
||||||
/temp
|
/temp
|
||||||
*.retry
|
*.retry
|
||||||
|
|
||||||
|
__pycache__
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ Ansible-based server automation for personal services. Playbooks provision Docke
|
|||||||
- Ansible lint: `ansible-lint .` (CI default).
|
- Ansible lint: `ansible-lint .` (CI default).
|
||||||
- Authelia config validation: `task authelia-validate-config` (renders with secrets then validates via docker).
|
- Authelia config validation: `task authelia-validate-config` (renders with secrets then validates via docker).
|
||||||
- Black formatting for Python helpers: `task format-py-files`.
|
- Black formatting for Python helpers: `task format-py-files`.
|
||||||
|
- Python types validation with mypy: `mypy <file.py>`.
|
||||||
|
|
||||||
## Operational Notes
|
## Operational Notes
|
||||||
- Deployments rely on `production.yml` inventory and per-app playbooks; run with `--diff` for visibility.
|
- Deployments rely on `production.yml` inventory and per-app playbooks; run with `--diff` for visibility.
|
||||||
|
|||||||
@@ -4,18 +4,28 @@ Backup script for all applications
|
|||||||
Automatically discovers and runs backup scripts for all users,
|
Automatically discovers and runs backup scripts for all users,
|
||||||
then creates restic backups and sends notifications.
|
then creates restic backups and sends notifications.
|
||||||
"""
|
"""
|
||||||
|
import itertools
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import subprocess
|
import subprocess
|
||||||
import logging
|
import logging
|
||||||
import pwd
|
import pwd
|
||||||
|
from abc import ABC
|
||||||
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Tuple, Optional
|
from typing import Dict, List, Optional, Any
|
||||||
import requests
|
import requests
|
||||||
import configparser
|
import tomllib
|
||||||
import itertools
|
|
||||||
|
|
||||||
|
# Default config path
|
||||||
|
CONFIG_PATH = Path("/etc/backup/config.toml")
|
||||||
|
|
||||||
|
# File name to store directories and files to back up
|
||||||
|
BACKUP_TARGETS_FILE = "backup-targets"
|
||||||
|
|
||||||
|
# Default directory fo backups (relative to app dir)
|
||||||
|
# Used when backup-targets file not exists
|
||||||
|
BACKUP_DEFAULT_DIR = "backups"
|
||||||
|
|
||||||
# Configure logging
|
# Configure logging
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@@ -28,44 +38,193 @@ logging.basicConfig(
|
|||||||
)
|
)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
config = configparser.ConfigParser()
|
|
||||||
config.read("/etc/backup/config.ini")
|
|
||||||
|
|
||||||
RESTIC_REPOSITORY = config.get("restic", "RESTIC_REPOSITORY")
|
@dataclass
|
||||||
RESTIC_PASSWORD = config.get("restic", "RESTIC_PASSWORD")
|
class Config:
|
||||||
AWS_ACCESS_KEY_ID = config.get("restic", "AWS_ACCESS_KEY_ID")
|
host_name: str
|
||||||
AWS_SECRET_ACCESS_KEY = config.get("restic", "AWS_SECRET_ACCESS_KEY")
|
roots: List[Path]
|
||||||
AWS_DEFAULT_REGION = config.get("restic", "AWS_DEFAULT_REGION")
|
|
||||||
TELEGRAM_BOT_TOKEN = config.get("telegram", "TELEGRAM_BOT_TOKEN")
|
|
||||||
TELEGRAM_CHAT_ID = config.get("telegram", "TELEGRAM_CHAT_ID")
|
@dataclass
|
||||||
NOTIFICATIONS_NAME = config.get("telegram", "NOTIFICATIONS_NAME")
|
class Application:
|
||||||
|
path: Path
|
||||||
|
owner: str
|
||||||
|
|
||||||
|
|
||||||
|
class Storage(ABC):
|
||||||
|
def backup(self, backup_dirs: List[str]) -> bool:
|
||||||
|
"""Backup directories"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
class ResticStorage(Storage):
|
||||||
|
TYPE_NAME = "restic"
|
||||||
|
|
||||||
|
def __init__(self, name: str, params: Dict[str, Any]):
|
||||||
|
self.name = name
|
||||||
|
self.restic_repository = str(params.get("restic_repository", ""))
|
||||||
|
self.restic_password = str(params.get("restic_password", ""))
|
||||||
|
self.aws_access_key_id = str(params.get("aws_access_key_id", ""))
|
||||||
|
self.aws_secret_access_key = str(params.get("aws_secret_access_key", ""))
|
||||||
|
self.aws_default_region = str(params.get("aws_default_region", ""))
|
||||||
|
|
||||||
|
if not all(
|
||||||
|
[
|
||||||
|
self.restic_repository,
|
||||||
|
self.restic_password,
|
||||||
|
self.aws_access_key_id,
|
||||||
|
self.aws_secret_access_key,
|
||||||
|
self.aws_default_region,
|
||||||
|
]
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
f"Missing storage configuration values for backend ResticStorage: '{self.name}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
def backup(self, backup_dirs: List[str]) -> bool:
|
||||||
|
if not backup_dirs:
|
||||||
|
logger.warning("No backup directories found")
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
return self.__backup_internal(backup_dirs)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
logger.error("Restic backup process failed: %s", exc)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def __backup_internal(self, backup_dirs: List[str]) -> bool:
|
||||||
|
logger.info("Starting restic backup")
|
||||||
|
logger.info("Destination: %s", self.restic_repository)
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.update(
|
||||||
|
{
|
||||||
|
"RESTIC_REPOSITORY": self.restic_repository,
|
||||||
|
"RESTIC_PASSWORD": self.restic_password,
|
||||||
|
"AWS_ACCESS_KEY_ID": self.aws_access_key_id,
|
||||||
|
"AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key,
|
||||||
|
"AWS_DEFAULT_REGION": self.aws_default_region,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs
|
||||||
|
result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error("Restic backup failed: %s", result.stderr)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("Restic backup completed successfully")
|
||||||
|
|
||||||
|
check_cmd = ["restic", "check"]
|
||||||
|
result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error("Restic check failed: %s", result.stderr)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("Restic check completed successfully")
|
||||||
|
|
||||||
|
forget_cmd = [
|
||||||
|
"restic",
|
||||||
|
"forget",
|
||||||
|
"--compact",
|
||||||
|
"--prune",
|
||||||
|
"--keep-daily",
|
||||||
|
"90",
|
||||||
|
"--keep-monthly",
|
||||||
|
"36",
|
||||||
|
]
|
||||||
|
result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error("Restic forget/prune failed: %s", result.stderr)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("Restic forget/prune completed successfully")
|
||||||
|
|
||||||
|
result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error("Final restic check failed: %s", result.stderr)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("Final restic check completed successfully")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class Notifier(ABC):
|
||||||
|
def send(self, html_message: str):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
class TelegramNotifier(Notifier):
|
||||||
|
TYPE_NAME = "telegram"
|
||||||
|
|
||||||
|
def __init__(self, name: str, params: Dict[str, Any]):
|
||||||
|
self.name = name
|
||||||
|
self.telegram_bot_token = str(params.get("telegram_bot_token", ""))
|
||||||
|
self.telegram_chat_id = str(params.get("telegram_chat_id", ""))
|
||||||
|
if not all(
|
||||||
|
[
|
||||||
|
self.telegram_bot_token,
|
||||||
|
self.telegram_chat_id,
|
||||||
|
]
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
f"Missing notification configuration values for backend {name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def send(self, html_message: str):
|
||||||
|
url = f"https://api.telegram.org/bot{self.telegram_bot_token}/sendMessage"
|
||||||
|
data = {
|
||||||
|
"chat_id": self.telegram_chat_id,
|
||||||
|
"parse_mode": "HTML",
|
||||||
|
"text": html_message,
|
||||||
|
}
|
||||||
|
|
||||||
|
response = requests.post(url, data=data, timeout=30)
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
logger.info("Telegram notification sent successfully")
|
||||||
|
else:
|
||||||
|
logger.error(
|
||||||
|
f"Failed to send Telegram notification: {response.status_code} - {response.text}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class BackupManager:
|
class BackupManager:
|
||||||
def __init__(self):
|
def __init__(
|
||||||
self.errors = []
|
self,
|
||||||
self.warnings = []
|
config: Config,
|
||||||
self.successful_backups = []
|
roots: List[Path],
|
||||||
|
storages: List[Storage],
|
||||||
|
notifiers: List[Notifier],
|
||||||
|
):
|
||||||
|
self.errors: List[str] = []
|
||||||
|
self.warnings: List[str] = []
|
||||||
|
self.successful_backups: List[str] = []
|
||||||
|
self.config = config
|
||||||
|
self.roots: List[Path] = roots
|
||||||
|
self.storages = storages
|
||||||
|
self.notifiers = notifiers
|
||||||
|
|
||||||
def get_application_directories(self) -> List[Tuple[str, str]]:
|
def find_applications(self) -> List[Application]:
|
||||||
"""Get all home directories and their owners"""
|
"""Get all application directories and their owners."""
|
||||||
app_dirs = []
|
applications: List[Application] = []
|
||||||
applications_path = Path("/mnt/applications")
|
source_dirs = itertools.chain(*(root.iterdir() for root in self.roots))
|
||||||
source_dirs = applications_path.iterdir()
|
|
||||||
|
|
||||||
for app_dir in source_dirs:
|
for app_dir in source_dirs:
|
||||||
if app_dir == "lost+found":
|
if "lost+found" in str(app_dir):
|
||||||
continue
|
continue
|
||||||
if app_dir.is_dir():
|
if app_dir.is_dir():
|
||||||
try:
|
try:
|
||||||
# Get the owner of the directory
|
|
||||||
stat_info = app_dir.stat()
|
stat_info = app_dir.stat()
|
||||||
owner = pwd.getpwuid(stat_info.st_uid).pw_name
|
owner = pwd.getpwuid(stat_info.st_uid).pw_name
|
||||||
app_dirs.append((str(app_dir), owner))
|
applications.append(Application(path=app_dir, owner=owner))
|
||||||
except (KeyError, OSError) as e:
|
except (KeyError, OSError) as e:
|
||||||
logger.warning(f"Could not get owner for {app_dir}: {e}")
|
logger.warning(f"Could not get owner for {app_dir}: {e}")
|
||||||
|
|
||||||
return app_dirs
|
return applications
|
||||||
|
|
||||||
def find_backup_script(self, app_dir: str) -> Optional[str]:
|
def find_backup_script(self, app_dir: str) -> Optional[str]:
|
||||||
"""Find backup script in user's home directory"""
|
"""Find backup script in user's home directory"""
|
||||||
@@ -126,113 +285,72 @@ class BackupManager:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def get_backup_directories(self) -> List[str]:
|
def get_backup_directories(self) -> List[str]:
|
||||||
"""Get all backup directories that exist"""
|
"""Collect backup targets according to backup-targets rules"""
|
||||||
backup_dirs = []
|
backup_dirs: List[str] = []
|
||||||
app_dirs = self.get_application_directories()
|
applications = self.find_applications()
|
||||||
|
|
||||||
for app_dir, _ in app_dirs:
|
def parse_targets_file(targets_file: Path) -> List[str]:
|
||||||
backup_path = os.path.join(app_dir, "backups")
|
"""Parse backup-targets file, skipping comments and empty lines."""
|
||||||
if os.path.exists(backup_path) and os.path.isdir(backup_path):
|
targets: List[str] = []
|
||||||
backup_dirs.append(backup_path)
|
try:
|
||||||
|
for raw_line in targets_file.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
targets.append(line)
|
||||||
|
except OSError as e:
|
||||||
|
warning_msg = f"Could not read backup targets file {targets_file}: {e}"
|
||||||
|
logger.warning(warning_msg)
|
||||||
|
self.warnings.append(warning_msg)
|
||||||
|
return targets
|
||||||
|
|
||||||
|
for app in applications:
|
||||||
|
app_dir = app.path
|
||||||
|
targets_file = app_dir / BACKUP_TARGETS_FILE
|
||||||
|
resolved_targets: List[Path] = []
|
||||||
|
|
||||||
|
if targets_file.exists():
|
||||||
|
# Read custom targets defined by the application.
|
||||||
|
for target_line in parse_targets_file(targets_file):
|
||||||
|
target_path = Path(target_line)
|
||||||
|
if not target_path.is_absolute():
|
||||||
|
target_path = (app_dir / target_path).resolve()
|
||||||
|
else:
|
||||||
|
target_path = target_path.resolve()
|
||||||
|
if target_path.exists():
|
||||||
|
resolved_targets.append(target_path)
|
||||||
|
else:
|
||||||
|
warning_msg = (
|
||||||
|
f"Backup target does not exist for {app_dir}: {target_path}"
|
||||||
|
)
|
||||||
|
logger.warning(warning_msg)
|
||||||
|
self.warnings.append(warning_msg)
|
||||||
|
else:
|
||||||
|
# Fallback to default backups directory when no list is provided.
|
||||||
|
default_target = (app_dir / BACKUP_DEFAULT_DIR).resolve()
|
||||||
|
if default_target.exists():
|
||||||
|
resolved_targets.append(default_target)
|
||||||
|
else:
|
||||||
|
warning_msg = f"Default backup path does not exist for {app_dir}: {default_target}"
|
||||||
|
logger.warning(warning_msg)
|
||||||
|
self.warnings.append(warning_msg)
|
||||||
|
|
||||||
|
for target in resolved_targets:
|
||||||
|
target_str = str(target)
|
||||||
|
if target_str not in backup_dirs:
|
||||||
|
backup_dirs.append(target_str)
|
||||||
|
|
||||||
return backup_dirs
|
return backup_dirs
|
||||||
|
|
||||||
def run_restic_backup(self, backup_dirs: List[str]) -> bool:
|
def send_notification(self, success: bool) -> None:
|
||||||
"""Run restic backup for all backup directories"""
|
"""Send notification to Notifiers"""
|
||||||
if not backup_dirs:
|
|
||||||
logger.warning("No backup directories found")
|
|
||||||
return True
|
|
||||||
|
|
||||||
try:
|
|
||||||
logger.info("Starting restic backup")
|
|
||||||
logger.info("Destination: %s", RESTIC_REPOSITORY)
|
|
||||||
|
|
||||||
# Set environment variables for restic
|
|
||||||
env = os.environ.copy()
|
|
||||||
env.update(
|
|
||||||
{
|
|
||||||
"RESTIC_REPOSITORY": RESTIC_REPOSITORY,
|
|
||||||
"RESTIC_PASSWORD": RESTIC_PASSWORD,
|
|
||||||
"AWS_ACCESS_KEY_ID": AWS_ACCESS_KEY_ID,
|
|
||||||
"AWS_SECRET_ACCESS_KEY": AWS_SECRET_ACCESS_KEY,
|
|
||||||
"AWS_DEFAULT_REGION": AWS_DEFAULT_REGION,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run backup
|
|
||||||
backup_cmd = ["restic", "backup", "--verbose"] + backup_dirs
|
|
||||||
result = subprocess.run(backup_cmd, env=env, capture_output=True, text=True)
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
error_msg = f"Restic backup failed: {result.stderr}"
|
|
||||||
logger.error(error_msg)
|
|
||||||
self.errors.append(f"Restic backup: {error_msg}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
logger.info("Restic backup completed successfully")
|
|
||||||
|
|
||||||
# Run check
|
|
||||||
check_cmd = ["restic", "check"]
|
|
||||||
result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
error_msg = f"Restic check failed: {result.stderr}"
|
|
||||||
logger.error(error_msg)
|
|
||||||
self.errors.append(f"Restic check: {error_msg}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
logger.info("Restic check completed successfully")
|
|
||||||
|
|
||||||
# Run forget and prune
|
|
||||||
forget_cmd = [
|
|
||||||
"restic",
|
|
||||||
"forget",
|
|
||||||
"--compact",
|
|
||||||
"--prune",
|
|
||||||
"--keep-daily",
|
|
||||||
"90",
|
|
||||||
"--keep-monthly",
|
|
||||||
"36",
|
|
||||||
]
|
|
||||||
result = subprocess.run(forget_cmd, env=env, capture_output=True, text=True)
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
error_msg = f"Restic forget/prune failed: {result.stderr}"
|
|
||||||
logger.error(error_msg)
|
|
||||||
self.errors.append(f"Restic forget/prune: {error_msg}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
logger.info("Restic forget/prune completed successfully")
|
|
||||||
|
|
||||||
# Final check
|
|
||||||
result = subprocess.run(check_cmd, env=env, capture_output=True, text=True)
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
error_msg = f"Final restic check failed: {result.stderr}"
|
|
||||||
logger.error(error_msg)
|
|
||||||
self.errors.append(f"Final restic check: {error_msg}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
logger.info("Final restic check completed successfully")
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
error_msg = f"Restic backup process failed: {str(e)}"
|
|
||||||
logger.error(error_msg)
|
|
||||||
self.errors.append(f"Restic: {error_msg}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
def send_telegram_notification(self, success: bool) -> None:
|
|
||||||
"""Send notification to Telegram"""
|
|
||||||
try:
|
|
||||||
if success and not self.errors:
|
if success and not self.errors:
|
||||||
message = f"<b>{NOTIFICATIONS_NAME}</b>: бекап успешно завершен!"
|
message = f"<b>{self.config.host_name}</b>: бекап успешно завершен!"
|
||||||
if self.successful_backups:
|
if self.successful_backups:
|
||||||
message += (
|
message += f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}"
|
||||||
f"\n\nУспешные бекапы: {', '.join(self.successful_backups)}"
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
message = f"<b>{NOTIFICATIONS_NAME}</b>: бекап завершен с ошибками!"
|
message = f"<b>{self.config.host_name}</b>: бекап завершен с ошибками!"
|
||||||
|
|
||||||
if self.successful_backups:
|
if self.successful_backups:
|
||||||
message += (
|
message += (
|
||||||
@@ -245,31 +363,24 @@ class BackupManager:
|
|||||||
if self.errors:
|
if self.errors:
|
||||||
message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors)
|
message += f"\n\n❌ Ошибки:\n" + "\n".join(self.errors)
|
||||||
|
|
||||||
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
for notificator in self.notifiers:
|
||||||
data = {"chat_id": TELEGRAM_CHAT_ID, "parse_mode": "HTML", "text": message}
|
try:
|
||||||
|
notificator.send(message)
|
||||||
response = requests.post(url, data=data, timeout=30)
|
|
||||||
|
|
||||||
if response.status_code == 200:
|
|
||||||
logger.info("Telegram notification sent successfully")
|
|
||||||
else:
|
|
||||||
logger.error(
|
|
||||||
f"Failed to send Telegram notification: {response.status_code} - {response.text}"
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to send Telegram notification: {str(e)}")
|
logger.error(f"Failed to send notification: {str(e)}")
|
||||||
|
|
||||||
def run_backup_process(self) -> bool:
|
def run_backup_process(self) -> bool:
|
||||||
"""Main backup process"""
|
"""Main backup process"""
|
||||||
logger.info("Starting backup process")
|
logger.info("Starting backup process")
|
||||||
|
|
||||||
# Get all home directories
|
# Get all home directories
|
||||||
app_dirs = self.get_application_directories()
|
applications = self.find_applications()
|
||||||
logger.info(f"Found {len(app_dirs)} application directories")
|
logger.info(f"Found {len(applications)} application directories")
|
||||||
|
|
||||||
# Process each user's backup
|
# Process each user's backup
|
||||||
for app_dir, username in app_dirs:
|
for app in applications:
|
||||||
|
app_dir = str(app.path)
|
||||||
|
username = app.owner
|
||||||
logger.info(f"Processing backup for app: {app_dir} (user {username})")
|
logger.info(f"Processing backup for app: {app_dir} (user {username})")
|
||||||
|
|
||||||
# Find backup script
|
# Find backup script
|
||||||
@@ -289,14 +400,18 @@ class BackupManager:
|
|||||||
backup_dirs = self.get_backup_directories()
|
backup_dirs = self.get_backup_directories()
|
||||||
logger.info(f"Found backup directories: {backup_dirs}")
|
logger.info(f"Found backup directories: {backup_dirs}")
|
||||||
|
|
||||||
# Run restic backup
|
overall_success = True
|
||||||
restic_success = self.run_restic_backup(backup_dirs)
|
|
||||||
|
for storage in self.storages:
|
||||||
|
backup_result = storage.backup(backup_dirs)
|
||||||
|
if not backup_result:
|
||||||
|
self.errors.append("Restic backup failed")
|
||||||
|
|
||||||
# Determine overall success
|
# Determine overall success
|
||||||
overall_success = restic_success and len(self.errors) == 0
|
overall_success = overall_success and backup_result
|
||||||
|
|
||||||
# Send notification
|
# Send notification
|
||||||
self.send_telegram_notification(overall_success)
|
self.send_notification(overall_success)
|
||||||
|
|
||||||
logger.info("Backup process completed")
|
logger.info("Backup process completed")
|
||||||
|
|
||||||
@@ -311,9 +426,53 @@ class BackupManager:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(config_path: Path) -> BackupManager:
|
||||||
|
try:
|
||||||
|
with config_path.open("rb") as config_file:
|
||||||
|
raw_config = tomllib.load(config_file)
|
||||||
|
except OSError as e:
|
||||||
|
logger.error(f"Failed to read config file {config_path}: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
host_name = str(raw_config.get("host_name", "unknown"))
|
||||||
|
|
||||||
|
roots_raw = raw_config.get("roots") or []
|
||||||
|
if not isinstance(roots_raw, list) or not roots_raw:
|
||||||
|
raise ValueError("roots must be a non-empty list of paths in config.toml")
|
||||||
|
roots = [Path(root) for root in roots_raw]
|
||||||
|
|
||||||
|
storage_raw = raw_config.get("storage") or {}
|
||||||
|
storages: List[Storage] = []
|
||||||
|
for name, params in storage_raw.items():
|
||||||
|
if not isinstance(params, dict):
|
||||||
|
raise ValueError(f"Storage config for {name} must be a table")
|
||||||
|
storage_type = params.get("type", "")
|
||||||
|
if storage_type == ResticStorage.TYPE_NAME:
|
||||||
|
storages.append(ResticStorage(name, params))
|
||||||
|
if not storages:
|
||||||
|
raise ValueError("At least one storage backend must be configured")
|
||||||
|
|
||||||
|
notifications_raw = raw_config.get("notifier") or {}
|
||||||
|
notifiers: List[Notifier] = []
|
||||||
|
for name, params in notifications_raw.items():
|
||||||
|
if not isinstance(params, dict):
|
||||||
|
raise ValueError(f"Notificator config for {name} must be a table")
|
||||||
|
notifier_type = params.get("type", "")
|
||||||
|
if notifier_type == TelegramNotifier.TYPE_NAME:
|
||||||
|
notifiers.append(TelegramNotifier(name, params))
|
||||||
|
if not notifiers:
|
||||||
|
raise ValueError("At least one notification backend must be configured")
|
||||||
|
|
||||||
|
config = Config(host_name=host_name, roots=roots)
|
||||||
|
|
||||||
|
return BackupManager(
|
||||||
|
config=config, roots=roots, storages=storages, notifiers=notifiers
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
try:
|
try:
|
||||||
backup_manager = BackupManager()
|
backup_manager = initialize(CONFIG_PATH)
|
||||||
success = backup_manager.run_backup_process()
|
success = backup_manager.run_backup_process()
|
||||||
if not success:
|
if not success:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
[restic]
|
|
||||||
RESTIC_REPOSITORY={{ restic_repository }}
|
|
||||||
RESTIC_PASSWORD={{ restic_password }}
|
|
||||||
AWS_ACCESS_KEY_ID={{ restic_s3_access_key }}
|
|
||||||
AWS_SECRET_ACCESS_KEY={{ restic_s3_access_secret }}
|
|
||||||
AWS_DEFAULT_REGION={{ restic_s3_region }}
|
|
||||||
|
|
||||||
[telegram]
|
|
||||||
TELEGRAM_BOT_TOKEN={{ notifications_tg_bot_token }}
|
|
||||||
TELEGRAM_CHAT_ID={{ notifications_tg_chat_id }}
|
|
||||||
NOTIFICATIONS_NAME={{ notifications_name }}
|
|
||||||
18
files/backups/config.template.toml
Normal file
18
files/backups/config.template.toml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
host_name = "{{ host_name }}"
|
||||||
|
|
||||||
|
roots = [
|
||||||
|
"{{ application_dir }}"
|
||||||
|
]
|
||||||
|
|
||||||
|
[storage.yandex_cloud_s3]
|
||||||
|
type = "restic"
|
||||||
|
restic_repository = "{{ restic_repository }}"
|
||||||
|
restic_password = "{{ restic_password }}"
|
||||||
|
aws_access_key_id = "{{ restic_s3_access_key }}"
|
||||||
|
aws_secret_access_key = "{{ restic_s3_access_secret }}"
|
||||||
|
aws_default_region = "{{ restic_s3_region }}"
|
||||||
|
|
||||||
|
[notifier.server_notifications_channel]
|
||||||
|
type = "telegram"
|
||||||
|
telegram_bot_token = "{{ notifications_tg_bot_token }}"
|
||||||
|
telegram_chat_id = "{{ notifications_tg_chat_id }}"
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
services:
|
services:
|
||||||
|
|
||||||
gitea_app:
|
gitea_app:
|
||||||
image: gitea/gitea:1.25.2
|
image: gitea/gitea:1.25.3
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
container_name: gitea_app
|
container_name: gitea_app
|
||||||
ports:
|
ports:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
services:
|
services:
|
||||||
|
|
||||||
gramps_app: &gramps_app
|
gramps_app: &gramps_app
|
||||||
image: ghcr.io/gramps-project/grampsweb:25.11.2
|
image: ghcr.io/gramps-project/grampsweb:25.12.0
|
||||||
container_name: gramps_app
|
container_name: gramps_app
|
||||||
depends_on:
|
depends_on:
|
||||||
- gramps_redis
|
- gramps_redis
|
||||||
|
|||||||
@@ -23,7 +23,3 @@ models:
|
|||||||
undo:
|
undo:
|
||||||
type: sqlite
|
type: sqlite
|
||||||
path: "{{ (data_dir, 'gramps_db/59a0f3d6-1c3d-4410-8c1d-1c9c6689659f/undo.db') | path_join }}"
|
path: "{{ (data_dir, 'gramps_db/59a0f3d6-1c3d-4410-8c1d-1c9c6689659f/undo.db') | path_join }}"
|
||||||
archive:
|
|
||||||
includes:
|
|
||||||
- "{{ data_dir }}"
|
|
||||||
- "{{ media_dir }}"
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
models:
|
models:
|
||||||
|
|
||||||
gramps:
|
memos:
|
||||||
compress_with:
|
compress_with:
|
||||||
type: 'tgz'
|
type: 'tgz'
|
||||||
storages:
|
storages:
|
||||||
@@ -14,8 +14,3 @@ models:
|
|||||||
users:
|
users:
|
||||||
type: sqlite
|
type: sqlite
|
||||||
path: "{{ (data_dir, 'memos_prod.db') | path_join }}"
|
path: "{{ (data_dir, 'memos_prod.db') | path_join }}"
|
||||||
archive:
|
|
||||||
includes:
|
|
||||||
- "{{ data_dir }}"
|
|
||||||
excludes:
|
|
||||||
- "{{ (data_dir, '.thumbnail_cache') | path_join }}"
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
services:
|
services:
|
||||||
|
|
||||||
netdata:
|
netdata:
|
||||||
image: netdata/netdata:v2.8.2
|
image: netdata/netdata:v2.8.4
|
||||||
container_name: netdata
|
container_name: netdata
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ services:
|
|||||||
outline_app:
|
outline_app:
|
||||||
image: outlinewiki/outline:1.1.0
|
image: outlinewiki/outline:1.1.0
|
||||||
container_name: outline_app
|
container_name: outline_app
|
||||||
|
user: "{{ user_create_result.uid }}:{{ user_create_result.group }}"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
depends_on:
|
depends_on:
|
||||||
- outline_postgres
|
- outline_postgres
|
||||||
@@ -12,6 +13,8 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- "outline_network"
|
- "outline_network"
|
||||||
- "web_proxy_network"
|
- "web_proxy_network"
|
||||||
|
volumes:
|
||||||
|
- "{{ media_dir }}:/var/lib/outline/data"
|
||||||
environment:
|
environment:
|
||||||
NODE_ENV: 'production'
|
NODE_ENV: 'production'
|
||||||
URL: 'https://outline.vakhrushev.me'
|
URL: 'https://outline.vakhrushev.me'
|
||||||
@@ -22,16 +25,8 @@ services:
|
|||||||
PGSSLMODE: 'disable'
|
PGSSLMODE: 'disable'
|
||||||
REDIS_URL: 'redis://outline_redis:6379'
|
REDIS_URL: 'redis://outline_redis:6379'
|
||||||
|
|
||||||
FILE_STORAGE: 's3'
|
FILE_STORAGE: 'local'
|
||||||
FILE_STORAGE_UPLOAD_MAX_SIZE: '262144000'
|
FILE_STORAGE_UPLOAD_MAX_SIZE: '262144000' # 250 MB
|
||||||
AWS_ACCESS_KEY_ID: '{{ outline_s3_access_key }}'
|
|
||||||
AWS_SECRET_ACCESS_KEY: '{{ outline_s3_secret_key }}'
|
|
||||||
AWS_REGION: '{{ outline_s3_region }}'
|
|
||||||
AWS_S3_ACCELERATE_URL: ''
|
|
||||||
AWS_S3_UPLOAD_BUCKET_URL: '{{ outline_s3_url }}'
|
|
||||||
AWS_S3_UPLOAD_BUCKET_NAME: '{{ outline_s3_bucket }}'
|
|
||||||
AWS_S3_FORCE_PATH_STYLE: 'true'
|
|
||||||
AWS_S3_ACL: 'private'
|
|
||||||
|
|
||||||
OIDC_CLIENT_ID: '{{ outline_oidc_client_id | replace("$", "$$") }}'
|
OIDC_CLIENT_ID: '{{ outline_oidc_client_id | replace("$", "$$") }}'
|
||||||
OIDC_CLIENT_SECRET: '{{ outline_oidc_client_secret | replace("$", "$$") }}'
|
OIDC_CLIENT_SECRET: '{{ outline_oidc_client_secret | replace("$", "$$") }}'
|
||||||
|
|||||||
11
lefthook.yml
11
lefthook.yml
@@ -1,6 +1,8 @@
|
|||||||
# Refer for explanation to following link:
|
# Refer for explanation to following link:
|
||||||
# https://lefthook.dev/configuration/
|
# https://lefthook.dev/configuration/
|
||||||
|
|
||||||
|
glob_matcher: doublestar
|
||||||
|
|
||||||
templates:
|
templates:
|
||||||
av-hooks-dir: "/home/av/projects/private/git-hooks"
|
av-hooks-dir: "/home/av/projects/private/git-hooks"
|
||||||
|
|
||||||
@@ -12,3 +14,12 @@ pre-commit:
|
|||||||
|
|
||||||
- name: "check secret files"
|
- name: "check secret files"
|
||||||
run: "python3 {av-hooks-dir}/pre-commit/check-secrets-encrypted-with-ansible-vault.py"
|
run: "python3 {av-hooks-dir}/pre-commit/check-secrets-encrypted-with-ansible-vault.py"
|
||||||
|
|
||||||
|
- name: "format python"
|
||||||
|
glob: "**/*.py"
|
||||||
|
run: "black --quiet {staged_files}"
|
||||||
|
stage_fixed: true
|
||||||
|
|
||||||
|
- name: "mypy"
|
||||||
|
glob: "**/*.py"
|
||||||
|
run: "mypy {staged_files}"
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
|
|
||||||
vars:
|
vars:
|
||||||
backup_config_dir: "/etc/backup"
|
backup_config_dir: "/etc/backup"
|
||||||
backup_config_file: "{{ (backup_config_dir, 'config.ini') | path_join }}"
|
backup_config_file: "{{ (backup_config_dir, 'config.toml') | path_join }}"
|
||||||
|
|
||||||
restic_shell_script: "{{ (bin_prefix, 'restic-shell.sh') | path_join }}"
|
restic_shell_script: "{{ (bin_prefix, 'restic-shell.sh') | path_join }}"
|
||||||
backup_all_script: "{{ (bin_prefix, 'backup-all.py') | path_join }}"
|
backup_all_script: "{{ (bin_prefix, 'backup-all.py') | path_join }}"
|
||||||
@@ -23,7 +23,7 @@
|
|||||||
|
|
||||||
- name: "Create backup config file"
|
- name: "Create backup config file"
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
src: "files/backups/config.template.ini"
|
src: "files/backups/config.template.toml"
|
||||||
dest: "{{ backup_config_file }}"
|
dest: "{{ backup_config_file }}"
|
||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
|
|||||||
@@ -57,6 +57,19 @@
|
|||||||
group: "{{ app_user }}"
|
group: "{{ app_user }}"
|
||||||
mode: "0750"
|
mode: "0750"
|
||||||
|
|
||||||
|
- name: "Create backup targets file"
|
||||||
|
ansible.builtin.lineinfile:
|
||||||
|
path: "{{ base_dir }}/backup-targets"
|
||||||
|
line: "{{ item }}"
|
||||||
|
create: true
|
||||||
|
owner: "{{ app_user }}"
|
||||||
|
group: "{{ app_user }}"
|
||||||
|
mode: "0750"
|
||||||
|
loop:
|
||||||
|
- "{{ data_dir }}"
|
||||||
|
- "{{ media_dir }}"
|
||||||
|
- "{{ backups_dir }}"
|
||||||
|
|
||||||
- name: "Copy rename script"
|
- name: "Copy rename script"
|
||||||
ansible.builtin.copy:
|
ansible.builtin.copy:
|
||||||
src: "files/{{ app_name }}/gramps_rename.py"
|
src: "files/{{ app_name }}/gramps_rename.py"
|
||||||
|
|||||||
@@ -53,6 +53,18 @@
|
|||||||
group: "{{ app_user }}"
|
group: "{{ app_user }}"
|
||||||
mode: "0750"
|
mode: "0750"
|
||||||
|
|
||||||
|
- name: "Create backup targets file"
|
||||||
|
ansible.builtin.lineinfile:
|
||||||
|
path: "{{ base_dir }}/backup-targets"
|
||||||
|
line: "{{ item }}"
|
||||||
|
create: true
|
||||||
|
owner: "{{ app_user }}"
|
||||||
|
group: "{{ app_user }}"
|
||||||
|
mode: "0750"
|
||||||
|
loop:
|
||||||
|
- "{{ data_dir }}"
|
||||||
|
- "{{ backups_dir }}"
|
||||||
|
|
||||||
- name: "Copy docker compose file"
|
- name: "Copy docker compose file"
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
src: "./files/{{ app_name }}/docker-compose.template.yml"
|
src: "./files/{{ app_name }}/docker-compose.template.yml"
|
||||||
|
|||||||
@@ -10,10 +10,15 @@
|
|||||||
app_user: "{{ app_name }}"
|
app_user: "{{ app_name }}"
|
||||||
app_owner_uid: 1007
|
app_owner_uid: 1007
|
||||||
app_owner_gid: 1008
|
app_owner_gid: 1008
|
||||||
|
|
||||||
base_dir: "{{ (application_dir, app_name) | path_join }}"
|
base_dir: "{{ (application_dir, app_name) | path_join }}"
|
||||||
data_dir: "{{ (base_dir, 'data') | path_join }}"
|
data_dir: "{{ (base_dir, 'data') | path_join }}"
|
||||||
postgres_data_dir: "{{ (base_dir, 'data', 'postgres') | path_join }}"
|
media_dir: "{{ (base_dir, 'media') | path_join }}"
|
||||||
postgres_backups_dir: "{{ (base_dir, 'backups', 'postgres') | path_join }}"
|
backups_dir: "{{ (base_dir, 'backups') | path_join }}"
|
||||||
|
|
||||||
|
postgres_data_dir: "{{ (data_dir, 'postgres') | path_join }}"
|
||||||
|
postgres_backups_dir: "{{ (backups_dir, 'postgres') | path_join }}"
|
||||||
|
|
||||||
|
|
||||||
tasks:
|
tasks:
|
||||||
- name: "Create user and environment"
|
- name: "Create user and environment"
|
||||||
@@ -35,17 +40,11 @@
|
|||||||
loop:
|
loop:
|
||||||
- "{{ base_dir }}"
|
- "{{ base_dir }}"
|
||||||
- "{{ data_dir }}"
|
- "{{ data_dir }}"
|
||||||
|
- "{{ media_dir }}"
|
||||||
|
- "{{ backups_dir }}"
|
||||||
- "{{ postgres_data_dir }}"
|
- "{{ postgres_data_dir }}"
|
||||||
- "{{ postgres_backups_dir }}"
|
- "{{ postgres_backups_dir }}"
|
||||||
|
|
||||||
- name: "Copy docker compose file"
|
|
||||||
ansible.builtin.template:
|
|
||||||
src: "./files/{{ app_name }}/docker-compose.template.yml"
|
|
||||||
dest: "{{ base_dir }}/docker-compose.yml"
|
|
||||||
owner: "{{ app_user }}"
|
|
||||||
group: "{{ app_user }}"
|
|
||||||
mode: "0640"
|
|
||||||
|
|
||||||
- name: "Copy backup script"
|
- name: "Copy backup script"
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
src: "./files/{{ app_name }}/backup.template.sh"
|
src: "./files/{{ app_name }}/backup.template.sh"
|
||||||
@@ -54,6 +53,26 @@
|
|||||||
group: "{{ app_user }}"
|
group: "{{ app_user }}"
|
||||||
mode: "0750"
|
mode: "0750"
|
||||||
|
|
||||||
|
- name: "Create backup targets file"
|
||||||
|
ansible.builtin.lineinfile:
|
||||||
|
path: "{{ base_dir }}/backup-targets"
|
||||||
|
line: "{{ item }}"
|
||||||
|
create: true
|
||||||
|
owner: "{{ app_user }}"
|
||||||
|
group: "{{ app_user }}"
|
||||||
|
mode: "0750"
|
||||||
|
loop:
|
||||||
|
- "{{ media_dir }}"
|
||||||
|
- "{{ backups_dir }}"
|
||||||
|
|
||||||
|
- name: "Copy docker compose file"
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: "./files/{{ app_name }}/docker-compose.template.yml"
|
||||||
|
dest: "{{ base_dir }}/docker-compose.yml"
|
||||||
|
owner: "{{ app_user }}"
|
||||||
|
group: "{{ app_user }}"
|
||||||
|
mode: "0640"
|
||||||
|
|
||||||
- name: "Run application with docker compose"
|
- name: "Run application with docker compose"
|
||||||
community.docker.docker_compose_v2:
|
community.docker.docker_compose_v2:
|
||||||
project_src: "{{ base_dir }}"
|
project_src: "{{ base_dir }}"
|
||||||
|
|||||||
@@ -51,13 +51,29 @@
|
|||||||
group: "{{ app_user }}"
|
group: "{{ app_user }}"
|
||||||
mode: "0640"
|
mode: "0640"
|
||||||
|
|
||||||
- name: "Copy backup script"
|
# - name: "Copy backup script"
|
||||||
ansible.builtin.template:
|
# ansible.builtin.template:
|
||||||
src: "files/{{ app_name }}/backup.template.sh"
|
# src: "files/{{ app_name }}/backup.template.sh"
|
||||||
|
# dest: "{{ base_dir }}/backup.sh"
|
||||||
|
# owner: "{{ app_user }}"
|
||||||
|
# group: "{{ app_user }}"
|
||||||
|
# mode: "0750"
|
||||||
|
|
||||||
|
- name: "Disable backup script"
|
||||||
|
ansible.builtin.file:
|
||||||
dest: "{{ base_dir }}/backup.sh"
|
dest: "{{ base_dir }}/backup.sh"
|
||||||
|
state: absent
|
||||||
|
|
||||||
|
- name: "Create backup targets file"
|
||||||
|
ansible.builtin.lineinfile:
|
||||||
|
path: "{{ base_dir }}/backup-targets"
|
||||||
|
line: "{{ item }}"
|
||||||
|
create: true
|
||||||
owner: "{{ app_user }}"
|
owner: "{{ app_user }}"
|
||||||
group: "{{ app_user }}"
|
group: "{{ app_user }}"
|
||||||
mode: "0750"
|
mode: "0750"
|
||||||
|
loop:
|
||||||
|
- "{{ data_dir }}"
|
||||||
|
|
||||||
- name: "Copy docker compose file"
|
- name: "Copy docker compose file"
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
|
|||||||
322
vars/secrets.yml
322
vars/secrets.yml
@@ -1,162 +1,162 @@
|
|||||||
$ANSIBLE_VAULT;1.1;AES256
|
$ANSIBLE_VAULT;1.1;AES256
|
||||||
66333736306664316131646233393639356436323832386638353237393761386631303639396432
|
35643431306561396237633439336230333563316364353462393534646565653163396362393566
|
||||||
6266616434663638306637356435393564303633613332640a663366336135373061396239653065
|
6636306532353662343437646631363766376166333934340a363062323332386261643832323762
|
||||||
32343339313734346261363461633735383538613033656138383835653661633334316533613738
|
33303235646434636636366532626331653865666465643561633863663938313535623162306334
|
||||||
3730656661393466370a653863353234323739346630323534333334306432646636383664313865
|
3332616236326436650a383636393434303165363964383966633138326361373134306563653363
|
||||||
39666133623032393638633239653665376336626638303334626164376663626666393439346661
|
37376465643762383761653132636336663762656136303730616463386134346436383539653661
|
||||||
32363364363066653933313135666566346138353163333639353062336331623938353131646334
|
35623933346565633030373730316331633236623561313166376335636464333439643637623338
|
||||||
32326466323030643364356530653862633332363338333633393730663536353363663165303838
|
32393433653339656135343766393565366530636132346339306339353438396337303534336266
|
||||||
66643331366231613733316139333233393636626162643861346437613062643133353035353566
|
34303635316363343031363263626434386139633762353038666266643338343631303530313661
|
||||||
38323062653135393364376265346361613163346337636634383932666137636133363864643264
|
61373239663739666535636466393937333437316334306631336334303961396134323062666339
|
||||||
39646634363037383662376166633335636365303734646465623833313766656235333835396263
|
62303437623131626130363838353736386463663734356530333061353031643861636437323431
|
||||||
63326237306337396465343065313263663466613336303934373337623364613839323931623934
|
62633036316639306366643633333366386132343337313832343133313235626162323266306165
|
||||||
35353665363339646365353139643032366164653133616562303930306264343633343439373830
|
39326434326235373536633464313234623464373465653061383834363266663435303461633834
|
||||||
66356635343363363361313633353134313837653163346537656465623832633561323833663763
|
36663031323634326436303138623262396332363934646337353166373163326132626437636138
|
||||||
32323537303931343934376165663466356466333533643237653038373866626439356334663539
|
38623864666434363939663234623732623962323836323532393031396638343737373764303239
|
||||||
36613336326633306331323763393230653236306337353139353135316566336661623464316237
|
31316639653066373166326432653930666630393061366461663337333936326239376165633064
|
||||||
37363930666539373837336466333265333866633733363032323862333239643764333766366134
|
32356530333338303636366234336135353130646134643432613965366430326164393563383135
|
||||||
33396330613463633261626334393262343031663131313737623233323535303965333864366631
|
63653237646237323832613363643732383436646539396239333632353566386135633961316132
|
||||||
65333535343062393434396361313433386463613630353765663363633431393831316233313162
|
62313630313065376465346338663661343466366364363761616363376535373031383736616136
|
||||||
39656333306265386437663863343638323262663734623830336562636366333534373735313932
|
37343634626366356261363030616330396263623965306166386238363039663461396431303733
|
||||||
61653739616635643661306165363364316262386434643033646430366235303334326236373262
|
61626164353466323762666236376536356364653565333266363730653539626531663639333864
|
||||||
34396261653934633337646338333364326237663739623965313039303134393238653634616431
|
32366333336234376635616361626635616265376361316439653230646537303031343564653631
|
||||||
39376539323238353033323333643337613935616335336562626361383164616432633331313536
|
63663237353766643766343130613739386636663534336536353936353561616538316535333064
|
||||||
63373633633966373237393131663064613332316536373163303163653638343264323934393365
|
62343835643636633831366265313939313433326239633130303863303136363732653766303663
|
||||||
62356636643434646465313734363936346435663237393064663138643135303237633965393965
|
33643231363761336463623339323831363034636532346237373236656266303063633365393636
|
||||||
31353564616136366237666139646561363364353738633862346262616637663362333137386133
|
36663066336233643236393837323762393435303665303661356237666237646464326536623731
|
||||||
63383662363866306366326232643462376532313632336662666666386562356436616236663430
|
64396431323561353832333834336232396366663439643161626338653930633665346566303435
|
||||||
37636661656237646431616331633833306266313434326461353561643835646236346539323337
|
66636466313936316530353537643637323833653561616339353465323466333332363436613461
|
||||||
30303065313165306233646135363766613931383433313165366662636137326162633363306431
|
38653837373637343532343662366435616366313831623239343363366131633062303036333135
|
||||||
31626564323566313036363666386338366462613164663735363333313830306238323365363737
|
36356136376236383431613266663164616135343062666136353430626663653631313064326131
|
||||||
65333137363631623139333964316464376662643661643038326162643630323938626332633130
|
63353066313433653838613934323833373237356230376363376132306663656461653264383866
|
||||||
62626334666131373731343432303461656537303062396638326134356166663936663364656139
|
37333863633536346538366665346666346364633733366434393135353664336330623865343132
|
||||||
32333864653231636235663165323936626135313838663866373132376437656236363235353533
|
31396534643633333134363465323063313435333236646630613066353232613039393235366630
|
||||||
39346532626666656233343433383434613238653833626436643566653462346330316565366564
|
36646664366339613535616437376437326561636130333063626637316431626330386137306162
|
||||||
33623831643265623966326638656462306161636366303266333734396433653861393363333332
|
65393533306439613032623061373331333762643336633863333061393734313335386633666338
|
||||||
61616463613931393835613463353039646164383435373330626134376339623861396266623430
|
31316637643031353462363363626662373462633065653738613733633933623432343535316438
|
||||||
30316365356337356531363263623362633332313536373333653964356534623861613232653932
|
31356530306263303566636535363465356562633439386365316461323339393339643133323032
|
||||||
35613064336131313632656663303631363664366163343362643365663932356138376235313466
|
32616662633565346564643737386266623836623830663265323439313762636437336436346430
|
||||||
30353333376461623435363931356665306333623736313562333836336563616137633863346635
|
65353266373766363166633363636231623031363337626631626466616637616630383632333565
|
||||||
37356137306361383134663535626130646135363661353438343961653766333538353330346236
|
35303737316531306237613832366565376364313435376266306263623139663964336635633030
|
||||||
64326639383864396336343062663965343964396162386266363639643962666431336237303864
|
39306662616233646136336237646139326239653435653436323066333332626266646564616631
|
||||||
66663861346139363335663362333032613637336266323439366566373136643133383361323061
|
31653433663539333633663433643261363065663030373265646533323832323366376230346266
|
||||||
37633134323933613665366665383962373935646137616139353661336665613661623834303465
|
65616633363938333037656663393334363564626136333162383162663437653565663532646366
|
||||||
30323264333137636261393535376438363134663734313662383533313130623365386335323662
|
31343063306434663231396464646232353335636138393038633737323339303033373539363739
|
||||||
63366139643238613632326165313835363964336237383936333737646239363365623030666364
|
64306163343534363131646135396234346166313039323833656636343235656238623333626534
|
||||||
64656432653164643565376666373262333839353139666561623731343234326637316636333765
|
30653264656166326562343534313063326332336330623530646563313262396233386335316538
|
||||||
63306331343538386433376566326239376232363434343838653864393562383063663566333263
|
64316363613865313236633437646139353433386234326537633961356133353361333833636339
|
||||||
35326333326462316134383139303534343263646530363266663933353834353138303435646339
|
65646232336137626130326631626436623933353333383438306237326132303838353833376432
|
||||||
33643361363062373735663430346636333431363736373463326439353437356530373935633962
|
31653936363131623535386331353066323462613238636130376562376361643166363332343537
|
||||||
63333031376163656565663536366230333731613833396266383465333461386161373337323863
|
63346366306161373666306536396164323437613066363537366562666130653638356564383235
|
||||||
36313032613534346230636566383930656330656133376431376462656536386263343831393862
|
66303838616266623335363338376535303665643363326662376166643865383039306236393730
|
||||||
32363365623061653837303736636664663361663862656562393661623330386435646336373531
|
34303130386631636636393736616237633933346136303530373033383739653531326437656463
|
||||||
35363037356638653831386261646235613337363066343632653632306631633138633235356139
|
39326239616163313463636232343036626435366261643030386530393763356133663863373264
|
||||||
39663936333262643061646330316538373862353030626632396336393030643239316634343730
|
62316164386464366663373635393365363161336530313734343633346563396666616262396563
|
||||||
35363564373865376533333466666439646633313932656665383930623531633038316363636332
|
37383638613930366635623962366662343232303635636330383534653861336264396132653538
|
||||||
37363466643835353132333532646163316636303662646234613038333334626365623964653235
|
65306539613333306339643063303063333165653734323333353461356536663661613463633632
|
||||||
36393739353164306666313537633538383934363330373235353262616165623132613330303735
|
61613530653734643761616636386238633535643961633765313863323961313362323265373132
|
||||||
65323362666263653937376562633833653264613439303236373466646362316462386632373038
|
65346530653962613534343062333039663837623130666261636135343663643036356261643333
|
||||||
35353462633730666430623638626439626364616335643964623933663233376433353233633235
|
64306430343136616438636231623236316264623533313439643434343333623030376338663365
|
||||||
34616233636335323365646538326639383033643832323139633064616635353331376230663738
|
36393932613161316137386136343939383035326231323561636337343833613635643261633165
|
||||||
35316665616230633335366233363332353432633937653335316662306166383337633262633634
|
38316233323862666439633833646235623031336638353834323633333562346463333436316437
|
||||||
35303766653733376362623436383663663437393461643266376530613533383038373035356234
|
36303237353138353763346166626465663137323062643633323762313137343661323839353231
|
||||||
32343962623930336566363164616361386134376362383138653963366339326431653832393664
|
38643137383739393866616364386262363733316133313934366165353761643164303439643534
|
||||||
66633566343666326339323536343738396532623735343034646130386332323466346365383135
|
37633636323535373964323939346339633138386239333466363239393430613139623466643531
|
||||||
33373732363833316138343938633262373066613930343162663163323331373466366631626237
|
66323662363834653166363835343363303066306262623432353165313239323130616235316264
|
||||||
34623062353139353734653366313363323634346564663861613233366537663732363134643638
|
65376532623530353935346238346261613062633439333630643962343233326234316130616538
|
||||||
65623964396665303765343430623636376637666364363835346462613763306135326463353631
|
64353932613038353866383263333561623632663863373231636132653365613836363730636263
|
||||||
33333834333464383437336531616662643461666138383435636530373761646564626438313433
|
65626666653962653266343134343638666437373532333464633163333861386632356562346637
|
||||||
64643037626637343433366631343266373436343864396663353039333231646462333932353436
|
62303433383464373165316561646633636130636434633430623031633364643336643835613030
|
||||||
33643837616639666363353237313137353133346636383231623634326335386537363030613537
|
64653937656262623536353764343661326632303964393862343032343330653534353530326238
|
||||||
33306333363638643533373237633833663333393566656466313832383636663031663433636663
|
34613236336339653364333565313862613531643763633166363734316363643534323838333531
|
||||||
37313034353734343966613934663530643537366562623137373331623632653466333839376331
|
36353137653535393862393534303366366464613035376439383134336534366263393661646564
|
||||||
61623866633937666365633763613138346365393934383163623730373134373531616138363733
|
39616233323638653337333461636465353935613236646430623163656334376463323135323761
|
||||||
65616230303337396561666462653866333438353463316235303331643834653165363033626537
|
36353064646532323738363837663238653766366634363232323933363033656332626436363166
|
||||||
37386636653337363666393163393031333461383331613965303262616530363133373732313362
|
36376366336236626265373530646664633335623035646662663261353165313634646463316535
|
||||||
33343033373963656333363064303035666663646536373764323833653037316231316430333634
|
38633862623433613937666162343936393337336537623663306439373830326434356366323664
|
||||||
65383236663035326636336535636462323438363165633437333337613439326433626466396365
|
61646337396562373535353834393665643037626539303932663831306535623733626239323939
|
||||||
38393362633266663962646565336539333239346133323434646632643537613435643434623631
|
61616165643439653235346566346631323035306437323339326665363232333338616133666366
|
||||||
63636536393936343165393966663438343261333966363639323462663566613437393838323036
|
61396136353432376263333961353738343463663937313136336536623539383835663361353232
|
||||||
33393163323034336533333632646230343138336333353236376136383664646466626666356234
|
38343737383361613930326265346562356136313539393464623935626566343137346430356261
|
||||||
61653466363933333331613539636431393934393235376433326665643263663638306463393837
|
62633230396665316533313831346236303265303232363935313138323537373933633232343732
|
||||||
34396339383536636461366230383938386339303334393038343239363361666565336237326465
|
35346363663938306562663433396138393861343332373934303864303963623739316239303264
|
||||||
63623231663861303436353533663661656431646165646662383065386362636633333631643335
|
62393634346231393534633861386364336639366663353065333930616335323438383537616666
|
||||||
38373464373432626538616234623638303734626237393566326463633765316365653837303433
|
63373836363839336634316338643935346634626437326431363132636335326238353930663662
|
||||||
34376438323439633237313733343836343733373930643138636333366166353666373966323231
|
62383464366263336430633338646263626565393030646230306230353837313965623430636138
|
||||||
63613466306365386137336538613837613264633735393937343166303736396162303230623430
|
38393334623661363532663464363163383365323736386134393634333332353263353763393636
|
||||||
33363233663761333063363134393935333530343133303138373934666236393732396330643438
|
35353535303030356665303262333334353630613832323733393638613035343833356662353565
|
||||||
61343231666363323834346132376135303339313766383365363837313163393636333563376436
|
64373334626565653631613863323866336261393533333639326266343435376666646131313535
|
||||||
64333832393361626663376161343033383763373537396264646239303736333263303739326436
|
39343734326162623831343730303734643830626338653232633834373565643133393562653336
|
||||||
31393665363734323364643032393031313135623563383639316163616535323938343765356264
|
35373432316161623061373238333932366161663137616337613632366263373830663464396537
|
||||||
35616433396236396662393930646431643063323163343366313030383766323733333865616235
|
37303363386336663663303161343963633834623564643661653332623434626466376637323631
|
||||||
65623737663763373363613030326665613333636366393464383139626462346333323162333537
|
64653265623861363837346462613164373264313766633034353531666561623732353838643639
|
||||||
32386635636532346539623534346364623532656435653638643962353836653330316638633661
|
37646262646362386331663334666131336335313833313463373463313439353934313733303763
|
||||||
38363330656161393766383237663737663835646565363966373563663832313263633238316165
|
65626339363732326565366464633230303933386438373539633338363439393630316164376565
|
||||||
64643536336232396436636261303866376132366261373132333631646166393764636130326335
|
63333132613739663461346361613761396161323863616263333638376437613139353733613963
|
||||||
39613037633239626666396262386533343764623233626335373139663937613433356134633330
|
32663062393435323061353738316132363438393762316666626231386138366639313566646461
|
||||||
32656138383563343734616536303265306661386362343939626332623763393136323138653937
|
36306664343433653836316436303736343436343762366133373066313232333834363536343462
|
||||||
39646537366461353364616538663361663465626634356338323166373837323339343061323261
|
30303866646131663530313463343261633261663932323562636135346664323761343765333730
|
||||||
33393765653034643166313639323962306265383336663265623832393038386334643661336663
|
39663231636465336361393062666135316534656637306236323137653939383565643637623866
|
||||||
66643330306239383736636330393266346537326436663432653031303765393930633761343363
|
33633536366163346466303864313732303036306665326664353064366134326539333831663632
|
||||||
61353163323830656538373366323230353830633534326133613861356132623938663565393130
|
64343533323166623661326536393437623432336435366435396165333431366532653936396636
|
||||||
63333764613761663162633035353036663736656138623234313833326337316438323835313465
|
30653263306461656230663731336266363832616539373333306238363834333838393063383432
|
||||||
31653136356638623965363266636461646465313065666137313931623063303337393963353437
|
61343238373861303932653162663330333964623834336235336430376562313963613134353834
|
||||||
39633166326662303834303261353965363235626535643564613064306636366461666263383833
|
63666537343837336531633536633038666232343165336663356639636537306333303434383231
|
||||||
66363966636538373435323365636630383561336533356631646563646536383333393864613164
|
64306633383338646362633238386263336632393562623330343532373839663238383965643739
|
||||||
38393638623263666239623835613933393564323132373630623734363334663735633438663137
|
39313363383331363339616431653434366131303666623263326662636161666635643162663730
|
||||||
35613563356161316632323562626639633931643762643232636136653731323337333236326363
|
62356136643364666164383932376536623266373535613230336165396163383830346265393738
|
||||||
36633338386230626665373666643831626132323866653430393531336466386463663865393962
|
30636139353336343630316132366365306234313064636537646262666463646437376336366562
|
||||||
37363633343632653963306531393537633762313565303631643064363531363839643363356139
|
31663731666664313339353331626534626462663431633530653365613762363666373532383762
|
||||||
33666162306331393237333966643735643731633331373839356637636462633836633762653039
|
34323866356239383332373632616464666230373339386235393838376333333464613038373337
|
||||||
32633630303437633130613366386633346637623266663361656365633737346432343263646536
|
33373231386263623664353231626331366565356662623934316632653763363739653862356265
|
||||||
35376433386464383533323231353830666363363437623033643238363438303565616137316461
|
39653539616232356333326463373666656133303161356630393839336638316338646238623138
|
||||||
64636233643365663666656431613966396561366236383363303135613433626466366437396465
|
61343833613130376562656465343438653861653431313762313062656563643665356664646465
|
||||||
37303562633336636635363133626238643430663132626331316437626532356237383730626665
|
33343430663066616536393935366132336666616164356463323663373930653266653238663336
|
||||||
38373633323531346366313631376231353966626637636262333936343066336639396262646436
|
35616263613461356333633430653661376239663363653434346437656161623961333761633663
|
||||||
31616165626466623066336138653966323334326236663439643561663863323130653631643231
|
35643031323063666163623639313864313937386339376235653033366136373333653239653238
|
||||||
31396335333066326166383337353033376335376664653162396466346638653531346239353936
|
36376636653766366632613262326336333430353534326464653931663366383039643466303739
|
||||||
33663862323135623432333036336162353061363537363463366435643461356562323935653962
|
66393065323965383962633266623033643330333437383262373461363539626333653433356436
|
||||||
62333130616466353032316462356233393037326438303064656235323966306535346132663232
|
64383738653735386630346264643462353336313830376139353166666330303638386639623039
|
||||||
63393238646534316433643638646536313934666361323061306561396132306431633932313031
|
34303137623936663636303864623932343766323235653934303263363164326534653137613863
|
||||||
31356430313463613331363064363265646361353430646563396238326330323038666264653935
|
39353739633466376136343938303634643565613537646166326536303739316536376162353838
|
||||||
63306635323439326665306135643730386239356661303232353737383532353363303163313437
|
61326235373733623465313530393831623438343530616339326561386239646562343130376336
|
||||||
38623064383035373239623966336563626130636335383833383366343130393939316535333466
|
63373430313332613561623636393338353933613161656264646366633665373336343530303635
|
||||||
65353062396632373330663365373761656161653837396666376531376663313466393230646461
|
35336561336334626333343030366264356264343062623531363331313466666136666332376265
|
||||||
62306465363166663962316634663763626666306631363731633834366433363630383833393361
|
62363862623635376631336235353531343931663131333861616566356438396664633962646236
|
||||||
36393932326138643432666437373463343061313135613163343034373464356666333134366664
|
61343063383238376261656465386635623263386430656132393631623330326462326166643631
|
||||||
66343461393733343130636266623661323066623332633930326361626436366237323664336637
|
34333238663531366232393764626665376465386336653133643335656161343261616532386135
|
||||||
61316135613330663835353139613039613264383838313337373432623133363735333732376566
|
63333931646435373630323931396564636131616234346130393566663131353835323833666338
|
||||||
38633763336438663737653736326466646237313130623934396531306430363461653962336132
|
63653332383130666164376534656430366664356339323463393338353066663537313336653864
|
||||||
64313437323766613632376439336234393165646338396334373037323237633737393866303231
|
66323361383737323930663732376261323238653537623335636638363966343264346537383264
|
||||||
33646338333532613432393234316235653466633639306465363062656335353034666665623631
|
64663334366538326365663266613662356132646230363461643862643337633531333233363134
|
||||||
66386437616462633336636566393537323831313566326637386466616331396464653438396562
|
66313565313733326363623935363535336166623761346162663964396266613932356462376664
|
||||||
33343132626463393364386133323163393336313065316433613133663961633033613232343337
|
63313339333035333234353765383735303830653162346565303465363762623465326431353431
|
||||||
37313334386533366465353461633930643662326235366139306335656163313864636161623239
|
65616335313733353863626561613236616339393038383562313263376633363765306262396230
|
||||||
38336238663630313836363739623334633130616433393536303431333735643565326265613561
|
38373963306637316534613964376538666536383235393663616236653338373034363765636135
|
||||||
30613762396162633964336165666137626333643735323330646266666563313935623230643262
|
65666239356563363538666162313038633861353732386630633930616439323838666536383165
|
||||||
30383635663035653437333339303730346366333765353739663231643433353764363966353435
|
34353761613738616361613862656236623633356364383436383435383632303635343233356435
|
||||||
63663466343864363033646663376261363562636630643038613365333936653165633733623134
|
35353466636461376436643965653837633461643430386565353530323333313964376233666665
|
||||||
36626135313330303463336535663235323536613661353332653139336165613261316638666563
|
63353838323363666435393634643237626663356331653461303936366533343435373732613861
|
||||||
63346131306536353630363236613864393935333431333864333464353664366134313861633463
|
33393231636235613234383737633430343461616261373265313163363730653237346563393835
|
||||||
34336364663030376264663961396235643438653730396661643032623762623965623737326430
|
38646566313730633866636162393434316365396663333938656235613635613161363938653531
|
||||||
34333236316335373863663232356136333431666233353138353139313466343762356632396561
|
65316233643561623837633362383232313664313737643231363838616434666635343466653435
|
||||||
37356162303636343833316432353831373532326436363538333466333636306666626465666333
|
62653765303832353761346563363338656431313439346234333432363535626461336163616330
|
||||||
36653065363538653266643831326234356534303133343564646564323762653238303161336131
|
33306633313764616533666263633634306633633261663637633133356430313231323331353535
|
||||||
35396235343837333331396535646564656433343766323765333465356134323536636639623962
|
31316663373063376665373732366365386464393032313335376463353435313133376161336263
|
||||||
65633233376136316334653335623666313666376661326362663338633862643963353536616563
|
33363835363062646536343833383232386133666137313539663534653138333236626438333866
|
||||||
66323939643332663665306536356566383164356561303935313432373264353738643131653831
|
39316433626133666636376532366332346134646233383737386337356464363263343334363663
|
||||||
31346231633037616139373330316231363938386536303432386638323139326132663539663738
|
32363538343566666263396265636261643764383736643663386333623266363535333766353037
|
||||||
64636231663538333932363332663137323764346336633336363965616535373030663363363861
|
33333864636539636165373433343532326431316138666164353262643236623237636539646636
|
||||||
65363632653831353761306231663562623732353433656637353966303033636666613739306230
|
30356361663863646366303238643335626434353832306338346231383764616566626163313931
|
||||||
64363465623031386431663565623966643836383532626134366363656265646563343635383538
|
64663663613363396661396564383531653231663735653535393437356339353466353737366266
|
||||||
32346166363832626664613335383731616135376635336266326531663731373633303131373965
|
37353739646632393837353032636331316338393564643261383536383536323036333238633237
|
||||||
66663666333739373033616363313132643938656335353164343764383933636265656665663433
|
38643735663339316263633964336137303939303531626633646236656430373132316432343436
|
||||||
65386235636531653236333465316331353938323331623733623731303462306566333365383134
|
66383135663130323462373934656666653837336332626137303931303263613038646235623631
|
||||||
31356337343732613764633731306335366136346633393265666331636465323566656337323838
|
39383936393665316561373637333935643565656433316462333832323034323533316232656164
|
||||||
37393031316630623466393138636666303065623430656336386430363962336539616262633030
|
63656630356363336231326364656531623839316236616266363037303138306537376131616134
|
||||||
62616365623765366236303539353166306630393362386430363736313438346266366336323839
|
31323132613533663664376136376437623837303835613331623339623531653563386464306339
|
||||||
34376462376633366463636339646337623965633663363862393261373535636230613532386464
|
62346465396362303262356239326636666435343131333566653661613463363461633631383030
|
||||||
34393564383130306138366564366666646132363732653036353135656132656339353932383530
|
63303738613735313262656362383432356236613339646462393836633861303562663262333561
|
||||||
61636234393165326631303731633062333735306635303566373838393164306538303664313266
|
62323562656461663764336462353230653537383038323931353831643731343837323234643565
|
||||||
33346337613836636337316638323631396235643363333836323135303738366235326465376630
|
36386531613931623036636332663561663438333364616232626461333639326564313335376134
|
||||||
3363
|
3935
|
||||||
|
|||||||
Reference in New Issue
Block a user