GoAccess: add for caddy logs monitoring

AGENTS.md

@@ -68,6 +68,7 @@ uv run ansible-galaxy install --role-file requirements.yml
 - `playbook-rssbridge.yml` — RSS-агрегатор.
 - `playbook-netdata.yml` — мониторинг.
 - `playbook-dozzle.yml` — просмотр Docker-логов.
+- `playbook-goaccess.yml` — аналитика веб-логов Caddy в реальном времени.
 - `playbook-gramps.yml` — генеалогия.
 - `playbook-calibre.yml` — управление электронными книгами.
 - `playbook-transcriber.yml` — транскрибация (образ из Yandex Registry).

files/authelia/configuration.template.yml

@@ -731,6 +731,10 @@ access_control:
       subject: 'group:admins'
       policy: 'two_factor'
 
+    - domain: 'goaccess.vakhrushev.me'
+      subject: 'group:admins'
+      policy: 'two_factor'
+
     - domain: 'wanderbase.vakhrushev.me'
       subject: 'group:admins'
       policy: 'two_factor'

files/caddyproxy/Caddyfile.j2

@@ -12,12 +12,32 @@
 }
 
 # -------------------------------------------------------------------
+# Snippets
+# -------------------------------------------------------------------
+
+# Shared access log for all sites; consumed by GoAccess.
+# Mode 644 lets read-only consumers (goaccess and ad-hoc host-side tail)
+# read the file; lumberjack would otherwise default to 0600.
+(access_log) {
+    log {
+        output file /var/log/caddy/access.log {
+            mode 644
+            roll_size 100mib
+            roll_keep 10
+            roll_keep_for 720h
+        }
+        format json
+    }
+}
+
+# -------------------------------------------------------------------
 # Applications
 # -------------------------------------------------------------------
 
 vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     # Matrix federation delegation: tells other servers/clients that the
     # homeserver for vakhrushev.me lives at matrix.vakhrushev.me.
     # https://spec.matrix.org/latest/server-server-api/#server-discovery
@@ -43,6 +63,7 @@ vakhrushev.me {
 matrix.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to tuwunel_app:6167
     }
@@ -51,12 +72,14 @@ matrix.vakhrushev.me {
 auth.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+    
     reverse_proxy authelia_app:9091
 }
 
 status.vakhrushev.me, :29999 {
     tls anwinged@ya.ru
     import access_log
+
     forward_auth authelia_app:9091 {
         uri /api/authz/forward-auth
         copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
@@ -68,6 +91,7 @@ status.vakhrushev.me, :29999 {
 git.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to gitea_app:3000
     }
@@ -76,6 +100,7 @@ git.vakhrushev.me {
 outline.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to outline_app:3000
     }
@@ -84,6 +109,7 @@ outline.vakhrushev.me {
 gramps.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to gramps_app:5000
     }
@@ -92,6 +118,7 @@ gramps.vakhrushev.me {
 miniflux.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to miniflux_app:8080
     }
@@ -100,6 +127,7 @@ miniflux.vakhrushev.me {
 wakapi.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to wakapi_app:3000
     }
@@ -108,6 +136,7 @@ wakapi.vakhrushev.me {
 wanderer.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to wanderer_web:3000
     }
@@ -116,6 +145,7 @@ wanderer.vakhrushev.me {
 memos.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to memos_app:5230
     }
@@ -124,6 +154,7 @@ memos.vakhrushev.me {
 remembos.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     forward_auth authelia_app:9091 {
         uri /api/authz/forward-auth
         copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
@@ -137,6 +168,7 @@ remembos.vakhrushev.me {
 calibre.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     reverse_proxy {
         to calibre_web_app:8083
     }
@@ -145,6 +177,7 @@ calibre.vakhrushev.me {
 wanderbase.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     forward_auth authelia_app:9091 {
         uri /api/authz/forward-auth
         copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
@@ -158,6 +191,7 @@ wanderbase.vakhrushev.me {
 rssbridge.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     forward_auth authelia_app:9091 {
         uri /api/authz/forward-auth
         copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
@@ -171,6 +205,7 @@ rssbridge.vakhrushev.me {
 dozzle.vakhrushev.me {
     tls anwinged@ya.ru
     import access_log
+
     forward_auth authelia_app:9091 {
         uri /api/authz/forward-auth
         copy_headers Remote-User Remote-Groups Remote-Email Remote-Name Remote-Filter
@@ -180,3 +215,21 @@ dozzle.vakhrushev.me {
 }
 
 goaccess.vakhrushev.me {
+    tls anwinged@ya.ru
+    import access_log
+
+    forward_auth authelia_app:9091 {
+        uri /api/authz/forward-auth
+        copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
+    }
+
+    @websocket {
+        header Connection *Upgrade*
+        header Upgrade websocket
+    }
+    reverse_proxy @websocket goaccess_processor:7890
+
+    reverse_proxy goaccess_app:8080
+}
+
+

files/caddyproxy/docker-compose.yml.j2

@@ -14,6 +14,7 @@ services:
       - {{ caddy_file_dir }}:/etc/caddy
       - {{ data_dir }}:/data
       - {{ config_dir }}:/config
+      - {{ caddy_logs_dir }}:/var/log/caddy
     networks:
       - "web_proxy_network"
 

files/goaccess/docker-compose.template.yml

@@ -0,0 +1,40 @@
+services:
+
+  goaccess_processor:
+    image: allinurl/goaccess:1.10.2
+    container_name: goaccess_processor
+    restart: unless-stopped
+    user: "{{ app_owner_uid }}:{{ app_owner_gid }}"
+    command: >
+      --log-format=CADDY
+      --enable-panel=VIRTUAL_HOSTS
+      --real-time-html
+      --port=7890
+      --ws-url=wss://goaccess.vakhrushev.me:443
+      --output=/srv/report/index.html
+      --persist
+      --restore
+      --db-path=/srv/db
+      --no-global-config
+      /srv/logs/access.log
+    volumes:
+      - "{{ caddy_logs_dir }}:/srv/logs:ro"
+      - "{{ db_dir }}:/srv/db"
+      - "{{ report_dir }}:/srv/report"
+    networks:
+      - "web_proxy_network"
+
+  goaccess_app:
+    image: caddy:2.11.2
+    container_name: goaccess_app
+    restart: unless-stopped
+    user: "{{ app_owner_uid }}:{{ app_owner_gid }}"
+    command: caddy file-server --listen :8080 --root /srv --browse
+    volumes:
+      - "{{ report_dir }}:/srv:ro"
+    networks:
+      - "web_proxy_network"
+
+networks:
+  web_proxy_network:
+    external: true

playbook-all-applications.yml

@@ -7,6 +7,9 @@
 - name: 'Configure dozzle'
   ansible.builtin.import_playbook: playbook-dozzle.yml
 
+- name: 'Configure goaccess'
+  ansible.builtin.import_playbook: playbook-goaccess.yml
+
 - name: 'Configure gitea'
   ansible.builtin.import_playbook: playbook-gitea.yml
 

playbook-caddyproxy.yml

@@ -4,6 +4,7 @@
 
   vars_files:
     - vars/secrets.yml
+    - vars/vars.yml
 
   vars:
     app_name: "caddyproxy"
@@ -41,6 +42,35 @@
         - "{{ config_dir }}"
         - "{{ caddy_file_dir }}"
 
+    # Shared HTTP access log directory: caddy writes here, other
+    # containers (goaccess, etc.) mount it read-only. Dir mode 0755
+    # so anyone can list/read; the file mode itself comes from the
+    # `mode 644` option in the Caddyfile log snippet.
+    - name: "Create shared caddy logs directory"
+      ansible.builtin.file:
+        path: "{{ caddy_logs_dir }}"
+        state: "directory"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0755"
+
+    - name: "Find pre-existing caddy log files"
+      ansible.builtin.find:
+        paths: "{{ caddy_logs_dir }}"
+        file_type: "file"
+      register: caddy_log_files
+
+    # Lumberjack created earlier files with 0600 before we set `mode`
+    # in the Caddyfile; relax them so existing rotated archives stay
+    # readable to consumers.
+    - name: "Relax mode on pre-existing caddy log files"
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        mode: "0644"
+      loop: "{{ caddy_log_files.files }}"
+      loop_control:
+        label: "{{ item.path }}"
+
     - name: "Copy caddy file"
       ansible.builtin.template:
         src: "./files/{{ app_name }}/Caddyfile.j2"

playbook-goaccess.yml

@@ -0,0 +1,78 @@
+---
+- name: "Configure goaccess application"
+  hosts: all
+
+  vars_files:
+    - vars/secrets.yml
+    - vars/vars.yml
+
+  vars:
+    app_name: "goaccess"
+    app_user: "{{ app_name }}"
+    app_owner_uid: 1106
+    app_owner_gid: 1106
+    base_dir: "{{ (application_dir, app_name) | path_join }}"
+
+    db_dir: "{{ (base_dir, 'db') | path_join }}"
+    report_dir: "{{ (base_dir, 'report') | path_join }}"
+
+  tasks:
+    - name: "Create user and environment"
+      ansible.builtin.import_role:
+        name: owner
+      vars:
+        owner_name: "{{ app_user }}"
+        owner_uid: "{{ app_owner_uid }}"
+        owner_gid: "{{ app_owner_gid }}"
+        owner_extra_groups: ["docker"]
+
+    - name: "Create internal application directories"
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: "directory"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0770"
+      loop:
+        - "{{ base_dir }}"
+        - "{{ db_dir }}"
+        - "{{ report_dir }}"
+
+    # Earlier runs left root-owned files inside db/report (the
+    # containers used to start as root). Recurse-chown realigns them
+    # so the now-non-root processor can rewrite/restore them.
+    - name: "Realign ownership of generated artefacts"
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: "directory"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        recurse: true
+      loop:
+        - "{{ db_dir }}"
+        - "{{ report_dir }}"
+
+    - name: "Ensure caddy access log exists before goaccess starts"
+      ansible.builtin.copy:
+        content: ""
+        dest: "{{ (caddy_logs_dir, 'access.log') | path_join }}"
+        force: false
+        owner: "root"
+        group: "root"
+        mode: "0644"
+
+    - name: "Copy docker compose file"
+      ansible.builtin.template:
+        src: "./files/{{ app_name }}/docker-compose.template.yml"
+        dest: "{{ base_dir }}/docker-compose.yml"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0640"
+
+    - name: "Run application with docker compose"
+      community.docker.docker_compose_v2:
+        project_src: "{{ base_dir }}"
+        state: "present"
+        remove_orphans: true
+      tags:
+        - run-app

vars/vars.yml

@@ -1,3 +1,8 @@
 ---
 apprise_external_port: 8000
 apprise_external_url: "http://127.0.0.1:{{ apprise_external_port }}"
+
+# Shared HTTP access log written by caddyproxy and consumed by analytics
+# tools (goaccess and so on). Lives under the system log path so it is
+# decoupled from any individual application's data directory.
+caddy_logs_dir: "/var/log/caddy"