Netdata + Caddy: add monitoring for http-server

files/caddyproxy/Caddyfile.j2

@@ -4,6 +4,12 @@
 {
     grace_period 15s
 
+    admin :2019
+
+    # Enable metrics in Prometheus format
+    # https://caddyserver.com/docs/metrics
+    metrics
+}
 
 # -------------------------------------------------------------------
 # Applications

files/netdata/docker-compose.yml.j2

@@ -0,0 +1,35 @@
+
+services:
+
+  netdata:
+    image: netdata/netdata:v2.5.0
+    container_name: netdata
+    restart: unless-stopped
+    cap_add:
+      - SYS_PTRACE
+      - SYS_ADMIN
+    security_opt:
+      - apparmor:unconfined
+    networks:
+      - "{{ web_proxy_network }}"
+    volumes:
+      - "{{ config_dir }}:/etc/netdata"
+      - "{{ (data_dir, 'lib') | path_join }}:/var/lib/netdata"
+      - "{{ (data_dir, 'cache') | path_join }}:/var/cache/netdata"
+      # Netdata system volumes 
+      - "/:/host/root:ro,rslave"
+      - "/etc/group:/host/etc/group:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "/etc/os-release:/host/etc/os-release:ro"
+      - "/etc/passwd:/host/etc/passwd:ro"
+      - "/proc:/host/proc:ro"
+      - "/run/dbus:/run/dbus:ro"
+      - "/sys:/host/sys:ro"
+      - "/var/log:/host/var/log:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+    environment:
+      PGID: "{{ netdata_docker_group_output.stdout | default(999) }}"
+
+networks:
+  {{ web_proxy_network }}:
+    external: true

files/netdata/go.d/prometheus.conf

@@ -0,0 +1,9 @@
+update_every: 5
+autodetection_retry: 0
+
+jobs:
+  - name: caddyproxy
+    url: http://caddyproxy:2019/metrics
+    selector:
+      allow:
+        - "caddy_http_*"

files/netdata/netdata.conf.j2

@@ -0,0 +1,687 @@
+# netdata configuration
+#
+# You can download the latest version of this file, using:
+#
+#  wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
+# or
+#  curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
+#
+# You can uncomment and change any of the options below.
+# The value shown in the commented settings, is the default value.
+#
+
+# global netdata configuration
+
+[global]
+	# run as user = netdata
+	# host access prefix = /host
+	# pthread stack size = 8MiB
+	# cpu cores = 2
+	# libuv worker threads = 16
+	# profile = standalone
+	hostname = {{ host_name }}
+	# glibc malloc arena max for plugins = 1
+	# glibc malloc arena max for netdata = 1
+	# crash reports = all
+	# timezone = Etc/UTC
+	# OOM score = 0
+	# process scheduling policy = keep
+	# is ephemeral node = no
+	# has unstable connection = no
+
+[db]
+	# enable replication = yes
+	# replication period = 1d
+	# replication step = 1h
+	# replication threads = 1
+	# replication prefetch = 10
+	# update every = 1s
+	# db = dbengine
+	# memory deduplication (ksm) = auto
+	# cleanup orphan hosts after = 1h
+	# cleanup ephemeral hosts after = off
+	# cleanup obsolete charts after = 1h
+	# gap when lost iterations above = 1
+	# dbengine page type = gorilla
+	# dbengine page cache size = 32MiB
+	# dbengine extent cache size = off
+	# dbengine enable journal integrity check = no
+	# dbengine use all ram for caches = no
+	# dbengine out of memory protection = 391.99MiB
+	# dbengine use direct io = yes
+	# dbengine journal v2 unmount time = 2m
+	# dbengine pages per extent = 109
+	# storage tiers = 3
+	# dbengine tier backfill = new
+	# dbengine tier 1 update every iterations = 60
+	# dbengine tier 2 update every iterations = 60
+	# dbengine tier 0 retention size = 1024MiB
+	# dbengine tier 0 retention time = 14d
+	# dbengine tier 1 retention size = 1024MiB
+	# dbengine tier 1 retention time = 3mo
+	# dbengine tier 2 retention size = 1024MiB
+	# dbengine tier 2 retention time = 2y
+	# extreme cardinality protection = yes
+	# extreme cardinality keep instances = 1000
+	# extreme cardinality min ephemerality = 50
+
+[directories]
+	# config = /etc/netdata
+	# stock config = /usr/lib/netdata/conf.d
+	# log = /var/log/netdata
+	# web = /usr/share/netdata/web
+	# cache = /var/cache/netdata
+	# lib = /var/lib/netdata
+	# cloud.d = /var/lib/netdata/cloud.d
+	# plugins = "/usr/libexec/netdata/plugins.d" "/etc/netdata/custom-plugins.d"
+	# registry = /var/lib/netdata/registry
+	# home = /etc/netdata
+	# stock health config = /usr/lib/netdata/conf.d/health.d
+	# health config = /etc/netdata/health.d
+
+[logs]
+	# facility = daemon
+	# logs flood protection period = 1m
+	# logs to trigger flood protection = 1000
+	# level = info
+	# debug = /var/log/netdata/debug.log
+	# daemon = /var/log/netdata/daemon.log
+	# collector = /var/log/netdata/collector.log
+	# access = /var/log/netdata/access.log
+	# health = /var/log/netdata/health.log
+	# debug flags = 0x0000000000000000
+
+[environment variables]
+	# PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin
+	# PYTHONPATH = 
+	# TZ = :/etc/localtime
+
+[host labels]
+	# name = value
+
+[cloud]
+	# conversation log = no
+	# scope = full
+	# query threads = 6
+	# proxy = env
+
+[ml]
+	# enabled = auto
+	# maximum num samples to train = 21600
+	# minimum num samples to train = 900
+	# train every = 3h
+	# number of models per dimension = 18
+	# delete models older than = 7d
+	# num samples to diff = 1
+	# num samples to smooth = 3
+	# num samples to lag = 5
+	# random sampling ratio = 0.20000
+	# maximum number of k-means iterations = 1000
+	# dimension anomaly score threshold = 0.99000
+	# host anomaly rate threshold = 1.00000
+	# anomaly detection grouping method = average
+	# anomaly detection grouping duration = 5m
+	# num training threads = 1
+	# flush models batch size = 256
+	# dimension anomaly rate suppression window = 15m
+	# dimension anomaly rate suppression threshold = 450
+	# enable statistics charts = yes
+	# hosts to skip from training = !*
+	# charts to skip from training = netdata.*
+	# stream anomaly detection charts = yes
+
+[health]
+	# silencers file = /var/lib/netdata/health.silencers.json
+	# enabled = yes
+	# enable stock health configuration = yes
+	# use summary for notifications = yes
+	# default repeat warning = off
+	# default repeat critical = off
+	# in memory max health log entries = 1000
+	# health log retention = 5d
+	# script to execute on alarm = /usr/libexec/netdata/plugins.d/alarm-notify.sh
+	# enabled alarms = *
+	# run at least every = 10s
+	# postpone alarms during hibernation for = 1m
+
+[web]
+	#| >>> [web].default port <<<
+	#| migrated from: [global].default port
+	# default port = 19999
+
+	# ssl key = /etc/netdata/ssl/key.pem
+	# ssl certificate = /etc/netdata/ssl/cert.pem
+	# tls version = 1.3
+	# tls ciphers = none
+	# ses max tg_des_window = 15
+	# des max tg_des_window = 15
+	# mode = static-threaded
+	# listen backlog = 4096
+	# bind to = *
+	# bearer token protection = no
+	# disconnect idle clients after = 1m
+	# timeout for first request = 1m
+	# accept a streaming request every = off
+	# respect do not track policy = no
+	# x-frame-options response header = 
+	# allow connections from = localhost *
+	# allow connections by dns = heuristic
+	# allow dashboard from = localhost *
+	# allow dashboard by dns = heuristic
+	# allow badges from = *
+	# allow badges by dns = heuristic
+	# allow streaming from = *
+	# allow streaming by dns = heuristic
+	# allow netdata.conf from = localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.* UNKNOWN
+	# allow netdata.conf by dns = no
+	# allow management from = localhost
+	# allow management by dns = heuristic
+	# enable gzip compression = yes
+	# gzip compression strategy = default
+	# gzip compression level = 3
+	# ssl skip certificate verification = no
+	# web server threads = 6
+	# web server max sockets = 262144
+
+[registry]
+	# enabled = no
+	# registry db file = /var/lib/netdata/registry/registry.db
+	# registry log file = /var/lib/netdata/registry/registry-log.db
+	# registry save db every new entries = 1000000
+	# registry expire idle persons = 1y
+	# registry domain = 
+	# registry to announce = https://registry.my-netdata.io
+	# registry hostname = 7171b7f9fc69
+	# verify browser cookies support = yes
+	# enable cookies SameSite and Secure = yes
+	# max URL length = 1024
+	# max URL name length = 50
+	# netdata management api key file = /var/lib/netdata/netdata.api.key
+	# allow from = *
+	# allow by dns = heuristic
+
+[pulse]
+	# extended = no
+	# update every = 1s
+
+[plugins]
+	# idlejitter = yes
+	# netdata pulse = yes
+	# profile = no
+	# tc = yes
+	# diskspace = yes
+	# proc = yes
+	# cgroups = yes
+	# timex = yes
+	# statsd = yes
+	# enable running new plugins = yes
+	# check for new plugins every = 1m
+	# slabinfo = no
+	# freeipmi = no
+	# python.d = yes
+	# go.d = yes
+	# apps = yes
+	# systemd-journal = yes
+	# network-viewer = yes
+	# charts.d = yes
+	# debugfs = yes
+	# perf = yes
+	# ioping = yes
+
+[statsd]
+	# update every (flushInterval) = 1s
+	# udp messages to process at once = 10
+	# create private charts for metrics matching = *
+	# max private charts hard limit = 1000
+	# set charts as obsolete after = off
+	# decimal detail = 1000
+	# disconnect idle tcp clients after = 10m
+	# private charts hidden = no
+	# histograms and timers percentile (percentThreshold) = 95.00000
+	# dictionaries max unique dimensions = 200
+	# add dimension for number of events received = no
+	# gaps on gauges (deleteGauges) = no
+	# gaps on counters (deleteCounters) = no
+	# gaps on meters (deleteMeters) = no
+	# gaps on sets (deleteSets) = no
+	# gaps on histograms (deleteHistograms) = no
+	# gaps on timers (deleteTimers) = no
+	# gaps on dictionaries (deleteDictionaries) = no
+	# statsd server max TCP sockets = 262144
+	# listen backlog = 4096
+	# default port = 8125
+	# bind to = udp:localhost tcp:localhost
+
+[plugin:idlejitter]
+	# loop time = 20ms
+
+[plugin:timex]
+	# update every = 10s
+	# clock synchronization state = yes
+	# time offset = yes
+
+[plugin:proc]
+	# /proc/net/dev = yes
+	# /proc/pagetypeinfo = no
+	# /proc/stat = yes
+	# /proc/uptime = yes
+	# /proc/loadavg = yes
+	# /proc/sys/fs/file-nr = yes
+	# /proc/sys/kernel/random/entropy_avail = yes
+	# /run/reboot_required = yes
+	# /proc/pressure = yes
+	# /proc/interrupts = yes
+	# /proc/softirqs = yes
+	# /proc/vmstat = yes
+	# /proc/meminfo = yes
+	# /sys/kernel/mm/ksm = yes
+	# /sys/block/zram = yes
+	# /sys/devices/system/edac/mc = yes
+	# /sys/devices/pci/aer = yes
+	# /sys/devices/system/node = yes
+	# /proc/net/wireless = yes
+	# /proc/net/sockstat = yes
+	# /proc/net/sockstat6 = yes
+	# /proc/net/netstat = yes
+	# /proc/net/sctp/snmp = yes
+	# /proc/net/softnet_stat = yes
+	# /proc/net/ip_vs/stats = yes
+	# /sys/class/infiniband = yes
+	# /proc/net/stat/conntrack = yes
+	# /proc/net/stat/synproxy = yes
+	# /proc/diskstats = yes
+	# /proc/mdstat = yes
+	# /proc/net/rpc/nfsd = yes
+	# /proc/net/rpc/nfs = yes
+	# /proc/spl/kstat/zfs/arcstats = yes
+	# /sys/fs/btrfs = yes
+	# ipc = yes
+	# /sys/class/power_supply = yes
+	# /sys/class/drm = yes
+
+[plugin:cgroups]
+	# update every = 1s
+	# check for new cgroups every = 10s
+	# use unified cgroups = auto
+	# max cgroups to allow = 1000
+	# max cgroups depth to monitor = 0
+	# enable by default cgroups matching =  !*/init.scope  !/system.slice/run-*.scope  *user.slice/docker-* !*user.slice* *.scope  !/machine.slice/*/.control  !/machine.slice/*/payload*  !/machine.slice/*/supervisor  /machine.slice/*.service  */kubepods/pod*/*  */kubepods/*/pod*/*  */*-kubepods-pod*/*  */*-kubepods-*-pod*/*  !*kubepods* !*kubelet*  !*/vcpu*  !*/emulator  !*.mount  !*.partition  !*.service  !*.service/udev  !*.socket  !*.slice  !*.swap  !*.user  !/  !/docker  !*/libvirt  !/lxc  !/lxc/*/*  !/lxc.monitor*  !/lxc.pivot  !/lxc.payload  !*lxcfs.service/.control !/machine  !/qemu  !/system  !/systemd  !/user  * 
+	# enable by default cgroups names matching =  * 
+	# search for cgroups in subpaths matching =  !*/init.scope  !*-qemu  !*.libvirt-qemu  !/init.scope  !/system  !/systemd  !/user  !/lxc/*/*  !/lxc.monitor  !/lxc.payload/*/*  !/lxc.payload.*  * 
+	# script to get cgroup names = /usr/libexec/netdata/plugins.d/cgroup-name.sh
+	# script to get cgroup network interfaces = /usr/libexec/netdata/plugins.d/cgroup-network
+	# run script to rename cgroups matching =  !/  !*.mount  !*.socket  !*.partition  /machine.slice/*.service  !*.service  !*.slice  !*.swap  !*.user  !init.scope  !*.scope/vcpu*  !*.scope/emulator  *.scope  *docker*  *lxc*  *qemu*  */kubepods/pod*/*  */kubepods/*/pod*/*  */*-kubepods-pod*/*  */*-kubepods-*-pod*/*  !*kubepods* !*kubelet*  *.libvirt-qemu  * 
+	# cgroups to match as systemd services =  !/system.slice/*/*.service  /system.slice/*.service 
+
+[plugin:proc:diskspace]
+	# remove charts of unmounted disks = yes
+	# update every = 1s
+	# check for new mount points every = 15s
+	# exclude space metrics on paths = /dev /dev/shm /proc/* /sys/* /var/run/user/* /run/lock /run/user/* /snap/* /var/lib/docker/* /var/lib/containers/storage/* /run/credentials/* /run/containerd/*  /rpool /rpool/*
+	# exclude space metrics on filesystems = *gvfs *gluster* *s3fs *ipfs *davfs2 *httpfs *sshfs *gdfs *moosefs fusectl autofs cgroup cgroup2 hugetlbfs devtmpfs fuse.lxcfs
+	# exclude inode metrics on filesystems = msdosfs msdos vfat overlayfs aufs* *unionfs
+	# space usage for all disks = auto
+	# inodes usage for all disks = auto
+
+[plugin:tc]
+	# script to run to get tc values = /usr/libexec/netdata/plugins.d/tc-qos-helper.sh
+
+[plugin:python.d]
+	# update every = 1s
+	# command options = 
+
+[plugin:go.d]
+	# update every = 1s
+	# command options = 
+
+[plugin:apps]
+	# update every = 1s
+	# command options = 
+
+[plugin:systemd-journal]
+	# update every = 1s
+	# command options = 
+
+[plugin:network-viewer]
+	# update every = 1s
+	# command options = 
+
+[plugin:charts.d]
+	# update every = 1s
+	# command options = 
+
+[plugin:debugfs]
+	# update every = 1s
+	# command options = 
+
+[plugin:perf]
+	# update every = 1s
+	# command options = 
+
+[plugin:ioping]
+	# update every = 1s
+	# command options = 
+
+[plugin:proc:/proc/net/dev]
+	# compressed packets for all interfaces = no
+	# disable by default interfaces matching = lo fireqos* *-ifb fwpr* fwbr* fwln* ifb4*
+
+[plugin:proc:/proc/stat]
+	# cpu utilization = yes
+	# per cpu core utilization = no
+	# cpu interrupts = yes
+	# context switches = yes
+	# processes started = yes
+	# processes running = yes
+	# keep per core files open = yes
+	# keep cpuidle files open = yes
+	# core_throttle_count = auto
+	# package_throttle_count = no
+	# cpu frequency = yes
+	# cpu idle states = no
+	# core_throttle_count filename to monitor = /host/sys/devices/system/cpu/%s/thermal_throttle/core_throttle_count
+	# package_throttle_count filename to monitor = /host/sys/devices/system/cpu/%s/thermal_throttle/package_throttle_count
+	# scaling_cur_freq filename to monitor = /host/sys/devices/system/cpu/%s/cpufreq/scaling_cur_freq
+	# time_in_state filename to monitor = /host/sys/devices/system/cpu/%s/cpufreq/stats/time_in_state
+	# schedstat filename to monitor = /host/proc/schedstat
+	# cpuidle name filename to monitor = /host/sys/devices/system/cpu/cpu%zu/cpuidle/state%zu/name
+	# cpuidle time filename to monitor = /host/sys/devices/system/cpu/cpu%zu/cpuidle/state%zu/time
+	# filename to monitor = /host/proc/stat
+
+[plugin:proc:/proc/uptime]
+	# filename to monitor = /host/proc/uptime
+
+[plugin:proc:/proc/loadavg]
+	# filename to monitor = /host/proc/loadavg
+	# enable load average = yes
+	# enable total processes = yes
+
+[plugin:proc:/proc/sys/fs/file-nr]
+	# filename to monitor = /host/proc/sys/fs/file-nr
+
+[plugin:proc:/proc/sys/kernel/random/entropy_avail]
+	# filename to monitor = /host/proc/sys/kernel/random/entropy_avail
+
+[plugin:proc:/proc/pressure]
+	# base path of pressure metrics = /proc/pressure
+	# enable cpu some pressure = yes
+	# enable cpu full pressure = no
+	# enable memory some pressure = yes
+	# enable memory full pressure = yes
+	# enable io some pressure = yes
+	# enable io full pressure = yes
+	# enable irq some pressure = no
+	# enable irq full pressure = yes
+
+[plugin:proc:/proc/interrupts]
+	# interrupts per core = no
+	# filename to monitor = /host/proc/interrupts
+
+[plugin:proc:/proc/softirqs]
+	# interrupts per core = no
+	# filename to monitor = /host/proc/softirqs
+
+[plugin:proc:/proc/vmstat]
+	# filename to monitor = /host/proc/vmstat
+	# swap i/o = auto
+	# disk i/o = yes
+	# memory page faults = yes
+	# out of memory kills = yes
+	# system-wide numa metric summary = auto
+	# transparent huge pages = auto
+	# zswap i/o = auto
+	# memory ballooning = auto
+	# kernel same memory = auto
+
+[plugin:proc:/sys/devices/system/node]
+	# directory to monitor = /host/sys/devices/system/node
+	# enable per-node numa metrics = auto
+
+[plugin:proc:/proc/meminfo]
+	# system ram = yes
+	# system swap = auto
+	# hardware corrupted ECC = auto
+	# committed memory = yes
+	# writeback memory = yes
+	# kernel memory = yes
+	# slab memory = yes
+	# hugepages = auto
+	# transparent hugepages = auto
+	# memory reclaiming = yes
+	# high low memory = yes
+	# cma memory = auto
+	# direct maps = yes
+	# filename to monitor = /host/proc/meminfo
+
+[plugin:proc:/sys/kernel/mm/ksm]
+	# /sys/kernel/mm/ksm/pages_shared = /host/sys/kernel/mm/ksm/pages_shared
+	# /sys/kernel/mm/ksm/pages_sharing = /host/sys/kernel/mm/ksm/pages_sharing
+	# /sys/kernel/mm/ksm/pages_unshared = /host/sys/kernel/mm/ksm/pages_unshared
+	# /sys/kernel/mm/ksm/pages_volatile = /host/sys/kernel/mm/ksm/pages_volatile
+
+[plugin:proc:/sys/devices/system/edac/mc]
+	# directory to monitor = /host/sys/devices/system/edac/mc
+
+[plugin:proc:/sys/class/pci/aer]
+	# enable root ports = no
+	# enable pci slots = no
+
+[plugin:proc:/proc/net/wireless]
+	# filename to monitor = /host/proc/net/wireless
+	# status for all interfaces = auto
+	# quality for all interfaces = auto
+	# discarded packets for all interfaces = auto
+	# missed beacon for all interface = auto
+
+[plugin:proc:/proc/net/sockstat]
+	# ipv4 sockets = auto
+	# ipv4 TCP sockets = auto
+	# ipv4 TCP memory = auto
+	# ipv4 UDP sockets = auto
+	# ipv4 UDP memory = auto
+	# ipv4 UDPLITE sockets = auto
+	# ipv4 RAW sockets = auto
+	# ipv4 FRAG sockets = auto
+	# ipv4 FRAG memory = auto
+	# update constants every = 1m
+	# filename to monitor = /host/proc/net/sockstat
+
+[plugin:proc:/proc/net/sockstat6]
+	# ipv6 TCP sockets = auto
+	# ipv6 UDP sockets = auto
+	# ipv6 UDPLITE sockets = auto
+	# ipv6 RAW sockets = auto
+	# ipv6 FRAG sockets = auto
+	# filename to monitor = /host/proc/net/sockstat6
+
+[plugin:proc:/proc/net/netstat]
+	# bandwidth = auto
+	# input errors = auto
+	# multicast bandwidth = auto
+	# broadcast bandwidth = auto
+	# multicast packets = auto
+	# broadcast packets = auto
+	# ECN packets = auto
+	# TCP reorders = auto
+	# TCP SYN cookies = auto
+	# TCP out-of-order queue = auto
+	# TCP connection aborts = auto
+	# TCP memory pressures = auto
+	# TCP SYN queue = auto
+	# TCP accept queue = auto
+	# filename to monitor = /host/proc/net/netstat
+
+[plugin:proc:/proc/net/snmp]
+	# ipv4 packets = auto
+	# ipv4 fragments sent = auto
+	# ipv4 fragments assembly = auto
+	# ipv4 errors = auto
+	# ipv4 TCP connections = auto
+	# ipv4 TCP packets = auto
+	# ipv4 TCP errors = auto
+	# ipv4 TCP opens = auto
+	# ipv4 TCP handshake issues = auto
+	# ipv4 UDP packets = auto
+	# ipv4 UDP errors = auto
+	# ipv4 ICMP packets = auto
+	# ipv4 ICMP messages = auto
+	# ipv4 UDPLite packets = auto
+	# filename to monitor = /host/proc/net/snmp
+
+[plugin:proc:/proc/net/snmp6]
+	# ipv6 packets = auto
+	# ipv6 fragments sent = auto
+	# ipv6 fragments assembly = auto
+	# ipv6 errors = auto
+	# ipv6 UDP packets = auto
+	# ipv6 UDP errors = auto
+	# ipv6 UDPlite packets = auto
+	# ipv6 UDPlite errors = auto
+	# bandwidth = auto
+	# multicast bandwidth = auto
+	# broadcast bandwidth = auto
+	# multicast packets = auto
+	# icmp = auto
+	# icmp redirects = auto
+	# icmp errors = auto
+	# icmp echos = auto
+	# icmp group membership = auto
+	# icmp router = auto
+	# icmp neighbor = auto
+	# icmp mldv2 = auto
+	# icmp types = auto
+	# ect = auto
+	# filename to monitor = /host/proc/net/snmp6
+
+[plugin:proc:/proc/net/sctp/snmp]
+	# established associations = auto
+	# association transitions = auto
+	# fragmentation = auto
+	# packets = auto
+	# packet errors = auto
+	# chunk types = auto
+	# filename to monitor = /host/proc/net/sctp/snmp
+
+[plugin:proc:/proc/net/softnet_stat]
+	# softnet_stat per core = no
+	# filename to monitor = /host/proc/net/softnet_stat
+
+[plugin:proc:/proc/net/ip_vs_stats]
+	# IPVS bandwidth = yes
+	# IPVS connections = yes
+	# IPVS packets = yes
+	# filename to monitor = /host/proc/net/ip_vs_stats
+
+[plugin:proc:/sys/class/infiniband]
+	# dirname to monitor = /host/sys/class/infiniband
+	# bandwidth counters = yes
+	# packets counters = yes
+	# errors counters = yes
+	# hardware packets counters = auto
+	# hardware errors counters = auto
+	# monitor only active ports = auto
+	# disable by default interfaces matching = 
+	# refresh ports state every = 30s
+
+[plugin:proc:/proc/net/stat/nf_conntrack]
+	# filename to monitor = /host/proc/net/stat/nf_conntrack
+	# netfilter new connections = no
+	# netfilter connection changes = no
+	# netfilter connection expectations = no
+	# netfilter connection searches = no
+	# netfilter errors = no
+	# netfilter connections = yes
+
+[plugin:proc:/proc/sys/net/netfilter/nf_conntrack_max]
+	# filename to monitor = /host/proc/sys/net/netfilter/nf_conntrack_max
+	# read every seconds = 10
+
+[plugin:proc:/proc/sys/net/netfilter/nf_conntrack_count]
+	# filename to monitor = /host/proc/sys/net/netfilter/nf_conntrack_count
+
+[plugin:proc:/proc/net/stat/synproxy]
+	# SYNPROXY cookies = auto
+	# SYNPROXY SYN received = auto
+	# SYNPROXY connections reopened = auto
+	# filename to monitor = /host/proc/net/stat/synproxy
+
+[plugin:proc:/proc/diskstats]
+	# enable new disks detected at runtime = yes
+	# performance metrics for physical disks = auto
+	# performance metrics for virtual disks = auto
+	# performance metrics for partitions = no
+	# bandwidth for all disks = auto
+	# operations for all disks = auto
+	# merged operations for all disks = auto
+	# i/o time for all disks = auto
+	# queued operations for all disks = auto
+	# utilization percentage for all disks = auto
+	# extended operations for all disks = auto
+	# backlog for all disks = auto
+	# bcache for all disks = auto
+	# bcache priority stats update every = off
+	# remove charts of removed disks = yes
+	# path to get block device = /host/sys/block/%s
+	# path to get block device bcache = /host/sys/block/%s/bcache
+	# path to get virtual block device = /host/sys/devices/virtual/block/%s
+	# path to get block device infos = /host/sys/dev/block/%lu:%lu/%s
+	# path to device mapper = /host/dev/mapper
+	# path to /dev/disk = /host/dev/disk
+	# path to /sys/block = /host/sys/block
+	# path to /dev/disk/by-label = /host/dev/disk/by-label
+	# path to /dev/disk/by-id = /host/dev/disk/by-id
+	# path to /dev/vx/dsk = /host/dev/vx/dsk
+	# name disks by id = no
+	# preferred disk ids = *
+	# exclude disks = loop* ram*
+	# filename to monitor = /host/proc/diskstats
+	# performance metrics for disks with major 252 = yes
+
+[plugin:proc:/proc/mdstat]
+	# faulty devices = yes
+	# nonredundant arrays availability = yes
+	# mismatch count = auto
+	# disk stats = yes
+	# operation status = yes
+	# make charts obsolete = yes
+	# filename to monitor = /host/proc/mdstat
+	# mismatch_cnt filename to monitor = /host/sys/block/%s/md/mismatch_cnt
+
+[plugin:proc:/proc/net/rpc/nfsd]
+	# filename to monitor = /host/proc/net/rpc/nfsd
+
+[plugin:proc:/proc/net/rpc/nfs]
+	# filename to monitor = /host/proc/net/rpc/nfs
+
+[plugin:proc:/proc/spl/kstat/zfs/arcstats]
+	# filename to monitor = /host/proc/spl/kstat/zfs/arcstats
+
+[plugin:proc:/sys/fs/btrfs]
+	# path to monitor = /host/sys/fs/btrfs
+	# check for btrfs changes every = 1m
+	# physical disks allocation = auto
+	# data allocation = auto
+	# metadata allocation = auto
+	# system allocation = auto
+	# commit stats = auto
+	# error stats = auto
+
+[plugin:proc:ipc]
+	# message queues = yes
+	# semaphore totals = yes
+	# shared memory totals = yes
+	# msg filename to monitor = /host/proc/sysvipc/msg
+	# shm filename to monitor = /host/proc/sysvipc/shm
+	# max dimensions in memory allowed = 50
+
+[plugin:proc:/sys/class/power_supply]
+	# battery capacity = yes
+	# battery power = yes
+	# battery charge = no
+	# battery energy = no
+	# power supply voltage = no
+	# keep files open = auto
+	# directory to monitor = /host/sys/class/power_supply
+
+[plugin:proc:/sys/class/drm]
+	# directory to monitor = /host/sys/class/drm

playbook-caddyproxy.yml

@@ -60,8 +60,13 @@
         state: "present"
         remove_orphans: true
 
-    - name: "Reload caddy"
+    # - name: "Reload caddy"
-      community.docker.docker_compose_v2_exec:
+    #   community.docker.docker_compose_v2_exec:
-        project_src: '{{ base_dir }}'
+    #     project_src: '{{ base_dir }}'
-        service: "{{ service_name }}"
+    #     service: "{{ service_name }}"
-        command: caddy reload --config /etc/caddy/Caddyfile
+    #     command: caddy reload --config /etc/caddy/Caddyfile
+
+    - name: "Restart application with docker compose"
+      community.docker.docker_compose_v2:
+        project_src: "{{ base_dir }}"
+        state: "restarted"

playbook-netdata.yml

@@ -7,11 +7,49 @@
     - vars/vars.yml
 
   vars:
-    netdata_version: "v2.4.0"
+    app_name: "netdata"
-    netdata_image: "netdata/netdata:{{ netdata_version }}"
+    app_user: "{{ app_name }}"
-    netdata_exposed_port: "{{ netdata_port }}"
+    base_dir: "/home/{{ app_user }}"
+    config_dir: "{{ (base_dir, 'config') | path_join }}"
+    config_go_d_dir: "{{ (config_dir, 'go.d') | path_join }}"
+    data_dir: "{{ (base_dir, 'data') | path_join }}"
 
   tasks:
+    - name: "Create user and environment"
+      ansible.builtin.import_role:
+        name: owner
+      vars:
+        owner_name: "{{ app_user }}"
+        owner_extra_groups: ["docker"]
+
+    - name: "Create internal application directories"
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: "directory"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0770"
+      loop:
+        - "{{ config_dir }}"
+        - "{{ config_go_d_dir }}"
+        - "{{ data_dir }}"
+
+    - name: "Copy netdata config file"
+      ansible.builtin.template:
+        src: "files/{{ app_name }}/netdata.conf.j2"
+        dest: "{{ config_dir }}/netdata.conf"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0640"
+
+    - name: "Copy prometheus plugin config file"
+      ansible.builtin.copy:
+        src: "files/{{ app_name }}/go.d/prometheus.conf"
+        dest: "{{ config_go_d_dir }}/prometheus.conf"
+        owner: "{{ app_user }}"
+        group: "{{ app_user }}"
+        mode: "0640"
+
     - name: "Grab docker group id."
       ansible.builtin.shell:
         cmd: |
@@ -21,31 +59,21 @@
       register: netdata_docker_group_output
       changed_when: netdata_docker_group_output.rc != 0
 
-    - name: "Create NetData container from {{ netdata_image }}"
+    - name: "Copy docker compose file"
-      community.docker.docker_container:
+      ansible.builtin.template:
-        name: netdata
+        src: "./files/{{ app_name }}/docker-compose.yml.j2"
-        image: "{{ netdata_image }}"
+        dest: "{{ base_dir }}/docker-compose.yml"
-        image_name_mismatch: "recreate"
+        owner: "{{ app_user }}"
-        restart_policy: "unless-stopped"
+        group: "{{ app_user }}"
-        published_ports:
+        mode: "0640"
-          - "127.0.0.1:{{ netdata_exposed_port }}:19999"
+
-        networks:
+    - name: "Run application with docker compose"
-          - name: "{{ web_proxy_network }}"
+      community.docker.docker_compose_v2:
-        volumes:
+        project_src: "{{ base_dir }}"
-          - "/:/host/root:ro,rslave"
+        state: "present"
-          - "/etc/group:/host/etc/group:ro"
+        remove_orphans: true
-          - "/etc/localtime:/etc/localtime:ro"
+
-          - "/etc/os-release:/host/etc/os-release:ro"
+    - name: "Restart application with docker compose"
-          - "/etc/passwd:/host/etc/passwd:ro"
+      community.docker.docker_compose_v2:
-          - "/proc:/host/proc:ro"
+        project_src: "{{ base_dir }}"
-          - "/run/dbus:/run/dbus:ro"
+        state: "restarted"
-          - "/sys:/host/sys:ro"
-          - "/var/log:/host/var/log:ro"
-          - "/var/run/docker.sock:/var/run/docker.sock:ro"
-        capabilities:
-          - "SYS_PTRACE"
-          - "SYS_ADMIN"
-        security_opts:
-          - "apparmor:unconfined"
-        env:
-          PGID: "{{ netdata_docker_group_output.stdout | default(999) }}"

vars/vars.yml

@@ -1,118 +1,119 @@
 $ANSIBLE_VAULT;1.1;AES256
-33323766373637333861393930366564333561663231636433373131353535386464643763396633
+63353966366236363639666631633336386562316663666137376163353862626462363236643930
-6261626463656534366434613865336366323036643731650a306466306261303831356534343133
+6534306465623637303832393765346434303462613666650a343661323330643663653631383038
-36386134346137333762646363393532666239386339393131326164366363386531613935356634
+36666165356130616437343133616638636463376239663539643033643662386537303365626133
-6362633333613839380a323666643935313437613161623338373734323438613062623831613565
+3632633133626464630a623332356234353361353239393235643135313035653664303361306532
-64666231316139653463666234633262393430633231373564623862376566636134626333393034
+31303763633838633738646566336430643565323664303565356164623261343762383361646434
-61343461383739626130353931306536353739353165383737656133383738363235376331636135
+32626465643237623737666565393338613265333131353134333035653433333935633666656662
-64626561386362646334663232313132343534353533313663313836373435316634336539313562
+62626338333462623735336563366164383131323464623662393134383836333834373832623735
-66333732366666663163343263636664326663623135633338616638353532623832356464366137
+32386638643334336234663063656534323933343431363335366433333235633064613664326539
-32383165396562613162373430356331396631323630396263393637373566393935333632386465
+35623465356534363162363833643266666664363638646138393236626532386465323939306139
-38623135316164333563633262666361663234383935343735393637323834356561336633373333
+39346538643664616338363165616638613533326339303337306232373364613234626163653165
-31633333323132393339336365393433663266373363666339643632353232363762326633363661
+62623731326232636530333462646438666630393365663837356461656263316365663935316431
-66373861376561303038636135393132353639386530383338386237383433663334623533636162
+62316266313931663538333837643139326530373139303665616234376330613137656631356363
-32396562653031396463393862633062346463643466353865393933373533323661623039626538
+65653434313763656265373939613736343064616530373530656534396633653737613662356333
-30343438326462626137643835643466643236373836383665396362346135333466623264633033
+61383564666231393531363166393139623663346362643763666538346437353833636663326465
-66626430646632346139663431643930646361363262366436353861643838663764643363306633
+65336534333162666436323665376163663765346433333562323862363566323535303364336666
-30303462316666646261343439336131336234326236356666323336356131386263303333633863
+38373830316566343938626661646463346532326130613733386536336266306336303661633062
-66623834356263623766386431336165653034623264623263626536626333363534323064323037
+65623261383963366266643162376132383137653137393166363766396638376461653062633531
-34393066663664353837346333663162623864383730323566646534336537623162303339653235
+34653336633464626632306566333937393735383265333939396231366662346164323138323365
-34343262373364313361316539653661656435343532353237313465653832313366383338333639
+61386439393836656264613331326561666434386138383035373165623864643838663635306134
-61386230626534386438386262396464656533303630306432376133636132376635613939343838
+38313738343364656665376133356164333865343335613030663138326130313637653766313337
-36393539343431326132616561366466663966386537633537386138623937343932653937623939
+61646365373661646333313363303538326230356466303863306233323263306564633464663061
-39663366306635346233323830643066386565616461346132613835343264353662613233353231
+65346266343834303331616466373934346236393036643437383738613536373864356536653361
-63343066663731663630343034363632663964323831336532663831663065326462613264343665
+34373638313034616566633638373866316436643266623265373137383036613963353838396662
-32386339313563363730303231643166326635323137393134323732353936626438663161333338
+65313762316366333335666363653964623536326336383337626635343233636538363136316164
-64626237656132653364663661363466613861306430333039343030616631663735383836303436
+33303536386433353065383938383138653131316335653739363833333230646635313138613834
-65383334386337626633643135353830306639353162306463313036313263613330316534376266
+35383362643038373265313461666231333161656333653739663066643134306330653739636663
-33326463623539393531303033356237613331663830663636323630383666333964626130333966
+64636338313664643136666631316464626234353032633265343038663733616563383233613238
-31343731356234396165633330613362363339653837633038616330366635363863333332656133
+30653963353531666235393932366462363832363163616531646261663035666261323539666364
-38636238383664653566613737373731313264633632656536656234653766613036343134386636
+64616532656530343638643034633534656436643162323032346530646535326463303463306261
-38643735613762383530363732303738613062336363383038313337326266376461393663633333
+31363637383666353461633963356339616262613233616238386662643839303138396633646263
-30316433613338373362333333383538376631383335346536366338663438313736313564663531
+30623563643463613963363461663434353436356661333435353539376662616637623161323162
-64313331623238313438636563313662623231346439393066386361353236306265343865323066
+61353161623839383866616437626239666435313039313336623735623862326162323831663639
-66383463396134633731643332383064663761653430313935646236386162313366643937633938
+33343032383161666131663462656162303165323935343264623161313137646230653537623730
-62656461363635366139343463653435363161393862353034616562363237636638346565636362
+38313337653933643631323464356634313137613536386561663735336235376432626363353435
-37616365636338616430313638656562613162346163356238306532396333306564633232383535
+33373063356165373430643264643937313735303039626463323632653762303066333034313731
-38613561363664653333626561373661376232363930643934333566313662613835376163616461
+65393730336362316331393266336162366133643162663732373331366261376361306635313162
-36656164656563333662616339663333353466636266633130386334643338346332326334376630
+33613962633164663834623636313932383861386164356161623434653130633362633737616466
-66373031383637353562653837316561373338306438333534376165356663613330316661383137
+62613066333334346639643339303661336439663238643336646234303062633432376335623831
-34386634626663326135383434383762656165393065323136326432613533626338396161623631
+30386531633032633964333832386233623561636538343838366437663837343332333235386362
-31633565373065333834363363363430633162376334343763393639653862363537313432383037
+63393733366562626164646232346431623266663434346438306635616464333332633435386632
-33646333386237306662313066343635386636323964366463386434653263353539393732303134
+33396166343033363337383563623735313337393935633633353065316335316664313831346236
-33383735363331386431303537666139383539326237313535353466393436633061313761386534
+30373930366131656437326137373562326432383162306466646533396561626234346133373133
-65333936613665616139343965366239306132636131393964656136363232656630626234653962
+37316264613939333439343934653137623632653231313166393630613763646163633063646533
-62623363323331383035393266343563383730386166353464323837633063366536376631313166
+39343830666533636133643365376238313164643963363932316332333965653431373463656166
-35613236326232396239626238396161386232363261376366303637326239383330663736633766
+34313333636532356564333664613838393662346239636333323634383732393030613532656339
-63363562383464326138663738383464343131343636363735303866373937653839663264396431
+36396461363563656134373735346230626264366133386534653533383536396663316263666337
-63373232343366356339626466336262383538626366653738313564303034373537383863363166
+65333061623738666239373861666663383939306365323964346237363561346231643864616636
-36363039666534333237353535323438356135303730336233323636396539666436306661613236
+38383365336662306166626632326565326662623730656662366464386634393131613934633831
-62653966653965383233376537666562643736646261636336633661653931636536663765373330
+35396339363237653532306636323639343537306165353464393434363764343932333361356236
-33366536663536613536626466363165396461616231316663653739373233323763373631333161
+35376135346630653762613833303139363830663335336262353465616564626230363463366533
-31383337653634343465616566353665363163316534646132636133373633623866353236313931
+37306535623566623065393133613761386430333334616432646330343239366433616432356636
-66323939633737353037643365653531313131346366316234656464396131333939373362646361
+32613632376530353432373937666164613234616236643639633335363365656165326334336262
-65396230646535336638623066393734623964663933653930386438383237666636303135616334
+37373138626531653764343237393762376239636132656232363636666261343238333033353937
-38366333663334363733386237313233373664373534363930396237323837376332656364666261
+63363261613339393436623534653634646566376337386433656563656466353138633965636234
-31323866666564653861386464343335396663646234386362333562396564636434376261376531
+34613563343937386431313563383431666335393161383766353666373431386138663864393837
-66613437666131346262383861383139396665623264363639336664303863343665636239373331
+33333831343563613832343338343833353331343538326666353334343032663730623133666530
-66333361393539313535656532383032653130633565356534353034613036336566636237333164
+65316335633736376333656264343664376435393131616233303634396330336162373539643164
-31633831646433313930323665373165663639393662383561363133636134383237646235613035
+65323537336439306539366538666139616338613063323830396133373765353433373566616632
-65666438303236366435613562636133383365653630343563303036373033346135343064383433
+38623330306538386630663065373933653431366362333934613662626533333135383737646432
-34356666313264303763663665333761376539346462656635356537616532663339373532376362
+37313363636166306261336437656564396265366132333762643363303065633831623166653264
-64636233306665353066306632363737653034636535363533666464366264383963313937313833
+39316136386533656638303663386262366438663433626536336235323238656235393533633862
-65643064646463313763343366373066663130623866333432366135353132356231366436333733
+31376338376330373737376332376462346136666338336664626536663034366136373536613437
-37373535306134663239313035303832333438363233646239656266346132656364383361313066
+63333035366361353765353962373266333065643330323961333638616535326132663232343737
-32386161326336356331323330326139333938393739383362386239316237346137666231633164
+38663931656634643965326333393565343132373764633164383232393230633933383964613234
-37383638363864303131626461393936616332646239623434376161353736656231633233313964
+33613539616638333261393630316161323434383738363164633137666464383031353737386238
-38383136626337383339303835363130346236323131366564333836303564633735623530663934
+66333835616364636339303234663263663062356135373066646235613865643962663038613432
-35626430303631613565383266363163343563323232653838323365613131663163646432653332
+32626634363738633961373432386437623663373065323133376665306261363431333537336663
-35376335666463646661623864343639393035643636623963343936623963316339623864386538
+36386132333130633536616131656563386236326533393066393436363734343961303939643130
-33373737306437383861633131656131616461333432616235346335316162386534316263303639
+63393236396266633566366232353064633366323237626263373664623134326238386530653965
-37646538356163336636353864623830336532366139306532356133616165336461323737383062
+35366665333032613133366263643335303163386330636665346437633438343438393138396431
-32666164353237363237363232633431376437343832363838386264623965663131326534383461
+63353664376636323066623339306163363435653831353934663533376634633666623738623936
-63343764323231333036663733303634383030383836653462303336656337313662306564373132
+32346162313962393838656638656330356331306639313835356333613439303461393763383762
-35353930343734646633613131623230363662316434373337333434326230383364626165643066
+64313633373364363166383166313238623062613430313038356263353463363366326530343136
-66343461303732663938303966656434653737366338623966373538313936636363353733613931
+30613861633431613063373163343734653139306339323664376266353033343533393361636663
-34323764613131636632653066663533356636363636373064326536376163393231313763363632
+37653439656563663130393266373333623937336563326361663165626631336636306339373733
-37353964323763386233623438386636353930396562653863643161343439636437366535393434
+38633630353563353731306662316466313261323337366561343238323338313561383161313331
-66303763373537323039336430343564663666313131643036623064363837613937336161323562
+31393337656364386236646434396262366339616435333062383338383037396230613436383530
-65363238373462613730653363333566636361303264633461356432313830353238633766396566
+35626534613236336334323666326666616236353432633435633762636339653130333464383261
-38393333653038643836373331646332633364666630613762623163353434326165303038653635
+36373233363133616365336462393661623765666465626333613531343962363136336466316663
-30646231663636653637386138643233666136636438613234333535666161643035663863656466
+34643033653836316138663161373030626536623661346439323336303038323136306131333831
-32353166353165666161663830643331393466333031613233303262623863623861333431313436
+31373863303330383263333066333265343262316636306237356565393263616162656237643066
-30313361613832336532653537653632623633653838393736646539633738363263653165623430
+32613061393365646463313764636363643636363661316264663961333939386431613035343537
-65306366333936323262663365643734356332336430383961363863396134326430303563346462
+61366361363138363533646434653234623134313631396239633832636635663737366138666331
-34303363626435383534653963353636393330373839356438336139616662616462633764363034
+64663739643038633032663531393537626331363763626637306362666637333461366434613930
-65346530636530383163663136386363336366663230316264346664386534633837653436323630
+36373366306166363734313836653738386132653031393963343461323465613633636332366131
-34363666663764313463393635393863623865353035616433356263653664363838666164396234
+36343237313361356531383264666436383861663665396230366261653764333438323930623331
-30333730643162633832303533623531326561373935346234646433363335646562613066343062
+64383763656430653235616664663761343233353633643533643733633633633837346233656138
-36336234653333643830653062343831633033323561313566613866326135316338656133333239
+62383735303662386265393331383862666635356433326163363132343530376664323931316534
-65363664623261636239616439613361346433313133333831616532343130383833353332633331
+37666535363066396633396366323666306262613733356466373336636165616161623239303031
-34383437666335663266303161636162626663303263366261653966623761396235636264346666
+39333232646138386538313934643734623334346538623233313032313464643935313563323364
-35366431643535373462616235313034353864366662643837353163353763636564663862316332
+33643461633566653336633732323164343939313365656163313964356436666264336630636166
-65323133373539313531326137356362356334386332336438373763363836623863393239633030
+62363535366438396232643662303837336239613162333364303938383134363438386637353664
-66386637303430623836636235653138633033346166326335383934643433363563303939666133
+61633766363262626661313162393033393930613233353464366264323661323838316234336438
-62333232376335356564363135656339333337303565356565376337326632346130643631616132
+33663661373931393839386635303538373438386431653238623664323838643966386162323361
-64633037326562393236616236663434623939646535336634646637316632306138383834363032
+64666436323139663839633232346539336634663363633363313338633135653865303434613731
-65383336343533663638616535613638303236613563316635383238376636306266643037333432
+34393262323933626638373238326437646339326330366161346538633737393766643138376434
-62363864613230306335656562356130613966313263646566373864356135653830393431323664
+34623932356364376135383363343137393365656635653438656163333835643265323866653136
-38313234386630653834386531393862386533643339666365626466353332373539653736653333
+61356266656465656238666566373537663939636635326233376164663061303065343239373432
-35396265393736393939633663613965356263343234303437393862356161363832336334303764
+37366165623339316238383664643061313433373162356633373165306533373863313935646239
-65393136656630366262616264613464613436613938646433363864643838663835393638313731
+32663439396634366538353230373265376435396336346534363335393137643530363563653766
-32343835396361626366636362636630633330356263316265326632666434386365356234646131
+36613061343139373861393836396433623663303964633037393734313963616666393839323765
-38316338636339613138623232363361663430303136323734376135366431336364643063303566
+61653761386233373432306131363636396562316130616564356433643832373463653764343838
-35366263633433333537373733313830313433633162333166346435363539353862313464316263
+39373264356466373838616130623763653666383335666165616233653634653562323737653535
-63653263373133633237653633326363656330373838306432333936656564323231376235663839
+62633532663035396236353061373663326663303235653137306133616464396639356462323030
-63353331636531366138646539323565316362656365303139616364393264643130383137383361
+35643739646531376439383638656138633737633038613335343462336138663963343163656339
-61343935373138393632663031633865663733353064343232376565363831356263306532363866
+63333339643462366538616465303331336130616532313063376465383138393232623634313062
-38323366316432366632656237656564396636656331653334323763663035313735343361383833
+62356130633031616232373335326630393631306432396138333239653066343363393336653437
-65333961663766636635353137666239323636373761653864396433393866336432366435336537
+32336236316666376236343865383364373030353835666439663963346563323631346464646366
-35333935636537376363663166316630613331643231653038356362323039643731363436356362
+31363166396133313830356530353865356266373437376632383231383164393637306664616637
-33306131303432343432323566653561303036616631383737366338393761333737653635633463
+66663664633463616432623739393836343865393732626239656638323034393736346632333339
-61633235313831353337323333343062343865343235313538303034346233376466316437343834
+36646233376233616464306532343765613834366434343434353231613030643539363263633266
-38643338653166613262326230646364343362303731316431316432663530623436346665336137
+34623332643863663130666231616663316264623632373364373465306133346166643734343363
-63666230323035653734353431616364363733313033376532323633386630356665633732656662
+36333534383330313466623237336161383664643636353139363636636365343863396161363364
-35633366376237616536386631386433333561623832386236393062663033643439396535373866
+38623366346236393466353838333762333263623738646462616666633937333666386239396334
-64313663643734376632363661653433376464653933636333626530613964613234666530663063
+32326635656662383232333031353738626432313539656164653038353531623063353862653038
-36643335396561336161656562303333323866663466323437303731376561643666336439666563
+65383034396134623431653061646235383264656633333030306562303630316333323237653065
-6637
+66656431376538386339656236326434313166383661353230346562343363343039376666616539
+3861313565366261363864313136653263373037313438353730