VectoDE · Oct 14, 2026 · Oct 14, 2026 · Oct 14, 2026 · Oct 14, 2026 · Oct 14, 2026
@@ -31,7 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10]
-        group: [base, security, scanners, web, runtime, database, database-alt, container]
+        group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network]
         arch: [x64, arm64]
         include:
           - group: base
@@ -54,6 +54,18 @@ jobs:
           # k3s need a kernel of their own (bridges, iptables, cgroups) and run in the VM jobs below.
           - group: container
             modules: podman
+          # restic alone (local repository), then the database dumps of every engine; the conflicting engines in a job
+          # of their own, as above.
+          - group: backup
+            modules: restic mariadb,postgresql,redis,restic,db-dumps
+          - group: backup-alt
+            modules: mysql,valkey,mongodb,restic,db-dumps
+          # The Prometheus stack, Grafana below the admin port of nginx, logs from the journal to Loki, Netdata on the
+          # WireGuard address (LSS_NETDATA_ACCESS=vpn) and smartd without drives.
+          - group: monitoring
+            modules: node-exporter prometheus,alertmanager,grafana nginx,prometheus,grafana loki,alloy wireguard,netdata smartd
+          - group: network
+            modules: wireguard dns-resolver ntp-server
     steps:
       - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1
         with:
@@ -85,7 +97,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install, verify, reinstall, remove
-        run: tests/integration/run.sh host time swap firewall kernel-hardening mac auditd usbguard
+        run: tests/integration/run.sh host time swap firewall kernel-hardening mac auditd usbguard wireguard
 
   profile:
     name: ${{ matrix.runner }} VM / profile minimal-secure
@@ -121,9 +133,10 @@ jobs:
         # Every module at once (profiles/all.conf), on a clean system without the runner's preinstalled databases: on
         # Debian 13 for every change, on every distribution in the nightly and the manual runs. Docker with Portainer,
         # then k3s (after Docker is removed; they conflict) on every supported system, in a clean VM because the runner
-        # already has Docker; minimal-secure on the newest Ubuntu release; the Kubernetes node profile with a Lynis audit.
+        # already has Docker; minimal-secure on the newest Ubuntu release; the Kubernetes node profile with a Lynis audit;
+        # Docker with the monitoring stack, as on a DevOps server.
         include: >-
-          ${{ fromJSON(format('[{0},{1}]', '{"system":"debian-12","run":"containers"},{"system":"debian-13","run":"containers"},{"system":"ubuntu-22.04","run":"containers"},{"system":"ubuntu-24.04","run":"containers"},{"system":"ubuntu-26.04","run":"containers"},{"system":"rocky-9","run":"containers"},{"system":"almalinux-10","run":"containers"},{"system":"ubuntu-26.04","run":"profile minimal-secure"},{"system":"debian-13","run":"profile k8s-node"},{"system":"almalinux-10","run":"profile k8s-node"}',
+          ${{ fromJSON(format('[{0},{1}]', '{"system":"debian-12","run":"containers"},{"system":"debian-13","run":"containers"},{"system":"ubuntu-22.04","run":"containers"},{"system":"ubuntu-24.04","run":"containers"},{"system":"ubuntu-26.04","run":"containers"},{"system":"rocky-9","run":"containers"},{"system":"almalinux-10","run":"containers"},{"system":"ubuntu-26.04","run":"profile minimal-secure"},{"system":"debian-13","run":"profile k8s-node"},{"system":"almalinux-10","run":"profile k8s-node"},{"system":"debian-13","run":"profile devops"},{"system":"almalinux-10","run":"profile devops"}',
           (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
           && '{"system":"debian-12","run":"profile all"},{"system":"debian-13","run":"profile all"},{"system":"rocky-9","run":"profile all"},{"system":"almalinux-10","run":"profile all"},{"system":"ubuntu-22.04","run":"profile all"},{"system":"ubuntu-24.04","run":"profile all"},{"system":"ubuntu-26.04","run":"profile all"}'
           || '{"system":"debian-13","run":"profile all"}')) }}
@@ -153,7 +166,7 @@ jobs:
           RUN: ${{ matrix.run }}
         run: |
           if [[ $RUN == kernel ]]; then
-            tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard
+            tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard wireguard
           elif [[ $RUN == containers ]]; then
             tests/integration/run.sh "$SYSTEM" firewall,docker,portainer firewall,k3s
           else

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2025-2026 VectoDE
+#
+# SPDX-License-Identifier: MIT
+
+# Findings of gitleaks that are no secrets, by fingerprint (commit:file:rule:line).
+# WireGuard public keys from the wg(8) examples, used as peers in the unit tests; public keys are meant to be shared.
+78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371:tests/unit/network.bats:generic-api-key:34
+78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371:tests/unit/network.bats:generic-api-key:35
@@ -66,6 +66,23 @@ now or later.
 - Supported systems: Debian 12 and 13, Ubuntu 22.04, 24.04 and 26.04, Rocky Linux 9, AlmaLinux 10. CI installs,
   verifies, reinstalls and removes every module on each of them.
 - OpenSSF Scorecard, signed-off commits (DCO), REUSE licensing, security policy and governance.
+- Backup modules: `restic` (encrypted daily backups to a local path, SFTP or S3, retention, repository checks, the
+  password and keys in the lss secrets) and `db-dumps` (consistent dumps of every database engine lss installed,
+  taken before each backup).
+- Monitoring modules: `node-exporter`, `prometheus` (with alert rules for down targets, full disks, guard findings and
+  old or failed backups), `alertmanager` (e-mail and webhook), `grafana`, `loki`, `alloy` (Promtail's successor; it
+  ships the journal to Loki), `netdata` (never claimed to Netdata Cloud, no registry or telemetry) and `smartd`.
+  Every interface listens on 127.0.0.1 or, with `access=vpn`, on the WireGuard address, and can be published below
+  the web server's admin port.
+- Network modules: `wireguard` (server with peers from the settings, keys in the lss secrets), `dns-resolver` (Unbound
+  with DNSSEC on localhost, optionally for the WireGuard network, used by the server itself) and `ntp-server` (chrony of
+  the time module serves the WireGuard network or everyone, rate-limited).
+- Profile `devops` (minimal-secure, Docker and the monitoring stack); `webhosting` adds restic and db-dumps, `all`
+  every new module.
+- guard checks the age of the last backup (`LSS_GUARD_BACKUP_MAX_AGE`), runs a restore test every week and exports
+  its results for Prometheus through the node exporter.
+- The services of the backup, monitoring and network modules run with a systemd sandbox drop-in; a service that
+  listens on the WireGuard address starts after the interface.
 
 ### Removed
 

@@ -72,6 +72,22 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res
 | `podman`           | not measured yet   | not measured yet          | not measured yet | – (no daemon)                              |
 | `portainer`        | not measured yet   | not measured yet          | not measured yet | Portainer container, up to 512 MiB         |
 | `k3s`              | not measured yet   | not measured yet          | not measured yet | k3s server, about 1 GiB with Traefik       |
+| `restic`           | 35 s               | 700 MiB (dnf), else 220 MiB | 260 MiB (dnf), else 115 MiB | daily backup, weekly restore test                   |
+| `db-dumps`         | measured with the databases | –                         | –       | dumps before each backup                            |
+| `node-exporter`    | 20 s               | 255 MiB                   | 110 MiB | node exporter, about 20 MiB                         |
+| `prometheus`       | 100 s with alertmanager and grafana | 860 MiB together          | 2.2 GiB together | Prometheus about 75 MiB, and its metrics on disk    |
+| `alertmanager`     | see `prometheus`   | see `prometheus`          | see `prometheus` | Alertmanager about 40 MiB                           |
+| `grafana`          | see `prometheus`   | see `prometheus`          | see `prometheus` | Grafana about 350 MiB, its plugins about 30 MiB each |
+| `loki`             | 50 s with alloy    | 545 MiB together          | 700 MiB together | Loki about 155 MiB, and the logs on disk            |
+| `alloy`            | see `loki`         | see `loki`                | see `loki` | Alloy about 230 MiB                                 |
+| `netdata`          | 45 s               | 440 MiB                   | 680 MiB | Netdata and its plugins, about 220 MiB              |
+| `smartd`           | 15 s               | 140 MiB                   | 5 MiB   | smartd about 5 MiB                                  |
+| `wireguard`        | 15 s               | 260 MiB                   | 100 MiB | – (kernel)                                          |
+| `dns-resolver`     | 45 s               | 180 MiB                   | 40 MiB  | Unbound about 20 MiB                                |
+| `ntp-server`       | 15 s               | 195 MiB                   | 10 MiB  | – (the chronyd of `time`, about 10 MiB)             |
+
+Modules the tests install together are measured together. The Rocky 9 VM in CI is slow throughout (eight minutes even
+for `wireguard`); those outliers are left out.
 
 Web, runtime and database modules, measured the same way. Modules that only work together are tested together; their
 row is the cost of the whole set, including the web server or PHP it needs.
@@ -160,6 +176,9 @@ LSS_DOCKER_USERNS_REMAP=true
 LSS_PORTAINER_ACCESS=vpn
 LSS_K3S_API_ACCESS=vpn
 LSS_K3S_TRAEFIK=false
+LSS_RESTIC_REPOSITORY=sftp:backup@nas.example.com:/srv/restic/server1
+LSS_WIREGUARD_PEERS=laptop:<public key of the laptop>:10.66.0.2
+LSS_GRAFANA_ACCESS=vpn
 ```
 
 `--set` wins over the environment, which wins over the file, which wins over the profile, which wins over the
@@ -194,7 +213,16 @@ first password with `lss secrets show pgadmin-admin-password`), all on the admin
 together with `docker`). See [Containers](#containers).
 
 The [hardening guide](docs/hardening.md) lists what each security module changes, the CIS Benchmark areas it covers
-and where lss deliberately deviates. Backup, monitoring and mail follow as further module groups.
+and where lss deliberately deviates. Mail follows as a further module group.
+
+**Backup:** `restic` (daily, encrypted backups to a local path, SFTP or S3) and `db-dumps` (dumps of the databases
+before each backup). See [Backup](#backup).
+
+**Monitoring:** `node-exporter`, `prometheus`, `alertmanager`, `grafana`, `loki`, `alloy`, `netdata`, `smartd`. See
+[Monitoring](#monitoring).
+
+**Network:** `wireguard` (VPN server), `dns-resolver` (Unbound with DNSSEC), `ntp-server` (chrony serves the time).
+See [WireGuard, DNS and NTP](#wireguard-dns-and-ntp).
 
 ### Web sites
 
@@ -246,6 +274,74 @@ sudo lss install --profile k8s-node --set LSS_ADMIN_USER_NAME=alice --set LSS_AD
 With `docker` or `podman` installed, the guard scans every image daily with Trivy and reports HIGH and CRITICAL
 vulnerabilities per image.
 
+### Backup
+
+`restic` backs up `LSS_RESTIC_PATHS` (configuration, home directories, web roots, the database dumps) every night at
+`LSS_RESTIC_TIME`, keeps 7 daily, 4 weekly and 6 monthly snapshots and checks the repository after each run. The
+repository is a local path (`/var/backups/lss-restic` by default; put it on another disk), `sftp:USER@HOST:/PATH` or
+`s3:https://HOST/BUCKET/PATH`. lss generates the repository password; keep a copy outside the server, since no backup
+can be restored without it:
+
+```bash
+sudo lss secrets show restic-password
+```
+
+For SFTP lss creates an SSH key and prints its public half, which the backup host must accept. For S3,
+`LSS_RESTIC_S3_KEY_ID` and `LSS_RESTIC_S3_SECRET` go into the lss secrets. `/usr/local/lib/lss/backup restic
+snapshots` runs restic with the settings of the server, and `/usr/local/lib/lss/backup restore-test` restores the
+latest backup's test file and reads back 5 % of the data.
+
+`db-dumps` dumps every database engine lss installed (MariaDB, MySQL, PostgreSQL, MongoDB, Redis, Valkey) to
+`/var/backups/lss-db` before each backup. A database added later is dumped without further setup.
+
+### Monitoring
+
+Prometheus scrapes the node exporter, itself, Alertmanager and Loki, and alerts on unreachable targets, full disks,
+low memory, guard findings and backups that failed, are older than 30 hours or did not pass the restore test.
+Alertmanager sends the alerts to `LSS_ALERTMANAGER_EMAIL` or `LSS_ALERTMANAGER_WEBHOOK`. Grafana starts with
+Prometheus, Alertmanager and Loki as data sources; the user is `admin`, the first password `lss secrets show
+grafana-admin-password`. Alloy ships the systemd journal to Loki (Promtail reached its end of life in March 2026).
+
+| Module          | Address                                  | `LSS_<MODULE>_WEB_ADMIN=true` |
+| --------------- | ---------------------------------------- | ----------------------------- |
+| `prometheus`    | `http://127.0.0.1:9090/prometheus/`      | `/prometheus/` on port 8090   |
+| `alertmanager`  | `http://127.0.0.1:9093/alertmanager/`    | `/alertmanager/` on port 8090 |
+| `grafana`       | `http://127.0.0.1:3000/grafana/`         | `/grafana/` on port 8090      |
+| `netdata`       | `http://127.0.0.1:19999/`                | `/netdata/` on port 8090      |
+| `node-exporter` | `http://127.0.0.1:9100/metrics`          | –                             |
+| `loki`          | `http://127.0.0.1:3100`                  | –                             |
+| `alloy`         | `http://127.0.0.1:12345` (always local)  | –                             |
+
+With `LSS_<MODULE>_ACCESS=vpn` a module listens on the server's WireGuard address instead (the first address of
+`LSS_WIREGUARD_NETWORK`, 10.66.0.1 by default) and the firewall opens its port to the WireGuard network only; Netdata
+keeps 127.0.0.1 as well. That needs the `wireguard` module. Debian and Ubuntu get Prometheus, Alertmanager and the
+node exporter from the distribution, the RHEL family the checksum-verified upstream releases; Grafana, Loki and Alloy
+come from Grafana's repository and Netdata from Netdata's, both with pinned keys. `smartd` watches the drives, also in
+virtual machines, where it usually finds none.
+
+The profile `devops` installs `minimal-secure`, `docker` and the monitoring stack without Netdata:
+
+```bash
+sudo lss install --profile devops --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github
+```
+
+### WireGuard, DNS and NTP
+
+`wireguard` sets up `wg0` with the server's key in the lss secrets and one preshared key per peer. Each peer is
+`NAME:PUBLIC_KEY:ADDRESS` in `LSS_WIREGUARD_PEERS`; lss prints the settings a peer needs (the server's public key,
+endpoint and port) after every change, and `lss secrets show wireguard-psk-NAME` the peer's preshared key. Peers reach
+the server, not the networks behind it: IP forwarding stays off.
+
+`dns-resolver` runs Unbound on 127.0.0.1 as a validating (DNSSEC) resolver and makes the server use it, through
+systemd-resolved, NetworkManager or `/etc/resolv.conf`; removing the module restores the previous resolver. Where
+Unbound reaches no DNS server on the Internet (outgoing DNS blocked), the server keeps its previous resolver and lss
+warns; `LSS_DNS_RESOLVER_FORWARD` then names resolvers Unbound can reach.
+`LSS_DNS_RESOLVER_ACCESS=vpn` serves the WireGuard network too, `LSS_DNS_RESOLVER_FORWARD` forwards to other
+resolvers, over TLS with `LSS_DNS_RESOLVER_FORWARD_TLS=true`.
+
+`ntp-server` opens chrony of the `time` module to the WireGuard network (or to everyone with
+`LSS_NTP_SERVER_ACCESS=public`), with rate limiting.
+
 ## Guard
 
 The `guard` module checks the server every day: it runs `lss verify`, repairs modules that fail or whose files were
@@ -256,6 +352,10 @@ what the daily AIDE, rkhunter and ClamAV runs found, and reports more failed SSH
 `LSS_GUARD_LOGIN_THRESHOLD` with their main sources. Successful SSH logins and active fail2ban or CrowdSec bans are
 listed as notes. Findings go to a webhook (Slack, Teams, Mattermost, Discord, ntfy) or by e-mail.
 
+With `restic` installed, the guard reports a last successful backup older than `LSS_GUARD_BACKUP_MAX_AGE` hours (30)
+and runs a restore test every week. With `node-exporter` installed, the number of findings goes to Prometheus as
+well, where an alert rule picks it up.
+
 ## Documentation
 
 - [Architecture](docs/architecture.md): how a run works, where lss keeps its state

@@ -93,3 +93,13 @@ k3s-selinux-el10 any 7ab503adacf07c3235ba543b94b6a109ec7b192596b887c200c90ab7a8b
 # as the asset digests GitHub reports (2026-10-13). 0.75.0 was out for less than two weeks then.
 trivy-0.74.0 x86_64 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-64bit.tar.gz
 trivy-0.74.0 aarch64 b94ce1976bbf3c15b514b605ee88be7c6d94a29be2302847ff01cb794d47aad5 https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-ARM64.tar.gz
+
+# Prometheus 3.13.4 (the 3.13 long-term support release; 3.13.4 is younger than two weeks but fixes two security
+# issues), Alertmanager 0.34.1 and node_exporter 1.12.1 for the RHEL family, which has no packages for them: SHA-256 from
+# sha256sums.txt of each GitHub release, the same as the asset digests GitHub reports (2026-10-14).
+prometheus-3.13.4 x86_64 87f21a66f96c597a189cef8d640e8921b621fc17a9feff707c332c4f3b3ddd56 https://github.com/prometheus/prometheus/releases/download/v3.13.4/prometheus-3.13.4.linux-amd64.tar.gz
+prometheus-3.13.4 aarch64 ffa86c7d6e7f7e7dc8a79c81c5c2a6c9164464b79335f25613f4327c863b8489 https://github.com/prometheus/prometheus/releases/download/v3.13.4/prometheus-3.13.4.linux-arm64.tar.gz
+alertmanager-0.34.1 x86_64 265b9d1e55ef0d5306a436018af6d2b686c2ce051f03d968f7464ecb1372a7e8 https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-amd64.tar.gz
+alertmanager-0.34.1 aarch64 d98d6cbaf52151c7e76e24355fec88b11cebcb9875d4cdd8b76ddce7a7e5535c https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-arm64.tar.gz
+node_exporter-1.12.1 x86_64 b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz
+node_exporter-1.12.1 aarch64 ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-arm64.tar.gz