diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 84f9fc8..3382e3f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] - group: [base, security, scanners, web, runtime, database, database-alt, container] + group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network] arch: [x64, arm64] include: - group: base @@ -54,6 +54,18 @@ jobs: # k3s need a kernel of their own (bridges, iptables, cgroups) and run in the VM jobs below. - group: container modules: podman + # restic alone (local repository), then the database dumps of every engine; the conflicting engines in a job + # of their own, as above. + - group: backup + modules: restic mariadb,postgresql,redis,restic,db-dumps + - group: backup-alt + modules: mysql,valkey,mongodb,restic,db-dumps + # The Prometheus stack, Grafana below the admin port of nginx, logs from the journal to Loki, Netdata on the + # WireGuard address (LSS_NETDATA_ACCESS=vpn) and smartd without drives. + - group: monitoring + modules: node-exporter prometheus,alertmanager,grafana nginx,prometheus,grafana loki,alloy wireguard,netdata smartd + - group: network + modules: wireguard dns-resolver ntp-server steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -85,7 +97,7 @@ jobs: with: persist-credentials: false - name: Install, verify, reinstall, remove - run: tests/integration/run.sh host time swap firewall kernel-hardening mac auditd usbguard + run: tests/integration/run.sh host time swap firewall kernel-hardening mac auditd usbguard wireguard profile: name: ${{ matrix.runner }} VM / profile minimal-secure @@ -121,9 +133,10 @@ jobs: # Every module at once (profiles/all.conf), on a clean system without the runner's preinstalled databases: on # Debian 13 for every change, on every distribution in the nightly and the manual runs. Docker with Portainer, # then k3s (after Docker is removed; they conflict) on every supported system, in a clean VM because the runner - # already has Docker; minimal-secure on the newest Ubuntu release; the Kubernetes node profile with a Lynis audit. + # already has Docker; minimal-secure on the newest Ubuntu release; the Kubernetes node profile with a Lynis audit; + # Docker with the monitoring stack, as on a DevOps server. include: >- - ${{ fromJSON(format('[{0},{1}]', '{"system":"debian-12","run":"containers"},{"system":"debian-13","run":"containers"},{"system":"ubuntu-22.04","run":"containers"},{"system":"ubuntu-24.04","run":"containers"},{"system":"ubuntu-26.04","run":"containers"},{"system":"rocky-9","run":"containers"},{"system":"almalinux-10","run":"containers"},{"system":"ubuntu-26.04","run":"profile minimal-secure"},{"system":"debian-13","run":"profile k8s-node"},{"system":"almalinux-10","run":"profile k8s-node"}', + ${{ fromJSON(format('[{0},{1}]', '{"system":"debian-12","run":"containers"},{"system":"debian-13","run":"containers"},{"system":"ubuntu-22.04","run":"containers"},{"system":"ubuntu-24.04","run":"containers"},{"system":"ubuntu-26.04","run":"containers"},{"system":"rocky-9","run":"containers"},{"system":"almalinux-10","run":"containers"},{"system":"ubuntu-26.04","run":"profile minimal-secure"},{"system":"debian-13","run":"profile k8s-node"},{"system":"almalinux-10","run":"profile k8s-node"},{"system":"debian-13","run":"profile devops"},{"system":"almalinux-10","run":"profile devops"}', (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '{"system":"debian-12","run":"profile all"},{"system":"debian-13","run":"profile all"},{"system":"rocky-9","run":"profile all"},{"system":"almalinux-10","run":"profile all"},{"system":"ubuntu-22.04","run":"profile all"},{"system":"ubuntu-24.04","run":"profile all"},{"system":"ubuntu-26.04","run":"profile all"}' || '{"system":"debian-13","run":"profile all"}')) }} @@ -153,7 +166,7 @@ jobs: RUN: ${{ matrix.run }} run: | if [[ $RUN == kernel ]]; then - tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard + tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard wireguard elif [[ $RUN == containers ]]; then tests/integration/run.sh "$SYSTEM" firewall,docker,portainer firewall,k3s else diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..aab7c3e --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Findings of gitleaks that are no secrets, by fingerprint (commit:file:rule:line). +# WireGuard public keys from the wg(8) examples, used as peers in the unit tests; public keys are meant to be shared. +78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371:tests/unit/network.bats:generic-api-key:34 +78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371:tests/unit/network.bats:generic-api-key:35 diff --git a/CHANGELOG.md b/CHANGELOG.md index ff08c50..d9765ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,23 @@ now or later. - Supported systems: Debian 12 and 13, Ubuntu 22.04, 24.04 and 26.04, Rocky Linux 9, AlmaLinux 10. CI installs, verifies, reinstalls and removes every module on each of them. - OpenSSF Scorecard, signed-off commits (DCO), REUSE licensing, security policy and governance. +- Backup modules: `restic` (encrypted daily backups to a local path, SFTP or S3, retention, repository checks, the + password and keys in the lss secrets) and `db-dumps` (consistent dumps of every database engine lss installed, + taken before each backup). +- Monitoring modules: `node-exporter`, `prometheus` (with alert rules for down targets, full disks, guard findings and + old or failed backups), `alertmanager` (e-mail and webhook), `grafana`, `loki`, `alloy` (Promtail's successor; it + ships the journal to Loki), `netdata` (never claimed to Netdata Cloud, no registry or telemetry) and `smartd`. + Every interface listens on 127.0.0.1 or, with `access=vpn`, on the WireGuard address, and can be published below + the web server's admin port. +- Network modules: `wireguard` (server with peers from the settings, keys in the lss secrets), `dns-resolver` (Unbound + with DNSSEC on localhost, optionally for the WireGuard network, used by the server itself) and `ntp-server` (chrony of + the time module serves the WireGuard network or everyone, rate-limited). +- Profile `devops` (minimal-secure, Docker and the monitoring stack); `webhosting` adds restic and db-dumps, `all` + every new module. +- guard checks the age of the last backup (`LSS_GUARD_BACKUP_MAX_AGE`), runs a restore test every week and exports + its results for Prometheus through the node exporter. +- The services of the backup, monitoring and network modules run with a systemd sandbox drop-in; a service that + listens on the WireGuard address starts after the interface. ### Removed diff --git a/README.md b/README.md index 6bf6a08..9f72032 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,22 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `podman` | not measured yet | not measured yet | not measured yet | – (no daemon) | | `portainer` | not measured yet | not measured yet | not measured yet | Portainer container, up to 512 MiB | | `k3s` | not measured yet | not measured yet | not measured yet | k3s server, about 1 GiB with Traefik | +| `restic` | 35 s | 700 MiB (dnf), else 220 MiB | 260 MiB (dnf), else 115 MiB | daily backup, weekly restore test | +| `db-dumps` | measured with the databases | – | – | dumps before each backup | +| `node-exporter` | 20 s | 255 MiB | 110 MiB | node exporter, about 20 MiB | +| `prometheus` | 100 s with alertmanager and grafana | 860 MiB together | 2.2 GiB together | Prometheus about 75 MiB, and its metrics on disk | +| `alertmanager` | see `prometheus` | see `prometheus` | see `prometheus` | Alertmanager about 40 MiB | +| `grafana` | see `prometheus` | see `prometheus` | see `prometheus` | Grafana about 350 MiB, its plugins about 30 MiB each | +| `loki` | 50 s with alloy | 545 MiB together | 700 MiB together | Loki about 155 MiB, and the logs on disk | +| `alloy` | see `loki` | see `loki` | see `loki` | Alloy about 230 MiB | +| `netdata` | 45 s | 440 MiB | 680 MiB | Netdata and its plugins, about 220 MiB | +| `smartd` | 15 s | 140 MiB | 5 MiB | smartd about 5 MiB | +| `wireguard` | 15 s | 260 MiB | 100 MiB | – (kernel) | +| `dns-resolver` | 45 s | 180 MiB | 40 MiB | Unbound about 20 MiB | +| `ntp-server` | 15 s | 195 MiB | 10 MiB | – (the chronyd of `time`, about 10 MiB) | + +Modules the tests install together are measured together. The Rocky 9 VM in CI is slow throughout (eight minutes even +for `wireguard`); those outliers are left out. Web, runtime and database modules, measured the same way. Modules that only work together are tested together; their row is the cost of the whole set, including the web server or PHP it needs. @@ -160,6 +176,9 @@ LSS_DOCKER_USERNS_REMAP=true LSS_PORTAINER_ACCESS=vpn LSS_K3S_API_ACCESS=vpn LSS_K3S_TRAEFIK=false +LSS_RESTIC_REPOSITORY=sftp:backup@nas.example.com:/srv/restic/server1 +LSS_WIREGUARD_PEERS=laptop::10.66.0.2 +LSS_GRAFANA_ACCESS=vpn ``` `--set` wins over the environment, which wins over the file, which wins over the profile, which wins over the @@ -194,7 +213,16 @@ first password with `lss secrets show pgadmin-admin-password`), all on the admin together with `docker`). See [Containers](#containers). The [hardening guide](docs/hardening.md) lists what each security module changes, the CIS Benchmark areas it covers -and where lss deliberately deviates. Backup, monitoring and mail follow as further module groups. +and where lss deliberately deviates. Mail follows as a further module group. + +**Backup:** `restic` (daily, encrypted backups to a local path, SFTP or S3) and `db-dumps` (dumps of the databases +before each backup). See [Backup](#backup). + +**Monitoring:** `node-exporter`, `prometheus`, `alertmanager`, `grafana`, `loki`, `alloy`, `netdata`, `smartd`. See +[Monitoring](#monitoring). + +**Network:** `wireguard` (VPN server), `dns-resolver` (Unbound with DNSSEC), `ntp-server` (chrony serves the time). +See [WireGuard, DNS and NTP](#wireguard-dns-and-ntp). ### Web sites @@ -246,6 +274,74 @@ sudo lss install --profile k8s-node --set LSS_ADMIN_USER_NAME=alice --set LSS_AD With `docker` or `podman` installed, the guard scans every image daily with Trivy and reports HIGH and CRITICAL vulnerabilities per image. +### Backup + +`restic` backs up `LSS_RESTIC_PATHS` (configuration, home directories, web roots, the database dumps) every night at +`LSS_RESTIC_TIME`, keeps 7 daily, 4 weekly and 6 monthly snapshots and checks the repository after each run. The +repository is a local path (`/var/backups/lss-restic` by default; put it on another disk), `sftp:USER@HOST:/PATH` or +`s3:https://HOST/BUCKET/PATH`. lss generates the repository password; keep a copy outside the server, since no backup +can be restored without it: + +```bash +sudo lss secrets show restic-password +``` + +For SFTP lss creates an SSH key and prints its public half, which the backup host must accept. For S3, +`LSS_RESTIC_S3_KEY_ID` and `LSS_RESTIC_S3_SECRET` go into the lss secrets. `/usr/local/lib/lss/backup restic +snapshots` runs restic with the settings of the server, and `/usr/local/lib/lss/backup restore-test` restores the +latest backup's test file and reads back 5 % of the data. + +`db-dumps` dumps every database engine lss installed (MariaDB, MySQL, PostgreSQL, MongoDB, Redis, Valkey) to +`/var/backups/lss-db` before each backup. A database added later is dumped without further setup. + +### Monitoring + +Prometheus scrapes the node exporter, itself, Alertmanager and Loki, and alerts on unreachable targets, full disks, +low memory, guard findings and backups that failed, are older than 30 hours or did not pass the restore test. +Alertmanager sends the alerts to `LSS_ALERTMANAGER_EMAIL` or `LSS_ALERTMANAGER_WEBHOOK`. Grafana starts with +Prometheus, Alertmanager and Loki as data sources; the user is `admin`, the first password `lss secrets show +grafana-admin-password`. Alloy ships the systemd journal to Loki (Promtail reached its end of life in March 2026). + +| Module | Address | `LSS__WEB_ADMIN=true` | +| --------------- | ---------------------------------------- | ----------------------------- | +| `prometheus` | `http://127.0.0.1:9090/prometheus/` | `/prometheus/` on port 8090 | +| `alertmanager` | `http://127.0.0.1:9093/alertmanager/` | `/alertmanager/` on port 8090 | +| `grafana` | `http://127.0.0.1:3000/grafana/` | `/grafana/` on port 8090 | +| `netdata` | `http://127.0.0.1:19999/` | `/netdata/` on port 8090 | +| `node-exporter` | `http://127.0.0.1:9100/metrics` | – | +| `loki` | `http://127.0.0.1:3100` | – | +| `alloy` | `http://127.0.0.1:12345` (always local) | – | + +With `LSS__ACCESS=vpn` a module listens on the server's WireGuard address instead (the first address of +`LSS_WIREGUARD_NETWORK`, 10.66.0.1 by default) and the firewall opens its port to the WireGuard network only; Netdata +keeps 127.0.0.1 as well. That needs the `wireguard` module. Debian and Ubuntu get Prometheus, Alertmanager and the +node exporter from the distribution, the RHEL family the checksum-verified upstream releases; Grafana, Loki and Alloy +come from Grafana's repository and Netdata from Netdata's, both with pinned keys. `smartd` watches the drives, also in +virtual machines, where it usually finds none. + +The profile `devops` installs `minimal-secure`, `docker` and the monitoring stack without Netdata: + +```bash +sudo lss install --profile devops --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github +``` + +### WireGuard, DNS and NTP + +`wireguard` sets up `wg0` with the server's key in the lss secrets and one preshared key per peer. Each peer is +`NAME:PUBLIC_KEY:ADDRESS` in `LSS_WIREGUARD_PEERS`; lss prints the settings a peer needs (the server's public key, +endpoint and port) after every change, and `lss secrets show wireguard-psk-NAME` the peer's preshared key. Peers reach +the server, not the networks behind it: IP forwarding stays off. + +`dns-resolver` runs Unbound on 127.0.0.1 as a validating (DNSSEC) resolver and makes the server use it, through +systemd-resolved, NetworkManager or `/etc/resolv.conf`; removing the module restores the previous resolver. Where +Unbound reaches no DNS server on the Internet (outgoing DNS blocked), the server keeps its previous resolver and lss +warns; `LSS_DNS_RESOLVER_FORWARD` then names resolvers Unbound can reach. +`LSS_DNS_RESOLVER_ACCESS=vpn` serves the WireGuard network too, `LSS_DNS_RESOLVER_FORWARD` forwards to other +resolvers, over TLS with `LSS_DNS_RESOLVER_FORWARD_TLS=true`. + +`ntp-server` opens chrony of the `time` module to the WireGuard network (or to everyone with +`LSS_NTP_SERVER_ACCESS=public`), with rate limiting. + ## Guard The `guard` module checks the server every day: it runs `lss verify`, repairs modules that fail or whose files were @@ -256,6 +352,10 @@ what the daily AIDE, rkhunter and ClamAV runs found, and reports more failed SSH `LSS_GUARD_LOGIN_THRESHOLD` with their main sources. Successful SSH logins and active fail2ban or CrowdSec bans are listed as notes. Findings go to a webhook (Slack, Teams, Mattermost, Discord, ntfy) or by e-mail. +With `restic` installed, the guard reports a last successful backup older than `LSS_GUARD_BACKUP_MAX_AGE` hours (30) +and runs a restore test every week. With `node-exporter` installed, the number of findings goes to Prometheus as +well, where an alert rule picks it up. + ## Documentation - [Architecture](docs/architecture.md): how a run works, where lss keeps its state diff --git a/data/checksums.txt b/data/checksums.txt index 2e3464a..8b59d66 100644 --- a/data/checksums.txt +++ b/data/checksums.txt @@ -93,3 +93,13 @@ k3s-selinux-el10 any 7ab503adacf07c3235ba543b94b6a109ec7b192596b887c200c90ab7a8b # as the asset digests GitHub reports (2026-10-13). 0.75.0 was out for less than two weeks then. trivy-0.74.0 x86_64 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-64bit.tar.gz trivy-0.74.0 aarch64 b94ce1976bbf3c15b514b605ee88be7c6d94a29be2302847ff01cb794d47aad5 https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-ARM64.tar.gz + +# Prometheus 3.13.4 (the 3.13 long-term support release; 3.13.4 is younger than two weeks but fixes two security +# issues), Alertmanager 0.34.1 and node_exporter 1.12.1 for the RHEL family, which has no packages for them: SHA-256 from +# sha256sums.txt of each GitHub release, the same as the asset digests GitHub reports (2026-10-14). +prometheus-3.13.4 x86_64 87f21a66f96c597a189cef8d640e8921b621fc17a9feff707c332c4f3b3ddd56 https://github.com/prometheus/prometheus/releases/download/v3.13.4/prometheus-3.13.4.linux-amd64.tar.gz +prometheus-3.13.4 aarch64 ffa86c7d6e7f7e7dc8a79c81c5c2a6c9164464b79335f25613f4327c863b8489 https://github.com/prometheus/prometheus/releases/download/v3.13.4/prometheus-3.13.4.linux-arm64.tar.gz +alertmanager-0.34.1 x86_64 265b9d1e55ef0d5306a436018af6d2b686c2ce051f03d968f7464ecb1372a7e8 https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-amd64.tar.gz +alertmanager-0.34.1 aarch64 d98d6cbaf52151c7e76e24355fec88b11cebcb9875d4cdd8b76ddce7a7e5535c https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-arm64.tar.gz +node_exporter-1.12.1 x86_64 b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz +node_exporter-1.12.1 aarch64 ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-arm64.tar.gz diff --git a/data/keys/grafana.asc b/data/keys/grafana.asc new file mode 100644 index 0000000..2c6e93d --- /dev/null +++ b/data/keys/grafana.asc @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGTnhmkBDADUE+SzjRRyitIm1siGxiHlIlnn6KO4C4GfEuV+PNzqxvwYO+1r +mcKlGDU0ugo8ohXruAOC77Kwc4keVGNU89BeHvrYbIftz/yxEneuPsCbGnbDMIyC +k44UOetRtV9/59Gj5YjNqnsZCr+e5D/JfrHUJTTwKLv88A9eHKxskrlZr7Un7j3i +Ef3NChlOh2Zk9Wfk8IhAqMMTferU4iTIhQk+5fanShtXIuzBaxU3lkzFSG7VuAH4 +CBLPWitKRMn5oqXUE0FZbRYL/6Qz0Gt6YCJsZbaQ3Am7FCwWCp9+ZHbR9yU+bkK0 +Dts4PNx4Wr9CktHIvbypT4Lk2oJEPWjcCJQHqpPQZXbnclXRlK5Ea0NVpaQdGK+v +JS4HGxFFjSkvTKAZYgwOk93qlpFeDML3TuSgWxuw4NIDitvewudnaWzfl9tDIoVS +Bb16nwJ8bMDzovC/RBE14rRKYtMLmBsRzGYHWd0NnX+FitAS9uURHuFxghv9GFPh +eTaXvc4glM94HBUAEQEAAbQmR3JhZmFuYSBMYWJzIDxlbmdpbmVlcmluZ0BncmFm +YW5hLmNvbT6JAdQEEwEKAD4CGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQS1 +Oud7rbYwpoMEYAWWP6J3EEWFRQUCaKhvPQUJB4NP1AAKCRCWP6J3EEWFRUjOC/9Y +dWOWJLJVKzLx8uv5YVzebyw15HevhKahbznJX5fHnE8irjkiPFltVEZ4T37s5afR +GBEJnR1UFd80s7jzwbuoZh/zEB3jN8q50g64AznuzDa0PWKzaY7Tgkssx3+hs6TS +vIwV4z8T7f56lDudeHxHXx+htRnZ3ebKNPCJS7+G12GF6W3C3znpdjgvhVUB0uxd ++42V0fRqk2GLNZeKS9988fi5dYRAy9Ozwced7ByCFjde9FBgUtrH3mG1/ibzLEh0 +4k02nYjc8mrH32t4UCWpxQEJ1vZA2vT2HN3/cH/4uyFdyU6OHkMyMbz6lmeXe71d +F5hOB4+/RP6Ndyj7ViRNDbm70NRBaFne/+YOJvmMfJTCh7YbF5qEn1ihGkJJ0ohE +u2IB+EGEhyiDm8SIsj1uMw7n17iIPNtbsU5GgnmLtfguP/WbwKV2UeuxTpiOeYb6 +blDwRlh48uHMlA5HBW+487Jktw3iPj1IKhdtAC9CU3xAvzDcseMbgmM6Xj2bSQG5 +AY0EZOeGaQEMALNIFUricEIwtZiX7vSDjwxobbqPKqzdek8x3ud0CyYlrbGHy0k+ +FDEXstjJQQ1s9rjJSu3sv5wyg9GDAUH3nzO976n/ZZvKPti3p2XU2UFx5gYkaaFV +D56yYxqGY0YU5ft6BG+RUz3iEPg3UBUzt0sCIYnG9+CsDqGOnRYIIa46fu2/H9Vu +8JvvSq9xbsK9CfoQDkIcoQOixPuI4P7eHtswCeYR/1LUTWEnYQWsBCf57cEpzR6t +7mlQnzQo9z4i/kp4S0ybDB77wnn+isMADOS+/VpXO+M7Zj5tpfJ6PkKch3SGXdUy +3zht8luFOYpJr2lVzp7n3NwB4zW08RptTzTgFAaW/NH2JjYI+rDvQm4jNs08Dtsp +nm4OQvBA9Df/6qwMEOZ9i10ixqk+55UpQFJ3nf4uKlSUM7bKXXVcD/odq804Y/K4 +y3csE059YVIyaPexEvYSYlHE2odJWRg2Q1VehmrOSC8Qps3xpU7dTHXD74ZpaYbr +haViRS5v/lCsiwARAQABiQG8BBgBCgAmAhsMFiEEtTrne622MKaDBGAFlj+idxBF +hUUFAmiobzkFCQeDT9AACgkQlj+idxBFhUVsmQwA0PA/zd7NqtnZ/Z8857gp2Wq2 +/e4EX8nRjsW2ZlrZfbU5oMQv9OZZ4z1UjIKEUV+TnCwXEKXTMJomdekQSSayVVx/ +u5w+0YM8gRuQGrG8hW0GRR8sHIeuwBFlyQrlwxUwXvDOPDYyieETjaQqMucupIKo +IPm3CjFySvfizvSWUVSWBnGmQfpv6OiGYawvwfewcQHUdLMgWN3lYlzGQJL4+OMm +7XcB8VNTa586Q00fmjDfktHYvGpmhqr3gsd4gS3AjTk0zI65qXBRJkdqVnwUrMUD +8TcxXYNXf90mhR0NWkLmp6kBYiW8+QY6ndMmRVpodg1A87qgMYaZUAAlxCS4XKTU +r+/YMDYOWgLN6i4UeYG/3/hsnAEHm5ITojfh6cLfdlhjohFTnD0IYw3AsNJXRzKB +1g5FTBKLLLIdXgS/3rWV1qjAd3drQVIMCku6HKl/vT4ftrBHeSyV7eLwOYbe3/bw +8VMx+lmMheD8/qJMia1om0iBBRSXRjY//f+Lllqm +=TH3J +-----END PGP PUBLIC KEY BLOCK----- diff --git a/data/keys/netdata.asc b/data/keys/netdata.asc new file mode 100644 index 0000000..53e5093 --- /dev/null +++ b/data/keys/netdata.asc @@ -0,0 +1,93 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGfW4AkBEACc94RKVQuMuw/T52gD5g527H5A97wIUMIARDKswAAHhLWlGtgl +0JrmS6mzBjWGW80aAe1sfsxJE6fntqwri8u4U/kOY3X7e6yk+Gverzkeu1pcgKL8 +eD9o9U4l3l6/CtPjYH/1HNyTED8VN9iJzzn0Snkq0GDJHd4pyevlE5r3lHdYY3PX +1aaIQAZSWby1eBONKKJzNj/pvSza5OWuGhWy3kHNaWypuwqhMczQhMNhLNUjQBUi +ci7vRVAiFp/DvJcMmNW/ZBbZZaVsgW8SxfMFBzKiGfINYzn3dEFL3H6w7KDs3EWu ++RaAXyBjgNqGZrDJMNWE4NP4rqgIwCgTuphJ95R9fEz+Gvv6uh5BjoAOUNJ0hE/K +t+XBit/u0ravcTuHzHt/1bN27wzWgBVixoNGu7jmVE+t5sU07IUQoj+q0i6byBmN +xgZUftGFwcHRraVViAX4BXNuBs+7b1liS4opuhAxDWhW3dHZqQBZCHDWm7ewIRVd +piEzowILaOCMmZiiJINKrSOEvs073sMcPMvY+RDxE+9kHcj92KO1bPDZHrnMKbLf +4IB8ZPMDtfhp/aX00dbAR3BCq5fcjqRMGgfLREer3FekMbusjbwAnlS8uA748HFu +uPWqbqYUo8cWJkmV2+Dd0eUtKRnmxFJ65r6lHvR0NDHi13m7v3RugL0y3wARAQAB +tD9OZXRkYXRhYm90IChOZXRkYXRhIFJlcG9zaXRvcnkgU2lnbmluZyBLZXkpIDxi +b3RAbmV0ZGF0YS5jbG91ZD6JAlQEEwEKAD4WIQRuFV3BU5Brc3ZadKmd1KdM7PqP +TwUCZ9bgCQIbAwUJEswDAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCd1KdM +7PqPT3ArEACBMqt3lnRJNnzwOaQomzVkooj4+tBt0/5noyQMGOfJlEQU8scMquqp +kMRuq+5q15ZWQh5K9vmdLfukw4fzwgdjYmQ9e924q7K1/8EvEYmPe4MAd+D4rMoC +0eTioSCUZx5H/8bkKrpx39w5gvQzonhxNjfmY1dYrcVgCFXRXweO6aq06Gw5YWS+ +EAuSTC0G3UN2mHdpJ37AyvkY5Blv36l3S7uSfGO68bQsoKiK493ysLMdFuWD2lu1 +Y18LB8Y0mxsdynVbaknhtXtS6iPtKr5yeIGd3KpdcW3ftLgJEnkj4o/Bhl6doeXx +dOyRsyb7H9YVNqstOZHlbjr+mOmFR/2NU+cO57wgfaGtfu6FyI2E/Fg7YBShQ37j +BBaZ3jKF3i/PpApO2mhoHiNs9VSKBmLuptL0w0Xdk4OhnSlAalTWHYX02eBYHMSH +a9YSupGSgiBYLA8S/1IJ1nJ8NGDeLKQpvphjjlP0/lFCEcEquqbW8WHGwEpUECQ6 +TLPOseUR4Xy+ji4wXVRB6S18m+j6NwAt0fp8sVOex3j4BGbvTAnC+9wefjQR+j3H +ZzDVCf/NehLHarIVAWN01lnXSgEUG9SgTQ6rqFGaxvX4k/zV4juhN0jhodL362DK +y/TGwSySlKhyNl1uP3D3xdTs+yd1fqEe+MpPbM6dfscqXnRTBZVOmbkCDQRn1uAJ +ARAAvpTd4NkjSUUHUlMEhOiZpAfEgBo/H5jS7Pj12weG5dFQ6Or/A1aNH0/NXUSs +lU/9IJwSegLTwze3/mXY7t0ix1UcPW5OLltmiOlj7arnFBe/wkxZzR+it18/EKkG +KbXSw410a79j8Xbz9hLfvIAb6CVebr93yljyFAYZENTM6DAa1UkdjeGno3afxeay +HLAk42mKYbHyHe77hhx0YaMhjvmqIyDITH/A+21UWaLbdCw1EzChID5MQWnifMqM +M/zIOciHRYSuey3OZ5h3UATZftkUCbK/p2hN94oHO48jRQZ+ATClG1owQTj7faU4 +cjC0RH4cPZ0EjoLnFq7dE41c+ANu+ky1i+OLaL0uwog1UdDsQjTnWS/ec2EBcQuv +WRllxRNK88x9TMr2ntfiMZumULumtlv8dgsSyu7S/94im4cNg0WkeKvKDv8tuMMN +/xUwTQ0N8NVzrGfHAljDwdgd0gEyiiz5f9LF/pwdl3Kt9HS/gOGSBkEJsTpSCNdU +lcE9k+wCYqFXuZXn+Ll/Ju4FkvXkZPGpN9hWtCVf4tDOu5FJPpo8+S8RbvxGbS7k +nfeyeGXjyuSiqvvm/2nAG0tfFeAZLt9WeyHaZRFiZmHkZc69cHudqcDpHcjUe82k +G22TuBie89mFOD13m6ItQ930xosyFmaAVEiHwobbZYPzBc0AEQEAAYkCPAQYAQoA +JhYhBG4VXcFTkGtzdlp0qZ3Up0zs+o9PBQJn1uAJAhsMBQkSzAMAAAoJEJ3Up0zs ++o9P8RYP/R81xloQl8YJ19d2DnaHCej10SXUBTeCY7u960B61UjAzCPu1SSY5/rr +EkWy/uqstpztDeXrzeSGFKyiZOahwoCjkf/d3nV9eagFugC2Za6Mdes1ekToU134 +28rwDLybUZSIFXlM210MDUlNVPxwCXxuPPt+I1KfHkVVH8K6XOnTngqYyaO89VLK +iZM4dI1yV34CZ4DYfzer2o2LQcTYilXrEdmuPoj9RSj75TOUtJUS21bQpvqtwL0o +ImTqiV2ZhhoBE60h8Cz2r0ALO1oTuo7hs2LaZpd9icwoP/ZlYobKmFRZ03fnnz/q +drYlGJt6uPNdfjGOrhdh0SSjoShInAIULbyxYmEuJEVx8CbTjd2NLH57TmlVN9P0 +u1PfeTWNnRxVeyxk8xov/JU/zyzDsKdOp/8GokAADfaBpbVPbT0/X30IetpO6O5q +ExGSvwhmUXmYYxbawigtHL6SFAPAbIJLhEguGrXm6mOmQh5LiRV3qoCUmoghUzk2 +yTidLDuQOCRN8bQ1me6Pt/VYT9TUbdUwEvs2zyChmVOfOzQloTGvybJohCp+uHYU +bshAmsA26Iw3y2ZRSVWF4YJyjLMyi/VV7qVYMTLdusgGJsOdlfGOUbZWGInX6o6S +9RApebBUDUvjM3Cz+iSof9/OBP7w/oEF1LmLUtoDKZZZPotHK7mk +=1wXu +-----END PGP PUBLIC KEY BLOCK----- +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGK+ooUBDACpmop8b+UQurHIEVOZ6YXyzcjC08LoPh4+PBxjHE+TuxFQkFXS +PRGp4Vi97NEMcEJbM+Hg/lqp1VNKBD2djHyQhmL6SHpM2ecCEjj2XUXlB9LDX5MR +ml+K4mhxKplmJ1a6Mr/sgdrG9dEMjnSEWM4Li0EWlv/9FXgujZDDnw5fG6ZHn6sj +NZgQRaskpbcoNK6btP3uFUF301w5ypoS9LtWVH49ixN3lHcmvJdtazpIcgmus65G +vGREYrqkVwFGGdbrcE5qYulfNIH1QK2a1PtCbJuqXA5pgz7FypmP+h2BOLfJf4G2 +04CNyrXCbyBzzK09s2QzJNL5kjsOzb31amGfUFfgjt+GYLMdZabnZfBBsY7Ujpkl +msymJXPR5TjHiSCKNBoN5ENJ3QjYlb61NcQ86v7eSoDOG+0io6zoTp+9cLgOq20Y +PpPNVHDm7tG95S3RChYUxAHmcgp/t7r4MbFgXdH68WN1PjDBAxUbskDtUVMBlL5d +DzHJZ6Psj8BvbTkAEQEAAbQeTmV0ZGF0YWJvdCA8Ym90QG5ldGRhdGEuY2xvdWQ+ +iQHOBBMBCgA4AhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAFiEEZYj917FHIf58 +MRXm+Rd7UmX1Y0YFAmK+tOUACgkQ+Rd7UmX1Y0Y2iwwAgAQE0FxqUVelGLfgAVPX +vG9yr13MWoQ6s13LX1BaxCpGajgcLBrrTjaJo3VDxrI2I8KN3o+8QQhnbtzHMLcG +1Hv3RAcBBVDDs7iPnj5MaucR8ZXPbeJVglO6Du3cLwXV19rQxJC1DUTL2IDezcWy +eaZOgK/PybHcbUZ5pfZcIP1vbpATGyHKD4sIVofkIS0/aaeNwyb+OnORF6C9XkoO +mMzbx5dX6hB2kOKiZeF2ayQq5Lk5grT0wS1L/TeG/SonFORUzjzbHe6mUHfSTt2m +dIBTdRyglv8paCW5TAbFMPxnUcDeCFlROZIE+TTFueizJOJ8czcGujbXiU30JgoF +lNNM2nJvts/C/mMj+A0YxddGvgy/K1vdBAFFod3FVH090Xi1YFw7ZDWGFXnW8gqa +HpVFg4ibtbH5HR22MoQYWWxaWl0sus8TlYIMlRvMSbVQhvTRBJ/JAaet6+9BEXwm +1WKQ8fkQP7GpXOssNIxf4co0r+rboIWIIGshcQ6sevzfuQGNBGK+ooUBDADIlkQ7 +PNVbQHP5rDAAYPFVUiJFlXbR6SeF2JqDk0QkG2+UlAGFulb/MUKwNW8SwTPxdaeP +ysEZ4GskK1OB/uN/5Bbyh0iI4Hwmbl91aWz/EZD8IE+zYmwVgHypQEBVjibeZWhy +dNMa7ZLYW7HVIG/OZOZqCPWSqLRKApHUIuCX82BVnT65+MpCicMfv9kXhzImQPyD +3Sxx1CC9t2FmRTtI+22tYcrayQLFj0GBEkYVD43X6eMdSWkHPv5kHiX0S8VKrZkk +dQbsm0Z3KPwgs8cXEDi0vnfHIVXbj5u3dR2m0L8eIXuBLSZEvsbLoG/beI6poELP +rhFLdk06XVGNvf+TUqJe5t//bV4WvOvy7+l7odUWNk+X0s7mYQdJDYYO+ju1iJ84 +PmXJaJ7N+Cp5aYLsEjpKOrLWwDaN2ZlisZm68Q5mdtLcRxWm3BPe1TK1ojFenMrX +g/qK4bRBHJDBVvZYh5mZHoDYmUcrqQBl4NQbwmXCEWBHtsi1+73UJRniuWMAEQEA +AYkBtgQYAQoAIAIbDBYhBGWI/dexRyH+fDEV5vkXe1Jl9WNGBQJivrT4AAoJEPkX +e1Jl9WNGWsUL/AksktdmVxrBRjuhjysfynwwt78PlZSdPPQ9K5Gn68x9xovht7tq +5/f7S7nUNUA6xJoR+VJB+e8R4NnopvI+4O+yK45V2JAry9+xOXWHuLDNjf9TdprP +ICNawkFzAAGAlbUXce+QU70ei2azuqpT5pes47yAJ2xp8VsVK6MMHcFfVLw9mQ9Y +Y8Zz9Q4UoIrN80GWAR0+8mObwMgSzm5XjfJKbjXyRoai6JXAQe5WCrVgW7g0EPKY +4JU3mPcNOxhTALC8xYGJGQ4MnVdGQEkbVk6byvUBRc6QaDRUz642OlK/6MrMaSEu +cYTaIOiGJ9nomyMmYR8/RVx49ZUeAVeyLpZQj3R10pNp7ntqPN/tzVOKanNOQpGV +YsCNLExxwo39uCsd2hFBw3nDC6QhOy6T6RLrjTGgsoJkiVm9LcKVwzrgTU3U20/7 +92tacDWvfpg6k/alEO+sXKEcxbJNP0LTUixHydNcCZx2srh3CclfiQUumnkokdTI +KvWTMBz97LPIMA== +=OUNZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/data/repos/grafana.conf b/data/repos/grafana.conf new file mode 100644 index 0000000..3541f40 --- /dev/null +++ b/data/repos/grafana.conf @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Grafana, Loki and Alloy from Grafana Labs' repositories (https://apt.grafana.com, https://rpm.grafana.com): one +# suite for every Debian and Ubuntu release, one repository for the whole RHEL family, x86_64 and arm64. Key fetched on +# 2026-10-14 from https://apt.grafana.com/gpg.key; https://rpm.grafana.com/gpg.key and https://packages.grafana.com/gpg.key +# carry the same key, and it signs the apt InRelease file and the rpm metadata. It expires on 2027-08-22; Grafana +# extends it before then. +KEY=grafana.asc +FINGERPRINTS=B53AE77BADB630A683046005963FA27710458545 +DEB_URL=https://apt.grafana.com +DEB_SUITE=stable +DEB_COMPONENTS=main +RPM_BASEURL=https://rpm.grafana.com +RPM_GPGCHECK=1 +RPM_REPO_GPGCHECK=1 diff --git a/data/repos/netdata.conf b/data/repos/netdata.conf new file mode 100644 index 0000000..132ada1 --- /dev/null +++ b/data/repos/netdata.conf @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Netdata from its own stable repository (https://repository.netdata.cloud/repos/stable/): Debian 13, Ubuntu 26.04 and +# EPEL 10 have no Netdata package, so every system uses Netdata's, which gives all of them the same version. Flat apt +# repositories per release (the suite ends in a slash, no components), one rpm repository per RHEL major release, for +# x86_64 and arm64; checked on 2026-10-14 for all seven releases. Keys fetched the same day: the current signing key +# (https://repository.netdata.cloud/netdatabot.gpg.key, the fingerprint Netdata's install guide names) and the previous +# one (netdatabot.old.gpg.key), which still signs the apt InRelease files next to it. +KEY=netdata.asc +FINGERPRINTS=6E155DC153906B73765A74A99DD4A74CECFA8F4F 6588FDD7B14721FE7C3115E6F9177B5265F56346 +DEB_URL=https://repository.netdata.cloud/repos/stable/{{id}}/ +DEB_SUITE={{codename}}/ +DEB_COMPONENTS= +RPM_BASEURL=https://repository.netdata.cloud/repos/stable/el/{{major}}/$basearch/ +RPM_GPGCHECK=1 +RPM_REPO_GPGCHECK=1 diff --git a/docs/module-guide.md b/docs/module-guide.md index 009ec8a..d0b49e2 100644 --- a/docs/module-guide.md +++ b/docs/module-guide.md @@ -102,6 +102,22 @@ All of `lib/` is loaded in every step. The ones modules use most: - `fw_trust NETWORK INTERFACE` accepts all traffic from NETWORK arriving on INTERFACE, recorded and removed like a port. Only for networks of the server itself (a pod network on its bridge). +### Monitoring and admin interfaces + +`lib/monitoring.sh` is for services with a web interface or an API, so they all bind, open and publish the same way: + +- `mon_address MODULE` is 127.0.0.1, or the server's WireGuard address (`fw_vpn_address`) with + `LSS__ACCESS=vpn`; `mon_vpn_dropin UNIT` starts the unit after `wg0` in that case, and `mon_firewall + PORT/PROTO...` opens the ports to the WireGuard network only. +- `mon_web_admin KIND URL` publishes the interface below the web server's admin port when `LSS__WEB_ADMIN` + is true (`proxy` keeps the path, `strip` removes the prefix); `mon_web_admin_remove` takes it away. +- `mon_target NAME JOB HOST:PORT [PATH]` adds a Prometheus scrape target, `mon_datasource NAME TYPE URL` a Grafana data + source; both have a `_remove` counterpart. +- `mon_upstream_install NAME VERSION PROGRAM...` installs programs from a release archive listed in + `data/checksums.txt`; `mon_install_stopped PKG...` installs Debian packages without letting them start their + service before lss configured it. +- `mon_wait_http URL [SECONDS]` and `mon_listen_check [--process NAME] PORT ADDRESS...` are for `mod_verify`. + ### Containers - `ctr_run NAME IMAGE [OPTION...] [-- COMMAND...]` runs a container with the engine of the server (docker or podman) diff --git a/lib/files.sh b/lib/files.sh index 4dcc0c4..1995adc 100644 --- a/lib/files.sh +++ b/lib/files.sh @@ -124,15 +124,26 @@ file_write() { log_debug "Wrote ${dest}" } -# file_remove PATH: deletes a managed file with backup. +# file_remove PATH: deletes a managed file with backup. lss verify no longer expects it, also when it was gone already +# (a drop-in a module stops writing, a file the web server took with it). file_remove() { local path=$1 + files_forget_path "${path}" [[ -e ${path} || -L ${path} ]] || return 0 files_backup "${path}" rm -rf -- "${path}" files_note_change "removed ${path}" } +# files_forget_path PATH: drops PATH from the checksums of the current module. +files_forget_path() { + local path=$1 list="${LSS_STATE_DIR}/managed/${LSS_MODULE:-core}" + [[ -f ${list} ]] || return 0 + awk -F '\t' -v path="${path}" '$2 != path' "${list}" >"${list}.tmp" + mv -f -- "${list}.tmp" "${list}" + return +} + # Prints managed files of MODULE whose content no longer matches what lss wrote. files_drifted() { local list="${LSS_STATE_DIR}/managed/$1" sum path diff --git a/lib/firewall.sh b/lib/firewall.sh index 80098c0..37d9cd0 100644 --- a/lib/firewall.sh +++ b/lib/firewall.sh @@ -25,6 +25,21 @@ fw_vpn_network() { return } +# fw_vpn_address: the address of this server in the WireGuard network, the first host of it (10.66.0.1 for +# 10.66.0.0/24). The wireguard module gives it to wg0; admin interfaces with access vpn listen on it. +fw_vpn_address() { + local network prefix base + network=$(fw_vpn_network) + [[ ${network} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]{1,2})$ ]] || + die "LSS_WIREGUARD_NETWORK '${network}' must be an IPv4 network such as 10.66.0.0/24." + prefix=$((10#${BASH_REMATCH[2]})) + ((prefix >= 8 && prefix <= 30)) || die "LSS_WIREGUARD_NETWORK needs a prefix from /8 to /30, got /${prefix}." + base=$(_fw_ipv4_int "${network%/*}") + base=$(((base >> (32 - prefix) << (32 - prefix)) + 1)) + printf '%s.%s.%s.%s' $((base >> 24 & 255)) $((base >> 16 & 255)) $((base >> 8 & 255)) $((base & 255)) + return +} + fw_is_enabled() { if [[ $(fw_backend) == ufw ]]; then command -v ufw >/dev/null 2>&1 && ufw status 2>/dev/null | grep -q '^Status: active' diff --git a/lib/module.sh b/lib/module.sh index a2b9cfc..d174373 100644 --- a/lib/module.sh +++ b/lib/module.sh @@ -231,8 +231,7 @@ module_remove() { # config_unknown_keys: prints the given settings that no module reads, usually typos, one per line. config_unknown_keys() { - # LSS_WIREGUARD_NETWORK is read by the firewall for the vpn scope before the wireguard module exists. - local -A known=([LSS_MODULES]=1 [LSS_MODE]=1 [LSS_WIREGUARD_NETWORK]=1) + local -A known=([LSS_MODULES]=1 [LSS_MODE]=1) local name key _default _description while IFS= read -r name; do known[$(config_key "${name}" mode)]=1 diff --git a/lib/monitoring.sh b/lib/monitoring.sh new file mode 100644 index 0000000..187d0ed --- /dev/null +++ b/lib/monitoring.sh @@ -0,0 +1,380 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Monitoring layer shared by node-exporter, prometheus, alertmanager, grafana, loki, alloy and netdata, and the +# metric files of the guard and the backup. +# +# Conventions: +# MON_TEXTFILE_DIR node-exporter's textfile collector; the guard and the backup write their *.prom files there +# MON_TARGETS_DIR one file per module with its scrape target, which Prometheus reads by itself (file_sd), so +# the modules can come and go in any order +# MON_ALERTMANAGERS_DIR the same for the Alertmanager that Prometheus sends its alerts to +# MON_DATASOURCES_DIR one Grafana data source per module, read by Grafana when it starts +# +# Every interface with a UI or an API for people serves below a path of its own name (/prometheus/, /alertmanager/, +# /grafana/), on 127.0.0.1 or, with LSS__ACCESS=vpn, on the WireGuard address of the server. The web server +# can then publish it below the same path on its admin port (LSS__WEB_ADMIN), and none of the other modules +# needs to know whether it does. + +# shellcheck disable=SC2034 # read by node-exporter; libexec/lss-guard and libexec/lss-backup use the same path +MON_TEXTFILE_DIR=/var/lib/prometheus/node-exporter +MON_PROMETHEUS_DIR=/etc/lss/prometheus +MON_TARGETS_DIR=${MON_PROMETHEUS_DIR}/targets +MON_ALERTMANAGERS_DIR=${MON_PROMETHEUS_DIR}/alertmanagers +MON_GRAFANA_DIR=/etc/lss/grafana +MON_DATASOURCES_DIR=${MON_GRAFANA_DIR}/provisioning/datasources +MON_GRAFANA_UNIT=grafana-server +# shellcheck disable=SC2034 # read by the monitoring modules +MON_PROMETHEUS_PORT=9090 +# shellcheck disable=SC2034 # read by the monitoring modules +MON_ALERTMANAGER_PORT=9093 +# The unit of the WireGuard interface (modules/network/wireguard.sh). +MON_VPN_UNIT=wg-quick@wg0.service +MON_MANAGED_BY='# Managed by lss (module' +# Where the programs of upstream releases go (the RHEL family), and the policy file of Debian's package scripts. +MON_BIN_DIR=/usr/local/bin +MON_POLICY_RC=/usr/sbin/policy-rc.d +MON_CACHE_DIR=/var/cache/lss +# The interfaces speak plain HTTP; see mon_url. +MON_SCHEME=http + +# mon_vpn MODULE: true when MODULE listens on the WireGuard network (LSS__ACCESS=vpn). +mon_vpn() { + local module=$1 + [[ $(config_option "${module}" access local) == vpn ]] + return +} + +# mon_address MODULE: the address MODULE listens on, 127.0.0.1 or the WireGuard address of the server. +mon_address() { + local module=$1 + if mon_vpn "${module}"; then + fw_vpn_address + else + printf '127.0.0.1' + fi + return +} + +# mon_url MODULE PORT [PATH]: the URL of MODULE's interface. Plain HTTP is safe here: the interface listens on +# 127.0.0.1 or on the WireGuard address, and the WireGuard tunnel encrypts everything that leaves the server. +mon_url() { + local module=$1 port=$2 path=${3:-} + printf '%s://%s:%s%s' "${MON_SCHEME}" "$(mon_address "${module}")" "${port}" "${path}" + return +} + +# mon_vpn_ready: the WireGuard address must exist before a service of the current module can listen on it. Checked in +# mod_configure, because wireguard may be installed earlier in the same run. +mon_vpn_ready() { + local address + mon_vpn "${LSS_MODULE}" || return 0 + address=$(fw_vpn_address) + if ! ip -4 -o addr show 2>/dev/null | grep -qF " ${address}/"; then + die "$(config_key "${LSS_MODULE}" access)=vpn needs the WireGuard address ${address}; install the wireguard module first (lss install wireguard)." + fi + return +} + +# mon_vpn_dropin UNIT: with access vpn, a drop-in (lss-vpn.conf) starts UNIT after the WireGuard interface, whose +# address it listens on; with access local the drop-in goes away. Sets LSS_FILE_CHANGED. +mon_vpn_dropin() { + local unit=$1 + if mon_vpn "${LSS_MODULE}"; then + svc_dropin "${unit}" lss-vpn <_WEB_ADMIN=true and a web server, publishes the current module below +# /MODULE/ on the admin port (web_admin_write proxy or strip); otherwise removes that path. Reloads the web server +# only when something changed. +mon_web_admin() { + local kind=$1 url=$2 before server + if [[ -z $(web_server) ]]; then + # A web server removed since took the admin path with it; lss verify must not miss it. + for server in nginx apache caddy; do + files_forget_path "$(web_conf_dir "${server}")/admin/${LSS_MODULE}.$(_web_ext "${server}")" + done + return 0 + fi + before=$(files_changes_of "${LSS_MODULE}") + if config_bool "$(config_key "${LSS_MODULE}" web_admin)"; then + web_admin_write "${LSS_MODULE}" "${LSS_MODULE}" "${kind}" "${url}" + else + web_admin_remove "${LSS_MODULE}" + fi + if [[ $(files_changes_of "${LSS_MODULE}") != "${before}" ]]; then + web_reload + fi + return +} + +# mon_web_admin_remove: drops the admin path of the current module (mod_remove). +mon_web_admin_remove() { + local before + [[ -n $(web_server) ]] || return 0 + before=$(files_changes_of "${LSS_MODULE}") + web_admin_remove "${LSS_MODULE}" + if [[ $(files_changes_of "${LSS_MODULE}") != "${before}" ]]; then + web_reload + fi + return +} + +# mon_target NAME JOB HOST:PORT [METRICS_PATH]: the scrape target of the current module for Prometheus. +mon_target() { + local name=$1 job=$2 target=$3 path=${4:-/metrics} + install -d -m 0755 "${MON_TARGETS_DIR}" + file_write 0644 root:root "${MON_TARGETS_DIR}/${name}.yml" </dev/null || true + return +} + +# mon_datasource NAME TYPE URL [DEFAULT]: the Grafana data source of the current module. Grafana reads the files when it +# starts, so a running Grafana is restarted after a change; without Grafana the file waits for it. +mon_datasource() { + local name=$1 type=$2 url=$3 default=${4:-false} file + file="${MON_DATASOURCES_DIR}/${LSS_MODULE}.yaml" + install -d -m 0755 "${MON_DATASOURCES_DIR}" + file_write 0644 root:root "${file}" < <( + printf '%s %s). Grafana reads it when it starts.\napiVersion: 1\ndatasources:\n' "${MON_MANAGED_BY}" "${LSS_MODULE}" + printf ' - name: %s\n uid: lss-%s\n type: %s\n access: proxy\n url: %s\n' "${name}" "${LSS_MODULE}" "${type}" "${url}" + printf ' isDefault: %s\n editable: false\n' "${default}" + if [[ ${type} == alertmanager ]]; then + printf ' jsonData:\n implementation: prometheus\n handleGrafanaManagedAlerts: false\n' + fi + ) + if [[ ${LSS_FILE_CHANGED} == 1 ]] && svc_is_active "${MON_GRAFANA_UNIT}"; then + svc_restart "${MON_GRAFANA_UNIT}" || die "${LSS_MODULE}: Grafana does not start with the new data source." + fi + return +} + +# mon_datasource_remove: removes the data source file of the current module. Grafana keeps the data source until it is +# deleted in Grafana. +mon_datasource_remove() { + file_remove "${MON_DATASOURCES_DIR}/${LSS_MODULE}.yaml" + return +} + +# mon_install_stopped PACKAGE...: installs packages whose Debian scripts would start the service right away, listening on +# every address with the package defaults, before lss has written its configuration. A temporary policy-rc.d forbids +# those starts (exit 101: action not allowed); mod_configure starts the service with the lss settings. The RHEL family +# starts no service on installation. +mon_install_stopped() { + local policy=${MON_POLICY_RC} + if ! os_is_debian || [[ -e ${policy} || -L ${policy} ]] || [[ -z $(pkg_missing "$@") ]]; then + pkg_install "$@" + return + fi + ( + # The global name: when pkg_install dies, the trap runs after the locals of this function are gone. + trap 'rm -f -- "${MON_POLICY_RC}"' EXIT + printf '#!/bin/sh\n# Written by lss for one package installation.\nexit 101\n' >"${policy}" + chmod 0755 "${policy}" + pkg_install "$@" + ) + return +} + +# mon_grafana_repo_remove: grafana, loki and alloy share Grafana's repository; it goes with the last of them. +mon_grafana_repo_remove() { + local module + for module in grafana loki alloy; do + [[ ${module} != "${LSS_MODULE}" ]] || continue + state_installed "${module}" && return 0 + done + repo_remove grafana + return +} + +# mon_unit UNIT USER COMMAND DESCRIPTION: the unit of a program lss installs itself (the RHEL family, from the upstream +# release), running COMMAND as USER. Sets LSS_FILE_CHANGED. +mon_unit() { + local unit=$1 user=$2 command=$3 description=$4 + svc_unit "${unit}.service" <&1 || true) + [[ ${current} != *"version ${version} "* ]] || return 0 + archive=${MON_CACHE_DIR}/${name}-${version}.tar.gz + fetch_verified "${name}-${version}" "${archive}" + dir=$(mktemp -d "${MON_CACHE_DIR}/${name}.XXXXXX") + log_command tar -xzf "${archive}" -C "${dir}" --strip-components=1 --no-same-owner + for program in "$@"; do + [[ -f ${dir}/${program} && ! -L ${dir}/${program} ]] || + die "${LSS_MODULE}: the ${name} ${version} archive has no ${program}; its layout changed." + install -m 0755 -o root -g root "${dir}/${program}" "${MON_BIN_DIR}/${program}" + if command -v restorecon >/dev/null 2>&1; then + restorecon -F "${MON_BIN_DIR}/${program}" 2>/dev/null || true + fi + done + rm -rf -- "${dir}" "${archive}" + files_note_change "installed ${name} ${version} to ${MON_BIN_DIR}" + return +} + +# mon_upstream_remove PROGRAM... +mon_upstream_remove() { + local program + for program in "$@"; do + [[ -e ${MON_BIN_DIR}/${program} ]] || continue + rm -f -- "${MON_BIN_DIR}/${program}" + files_note_change "deleted ${MON_BIN_DIR}/${program}" + done + return +} + +# mon_user NAME HOME: a system user without login for a service (the RHEL family, where no package creates it). +mon_user() { + local name=$1 home=$2 + getent passwd "${name}" >/dev/null && return 0 + useradd --system --user-group --home-dir "${home}" --no-create-home --shell /usr/sbin/nologin "${name}" + files_note_change "created user ${name}" + return +} + +# mon_dir PATH OWNER MODE: creates a data directory, or fixes its owner and mode. +mon_dir() { + local path=$1 owner=$2 mode=$3 + [[ ! -L ${path} ]] || die "${LSS_MODULE}: ${path} is a symbolic link." + if [[ ! -d ${path} ]]; then + install -d -m "${mode}" -o "${owner%%:*}" -g "${owner#*:}" "${path}" + files_note_change "created ${path}" + elif [[ $(stat -c '%U:%G %a' "${path}") != "${owner} ${mode#0}" ]]; then + chown "${owner}" "${path}" + chmod "${mode}" "${path}" + files_note_change "owner and mode of ${path}" + fi + return +} + +# mon_delete PATH...: deletes data directories of the current module (mod_remove with --purge). Each PATH must be a +# directory below /var/lib, and never a symbolic link. +mon_delete() { + local path + for path in "$@"; do + [[ ${path} == /var/lib/?* && ${path} != *..* ]] || die "${LSS_MODULE}: refusing to delete ${path}." + [[ ! -L ${path} ]] || die "${LSS_MODULE}: refusing to delete ${path}: it is a symbolic link." + [[ -d ${path} ]] || continue + rm -rf -- "${path}" + files_note_change "deleted ${path}" + done + return +} + +# mon_http URL: the HTTP status code of URL, 000 while nothing answers. +mon_http() { + local url=$1 + curl -s --max-time 10 -o /dev/null -w '%{http_code}' "${url}" 2>/dev/null || true + return +} + +# mon_wait_http URL [SECONDS]: waits until URL answers with 200. +mon_wait_http() { + local url=$1 seconds=${2:-60} i status='' + for ((i = 0; i < seconds; i += 2)); do + status=$(mon_http "${url}") + [[ ${status} != 200 ]] || return 0 + sleep 2 + done + die "${LSS_MODULE}: ${url} does not answer (${status})." +} + +# mon_listen_check [--process NAME] PORT ADDRESS...: every TCP socket listening on PORT (of the process NAME only) is on +# one of the ADDRESSes. +mon_listen_check() { + local process='' port listeners address first=$1 + if [[ ${first} == --process ]]; then + process=$2 + shift 2 + fi + port=$1 + shift + listeners=$(ss -Hltnp "sport = :${port}" | awk -v process="${process}" ' + process == "" || index($0, "((\"" process "\"") { address = $4; sub(/:[0-9]+$/, "", address) + sub(/%.*/, "", address); gsub(/[][]/, "", address); print address }') + [[ -n ${listeners} ]] || die "${LSS_MODULE}: nothing listens on port ${port}." + while IFS= read -r address; do + [[ " $* " == *" ${address} "* ]] || die "${LSS_MODULE}: port ${port} is open on ${address}, expected $* only." + done <<<"${listeners}" + return +} diff --git a/lib/pkg.sh b/lib/pkg.sh index 23e9d73..b399070 100644 --- a/lib/pkg.sh +++ b/lib/pkg.sh @@ -10,11 +10,12 @@ LSS_PKG_UPDATED=${LSS_PKG_UPDATED:-0} # DPkg::Lock::Timeout waits for unattended-upgrades or apt-daily instead of failing on their lock; they start on # their own on a fresh server and as soon as the auto-updates module enables them. # The filter drops the notices that report nothing to do: the needrestart summaries and the man-db trigger on images -# that switched off man-db/auto-update. Errors and everything else stay in the log. +# that switched off man-db/auto-update, and adduser skipping the home directory of a package's system user. Errors and +# everything else stay in the log. _pkg_apt() { DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get -y -q -o DPkg::Lock::Timeout=600 \ -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold "$@" 2>&1 | - sed -E '/^(No (services|containers) need to be restarted\.|No user sessions are running outdated binaries\.|No VM guests are running outdated hypervisor|Not building database; man-db\/auto-update is not)/d' + sed -E '/^(No (services|containers) need to be restarted\.|No user sessions are running outdated binaries\.|No VM guests are running outdated hypervisor|Not building database; man-db\/auto-update is not|Not creating (home directory )?`)/d' return "${PIPESTATUS[0]}" } diff --git a/lib/repo.sh b/lib/repo.sh index 8cf3445..4be8d6e 100644 --- a/lib/repo.sh +++ b/lib/repo.sh @@ -81,15 +81,12 @@ repo_add() { mkdir -p /etc/apt/keyrings file_write 0644 root:root "${keyring}" < <(gpg --batch --dearmor <"${key}") local changed=${LSS_FILE_CHANGED} - file_write 0644 root:root "/etc/apt/sources.list.d/lss-${name}.sources" < +EOF + elif [[ ${kind} == strip ]]; then + cat < + ProxyPass ${target%/}/ upgrade=websocket + ProxyPassReverse ${target%/}/ + EOF else cat <&2 + return 1 + } + REPOSITORY=${repository} + BACKUP_TMP=${CACHE}/tmp + mkdir -p "${BACKUP_TMP}" + chmod 0700 "${CACHE}" "${BACKUP_TMP}" + RESTIC_ENV=("RESTIC_REPOSITORY=${repository}" "RESTIC_PASSWORD_FILE=$(secret_file restic-password)" + "RESTIC_CACHE_DIR=${CACHE}" "TMPDIR=${BACKUP_TMP}") + RESTIC_OPTIONS=(--quiet) + case ${repository} in + s3:*) + key_id=$(<"$(secret_file restic-s3-key-id)") + secret=$(<"$(secret_file restic-s3-secret)") + RESTIC_ENV+=("AWS_ACCESS_KEY_ID=${key_id}" "AWS_SECRET_ACCESS_KEY=${secret}") + region=$(setting BACKUP_S3_REGION) + if [[ -n ${region} ]]; then + RESTIC_ENV+=("AWS_DEFAULT_REGION=${region}") + fi + ;; + sftp:*) + target=$(sftp_target "${repository}") || { + say "BACKUP_REPOSITORY '${repository}' names no host." >&2 + return 1 + } + # The key in the secrets, the host key pinned on first contact, never a password prompt. + RESTIC_OPTIONS+=(-o "sftp.command=ssh -i $(secret_file restic-ssh-key) -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=${STATE}/known_hosts ${target} -s sftp") + ;; + *) ;; + esac + return +} + +# restic_env ARGS: restic with RESTIC_ENV exported in a subshell of its own, so no other process inherits it. +restic_env() { + ( + export "${RESTIC_ENV[@]}" + "${RESTIC}" "$@" + ) + return +} + +# run_restic ARGS: restic_env with the options of the repository, quiet. +run_restic() { + restic_env "${RESTIC_OPTIONS[@]}" "$@" + return +} + +stamp() { + local name=$1 value=$2 + mkdir -p "${STATE}" + printf '%s\n' "${value}" >"${STATE}/${name}.tmp" + mv -f -- "${STATE}/${name}.tmp" "${STATE}/${name}" + return +} + +stamp_get() { + local file=${STATE}/$1 + if [[ -f ${file} ]]; then + printf '%s' "$(<"${file}")" + fi + return +} + +# metrics: the stamps as Prometheus metrics for the node exporter's textfile collector (when it is installed). +metrics() { + local file name value + [[ -d ${TEXTFILE_DIR} ]] || return 0 + file=${TEXTFILE_DIR}/lss_backup.prom + for name in last_run_timestamp_seconds:last-run last_run_success:last-run-success \ + last_success_timestamp_seconds:last-success restore_test_timestamp_seconds:restore-test \ + restore_test_success:restore-test-success; do + value=$(stamp_get "${name#*:}") + [[ ${value} =~ ^[0-9]+$ ]] || continue + printf '# TYPE lss_backup_%s gauge\nlss_backup_%s %s\n' "${name%%:*}" "${name%%:*}" "${value}" + done >"${file}.tmp" + chmod 0644 "${file}.tmp" + mv -f -- "${file}.tmp" "${file}" + return +} + +# backup_paths: the configured paths that exist (restic counts a missing one as an error). +backup_paths() { + local path + local -a paths=() + read -ra paths <<<"$(setting BACKUP_PATHS /etc)" + for path in "${paths[@]}"; do + if [[ -e ${path} ]]; then + printf '%s\n' "${path}" + fi + done + printf '%s\n' "${CANARY_DIR}" + return +} + +# canary: a file with fresh content in every backup; the restore test compares it with what it gets back. +canary() { + local value + mkdir -p "${CANARY_DIR}" + chmod 0700 "${STATE}" "${CANARY_DIR}" + value="$(date +%s) $(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" + printf '%s\n' "${value}" >"${CANARY}" + printf '%s' "${value}" + return +} + +cmd_init() { + prepare + if run_restic cat config >/dev/null 2>&1; then + return 0 + fi + run_restic init >/dev/null + say "created the repository ${REPOSITORY%%\?*}." + return +} + +cmd_run() { + local status=0 code=0 value now pattern + local -a paths=() excludes=() patterns=() + now=$(date +%s) + stamp last-run "${now}" + # The dumps run before prepare reads any credential. + if [[ -x ${DB_DUMPS} && -f ${DB_DUMPS_CONF} ]]; then + "${DB_DUMPS}" || { + status=1 + say "the database dumps failed; the backup goes on with the files." + } + fi + prepare + cmd_init + mapfile -t paths < <(backup_paths) + # Read as words, never expanded as globs here: the patterns are restic's. + read -ra patterns <<<"$(setting BACKUP_EXCLUDE)" + for pattern in "${patterns[@]}"; do + excludes+=(--exclude "${pattern}") + done + value=$(canary) + run_restic backup --tag "${TAG}" --exclude-caches "${excludes[@]}" "${paths[@]}" || code=$? + case ${code} in + 0) stamp canary-last "${value}" ;; + 3) + # Some files could not be read; the snapshot exists without them. + stamp canary-last "${value}" + status=1 + say "restic could not read some files; see the lines above." + ;; + *) + status=1 + say "restic backup failed (exit code ${code})." + ;; + esac + if ! run_restic forget --tag "${TAG}" --group-by host,tags --prune \ + --keep-daily "$(setting BACKUP_KEEP_DAILY 7)" --keep-weekly "$(setting BACKUP_KEEP_WEEKLY 4)" \ + --keep-monthly "$(setting BACKUP_KEEP_MONTHLY 6)"; then + status=1 + say "restic forget and prune failed." + fi + if ! run_restic check; then + status=1 + say "restic check reports errors in the repository." + fi + if ((status == 0)); then + stamp last-success "${now}" + stamp last-run-success 1 + say "backup complete." + else + stamp last-run-success 0 + fi + metrics + return "${status}" +} + +# restore_test [TAG]: restores the canary of the latest snapshot of this host and compares it. +restore_test() { + local tag=${1:-${TAG}} target expected actual status=0 subset + prepare + expected=$(stamp_get canary-last) + target=$(mktemp -d "${BACKUP_TMP}/restore.XXXXXX") + if ! run_restic restore latest --host "$(hostname)" --tag "${tag}" --target "${target}" --include "${CANARY}"; then + say "restic restore of the latest snapshot failed." + status=1 + fi + actual='' + if [[ -f ${target}${CANARY} ]]; then + actual=$(<"${target}${CANARY}") + fi + rm -rf -- "${target}" + if ((status == 0)) && [[ -z ${actual} || ${actual} != "${expected}" ]]; then + say "the restored canary file differs from the one backed up last." + status=1 + fi + # Reads a share of the packs, so damage on the storage side shows up without downloading everything every week. + subset=$(setting BACKUP_CHECK_SUBSET 5%) + if ((status == 0)) && ! run_restic check --read-data-subset="${subset}"; then + say "restic check --read-data-subset=${subset} reports errors." + status=1 + fi + return "${status}" +} + +cmd_restore_test() { + local status=0 + restore_test || status=$? + stamp restore-test "$(date +%s)" + if ((status == 0)); then + stamp restore-test-success 1 + say "restore test passed." + else + stamp restore-test-success 0 + fi + metrics + return "${status}" +} + +# cmd_probe: proves that the repository takes a backup and gives it back, with the canary file only. +cmd_probe() { + local value + cmd_init + value=$(canary) + run_restic backup --tag "${PROBE_TAG}" "${CANARY_DIR}" + stamp canary-last "${value}" + restore_test "${PROBE_TAG}" + run_restic forget --tag "${PROBE_TAG}" --keep-last 1 >/dev/null + say "probe passed: the repository takes a backup and restores it." + return +} + +# dispatch COMMAND [ARGS]: runs one subcommand. +dispatch() { + local command=${1:-run} + shift || true + case ${command} in + run) cmd_run ;; + init) cmd_init ;; + restore-test) cmd_restore_test ;; + probe) cmd_probe ;; + restic) + prepare + restic_env "${RESTIC_OPTIONS[@]:1}" "$@" + ;; + *) + say "usage: backup run|init|restore-test|probe|restic ARGS" >&2 + exit 2 + ;; + esac +} + +main() { + [[ ${EUID} -eq 0 ]] || { + say "must run as root." >&2 + exit 1 + } + umask 077 + dispatch "$@" +} + +# The unit tests source this file without running it. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + main "$@" +fi diff --git a/libexec/lss-db-dumps b/libexec/lss-db-dumps new file mode 100755 index 0000000..e6998d8 --- /dev/null +++ b/libexec/lss-db-dumps @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lss db-dumps: consistent dumps of the databases lss installed, for the restic backup. Installed by the db-dumps +# module as /usr/local/lib/lss/db-dumps; lss-backup runs it before every backup. Settings come from +# /etc/lss/db-dumps.conf (KEY=value lines, parsed, never sourced). +# +# Which engines exist is read at run time from the lss module state (/var/lib/lss/modules), so a database installed +# later is dumped without touching this module: +# mariadb, mysql mariadb-dump / mysqldump of every database in one transaction, with routines, events and triggers +# postgresql pg_dumpall (roles, every database) as the postgres user +# mongodb mongodump of every database as an archive, with the administrator from the lss secrets +# redis, valkey BGSAVE, then a copy of the RDB snapshot +# One file per engine in /var/backups/lss-db (root only), replaced on every run; restic keeps the history. +set -Eeuo pipefail +shopt -s inherit_errexit + +CONF=${LSS_DB_DUMPS_CONF:-/etc/lss/db-dumps.conf} +SECRETS_DIR=${LSS_SECRETS_DIR:-/etc/lss/secrets} +MODULES_DIR=${LSS_STATE_DIR:-/var/lib/lss}/modules +STATUS_DIR=${LSS_DB_DUMPS_STATE:-${LSS_STATE_DIR:-/var/lib/lss}/db-dumps} +# The password files of Redis and Valkey (their modules write them) and the parent of their data directories. +ETC_DIR=${LSS_DB_DUMPS_ETC:-/etc/lss} +DATA_ROOT=${LSS_DB_DUMPS_DATA:-/var/lib} +ENGINES="mariadb mysql postgresql mongodb redis valkey" +# The dump programs start with an empty environment and this PATH: nothing of the unit's or the caller's environment +# reaches them, and nothing at all a process of another user (postgres). +CLEAN_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +say() { + printf 'lss-db-dumps: %s\n' "$*" + return +} + +setting() { + local key=$1 default=${2-} line + if [[ -r ${CONF} ]]; then + while IFS= read -r line || [[ -n ${line} ]]; do + if [[ ${line} == "${key}="* ]]; then + printf '%s' "${line#*=}" + return 0 + fi + done <"${CONF}" + fi + printf '%s' "${default}" +} + +# installed ENGINE: lss installed the module ENGINE. +installed() { + local file=${MODULES_DIR}/$1 + [[ -f ${file} ]] && grep -qx 'status=installed' "${file}" + return +} + +dump_dir() { + setting DB_DUMPS_DIR /var/backups/lss-db + return +} + +# first_program NAME...: the first program that exists, by path or in the PATH. +first_program() { + local program + for program in "$@"; do + if [[ ${program} == /* && -x ${program} ]] || { [[ ${program} != /* ]] && command -v "${program}" >/dev/null 2>&1; }; then + printf '%s' "${program}" + return 0 + fi + done + return 1 +} + +# The SQL servers: root logs in through the Unix socket (MariaDB, MySQL on Debian and Ubuntu) or with the password in +# /root/.my.cnf (MySQL on the RHEL family), which a service does not find on its own (no HOME). +dump_mysql() { + local engine=$1 out=$2 dump socket='' + local -a options=() + if [[ ${engine} == mariadb ]]; then + dump=$(first_program mariadb-dump mysqldump) || return 1 + else + dump=$(first_program /opt/mysql/bin/mysqldump mysqldump) || return 1 + # GTID metadata would make the dump fail to load on a server with its own GTIDs. + options+=(--set-gtid-purged=OFF) + if [[ -S /run/mysqld/mysqld.sock ]]; then socket=/run/mysqld/mysqld.sock; else socket=/var/lib/mysql/mysql.sock; fi + options+=(--socket="${socket}") + fi + if [[ -f /root/.my.cnf ]]; then + options=(--defaults-extra-file=/root/.my.cnf "${options[@]}") + fi + env -i "PATH=${CLEAN_PATH}" "${dump}" "${options[@]}" --user=root --all-databases --single-transaction --routines --events --triggers \ + --hex-blob >"${out}" + return +} + +dump_postgresql() { + local out=$1 dumpall port + dumpall=$(first_program pg_dumpall /usr/pgsql-*/bin/pg_dumpall) || return 1 + port=$(setting DB_POSTGRESQL_PORT 5432) + # As postgres from /, so the dump runs with the peer login and never inside a directory a user controls. + (cd / && runuser -u postgres -- env -i "PATH=${CLEAN_PATH}" "${dumpall}" -p "${port}" --clean --if-exists) >"${out}" + return +} + +# MongoDB: the password goes to mongodump in a configuration file of root's, never on the command line. +dump_mongodb() { + local out=$1 port password config status=0 + command -v mongodump >/dev/null 2>&1 || return 1 + port=$(setting DB_MONGODB_PORT 27017) + password=$(<"${SECRETS_DIR}/mongodb-admin") + config=$(mktemp) + printf 'password: "%s"\n' "${password}" >"${config}" + env -i "PATH=${CLEAN_PATH}" mongodump --quiet --config="${config}" --host=127.0.0.1 --port="${port}" --username=lss-admin \ + --authenticationDatabase=admin --archive="${out}" || status=$? + rm -f -- "${config}" + return "${status}" +} + +# Redis and Valkey: a fresh snapshot (BGSAVE), copied once LASTSAVE moves. The password goes through the environment. +dump_rdb() { + local engine=$1 out=$2 cli password port before i now + if [[ ${engine} == redis ]]; then + cli=$(first_program redis-cli) || return 1 + else + cli=$(first_program /opt/valkey/bin/valkey-cli valkey-cli) || return 1 + fi + password=$(<"${ETC_DIR}/${engine}-password") + port=$(setting "DB_${engine^^}_PORT" 6379) + before=$(REDISCLI_AUTH=${password} VALKEYCLI_AUTH=${password} "${cli}" -p "${port}" --no-auth-warning lastsave) + [[ ${before} =~ ^[0-9]+$ ]] || return 1 + REDISCLI_AUTH=${password} VALKEYCLI_AUTH=${password} "${cli}" -p "${port}" --no-auth-warning bgsave >/dev/null + for ((i = 0; i < 300; i++)); do + now=$(REDISCLI_AUTH=${password} VALKEYCLI_AUTH=${password} "${cli}" -p "${port}" --no-auth-warning lastsave) + [[ ${now} == "${before}" ]] || break + sleep 1 + done + [[ ${now} != "${before}" ]] || return 1 + cp -- "${DATA_ROOT}/${engine}/dump.rdb" "${out}" + return +} + +dump() { + local engine=$1 out=$2 + case ${engine} in + mariadb | mysql) dump_mysql "${engine}" "${out}" ;; + postgresql) dump_postgresql "${out}" ;; + mongodb) dump_mongodb "${out}" ;; + redis | valkey) dump_rdb "${engine}" "${out}" ;; + *) return 1 ;; + esac + return +} + +file_of() { + case $1 in + mariadb | mysql | postgresql) printf '%s.sql' "$1" ;; + mongodb) printf 'mongodb.archive' ;; + *) printf '%s.rdb' "$1" ;; + esac + return +} + +# run_dumps: one dump per installed engine; fails when one of them failed. +run_dumps() { + local dir engine out failed=0 count=0 + dir=$(dump_dir) + mkdir -p "${dir}" "${STATUS_DIR}" + chmod 0700 "${dir}" "${STATUS_DIR}" + : >"${STATUS_DIR}/status.tmp" + for engine in ${ENGINES}; do + installed "${engine}" || continue + count=$((count + 1)) + out=${dir}/$(file_of "${engine}") + # The dump goes to a temporary file first, so a failed run never replaces the last good dump. + if dump "${engine}" "${out}.part" && [[ -s ${out}.part ]]; then + mv -f -- "${out}.part" "${out}" + printf '%s ok %s\n' "${engine}" "$(date +%s)" >>"${STATUS_DIR}/status.tmp" + say "dumped ${engine} to ${out}." + else + rm -f -- "${out}.part" + printf '%s failed %s\n' "${engine}" "$(date +%s)" >>"${STATUS_DIR}/status.tmp" + say "the dump of ${engine} failed; ${out} keeps the previous one." + failed=1 + fi + done + mv -f -- "${STATUS_DIR}/status.tmp" "${STATUS_DIR}/status" + ((count > 0)) || say "lss installed 0 databases here; nothing to dump." + return "${failed}" +} + +main() { + [[ ${EUID} -eq 0 ]] || { + say "must run as root." >&2 + exit 1 + } + umask 077 + run_dumps +} + +# The unit tests source this file without running it. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + main "$@" +fi diff --git a/libexec/lss-guard b/libexec/lss-guard index 274e055..3a8f825 100755 --- a/libexec/lss-guard +++ b/libexec/lss-guard @@ -10,7 +10,8 @@ # when GUARD_AUTOFIX=true), failed systemd units, disk space, pending security updates, vulnerable packages (debsecan # on Debian, dnf updateinfo on the RHEL family), failed SSH logins and active bans, the results of the daily AIDE, # rkhunter and ClamAV runs, the Lynis hardening index, certificates that expire soon, HIGH and CRITICAL vulnerabilities -# in the container images of Docker and Podman (Trivy). +# in the container images of Docker and Podman (Trivy), the age of the last backup and a weekly restore test (restic). +# The numbers of findings, fixes and notes also go to the node exporter for Prometheus (lss_guard_*). # Every finding goes to the journal (identifier lss-guard); a summary goes to the webhook and the e-mail address. # Notes (logins, bans) are part of the summary but do not make the run fail. set -Eeuo pipefail @@ -24,6 +25,13 @@ TLS_DIR=${LSS_TLS_DIR:-/etc/lss/tls} # Installed by the docker and podman modules (lib/container.sh). TRIVY=${LSS_GUARD_TRIVY:-/usr/local/bin/trivy} TRIVY_CACHE=${LSS_GUARD_TRIVY_CACHE:-/var/cache/lss-trivy} +# Installed by the restic module (libexec/lss-backup), with its state next to it. +BACKUP=${LSS_GUARD_BACKUP:-/usr/local/lib/lss/backup} +BACKUP_STATE=${LSS_BACKUP_STATE:-${STATE_DIR}/restic} +# The node exporter's textfile collector (lib/monitoring.sh); the guard's metrics go there when it exists. +TEXTFILE_DIR=${LSS_TEXTFILE_DIR:-/var/lib/prometheus/node-exporter} +# A restore test every week, a little earlier so a daily run at the same time does not slip to the next day. +RESTORE_TEST_INTERVAL=$((7 * 86400 - 3600)) # Global also when the script is sourced inside a function (the unit tests). declare -ga FINDINGS=() FIXED=() NOTES=() @@ -326,6 +334,64 @@ check_containers() { return } +# backup_stamp NAME: a time or result the backup script recorded (libexec/lss-backup); nothing when it never did. +backup_stamp() { + local file=${BACKUP_STATE}/$1 value='' + [[ -f ${file} ]] && value=$(<"${file}") + [[ ${value} =~ ^[0-9]+$ ]] && printf '%s' "${value}" + return 0 +} + +# The restic backup (when the restic module is installed): the last successful backup must be younger than +# GUARD_BACKUP_MAX_AGE hours (counted from the installation until the first one), and once a week a restore test +# proves that the latest backup comes back. +check_backup() { + local max now last installed age tested + [[ -x ${BACKUP} ]] || return 0 + max=$(setting GUARD_BACKUP_MAX_AGE 30) + now=$(date +%s) + last=$(backup_stamp last-success) + if [[ -z ${last} ]]; then + installed=$(sed -n 's/^installed_at=//p' "${STATE_DIR}/modules/restic" 2>/dev/null || true) + last=$(date -d "${installed:-now}" +%s 2>/dev/null || printf '%s' "${now}") + fi + age=$(((now - last) / 3600)) + if ((age >= max)); then + finding "The last successful backup is ${age} hours old, more than ${max} (journalctl -u lss-backup)." + fi + [[ -n $(backup_stamp last-success) ]] || return 0 + tested=$(backup_stamp restore-test) + if [[ -z ${tested} ]] || ((now - tested >= RESTORE_TEST_INTERVAL)); then + if "${BACKUP}" restore-test >/dev/null 2>&1; then + logger -t lss-guard -p auth.info -- "Backup restore test passed" + else + finding "The weekly restore test of the backup failed (${BACKUP} restore-test shows why)." + fi + elif [[ $(backup_stamp restore-test-success) == 0 ]]; then + finding "The last restore test of the backup failed (${BACKUP} restore-test shows why)." + fi + return +} + +# export_metrics: the result of this run for Prometheus, through the node exporter's textfile collector. +export_metrics() { + local file=${TEXTFILE_DIR}/lss_guard.prom + [[ -d ${TEXTFILE_DIR} ]] || return 0 + { + printf '# HELP lss_guard_findings Findings of the last lss guard run.\n# TYPE lss_guard_findings gauge\n' + printf 'lss_guard_findings %s\n' "${#FINDINGS[@]}" + printf '# HELP lss_guard_fixed Modules the last lss guard run restored.\n# TYPE lss_guard_fixed gauge\n' + printf 'lss_guard_fixed %s\n' "${#FIXED[@]}" + printf '# HELP lss_guard_notes Notes of the last lss guard run.\n# TYPE lss_guard_notes gauge\n' + printf 'lss_guard_notes %s\n' "${#NOTES[@]}" + printf '# HELP lss_guard_last_run_timestamp_seconds End of the last lss guard run.\n' + printf '# TYPE lss_guard_last_run_timestamp_seconds gauge\nlss_guard_last_run_timestamp_seconds %s\n' "$(date +%s)" + } >"${file}.tmp" + chmod 0644 "${file}.tmp" + mv -f -- "${file}.tmp" "${file}" + return +} + json_escape() { local text=$1 text=${text//\\/\\\\} @@ -375,6 +441,8 @@ main() { check_lynis check_certificates check_containers + check_backup + export_metrics local summary host host=$(hostname -f 2>/dev/null || hostname) if [[ ${#FINDINGS[@]} -eq 0 && ${#FIXED[@]} -eq 0 ]]; then diff --git a/lss b/lss index 64f3393..532763c 100755 --- a/lss +++ b/lss @@ -14,7 +14,7 @@ LSS_ROOT=$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd) LSS_VERSION=$(<"${LSS_ROOT}/VERSION") export LSS_ROOT LSS_VERSION -for library in log os config files template secrets state pkg repo fetch service firewall container web module ui; do +for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring module ui; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/modules/backup/db-dumps.sh b/modules/backup/db-dumps.sh new file mode 100644 index 0000000..ee1c104 --- /dev/null +++ b/modules/backup/db-dumps.sh @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="db-dumps" +MOD_GROUP="backup" +MOD_DESCRIPTION="Consistent dumps of the MariaDB, MySQL, PostgreSQL, MongoDB, Redis and Valkey servers lss installed, before every restic backup" +MOD_REQUIRES="restic" +MOD_OPTIONS="dir|/var/backups/lss-db|Directory of the dumps (root only); restic backs it up when it is in LSS_RESTIC_PATHS" +MOD_PORTS="none" + +# A copy of a database's files taken while it runs may not load again; a dump does. libexec/lss-db-dumps finds the +# installed engines at run time, so databases added later are dumped without lss configure db-dumps. +DUMPS_SCRIPT=/usr/local/lib/lss/db-dumps +DUMPS_CONF=/etc/lss/db-dumps.conf +DUMPS_STATUS=/var/lib/lss/db-dumps/status + +mod_check() { + [[ $(opt dir) =~ ^/var/backups/[A-Za-z0-9._-]+$ ]] || die "LSS_DB_DUMPS_DIR must be a directory below /var/backups." + if [[ " $(config_option restic paths '') " != *" $(opt dir) "* && -n $(config_option restic paths '') ]]; then + log_warn "db-dumps: $(opt dir) is not in LSS_RESTIC_PATHS, so the dumps stay on this server only." + fi + return +} + +# mongodump is a separate package from the MongoDB repository the mongodb module set up; configure installs it too, +# for a mongodb module that came later. +mod_install() { + if state_installed mongodb; then + pkg_install mongodb-database-tools + fi + return +} + +mod_configure() { + mod_install + install -d -m 0700 "$(opt dir)" + file_write 0755 root:root "${DUMPS_SCRIPT}" <"${LSS_ROOT}/libexec/lss-db-dumps" + file_write 0600 root:root "${DUMPS_CONF}" </dev/null 2>&1; then + die "db-dumps: mongodump is missing; run lss configure db-dumps." + fi + # Dumps run with the backup; here only for engines that have never been dumped, so a second verify stays quiet. + if [[ -n $(_dumps_missing) ]]; then + "${DUMPS_SCRIPT}" || die "db-dumps: the dump of $(_dumps_missing)failed; see above." + fi + return +} + +mod_remove() { + file_remove "${DUMPS_SCRIPT}" + file_remove "${DUMPS_CONF}" + pkg_remove mongodb-database-tools + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + rm -rf -- "$(opt dir)" + mon_delete /var/lib/lss/db-dumps + files_note_change "deleted the database dumps" + return +} diff --git a/modules/backup/restic.sh b/modules/backup/restic.sh new file mode 100644 index 0000000..14f9261 --- /dev/null +++ b/modules/backup/restic.sh @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="restic" +MOD_GROUP="backup" +MOD_DESCRIPTION="Encrypted, deduplicated daily backups with restic to a local path, SFTP or S3, with retention, repository checks and a weekly restore test (guard)" +MOD_REQUIRES="system" +MOD_OPTIONS="repository|/var/backups/lss-restic|Where the backups go: a local path (better another disk), sftp:USER@HOST:/PATH or s3:https://HOST/BUCKET/PATH +paths|/etc /root /home /srv /opt /usr/local /var/www /var/spool/cron /var/lib/lss /var/backups/lss-db|What to back up, separated by spaces; paths that do not exist are skipped +exclude|/var/lib/lss/runs|Paths or patterns restic leaves out, separated by spaces (restic --exclude) +time|03:00|Time of the daily backup (HH:MM) +keep_daily|7|Daily snapshots to keep +keep_weekly|4|Weekly snapshots to keep +keep_monthly|6|Monthly snapshots to keep +check_subset|5%|Share of the data the weekly restore test reads back (restic check --read-data-subset) +s3_key_id||S3 access key ID, stored in the lss secrets (restic-s3-key-id); s3 repositories only +s3_secret||S3 secret access key, stored in the lss secrets (restic-s3-secret); s3 repositories only +s3_region||S3 region, when the provider needs one" +MOD_SECRETS="restic-password restic-ssh-key restic-s3-key-id restic-s3-secret" +MOD_PORTS="none (outgoing SSH or HTTPS to the repository)" + +# Every supported release packages restic (0.12 on Ubuntu 22.04 up to 0.19 in EPEL); the commands lss uses work with +# all of them. libexec/lss-backup does the work; the unit runs it daily. +BACKUP_SCRIPT=/usr/local/lib/lss/backup +BACKUP_CONF=/etc/lss/backup.conf +BACKUP_UNIT=lss-backup +BACKUP_STATE=/var/lib/lss/restic +BACKUP_CACHE=/var/cache/restic +BACKUP_PUBLIC_KEY=/etc/lss/restic-ssh-key.pub + +_restic_kind() { + case $(opt repository) in + sftp:*) printf 'sftp' ;; + s3:*) printf 's3' ;; + *) printf 'local' ;; + esac + return +} + +mod_check() { + local repository number + repository=$(opt repository) + case ${repository} in + sftp:*) [[ ${repository} =~ ^sftp:(//)?[A-Za-z0-9._-]+@[A-Za-z0-9.-]+(:[0-9]+)?[:/].+$ ]] || + die "LSS_RESTIC_REPOSITORY: an sftp repository looks like sftp:USER@HOST:/PATH." ;; + s3:*) [[ ${repository} =~ ^s3:https://[A-Za-z0-9.:-]+/[A-Za-z0-9._/-]+$ ]] || + die "LSS_RESTIC_REPOSITORY: an s3 repository looks like s3:https://HOST/BUCKET/PATH." ;; + /*) [[ ${repository} =~ ^/[A-Za-z0-9._/-]+$ && ${repository} != *..* ]] || + die "LSS_RESTIC_REPOSITORY: '${repository}' is not a plain absolute path." ;; + *) die "LSS_RESTIC_REPOSITORY must be an absolute path, sftp:USER@HOST:/PATH or s3:https://HOST/BUCKET/PATH." ;; + esac + [[ $(opt paths) =~ ^/[A-Za-z0-9._/[:space:]-]*$ ]] || die "LSS_RESTIC_PATHS must be absolute paths separated by spaces." + [[ $(opt exclude) =~ ^[A-Za-z0-9._/*?[:space:]-]*$ ]] || die "LSS_RESTIC_EXCLUDE holds characters restic patterns do not need." + [[ $(opt time) =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || die "LSS_RESTIC_TIME must look like 03:00." + for number in keep_daily keep_weekly keep_monthly; do + opt_int "${number}" >/dev/null + done + [[ $(opt check_subset) =~ ^([1-9][0-9]?|100)%$ ]] || die "LSS_RESTIC_CHECK_SUBSET must be a percentage such as 5%." + if [[ $(_restic_kind) == s3 ]] && ! secret_exists restic-s3-key-id && [[ -z $(opt s3_key_id) || -z $(opt s3_secret) ]]; then + die "LSS_RESTIC_REPOSITORY is an s3 repository: set LSS_RESTIC_S3_KEY_ID and LSS_RESTIC_S3_SECRET." + fi + [[ $(opt s3_region) =~ ^[a-z0-9-]*$ ]] || die "LSS_RESTIC_S3_REGION must look like eu-central-1." + return +} + +mod_install() { + if os_is_rhel; then + pkg_enable_epel + fi + pkg_install restic + return +} + +# _restic_secrets: the repository password (generated once), the S3 keys from the settings, the SSH key for sftp. +_restic_secrets() { + local value key + if ! secret_exists restic-password; then + secret_get restic-password 48 >/dev/null + log_warn "restic: generated the repository password. Keep a copy outside this server (lss secrets show restic-password); without it no backup can be restored." + files_note_change "generated the restic repository password" + fi + for key in s3_key_id:restic-s3-key-id s3_secret:restic-s3-secret; do + value=$(opt "${key%%:*}") + [[ -n ${value} ]] || continue + log_mask "${value}" + if ! secret_exists "${key#*:}" || [[ $(secret_get "${key#*:}") != "${value}" ]]; then + secret_set "${key#*:}" "${value}" + files_note_change "stored ${key#*:}" + fi + done + if [[ $(_restic_kind) == sftp && ! -s $(secret_path restic-ssh-key) ]]; then + ssh-keygen -q -t ed25519 -N '' -C "lss-restic@$(hostname)" -f "$(secret_path restic-ssh-key)" + mv -f -- "$(secret_path restic-ssh-key).pub" "${BACKUP_PUBLIC_KEY}" + chmod 0644 "${BACKUP_PUBLIC_KEY}" + files_note_change "generated the SSH key for the sftp repository" + fi + if [[ -s ${BACKUP_PUBLIC_KEY} ]]; then + log_info "restic: the sftp server must accept this key for the repository user: $(<"${BACKUP_PUBLIC_KEY}")" + fi + return +} + +_restic_conf() { + printf '# Managed by lss (module restic). Change LSS_RESTIC_* in /etc/lss/lss.conf and run lss configure restic.\n' + printf 'BACKUP_REPOSITORY=%s\nBACKUP_PATHS=%s\nBACKUP_EXCLUDE=%s\n' "$(opt repository)" "$(opt paths)" "$(opt exclude)" + printf 'BACKUP_KEEP_DAILY=%s\nBACKUP_KEEP_WEEKLY=%s\nBACKUP_KEEP_MONTHLY=%s\n' "$(opt_int keep_daily)" \ + "$(opt_int keep_weekly)" "$(opt_int keep_monthly)" + printf 'BACKUP_CHECK_SUBSET=%s\nBACKUP_S3_REGION=%s\n' "$(opt check_subset)" "$(opt s3_region)" + return +} + +mod_configure() { + local repository writable='' + repository=$(opt repository) + _restic_secrets + install -d -m 0700 "${BACKUP_STATE}" "${BACKUP_CACHE}" + if [[ $(_restic_kind) == local ]]; then + install -d -m 0700 "${repository}" + writable=${repository} + fi + file_write 0755 root:root "${BACKUP_SCRIPT}" <"${LSS_ROOT}/libexec/lss-backup" + _restic_conf | file_write 0600 root:root "${BACKUP_CONF}" + # Reads everything, writes only its state, its cache, the dumps, the metrics and a local repository. + svc_unit "${BACKUP_UNIT}.service" < <( + printf '# Managed by lss (module restic).\n[Unit]\nDescription=lss backup with restic\n' + printf 'After=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\n' + printf 'ExecStart=%s run\nNice=10\nIOSchedulingClass=idle\n' "${BACKUP_SCRIPT}" + svc_scanner_sandbox strict "${BACKUP_STATE}" "${BACKUP_CACHE}" /var/backups/lss-db "${MON_TEXTFILE_DIR}" \ + ${writable:+"${writable}"} + ) + svc_unit "${BACKUP_UNIT}.timer" </dev/null 2>&1 || + die "restic: the repository $(opt repository) does not open with the password from the lss secrets." + # Before the first scheduled backup a small probe proves that a backup goes in and comes back out. + if [[ ! -s ${BACKUP_STATE}/last-success && ! -s ${BACKUP_STATE}/probe ]]; then + "${BACKUP_SCRIPT}" probe || die "restic: the probe backup and its restore test fail." + date +%s >"${BACKUP_STATE}/probe" + fi + return +} + +mod_remove() { + svc_disable "${BACKUP_UNIT}.timer" + file_remove "/etc/systemd/system/${BACKUP_UNIT}.timer" + file_remove "/etc/systemd/system/${BACKUP_UNIT}.service" + svc_daemon_reload + file_remove "${BACKUP_SCRIPT}" + file_remove "${BACKUP_CONF}" + file_remove "${BACKUP_PUBLIC_KEY}" + rm -f -- "${MON_TEXTFILE_DIR}/lss_backup.prom" + pkg_remove restic + # The repository holds the backups and is never deleted by lss, not even with --purge. + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + rm -rf -- "${BACKUP_CACHE}" + mon_delete "${BACKUP_STATE}" + log_info "restic: the repository $(opt repository) and the password (lss secrets show restic-password) stay; delete them yourself when the backups are no longer needed." + return +} diff --git a/modules/base/time.sh b/modules/base/time.sh index b8c1676..29db971 100644 --- a/modules/base/time.sh +++ b/modules/base/time.sh @@ -41,8 +41,37 @@ mod_install() { return } +# The directory chrony reads further settings from, where ntp-server puts its own: inside chrony's directory of /etc, +# which the AppArmor profile of chronyd (Debian, Ubuntu) lets it read. The RHEL family uses /etc/chrony.d, except under +# an AppArmor kernel (a container on a Debian or Ubuntu host), where the host's profile for /usr/sbin/chronyd applies. +_time_confdir() { + if os_is_debian || [[ $(cat "${SVC_APPARMOR_ENABLED:-/sys/module/apparmor/parameters/enabled}" 2>/dev/null) == Y ]]; then + printf '/etc/chrony/conf.d' + else + printf '/etc/chrony.d' + fi + return +} + +# _time_chrony_config: writes chrony's configuration (sets LSS_FILE_CHANGED). ntp-server calls it as well, inside a +# container, where this module leaves chrony out. +_time_chrony_config() { + local server suffix='' + opt_bool nts && suffix=' nts' + TIME_SOURCES='' + for server in $(opt servers); do + TIME_SOURCES+="server ${server} iburst${suffix}"$'\n' + done + TIME_SOURCES=${TIME_SOURCES%$'\n'} + if os_is_debian; then TIME_DRIFTFILE=/var/lib/chrony/chrony.drift; else TIME_DRIFTFILE=/var/lib/chrony/drift; fi + TIME_CONFDIR=$(_time_confdir) + install -d -m 0755 "${TIME_CONFDIR}" + template_install time chrony.conf 0644 root:root "$(_time_config)" + return +} + mod_configure() { - local timezone zone_file server suffix='' + local timezone zone_file timezone=$(opt timezone) zone_file="/usr/share/zoneinfo/${timezone}" [[ -f ${zone_file} ]] || die "Unknown time zone '${timezone}' (no ${zone_file})." @@ -60,14 +89,7 @@ mod_configure() { return 0 fi - opt_bool nts && suffix=' nts' - TIME_SOURCES='' - for server in $(opt servers); do - TIME_SOURCES+="server ${server} iburst${suffix}"$'\n' - done - TIME_SOURCES=${TIME_SOURCES%$'\n'} - if os_is_debian; then TIME_DRIFTFILE=/var/lib/chrony/chrony.drift; else TIME_DRIFTFILE=/var/lib/chrony/drift; fi - template_install time chrony.conf 0644 root:root "$(_time_config)" + _time_chrony_config local changed=${LSS_FILE_CHANGED} # systemd-timesyncd must not fight chrony for the clock. svc_disable systemd-timesyncd diff --git a/modules/monitoring/alertmanager.sh b/modules/monitoring/alertmanager.sh new file mode 100644 index 0000000..1224237 --- /dev/null +++ b/modules/monitoring/alertmanager.sh @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="alertmanager" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Prometheus Alertmanager: sends the alerts of Prometheus by e-mail or webhook; UI and API below /alertmanager/ on 127.0.0.1:9093 or the WireGuard network" +MOD_REQUIRES="system" +MOD_OPTIONS="access|local|Who reaches http://SERVER:9093/alertmanager/: local (127.0.0.1 only; SSH tunnel) or vpn (WireGuard network only) +web_admin|false|Also publish Alertmanager below /alertmanager/ on the admin port of the web server (needs nginx, apache or caddy) +email||Send alerts to this address (needs the mail-relay module or another mail server on smarthost) +smarthost|localhost:25|Mail server Alertmanager hands e-mails to, HOST:PORT without authentication +webhook||Send alerts as JSON to this HTTPS URL (Alertmanager's webhook format; ntfy, n8n, a chat bridge, ...)" +MOD_PORTS="9093/tcp from the WireGuard network with access vpn; none with access local" + +# Debian and Ubuntu package Alertmanager (0.23 to 0.28); the RHEL family gets the checksum-verified upstream release +# with a unit of the same name. The configuration below works with all of these versions. +AM_VERSION=0.34.1 +AM_UNIT=prometheus-alertmanager +AM_CONF=/etc/lss/alertmanager/alertmanager.yml +AM_PATH=alertmanager + +_am_user() { + if os_is_debian; then printf 'prometheus'; else printf 'alertmanager'; fi + return +} + +_am_data() { + if os_is_debian; then printf '/var/lib/prometheus/alertmanager'; else printf '/var/lib/alertmanager'; fi + return +} + +_am_binary() { + if os_is_debian; then printf '/usr/bin/prometheus-alertmanager'; else printf '/usr/local/bin/alertmanager'; fi + return +} + +_am_url() { + mon_url alertmanager "${MON_ALERTMANAGER_PORT}" "/${AM_PATH}" + return +} + +mod_check() { + local email webhook + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_ALERTMANAGER_ACCESS must be local or vpn." + opt_bool web_admin || true + email=$(opt email) + webhook=$(opt webhook) + [[ -z ${email} || ${email} =~ ^[^@[:space:]\'\"]+@[^@[:space:]\'\"]+\.[^@[:space:]\'\"]+$ ]] || + die "LSS_ALERTMANAGER_EMAIL '${email}' is not an e-mail address." + [[ $(opt smarthost) =~ ^[A-Za-z0-9.:-]+:[0-9]{1,5}$ ]] || die "LSS_ALERTMANAGER_SMARTHOST must look like HOST:PORT." + [[ -z ${webhook} || ${webhook} =~ ^https://[^[:space:]\'\"]+$ ]] || die "LSS_ALERTMANAGER_WEBHOOK must be an https:// URL." + return +} + +mod_install() { + if os_is_debian; then + mon_install_stopped prometheus-alertmanager + else + mon_user alertmanager "$(_am_data)" + mon_upstream_install alertmanager "${AM_VERSION}" alertmanager amtool + fi + return +} + +# _am_config: one receiver for every alert. Without e-mail and webhook the alerts are only shown (Alertmanager UI, +# Grafana). +_am_config() { + local email webhook host + email=$(opt email) + webhook=$(opt webhook) + host=$(hostname -f 2>/dev/null || hostname) + printf '%s alertmanager). Change LSS_ALERTMANAGER_* in /etc/lss/lss.conf and run lss configure alertmanager.\n' "${MON_MANAGED_BY}" + cat <<'EOF' +route: + receiver: lss + group_by: [alertname, instance] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h +receivers: + - name: lss +EOF + if [[ -n ${email} ]]; then + printf " email_configs:\n - to: '%s'\n from: 'alertmanager@%s'\n smarthost: '%s'\n require_tls: false\n send_resolved: true\n" \ + "${email}" "${host}" "$(opt smarthost)" + fi + if [[ -n ${webhook} ]]; then + printf " webhook_configs:\n - url: '%s'\n send_resolved: true\n" "${webhook}" + fi + return +} + +_am_command() { + # An empty cluster address turns the gossip port 9094 off: a single Alertmanager has no peers. + printf '%s --config.file=%s --storage.path=%s --web.listen-address=%s:%s --web.external-url=%s/ --cluster.listen-address=' \ + "$(_am_binary)" "${AM_CONF}" "$(_am_data)" "$(mon_address alertmanager)" "${MON_ALERTMANAGER_PORT}" "$(_am_url)" + return +} + +_am_amtool() { + if os_is_debian; then printf '/usr/bin/amtool'; else printf '/usr/local/bin/amtool'; fi + return +} + +mod_configure() { + local before changed config reload=0 + before=$(files_changes_of alertmanager) + mon_vpn_ready + mon_dir "$(_am_data)" "$(_am_user):$(_am_user)" 0750 + install -d -m 0755 "$(dirname "${AM_CONF}")" + config=$(_am_config) + # The webhook URL is a credential (whoever has it can post), so only Alertmanager reads the file. + file_write 0640 "root:$(_am_user)" "${AM_CONF}" <<<"${config}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || reload=1 + "$(_am_amtool)" check-config "${AM_CONF}" >/dev/null || die "alertmanager: amtool rejects ${AM_CONF}." + install -d -m 0755 "${MON_ALERTMANAGERS_DIR}" + file_write 0644 root:root "${MON_ALERTMANAGERS_DIR}/alertmanager.yml" </dev/null || true + svc_disable "${AM_UNIT}" + svc_dropin_remove "${AM_UNIT}.service" lss-sandbox + svc_dropin_remove "${AM_UNIT}.service" lss-vpn + file_remove "${AM_CONF}" + rmdir "$(dirname "${AM_CONF}")" 2>/dev/null || true + if os_is_debian; then + svc_dropin_remove "${AM_UNIT}.service" lss-config + pkg_remove prometheus-alertmanager + else + file_remove "/etc/systemd/system/${AM_UNIT}.service" + svc_daemon_reload + mon_upstream_remove alertmanager amtool + fi + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete "$(_am_data)" + return +} diff --git a/modules/monitoring/alloy.sh b/modules/monitoring/alloy.sh new file mode 100644 index 0000000..779a5af --- /dev/null +++ b/modules/monitoring/alloy.sh @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="alloy" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Grafana Alloy ships the systemd journal of the server to Loki (Promtail's successor); its UI on 127.0.0.1:12345" +MOD_REQUIRES="system loki" +MOD_OPTIONS="max_age|12h|How far back Alloy reads the journal when it starts without a saved position (Go duration: 12h, 30m)" +MOD_PORTS="none (UI on 127.0.0.1:12345)" + +# Promtail reached its end of life on 2 March 2026 (https://grafana.com/docs/loki/latest/send-data/promtail/, checked +# 2026-10-14); Grafana Alloy replaces it and comes from the same repository (data/repos/grafana.conf) for every +# supported system. The package starts nothing on installation. +ALLOY_UNIT=alloy +ALLOY_CONF=/etc/lss/alloy/config.alloy +ALLOY_DATA=/var/lib/alloy/data +ALLOY_UI=127.0.0.1:12345 + +mod_check() { + [[ $(opt max_age) =~ ^[1-9][0-9]*(h|m)$ ]] || die "LSS_ALLOY_MAX_AGE must look like 12h or 30m." + return +} + +mod_install() { + repo_add grafana + pkg_install alloy + return +} + +# _alloy_config: the journal with the unit, the priority and the host as labels, pushed to Loki. +_alloy_config() { + cat </dev/null || true + pkg_remove alloy + mon_grafana_repo_remove + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete /var/lib/alloy + return +} diff --git a/modules/monitoring/grafana.sh b/modules/monitoring/grafana.sh new file mode 100644 index 0000000..ba113fa --- /dev/null +++ b/modules/monitoring/grafana.sh @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="grafana" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Grafana OSS from Grafana's repository with the data sources of the monitoring modules; below /grafana/ on 127.0.0.1:3000 or the WireGuard network" +MOD_REQUIRES="system prometheus" +MOD_OPTIONS="access|local|Who reaches http://SERVER:3000/grafana/: local (127.0.0.1 only; SSH tunnel) or vpn (WireGuard network only) +web_admin|false|Also publish Grafana below /grafana/ on the admin port of the web server (needs nginx, apache or caddy)" +MOD_PORTS="3000/tcp from the WireGuard network with access vpn; none with access local" +MOD_SECRETS="grafana-admin-password" + +# Grafana's own repository (data/repos/grafana.conf, pinned key) serves the same release to every supported system. +# The settings go in as GF_* variables of a drop-in, which win over grafana.ini; the package keeps its own file. +GRAFANA_PORT=3000 +GRAFANA_PATH=grafana +# The admin password for Grafana's first start, readable by Grafana only (the secrets directory is root's). +GRAFANA_PASSWORD_FILE=${MON_GRAFANA_DIR}/admin-password + +_grafana_url() { + mon_url grafana "${GRAFANA_PORT}" "/${GRAFANA_PATH}" + return +} + +mod_check() { + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_GRAFANA_ACCESS must be local or vpn." + opt_bool web_admin || true + return +} + +mod_install() { + repo_add grafana + pkg_install grafana + return +} + +_grafana_environment() { + cat <= 1)) || die "LSS_LOKI_RETENTION_DAYS must be at least 1." + return +} + +# _loki_config: one process with every component, an in-memory ring (no gossip port), the TSDB index and the chunks on +# the local disk; the compactor deletes what is older than the retention. +_loki_config() { + local days hours + days=$(opt_int retention_days) + hours=$((days * 24)) + cat </dev/null 2>&1 || + die "loki: Loki rejects ${LOKI_CONF} (loki -config.file=${LOKI_CONF} -verify-config shows why)." + # A start by the package's installation script that failed must not leave the unit at its start limit. + systemctl reset-failed "${LOKI_UNIT}.service" >/dev/null 2>&1 || true + svc_enable "${LOKI_UNIT}" + if [[ $(files_changes_of loki) != "${before}" ]] || ! svc_is_active "${LOKI_UNIT}"; then + svc_restart "${LOKI_UNIT}" || die "loki: Loki does not start." + fi + mon_firewall "${LOKI_PORT}/tcp" + mon_target loki loki "$(mon_address loki):${LOKI_PORT}" + mon_datasource Loki loki "$(_loki_url)" + return +} + +# _loki_probe: pushes one line and waits until a query returns it. +_loki_probe() { + local url token now i result='' + url=$(_loki_url) + token="lss-verify-$(date +%s)-${RANDOM}" + now=$(date +%s%N) + curl -s --max-time 10 -o /dev/null -H 'Content-Type: application/json' --data-binary @- "${url}/loki/api/v1/push" </dev/null || true + pkg_remove loki + mon_grafana_repo_remove + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete "${LOKI_DATA}" + return +} diff --git a/modules/monitoring/netdata.sh b/modules/monitoring/netdata.sh new file mode 100644 index 0000000..0644cba --- /dev/null +++ b/modules/monitoring/netdata.sh @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="netdata" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Netdata real-time dashboard, never claimed to Netdata Cloud, without registry or telemetry; on 127.0.0.1:19999 or the WireGuard network, optionally below /netdata/ on the admin port" +MOD_REQUIRES="system" +MOD_OPTIONS="access|local|Who reaches http://SERVER:19999: local (127.0.0.1 only; SSH tunnel) or vpn (also the WireGuard network) +web_admin|false|Also publish Netdata below /netdata/ on the admin port of the web server (needs nginx, apache or caddy) +ml|false|Machine-learning anomaly detection (costs CPU and memory on small servers)" +MOD_PORTS="19999/tcp from the WireGuard network with access vpn; none with access local" + +# Netdata's own repository (data/repos/netdata.conf, pinned keys) for every supported system: Debian 13, Ubuntu 26.04 +# and EPEL 10 have no Netdata package. Its packages need EPEL on the RHEL family. +NETDATA_UNIT=netdata +NETDATA_PORT=19999 +NETDATA_CONF=/etc/netdata/netdata.conf +NETDATA_OPT_OUT=/etc/netdata/.opt-out-from-anonymous-statistics + +mod_check() { + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_NETDATA_ACCESS must be local or vpn." + opt_bool web_admin || true + opt_bool ml || true + return +} + +_netdata_bind() { + local bind=127.0.0.1 + mon_vpn netdata && bind+=" $(fw_vpn_address)" + printf '%s' "${bind}" + return +} + +# _netdata_files: the settings that keep Netdata on this server, written before the package as well, so its first +# start already uses them. +_netdata_files() { + local ml=no + opt_bool ml && ml=yes + install -d -m 0755 /etc/netdata + file_write 0644 root:root "${NETDATA_CONF}" </dev/null || die "prometheus: promtool rejects ${PROM_CONF}." + mon_target prometheus prometheus "$(mon_address prometheus):${MON_PROMETHEUS_PORT}" "/${PROM_PATH}/metrics" + changed=$(files_changes_of prometheus) + if os_is_debian; then + mon_exec "${PROM_UNIT}" "$(_prom_command)" + else + mon_unit "${PROM_UNIT}" prometheus "$(_prom_command)" "Prometheus monitoring system" + fi + svc_sandbox "${PROM_UNIT}.service" "ReadWritePaths=$(_prom_data)" + mon_vpn_dropin "${PROM_UNIT}.service" + svc_enable "${PROM_UNIT}" + if [[ $(files_changes_of prometheus) != "${changed}" ]]; then + svc_restart "${PROM_UNIT}" || die "prometheus: Prometheus does not start." + elif [[ $(files_changes_of prometheus) != "${before}" && ${reload} == 1 ]]; then + # A new configuration or new rules only need a reload, which keeps the data in memory. + svc_reload "${PROM_UNIT}" || die "prometheus: Prometheus does not reload its configuration." + fi + mon_firewall "${MON_PROMETHEUS_PORT}/tcp" + mon_web_admin proxy "$(mon_url prometheus "${MON_PROMETHEUS_PORT}")" + mon_datasource Prometheus prometheus "$(_prom_url)" true + return +} + +mod_verify() { + local url rules + url=$(_prom_url) + svc_is_active "${PROM_UNIT}" || die "prometheus: ${PROM_UNIT} is not running." + mon_wait_http "${url}/-/ready" 120 + rules=$(curl -s --max-time 10 "${url}/api/v1/rules") + [[ ${rules} == *'"name":"LssBackupTooOld"'* ]] || die "prometheus: the lss alert rules are not loaded." + # Prometheus scrapes itself and the node exporter through the target files. + _prom_wait_target node + mon_listen_check --process prometheus "${MON_PROMETHEUS_PORT}" "$(mon_address prometheus)" + return +} + +# _prom_wait_target JOB: waits until Prometheus has scraped the target of JOB successfully. +_prom_wait_target() { + local job=$1 i result='' + for ((i = 0; i < 45; i++)); do + result=$(curl -s --max-time 10 -G "$(_prom_url)/api/v1/query" --data-urlencode "query=up{job=\"${job}\"}") + [[ ${result} != *'"value":['*',"1"]'* ]] || return 0 + sleep 2 + done + die "prometheus: the target ${job} is not up (${result:-no answer})." +} + +mod_remove() { + mon_web_admin_remove + mon_datasource_remove + mon_target_remove prometheus + svc_disable "${PROM_UNIT}" + svc_dropin_remove "${PROM_UNIT}.service" lss-sandbox + svc_dropin_remove "${PROM_UNIT}.service" lss-vpn + file_remove "${PROM_RULES_DIR}/lss.yml" + file_remove "${PROM_CONF}" + rmdir "${PROM_RULES_DIR}" 2>/dev/null || true + if os_is_debian; then + svc_dropin_remove "${PROM_UNIT}.service" lss-config + pkg_remove prometheus + else + file_remove "/etc/systemd/system/${PROM_UNIT}.service" + svc_daemon_reload + mon_upstream_remove prometheus promtool + fi + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete "$(_prom_data)" + return +} diff --git a/modules/monitoring/smartd.sh b/modules/monitoring/smartd.sh new file mode 100644 index 0000000..7f6aa32 --- /dev/null +++ b/modules/monitoring/smartd.sh @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="smartd" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="smartd watches the SMART health of every drive, runs short self-tests daily and long ones weekly, and logs failures and temperatures to the journal" +MOD_REQUIRES="system" +MOD_OPTIONS="temperature|45,55|Log a drive temperature above the first value (Celsius) and warn above the second" +MOD_PORTS="none" + +# Virtual machines and containers usually show no SMART drives. smartd then has nothing to watch; it keeps running +# (-q never) instead of failing, so the module behaves the same on every server and picks up drives that appear later +# on the next restart. +SMARTD_BINARY=/usr/sbin/smartd +# Self-tests: short ones daily at 02:00, long ones on Saturdays at 03:00 (smartd -s, a regular expression). +SMARTD_TESTS='(S/../.././02|L/../../6/03)' + +_smartd_conf() { + if os_is_debian; then printf '/etc/smartd.conf'; else printf '/etc/smartmontools/smartd.conf'; fi + return +} + +# The unit is smartmontools.service on Debian and Ubuntu (smartd.service is an alias there) and smartd.service on the +# RHEL family; drop-ins must go to the real name. +_smartd_unit() { + local unit + unit=$(systemctl show -p Id --value smartd.service 2>/dev/null || true) + printf '%s' "${unit:-smartd.service}" + return +} + +mod_check() { + [[ $(opt temperature) =~ ^[0-9]{2},[0-9]{2,3}$ ]] || die "LSS_SMARTD_TEMPERATURE must look like 45,55." + return +} + +mod_install() { + pkg_install smartmontools + return +} + +mod_configure() { + local before unit + before=$(files_changes_of smartd) + unit=$(_smartd_unit) + # Every drive that speaks SMART: health, attributes and error logs (-a), automatic offline tests and attribute + # autosave on, drives in standby are not woken up (and not counted as missed), self-tests by schedule, temperature. + file_write 0644 root:root "$(_smartd_conf)" </dev/null | grep -c '^/dev/' || true) + if ((devices == 0)); then + log_info "smartd: smartctl finds 0 drives here (virtual machine or container); smartd keeps running for drives added later." + fi + return +} + +mod_remove() { + local unit + unit=$(_smartd_unit) + svc_disable "${unit%.service}" + svc_dropin_remove "${unit}" lss-sandbox + svc_dropin_remove "${unit}" lss-config + file_remove "$(_smartd_conf)" + pkg_remove smartmontools + return +} diff --git a/modules/network/dns-resolver.sh b/modules/network/dns-resolver.sh new file mode 100644 index 0000000..321c268 --- /dev/null +++ b/modules/network/dns-resolver.sh @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="dns-resolver" +MOD_GROUP="network" +MOD_DESCRIPTION="Unbound as a validating (DNSSEC) caching resolver on 127.0.0.1, optionally for the WireGuard network; the server itself resolves through it" +MOD_REQUIRES="system" +MOD_OPTIONS="access|local|Who may ask: local (this server only) or vpn (also the WireGuard network, on the server's WireGuard address) +forward||Resolvers to forward every query to instead of asking the root servers, separated by spaces (1.1.1.1, or 1.1.1.1@853#cloudflare-dns.com with forward_tls); empty: Unbound resolves from the root servers itself +forward_tls|false|Forward over DNS over TLS; every forward entry then needs @853#NAME, the name its certificate must carry +system|true|Make this server resolve through Unbound (systemd-resolved, NetworkManager or /etc/resolv.conf); inside a container the engine owns /etc/resolv.conf, so it stays as it is" +MOD_PORTS="53/udp and 53/tcp from the WireGuard network with access vpn; none with access local" + +DNS_CONF=/etc/unbound/unbound.conf +DNS_ANCHOR=/var/lib/unbound/root.key +DNS_RESOLV=/etc/resolv.conf +# The resolv.conf found before lss changed it, put back when the module is removed. +DNS_SAVED_DIR=${LSS_STATE_DIR:-/var/lib/lss}/dns-resolver +DNS_SAVED=${DNS_SAVED_DIR}/resolv.conf +DNS_RESOLVED_DROPIN=/etc/systemd/resolved.conf.d/lss-dns-resolver.conf +DNS_NM_CONF=/etc/NetworkManager/conf.d/lss-dns-resolver.conf +# dhclient rewrites /etc/resolv.conf on every lease renewal unless a hook replaces its make_resolv_conf. +DNS_DHCLIENT_HOOK=/etc/dhcp/dhclient-enter-hooks.d/lss-dns-resolver +DNS_UNIT=unbound +DNS_LOOPBACK=127.0.0.1 + +mod_check() { + local entry + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_DNS_RESOLVER_ACCESS must be local or vpn." + opt_bool system || true + for entry in $(opt forward); do + [[ ${entry} =~ ^[0-9A-Fa-f.:]+(@[0-9]{1,5})?(#[A-Za-z0-9.-]+)?$ ]] || + die "LSS_DNS_RESOLVER_FORWARD: '${entry}' is not ADDRESS, ADDRESS@PORT or ADDRESS@PORT#NAME." + if opt_bool forward_tls && [[ ${entry} != *'#'* ]]; then + die "LSS_DNS_RESOLVER_FORWARD: '${entry}' needs #NAME (the name in its certificate) with forward_tls." + fi + done + if opt_bool forward_tls && [[ -z $(opt forward) ]]; then + die "LSS_DNS_RESOLVER_FORWARD_TLS=true needs resolvers in LSS_DNS_RESOLVER_FORWARD." + fi + return +} + +mod_install() { + # unbound-anchor creates and refreshes the DNSSEC trust anchor of the root zone; Debian ships it separately. + pkg_install_family "unbound unbound-anchor dns-root-data" unbound + return +} + +# IPv6 is used where the server has a default route for it; otherwise Unbound would try IPv6 root servers in vain. +_dns_ipv6() { + [[ -n $(ip -6 route show default 2>/dev/null) ]] && grep -q '^0\{31\}1 ' /proc/net/if_inet6 2>/dev/null + return +} + +# The control socket: in /run on Debian and Ubuntu, in Unbound's own directory under SELinux on the RHEL family. +_dns_control() { + if os_is_debian; then printf '/run/unbound.ctl'; else printf '/run/unbound/unbound.ctl'; fi + return +} + +_dns_ca_bundle() { + if os_is_debian; then printf '/etc/ssl/certs/ca-certificates.crt'; else printf '/etc/pki/tls/certs/ca-bundle.crt'; fi + return +} + +_dns_config() { + local ip6=no entry network + _dns_ipv6 && ip6=yes + network=$(fw_vpn_network) + printf '# Managed by lss (module dns-resolver). Change LSS_DNS_RESOLVER_* in /etc/lss/lss.conf and run lss configure dns-resolver.\n' + printf 'server:\n\tverbosity: 0\n\tuse-syslog: yes\n\tport: 53\n\tinterface: %s\n' "${DNS_LOOPBACK}" + [[ ${ip6} == no ]] || printf '\tinterface: ::1\n' + if [[ $(opt access) == vpn ]]; then + # The WireGuard address may come up after Unbound; freebind lets Unbound listen on it anyway. + printf '\tinterface: %s\n\tip-freebind: yes\n' "$(fw_vpn_address)" + fi + printf '\tdo-ip4: yes\n\tdo-ip6: %s\n\tdo-udp: yes\n\tdo-tcp: yes\n' "${ip6}" + printf '\taccess-control: 0.0.0.0/0 refuse\n\taccess-control: ::/0 refuse\n' + printf '\taccess-control: 127.0.0.0/8 allow\n\taccess-control: ::1/128 allow\n' + [[ $(opt access) != vpn ]] || printf '\taccess-control: %s allow\n' "${network}" + cat </dev/null 2>&1 || true + if [[ ! -s ${DNS_ANCHOR} && -s /usr/share/dns/root.key ]]; then + cp /usr/share/dns/root.key "${DNS_ANCHOR}" + fi + [[ -s ${DNS_ANCHOR} ]] || die "dns-resolver: unbound-anchor could not create ${DNS_ANCHOR}." + files_note_change "created the DNSSEC trust anchor" + fi + if [[ $(stat -c '%U' "${DNS_ANCHOR}") != unbound ]]; then + chown unbound:unbound "${DNS_ANCHOR}" + fi + if command -v restorecon >/dev/null 2>&1; then + restorecon -F "${DNS_ANCHOR}" 2>/dev/null || true + fi + return +} + +# How the server picks its resolver: systemd-resolved, NetworkManager, a plain /etc/resolv.conf, or nothing to change +# (inside a container, or with system=false). +_dns_system_mode() { + if ! opt_bool system || os_in_container; then + printf 'none' + elif svc_is_active systemd-resolved; then + printf 'resolved' + elif svc_is_active NetworkManager; then + printf 'networkmanager' + else + printf 'file' + fi + return +} + +# _dns_resolv_conf: /etc/resolv.conf pointing at Unbound, with the search domains of the original file. +_dns_resolv_conf() { + printf '# Managed by lss (module dns-resolver): this server resolves through Unbound.\nnameserver %s\n' "${DNS_LOOPBACK}" + if [[ -f ${DNS_SAVED} && ! -L ${DNS_SAVED} ]]; then + grep -E '^(search|domain)[[:space:]]' "${DNS_SAVED}" || true + fi + printf 'options edns0 trust-ad\n' + return +} + +_dns_save_original() { + [[ ! -e ${DNS_SAVED} && ! -L ${DNS_SAVED} ]] || return 0 + install -d -m 0700 "${DNS_SAVED_DIR}" + cp -a "${DNS_RESOLV}" "${DNS_SAVED}" + return +} + +_dns_system_apply() { + local mode=$1 changed=0 content + case ${mode} in + resolved) + # resolved keeps its stub on 127.0.0.53 and forwards everything to Unbound, which does the DNSSEC validation. + file_write 0644 root:root "${DNS_RESOLVED_DROPIN}" </dev/null || true + files_note_change "restored ${DNS_RESOLV}" + fi + if [[ -e ${DNS_NM_CONF} ]]; then + file_remove "${DNS_NM_CONF}" + systemctl reload NetworkManager || true + fi + [[ $(files_changes_of dns-resolver) == "${before}" ]] || log_info "dns-resolver: the server uses its previous resolvers again." + return +} + +mod_configure() { + local before config mode + before=$(files_changes_of dns-resolver) + mon_vpn_ready + _dns_anchor + config=$(_dns_config) + file_write 0644 root:root "${DNS_CONF}" <<<"${config}" + unbound-checkconf "${DNS_CONF}" >/dev/null || die "dns-resolver: unbound-checkconf rejects ${DNS_CONF}." + svc_sandbox "${DNS_UNIT}.service" ReadWritePaths=/var/lib/unbound + if os_is_rhel; then + # The unit of the RHEL family runs unbound-anchor before every start and fails when it cannot reach the root + # servers. Unbound keeps the anchor current itself (RFC 5011, _dns_anchor), so the unit's own update stays off. + svc_dropin "${DNS_UNIT}.service" lss-config <<'EOF' +# Managed by lss (module dns-resolver): Unbound updates its trust anchor itself. +[Service] +Environment=DISABLE_UNBOUND_ANCHOR=yes +EOF + fi + mon_vpn_dropin "${DNS_UNIT}.service" + svc_enable "${DNS_UNIT}" + if [[ $(files_changes_of dns-resolver) != "${before}" ]]; then + svc_restart "${DNS_UNIT}" || die "dns-resolver: Unbound does not start with the new configuration." + fi + mon_firewall 53/udp 53/tcp + # The server switches to Unbound only once it answers, and only when it reaches the DNS of the Internet: where + # outgoing DNS is blocked, the switch would leave the server without any name resolution. + _dns_wait + mode=$(_dns_system_mode) + if [[ ${mode} != none ]] && ! _dns_online; then + log_warn "dns-resolver: Unbound reaches no DNS server on the Internet, so this server keeps its previous resolvers; LSS_DNS_RESOLVER_FORWARD names resolvers that are reachable." + mode=none + fi + if [[ ${mode} == none ]]; then + _dns_system_undo + else + _dns_system_apply "${mode}" + fi + return +} + +# _dns_packet NAME TYPE: a DNS query (recursion desired, AD bit set to ask for the validation result) as printf %b +# escapes. NAME is a host name or "." for the root. +_dns_packet() { + local name=$1 type=$2 label packet='\x4c\x53\x01\x20\x00\x01\x00\x00\x00\x00\x00\x00' + local -a labels=() + IFS=. read -ra labels <<<"${name%.}" + for label in "${labels[@]}"; do + [[ -n ${label} ]] || continue + packet+=$(printf '\\x%02x%s' "${#label}" "${label}") + done + packet+=$(printf '\\x00\\x%02x\\x%02x\\x00\\x01' $((type >> 8)) $((type & 255))) + printf '%s' "${packet}" + return +} + +# _dns_query SERVER NAME TYPE: sends one query over UDP and prints the reply as hexadecimal bytes separated by +# spaces; nothing when no reply came within 5 seconds. +_dns_query() { + local server=$1 name=$2 type=$3 fd reply + exec {fd}<>"/dev/udp/${server}/53" || return 0 + printf '%b' "$(_dns_packet "${name}" "${type}")" >&"${fd}" + reply=$(timeout 5 dd bs=4096 count=1 status=none <&"${fd}" | od -An -v -tx1 | tr -s ' \n' ' ' || true) + exec {fd}>&- + printf '%s' "${reply# }" + return +} + +# _dns_reply_rcode REPLY / _dns_reply_ad REPLY / _dns_reply_answers REPLY: fields of a reply from _dns_query. +_dns_reply_rcode() { + local -a bytes=() + read -ra bytes <<<"$1" + [[ ${#bytes[@]} -ge 12 && ${bytes[0]}${bytes[1]} == 4c53 ]] || return 1 + printf '%s' $((0x${bytes[3]} & 15)) + return +} + +_dns_reply_ad() { + local -a bytes=() + read -ra bytes <<<"$1" + [[ ${#bytes[@]} -ge 12 ]] && (((0x${bytes[3]} & 0x20) != 0)) + return +} + +_dns_reply_answers() { + local -a bytes=() + read -ra bytes <<<"$1" + [[ ${#bytes[@]} -ge 12 ]] || return 1 + printf '%s' $((0x${bytes[6]} * 256 + 0x${bytes[7]})) + return +} + +# Waits until Unbound answers a local name (no network needed): the built-in zone localhost. +_dns_wait() { + local i reply='' + for ((i = 0; i < 15; i++)); do + reply=$(_dns_query "${DNS_LOOPBACK}" localhost 1) + if [[ $(_dns_reply_rcode "${reply}") == 0 && $(_dns_reply_answers "${reply}") -ge 1 ]]; then + return 0 + fi + sleep 1 + done + die "dns-resolver: Unbound does not answer on ${DNS_LOOPBACK}:53." +} + +# _dns_root_reply: Unbound's answer for the SOA of the signed root zone. A cold Unbound first fetches the root servers +# and keys, so it gets three tries. +_dns_root_reply() { + local reply='' + for _ in 1 2 3; do + reply=$(_dns_query "${DNS_LOOPBACK}" . 6) + [[ -z $(_dns_reply_rcode "${reply}" || true) ]] || break + done + printf '%s' "${reply}" + return +} + +# _dns_online: whether Unbound resolves the root zone, which needs the root servers or the forward resolvers. +_dns_online() { + [[ $(_dns_reply_rcode "$(_dns_root_reply)" || true) == 0 ]] + return +} + +mod_verify() { + local reply rcode address mode + local -a addresses=("${DNS_LOOPBACK}") + svc_is_active "${DNS_UNIT}" || die "dns-resolver: ${DNS_UNIT} is not running." + _dns_wait + unbound-control -c "${DNS_CONF}" status >/dev/null 2>&1 || die "dns-resolver: unbound-control gets no answer." + [[ $(unbound-control -c "${DNS_CONF}" get_option module-config 2>/dev/null) == *validator* ]] || + die "dns-resolver: Unbound does not validate DNSSEC." + [[ -s ${DNS_ANCHOR} ]] || die "dns-resolver: the DNSSEC trust anchor ${DNS_ANCHOR} is missing." + # The signed root zone must come back validated (AD). Without a reachable resolver upstream only the offline checks + # above apply, and the server keeps its previous resolvers (mod_configure). + reply=$(_dns_root_reply) + rcode=$(_dns_reply_rcode "${reply}" || true) + mode=$(_dns_system_mode) + if [[ ${rcode} == 0 ]]; then + _dns_reply_ad "${reply}" || die "dns-resolver: the answer for the root zone is not DNSSEC-validated." + else + log_warn "dns-resolver: the root zone did not resolve (rcode ${rcode:-none}); DNSSEC was checked offline only." + mode=none + fi + _dns_ipv6 && addresses+=(::1) + [[ $(opt access) != vpn ]] || addresses+=("$(fw_vpn_address)") + mon_listen_check --process unbound 53 "${addresses[@]}" + case ${mode} in + resolved) + resolvectl dns 2>/dev/null | grep -qE "^Global: .*${DNS_LOOPBACK}" || + die "dns-resolver: systemd-resolved does not forward to Unbound." + ;; + networkmanager | file) + address=$(awk '$1 == "nameserver" { print $2; exit }' "${DNS_RESOLV}") + [[ ${address} == "${DNS_LOOPBACK}" ]] || die "dns-resolver: ${DNS_RESOLV} does not point at Unbound." + ;; + *) ;; + esac + return +} + +mod_remove() { + _dns_system_undo + svc_disable "${DNS_UNIT}" + svc_dropin_remove "${DNS_UNIT}.service" lss-sandbox + svc_dropin_remove "${DNS_UNIT}.service" lss-vpn + svc_dropin_remove "${DNS_UNIT}.service" lss-config + file_remove "${DNS_CONF}" + pkg_remove_family "unbound unbound-anchor" unbound + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete /var/lib/unbound + return +} diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh new file mode 100644 index 0000000..730af9d --- /dev/null +++ b/modules/network/ntp-server.sh @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="ntp-server" +MOD_GROUP="network" +MOD_DESCRIPTION="chrony of the time module also serves NTP on port 123, to the WireGuard network or to everyone, rate-limited" +MOD_REQUIRES="time" +MOD_OPTIONS="access|vpn|Who may ask for the time: vpn (the WireGuard network) or public (everyone) +ratelimit|true|Answer each client at most about once a second, with short bursts, so the server cannot amplify floods" +MOD_PORTS="123/udp from the WireGuard network (access vpn) or from everyone (access public)" + +# chrony keeps the time module's configuration, which reads the files of this directory last; the file here turns the +# NTP port on and names who may ask. Inside a container the time module leaves chrony out (the host owns the clock), so +# this module sets chrony up there to serve the time without ever setting the clock (chronyd -x). +NTP_FILE_NAME=lss-ntp-server.conf +NTP_PORT=123 + +_ntp_unit() { + if os_is_debian; then printf 'chrony'; else printf 'chronyd'; fi + return +} + +# _ntp_conf: the file in the directory the time module's chrony configuration reads last. +_ntp_conf() { + local dir + dir=$( + # shellcheck source=/dev/null + source "$(module_file time)" + _time_confdir + ) + printf '%s/%s' "${dir}" "${NTP_FILE_NAME}" + return +} + +_ntp_main_conf() { + if os_is_debian; then printf '/etc/chrony/chrony.conf'; else printf '/etc/chrony.conf'; fi + return +} + +mod_check() { + [[ $(opt access) =~ ^(vpn|public)$ ]] || die "LSS_NTP_SERVER_ACCESS must be vpn or public." + opt_bool ratelimit || true + return +} + +mod_install() { + pkg_install chrony + return +} + +# _ntp_time_config: chrony's configuration as the time module writes it, from the time module's options. Inside a +# container only; elsewhere the time module has written it already. +_ntp_time_config() { + ( + # shellcheck source=/dev/null + source "$(module_file time)" + LSS_MODULE="time" + _time_chrony_config + ) + return +} + +_ntp_config() { + printf '# Managed by lss (module ntp-server). Change LSS_NTP_SERVER_* in /etc/lss/lss.conf and run lss configure ntp-server.\n' + printf 'port %s\n' "${NTP_PORT}" + if [[ $(opt access) == public ]]; then + printf 'allow all\n' + else + printf 'allow %s\n' "$(fw_vpn_network)" + fi + # lss verify asks over the loopback interface. + printf 'allow 127.0.0.1\nallow ::1\n' + if opt_bool ratelimit; then + printf 'ratelimit interval 1 burst 16 leak 2\n' + fi + return +} + +# _ntp_exec UNIT: the command line of the distribution's chronyd unit with -x added. It is taken over rather than +# written anew: Debian starts chronyd with "!" (as root although the unit says User=_chrony) and -n (Type=notify), +# the RHEL family lets it fork. +_ntp_exec() { + local unit=$1 fragment line + fragment=$(systemctl show -p FragmentPath --value "${unit}") + line=$(sed -n 's/^ExecStart=\(..*\)$/\1/p' "${fragment}" 2>/dev/null | tail -n 1) + [[ -n ${line} ]] || die "ntp-server: ${unit} has no command line in ${fragment:-its unit file}." + printf '%s -x' "${line}" + return +} + +# _ntp_container_dropin: inside a container chronyd serves the time but must not set the clock it shares with the host +# (-x), and the unit's conditions on CAP_SYS_TIME and on running outside a container no longer apply. +_ntp_container_dropin() { + local unit exec notify='' + unit="$(_ntp_unit).service" + if os_in_container; then + exec=$(_ntp_exec "${unit}") || return 1 + # chronyd in the foreground (-n) reports readiness to systemd; inside a container chrony 4.8 (Ubuntu 26.04) + # cannot reach the notification socket and stops. There systemd counts the started process as ready. + [[ " ${exec} " != *' -n '* ]] || notify=$'Type=simple\nNotifyAccess=none' + svc_dropin "${unit}" lss-ntp-server <"/dev/udp/${address}/${NTP_PORT}" || return 0 + # 48 bytes: LI 0, VN 4, mode 3, then zeros. + printf '\x23%047d' 0 | tr 0 '\000' >&"${fd}" + reply=$(timeout 5 dd bs=512 count=1 status=none <&"${fd}" | od -An -N1 -tu1 | tr -d ' \n' || true) + exec {fd}>&- + printf '%s' "${reply}" + return +} + +mod_verify() { + local unit i first='' + unit=$(_ntp_unit) + svc_is_active "${unit}" || die "ntp-server: ${unit} is not running." + # chronyd answers clients also before it is synchronised itself (with the leap indicator "unsynchronised"). + for ((i = 0; i < 10; i++)); do + first=$(_ntp_probe 127.0.0.1) + [[ -z ${first} ]] || break + sleep 1 + done + if [[ -z ${first} ]]; then + # What chronyd made of its configuration and which sockets it has, for the log. + # chrony's configuration holds no secrets (keys live in a separate key file). + chronyd -p 2>&1 | grep -v '^server ' | sed 's/^/ntp-server: chronyd -p: /' >&2 || true + grep -n '^confdir' "$(_ntp_main_conf)" 2>&1 | sed 's/^/ntp-server: main file: /' >&2 || true + find "$(dirname "$(_ntp_conf)")" -maxdepth 1 -printf '%M %u %s %f\n' 2>&1 | sed 's/^/ntp-server: confdir: /' >&2 || true + ps -o args= -C chronyd 2>&1 | sed 's/^/ntp-server: process: /' >&2 || true + chronyd -v 2>&1 | sed 's/^/ntp-server: version: /' >&2 || true + tail -c 40 "$(_ntp_main_conf)" | od -c | sed 's/^/ntp-server: main file end: /' >&2 || true + chronyd -p -f "$(_ntp_conf)" 2>&1 | sed 's/^/ntp-server: own file: /' >&2 || true + chronyd -p "confdir $(dirname "$(_ntp_conf)")" 2>&1 | sed 's/^/ntp-server: confdir alone: /' >&2 || true + ss -Hlunp 2>/dev/null | grep chronyd | sed 's/^/ntp-server: socket: /' >&2 || true + die "ntp-server: chronyd does not answer NTP requests on port ${NTP_PORT}." + fi + (((first & 7) == 4)) || die "ntp-server: the answer on port ${NTP_PORT} is no NTP server reply (mode $((first & 7)))." + ss -Hlun "sport = :${NTP_PORT}" | grep -q . || die "ntp-server: nothing listens on ${NTP_PORT}/udp." + return +} + +mod_remove() { + local unit + unit=$(_ntp_unit) + file_remove "$(_ntp_conf)" + svc_dropin_remove "${unit}.service" lss-sandbox + if os_in_container; then + svc_dropin_remove "${unit}.service" lss-ntp-server + svc_disable "${unit}" + pkg_remove chrony + elif svc_is_active "${unit}"; then + # The time module's chrony keeps the clock, with the NTP port closed again. + svc_restart "${unit}" || die "ntp-server: chronyd does not start without the NTP port." + fi + return +} diff --git a/modules/network/wireguard.sh b/modules/network/wireguard.sh new file mode 100644 index 0000000..b43e64d --- /dev/null +++ b/modules/network/wireguard.sh @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="wireguard" +MOD_GROUP="network" +MOD_DESCRIPTION="WireGuard VPN server on wg0 with the peers from the settings and its keys in the lss secrets; admin interfaces with access vpn listen on it" +MOD_REQUIRES="system" +MOD_OPTIONS="network|10.66.0.0/24|VPN network (IPv4, /8 to /30); the server takes its first address, the peers others; the vpn scope of the firewall is this network +port|51820|UDP port WireGuard listens on, open to everyone +peers||Peers as NAME:PUBLIC_KEY:ADDRESS, separated by spaces (laptop::10.66.0.2); each also gets a preshared key (lss secrets show wireguard-psk-NAME) +endpoint||Host name or address the peers connect to, for the peer settings lss prints; empty: the address of the default route" +MOD_PORTS="51820/udp from everyone" +MOD_SECRETS="wireguard-private-key wireguard-psk-" + +# Peers reach the server itself (its admin interfaces, SSH, the DNS resolver), not the networks behind it: lss does not +# turn on IP forwarding, which kernel-hardening keeps off. +WG_INTERFACE=wg0 +WG_CONF=/etc/wireguard/${WG_INTERFACE}.conf +WG_UNIT=wg-quick@${WG_INTERFACE} +WG_KEY_SECRET=wireguard-private-key +# The public key is no secret; peers need it. +WG_PUBLIC_KEY=/etc/lss/wireguard-public-key + +_wg_server_address() { + fw_vpn_address + return +} + +_wg_prefix() { + local network + network=$(opt network) + printf '%s' "${network##*/}" + return +} + +# _wg_in_network ADDRESS: ADDRESS lies in the VPN network. +_wg_in_network() { + local address=$1 network prefix + network=$(opt network) + prefix=$((10#${network##*/})) + (($(_fw_ipv4_int "${address}") >> (32 - prefix) == $(_fw_ipv4_int "${network%/*}") >> (32 - prefix))) + return +} + +mod_check() { + local port peer name key address server endpoint + local -A names=() keys=() addresses=() + server=$(_wg_server_address) + port=$(opt_int port) + ((port >= 1 && port <= 65535)) || die "LSS_WIREGUARD_PORT must be between 1 and 65535." + for peer in $(opt peers); do + IFS=: read -r name key address <<<"${peer}" + address=${address%/32} + [[ ${name} =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]] || + die "LSS_WIREGUARD_PEERS: '${name}' is not a peer name (a-z, 0-9 and -, up to 32 characters)." + # A Curve25519 key in base64: 43 characters and "=", the last one from the set a 256-bit value allows. + [[ ${key} =~ ^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$ ]] || + die "LSS_WIREGUARD_PEERS: the key of '${name}' is not a WireGuard public key." + if [[ ! ${address} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || ! _wg_in_network "${address}"; then + die "LSS_WIREGUARD_PEERS: the address of '${name}' must be an IPv4 address in $(opt network)." + fi + [[ ${address} != "${server}" ]] || die "LSS_WIREGUARD_PEERS: ${address} is the address of the server." + [[ -z ${names[${name}]+x} && -z ${keys[${key}]+x} && -z ${addresses[${address}]+x} ]] || + die "LSS_WIREGUARD_PEERS: every peer needs its own name, key and address ('${name}')." + names[${name}]=1 + keys[${key}]=1 + addresses[${address}]=1 + done + endpoint=$(opt endpoint) + [[ -z ${endpoint} || ${endpoint} =~ ^[A-Za-z0-9.:-]+$ ]] || die "LSS_WIREGUARD_ENDPOINT '${endpoint}' is not a host name." + return +} + +mod_install() { + pkg_install wireguard-tools + return +} + +# _wg_keys: the private key of the server and a preshared key per peer, made once by wg itself and kept in the lss +# secrets. +_wg_keys() { + local peer name _key _address value + if ! secret_exists "${WG_KEY_SECRET}"; then + value=$(wg genkey) + secret_set "${WG_KEY_SECRET}" "${value}" + log_info "Generated secret ${WG_KEY_SECRET}" + files_note_change "generated the WireGuard key" + fi + for peer in $(opt peers); do + IFS=: read -r name _key _address <<<"${peer}" + secret_exists "wireguard-psk-${name}" && continue + value=$(wg genpsk) + secret_set "wireguard-psk-${name}" "${value}" + log_info "Generated secret wireguard-psk-${name} (lss secrets show wireguard-psk-${name})" + files_note_change "generated the preshared key of ${name}" + done + return +} + +# _wg_config: wg-quick configuration without keys; PostUp hands the private and preshared keys to the interface from +# the secrets, so they exist in one place only. +_wg_config() { + local peer name key address + printf '# Managed by lss (module wireguard). Change LSS_WIREGUARD_* in /etc/lss/lss.conf and run lss configure wireguard.\n' + printf '[Interface]\nAddress = %s/%s\nListenPort = %s\n' "$(_wg_server_address)" "$(_wg_prefix)" "$(opt_int port)" + printf 'PostUp = wg set %%i private-key %s\n' "$(secret_path "${WG_KEY_SECRET}")" + for peer in $(opt peers); do + IFS=: read -r name key address <<<"${peer}" + printf 'PostUp = wg set %%i peer %s preshared-key %s\n' "${key}" "$(secret_path "wireguard-psk-${name}")" + done + for peer in $(opt peers); do + IFS=: read -r name key address <<<"${peer}" + printf '\n[Peer]\n# %s\nPublicKey = %s\nAllowedIPs = %s/32\n' "${name}" "${key}" "${address%/32}" + done + return +} + +# The settings a peer needs, printed after a change (no secret: the preshared key is named, not shown). +_wg_hints() { + local peer name _key address endpoint public + endpoint=$(opt endpoint) + if [[ -z ${endpoint} ]]; then + endpoint=$(_fw_host_addresses) + endpoint=${endpoint%%$'\n'*} + fi + public=$(<"${WG_PUBLIC_KEY}") + for peer in $(opt peers); do + IFS=: read -r name _key address <<<"${peer}" + log_info "wireguard: peer ${name}: Address = ${address%/32}/32; [Peer] PublicKey = ${public}, Endpoint = ${endpoint:-SERVER}:$(opt_int port), AllowedIPs = $(_wg_server_address)/32, PresharedKey from lss secrets show wireguard-psk-${name}" + done + return +} + +mod_configure() { + local before public config + before=$(files_changes_of wireguard) + _wg_keys + public=$(wg pubkey <"$(secret_path "${WG_KEY_SECRET}")") + file_write 0644 root:root "${WG_PUBLIC_KEY}" <<<"${public}" + install -d -m 0700 /etc/wireguard + config=$(_wg_config) + file_write 0600 root:root "${WG_CONF}" <<<"${config}" + # wg-quick is a root script that runs ip and wg; under SELinux these switch to their own domains (ifconfig_t), + # which NoNewPrivileges forbids, so the interface would never come up on the RHEL family. wg reads its settings + # from a pipe through /dev/fd, which SELinux denies wg in the private /dev of PrivateDevices. + svc_sandbox "${WG_UNIT}.service" NoNewPrivileges=no PrivateDevices=no + fw_allow "$(opt_int port)/udp" + svc_enable "${WG_UNIT}" + if [[ $(files_changes_of wireguard) != "${before}" ]]; then + svc_restart "${WG_UNIT}" || die "wireguard: ${WG_INTERFACE} does not come up with the new configuration." + _wg_hints + fi + return +} + +mod_verify() { + local peer _name key _address expected actual without + svc_is_active "${WG_UNIT}" || die "wireguard: ${WG_UNIT} is not running." + [[ $(wg show "${WG_INTERFACE}" listen-port) == "$(opt_int port)" ]] || + die "wireguard: ${WG_INTERFACE} does not listen on port $(opt_int port)." + [[ $(wg show "${WG_INTERFACE}" public-key) == "$(<"${WG_PUBLIC_KEY}")" ]] || + die "wireguard: ${WG_INTERFACE} does not use the key in the lss secrets." + ip -4 -o addr show dev "${WG_INTERFACE}" | grep -qF " $(_wg_server_address)/$(_wg_prefix) " || + die "wireguard: ${WG_INTERFACE} does not have the address $(_wg_server_address)/$(_wg_prefix)." + expected='' + for peer in $(opt peers); do + IFS=: read -r _name key _address <<<"${peer}" + expected+="${key}"$'\n' + done + expected=$(sort <<<"${expected%$'\n'}") + actual=$(wg show "${WG_INTERFACE}" peers | sort) + [[ ${actual} == "${expected}" ]] || die "wireguard: the peers of ${WG_INTERFACE} differ from LSS_WIREGUARD_PEERS." + # Only the peers without one are printed; the preshared keys themselves never leave wg. + without=$(wg show "${WG_INTERFACE}" preshared-keys | awk '$2 == "(none)" { print $1 }') + [[ -z ${without} ]] || die "wireguard: peers without a preshared key: ${without//$'\n'/ }." + return +} + +mod_remove() { + local secret + if svc_is_enabled "${WG_UNIT}" || svc_is_active "${WG_UNIT}"; then + systemctl disable --now "${WG_UNIT}" >/dev/null 2>&1 || die "wireguard: ${WG_UNIT} does not stop." + files_note_change "disabled ${WG_UNIT}" + fi + svc_dropin_remove "${WG_UNIT}.service" lss-sandbox + file_remove "${WG_CONF}" + file_remove "${WG_PUBLIC_KEY}" + pkg_remove wireguard-tools + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + for secret in "${LSS_SECRETS_DIR}"/wireguard-*; do + [[ -f ${secret} ]] || continue + secret_remove "${secret##*/}" + done + return +} diff --git a/modules/security/guard.sh b/modules/security/guard.sh index 307c78f..c4bcb81 100644 --- a/modules/security/guard.sh +++ b/modules/security/guard.sh @@ -13,6 +13,7 @@ webhook||Send findings to this HTTPS webhook (Slack, Teams, Mattermost, Discord disk_threshold|90|Report file systems that are fuller than this (percent) lynis_min|0|Report a Lynis hardening index below this (needs the lynis module) login_threshold|50|Report more failed SSH logins than this within 24 hours +backup_max_age|30|Report a last successful backup older than this (hours; needs the restic module, which the guard also restore-tests weekly) daily_summary|false|Also send a message when everything is fine" GUARD_SCRIPT=/usr/local/lib/lss/guard @@ -28,6 +29,7 @@ mod_check() { (($(opt_int disk_threshold) <= 100)) || die "LSS_GUARD_DISK_THRESHOLD must be at most 100." opt_int lynis_min >/dev/null opt_int login_threshold >/dev/null + (($(opt_int backup_max_age) >= 1)) || die "LSS_GUARD_BACKUP_MAX_AGE must be at least 1." email=$(opt email) webhook=$(opt webhook) [[ -z ${email} || ${email} =~ ^[^@[:space:]]+@[^@[:space:]]+\.[^@[:space:]]+$ ]] || die "LSS_GUARD_EMAIL '${email}' is not an e-mail address." @@ -59,7 +61,7 @@ mod_configure() { printf 'GUARD_DAILY_SUMMARY=%s\n' "${bool}" printf 'GUARD_EMAIL=%s\nGUARD_WEBHOOK=%s\n' "$(opt email)" "$(opt webhook)" printf 'GUARD_DISK_THRESHOLD=%s\nGUARD_LYNIS_MIN=%s\n' "$(opt_int disk_threshold)" "$(opt_int lynis_min)" - printf 'GUARD_LOGIN_THRESHOLD=%s\n' "$(opt_int login_threshold)" + printf 'GUARD_LOGIN_THRESHOLD=%s\nGUARD_BACKUP_MAX_AGE=%s\n' "$(opt_int login_threshold)" "$(opt_int backup_max_age)" ) svc_unit "${GUARD_UNIT}.service" <_ACCESS=vpn with the wireguard module or +# LSS__WEB_ADMIN=true with a web server says otherwise. netdata and restic are opt-in. A Git forge (Forgejo) +# and CI runners join this profile with their modules in a later stage. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard docker node-exporter prometheus alertmanager loki alloy grafana smartd diff --git a/profiles/webhosting.conf b/profiles/webhosting.conf index 116b30f..ac9e69b 100644 --- a/profiles/webhosting.conf +++ b/profiles/webhosting.conf @@ -4,6 +4,6 @@ # Profile: Web hosting: minimal-secure plus nginx, certificates, PHP, MariaDB, phpMyAdmin and sites # -# Sites go into LSS_VHOST_SITES, the host names of the server into LSS_TLS_DOMAINS. Backups (restic) join the profile -# with the backup modules. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mariadb phpmyadmin vhost modsecurity +# Sites go into LSS_VHOST_SITES, the host names of the server into LSS_TLS_DOMAINS. restic backs up the sites and +# the dumps of MariaDB daily; LSS_RESTIC_REPOSITORY should point to another disk or host. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mariadb phpmyadmin vhost modsecurity restic db-dumps diff --git a/templates/prometheus/rules.yml b/templates/prometheus/rules.yml new file mode 100644 index 0000000..f287cec --- /dev/null +++ b/templates/prometheus/rules.yml @@ -0,0 +1,58 @@ +# Managed by lss (module prometheus). Alert rules for the server lss set up; lss configure prometheus replaces local +# changes. Alertmanager (module alertmanager) sends them on; without it they show in Prometheus and Grafana. +groups: + - name: lss + rules: + - alert: LssTargetDown + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.job }} on {{ $labels.instance }} cannot be scraped" + - alert: LssDiskAlmostFull + expr: >- + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"} < 0.10 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.mountpoint }} has less than 10 % free space" + - alert: LssMemoryLow + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.05 + for: 15m + labels: + severity: warning + annotations: + summary: "less than 5 % of the memory is available" + - alert: LssGuardFindings + expr: lss_guard_findings > 0 + labels: + severity: warning + annotations: + summary: "the lss guard reports {{ $value }} finding(s); see journalctl -u lss-guard" + - alert: LssGuardStale + expr: time() - lss_guard_last_run_timestamp_seconds > 26 * 3600 + labels: + severity: warning + annotations: + summary: "the lss guard has not run for more than 26 hours" + - alert: LssBackupFailed + expr: lss_backup_last_run_success == 0 + labels: + severity: critical + annotations: + summary: "the last backup failed; see journalctl -u lss-backup" + - alert: LssBackupTooOld + expr: time() - lss_backup_last_success_timestamp_seconds > 30 * 3600 + labels: + severity: critical + annotations: + summary: "the last successful backup is more than 30 hours old" + - alert: LssRestoreTestFailed + expr: lss_backup_restore_test_success == 0 + labels: + severity: critical + annotations: + summary: "the last restore test of the backup failed" diff --git a/templates/time/chrony.conf b/templates/time/chrony.conf index f871278..b46b2b0 100644 --- a/templates/time/chrony.conf +++ b/templates/time/chrony.conf @@ -7,6 +7,9 @@ makestep 1.0 3 rtcsync leapsectz right/UTC logdir /var/log/chrony -# No NTP service for other hosts and no remote command port (chronyc uses the local socket). +# No NTP service for other hosts (unless the ntp-server module opens one in the confdir below) and no remote command +# port (chronyc uses the local socket). port 0 cmdport 0 +# Last, so that the files there win: the ntp-server module serves time to a network from there. +confdir {{TIME_CONFDIR}} diff --git a/tests/integration/settings.conf b/tests/integration/settings.conf index d8041aa..3c86208 100644 --- a/tests/integration/settings.conf +++ b/tests/integration/settings.conf @@ -15,7 +15,7 @@ LSS_TLS_SERVER=https://localhost:14000/dir LSS_TLS_CA_BUNDLE=/etc/lss-pebble/pebble.minica.pem LSS_TLS_DOMAINS=lss-test.example.com LSS_VHOST_SITES=demo:static:demo.example.com,www.demo.example.com app:proxy:app.example.com:http://127.0.0.1:9 -LSS_HAPROXY_SERVICES=echo:9100/local:127.0.0.1:9,127.0.0.1:10 +LSS_HAPROXY_SERVICES=echo:9190/local:127.0.0.1:9,127.0.0.1:10 # Database modules: one application user each, so verify also covers creating users, databases and their grants. # A small memory limit keeps redis and valkey from growing on the shared runner. @@ -33,3 +33,13 @@ LSS_MODSECURITY_EXCLUSIONS=wordpress nextcloud # Container modules: k3s without Traefik (fewer image pulls from Docker Hub on the shared runner). LSS_K3S_TRAEFIK=false + +# Monitoring: Prometheus, Grafana and Netdata also below the admin port when a web server is installed (nginx,grafana +# and the profile all). Netdata listens on the WireGuard address too, so its job installs wireguard first. +LSS_PROMETHEUS_WEB_ADMIN=true +LSS_GRAFANA_WEB_ADMIN=true +LSS_NETDATA_WEB_ADMIN=true +LSS_NETDATA_ACCESS=vpn + +# Network modules: one peer with a throwaway public key (any 32 bytes are a valid key; nobody holds a private half). +LSS_WIREGUARD_PEERS=ci:xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg=:10.66.0.2 diff --git a/tests/unit/backup.bats b/tests/unit/backup.bats new file mode 100644 index 0000000..472da90 --- /dev/null +++ b/tests/unit/backup.bats @@ -0,0 +1,336 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# The backup scripts (libexec/lss-backup, libexec/lss-db-dumps) against a restic stub, and the checks of the restic +# module. + +setup() { + load helpers + lss_test_setup + use_os debian-13 + export TMPDIR=${BATS_TEST_TMPDIR} + export LSS_BACKUP_CONF=${BATS_TEST_TMPDIR}/backup.conf + export LSS_BACKUP_STATE=${BATS_TEST_TMPDIR}/restic-state + export LSS_BACKUP_CACHE=${BATS_TEST_TMPDIR}/cache + export LSS_TEXTFILE_DIR=${BATS_TEST_TMPDIR}/textfile + export LSS_DB_DUMPS=${BATS_TEST_TMPDIR}/db-dumps + export LSS_DB_DUMPS_CONF=${BATS_TEST_TMPDIR}/db-dumps.conf + export LSS_DB_DUMPS_STATE=${BATS_TEST_TMPDIR}/db-dumps-state + export LSS_RESTIC=restic_stub + mkdir -p "${LSS_TEXTFILE_DIR}" "${LSS_SECRETS_DIR}" + printf 'BACKUP_REPOSITORY=/srv/repo\nBACKUP_PATHS=/etc /does-not-exist\nBACKUP_EXCLUDE=*.tmp\n' >"${LSS_BACKUP_CONF}" + printf 'password' >"${LSS_SECRETS_DIR}/restic-password" + RESTORED='' +} + +# restic stub: records the call (without --quiet), fails the subcommand named in $RESTIC_FAIL with $RESTIC_CODE, and +# "restore" writes $RESTORED as the canary below --target. +restic_stub() { + local args=" $* " target='' previous='' arg line + line=${args/ --quiet / } + printf 'restic%s\n' "${line% }" >>"${STUB_LOG}" + for arg in "$@"; do + [[ ${previous} != --target ]] || target=${arg} + previous=${arg} + done + if [[ -n ${RESTIC_FAIL:-} && ${args} == *" ${RESTIC_FAIL} "* ]]; then + return "${RESTIC_CODE:-1}" + fi + if [[ ${args} == *' restore '* && -n ${RESTORED} ]]; then + mkdir -p "${target}${LSS_BACKUP_STATE}/canary" + printf '%s\n' "${RESTORED}" >"${target}${LSS_BACKUP_STATE}/canary/canary" + fi + return 0 +} + +load_backup() { + # shellcheck source=/dev/null + source "${REPO_ROOT}/libexec/lss-backup" + export -f restic_stub + return +} + +@test "sftp repositories give restic an ssh command with the key and the pinned host" { + load_backup + [ "$(sftp_target sftp:backup@nas.example.com:/srv/restic)" = backup@nas.example.com ] + [ "$(sftp_target sftp://backup@nas.example.com:2222//srv/restic)" = 'backup@nas.example.com -p 2222' ] + run sftp_target 'sftp:;rm -rf /:/x' + [ "${status}" -eq 1 ] + printf 'BACKUP_REPOSITORY=sftp:backup@nas.example.com:/srv/restic\n' >"${LSS_BACKUP_CONF}" + prepare + [ "${RESTIC_ENV[0]}" = RESTIC_REPOSITORY=sftp:backup@nas.example.com:/srv/restic ] + [ "${RESTIC_ENV[1]}" = "RESTIC_PASSWORD_FILE=${LSS_SECRETS_DIR}/restic-password" ] + [[ ${RESTIC_OPTIONS[2]} == "sftp.command=ssh -i ${LSS_SECRETS_DIR}/restic-ssh-key "*"-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=${LSS_BACKUP_STATE}/known_hosts backup@nas.example.com -s sftp" ]] +} + +@test "s3 repositories take their keys from the secrets, through the environment of restic alone" { + load_backup + printf 'BACKUP_REPOSITORY=s3:https://s3.example.com/bucket/host\nBACKUP_S3_REGION=eu-central-1\n' >"${LSS_BACKUP_CONF}" + printf 'KEYID' >"${LSS_SECRETS_DIR}/restic-s3-key-id" + printf 'SECRET' >"${LSS_SECRETS_DIR}/restic-s3-secret" + prepare + [[ " ${RESTIC_ENV[*]} " == *' AWS_ACCESS_KEY_ID=KEYID AWS_SECRET_ACCESS_KEY=SECRET AWS_DEFAULT_REGION=eu-central-1 '* ]] + [ "${RESTIC_OPTIONS[*]}" = --quiet ] + # Nothing of it is exported: restic gets it, nothing else does. + [ -z "$(printenv AWS_SECRET_ACCESS_KEY)" ] + [ -z "$(printenv RESTIC_PASSWORD_FILE)" ] + # shellcheck disable=SC2329 # called by run_restic + env_stub() { + printenv AWS_SECRET_ACCESS_KEY + } + [ "$(RESTIC=env_stub run_restic snapshots)" = SECRET ] + printf 'BACKUP_REPOSITORY=\n' >"${LSS_BACKUP_CONF}" + run prepare + [ "${status}" -eq 1 ] + [[ ${output} == *"BACKUP_REPOSITORY is empty"* ]] +} + +@test "a run dumps the databases, backs up the existing paths, applies the retention, checks and records success" { + load_backup + printf '#!/bin/sh\necho dumps >>"%s"\n' "${STUB_LOG}" >"${LSS_DB_DUMPS}" + chmod +x "${LSS_DB_DUMPS}" + : >"${LSS_DB_DUMPS_CONF}" + cmd_run + grep -qxF 'dumps' "${STUB_LOG}" + grep -qxF 'restic cat config' "${STUB_LOG}" + grep -qxF "restic backup --tag lss --exclude-caches --exclude *.tmp /etc ${LSS_BACKUP_STATE}/canary" "${STUB_LOG}" + grep -qxF 'restic forget --tag lss --group-by host,tags --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6' "${STUB_LOG}" + grep -qxF 'restic check' "${STUB_LOG}" + [ "$(cat "${LSS_BACKUP_STATE}/last-run-success")" = 1 ] + [ "$(cat "${LSS_BACKUP_STATE}/canary-last")" = "$(cat "${LSS_BACKUP_STATE}/canary/canary")" ] + grep -q '^lss_backup_last_success_timestamp_seconds [0-9]*$' "${LSS_TEXTFILE_DIR}/lss_backup.prom" + grep -qxF 'lss_backup_last_run_success 1' "${LSS_TEXTFILE_DIR}/lss_backup.prom" +} + +@test "a repository that does not exist yet is created" { + load_backup + RESTIC_FAIL='cat' cmd_init + grep -qxF 'restic init' "${STUB_LOG}" +} + +@test "a backup with unreadable files or a failed check counts as failed and keeps the last success" { + load_backup + RESTIC_FAIL=backup RESTIC_CODE=3 run cmd_run + [ "${status}" -eq 1 ] + [[ ${output} == *"restic could not read some files"* ]] + [ "$(cat "${LSS_BACKUP_STATE}/last-run-success")" = 0 ] + [ ! -e "${LSS_BACKUP_STATE}/last-success" ] + [ -s "${LSS_BACKUP_STATE}/canary-last" ] + RESTIC_FAIL=backup RESTIC_CODE=1 run cmd_run + [[ ${output} == *"restic backup failed (exit code 1)."* ]] + RESTIC_FAIL=check run cmd_run + [[ ${output} == *"restic check reports errors in the repository."* ]] + RESTIC_FAIL=forget run cmd_run + [[ ${output} == *"restic forget and prune failed."* ]] + grep -qxF 'lss_backup_last_run_success 0' "${LSS_TEXTFILE_DIR}/lss_backup.prom" +} + +@test "the database dumps never see the credentials of the repository" { + load_backup + printf 'BACKUP_REPOSITORY=s3:https://s3.example.com/bucket/host\nBACKUP_PATHS=/etc\n' >"${LSS_BACKUP_CONF}" + printf 'KEYID' >"${LSS_SECRETS_DIR}/restic-s3-key-id" + printf 'SECRET' >"${LSS_SECRETS_DIR}/restic-s3-secret" + printf '#!/bin/sh\nenv >"%s/dumps.env"\n' "${BATS_TEST_TMPDIR}" >"${LSS_DB_DUMPS}" + chmod +x "${LSS_DB_DUMPS}" + : >"${LSS_DB_DUMPS_CONF}" + cmd_run + [ -s "${BATS_TEST_TMPDIR}/dumps.env" ] + run grep -E '^(AWS_|RESTIC_)' "${BATS_TEST_TMPDIR}/dumps.env" + [ "${status}" -eq 1 ] + run ! grep -q '=SECRET$' "${BATS_TEST_TMPDIR}/dumps.env" +} + +@test "failed database dumps fail the run but the files are still backed up" { + load_backup + printf '#!/bin/sh\nexit 1\n' >"${LSS_DB_DUMPS}" + chmod +x "${LSS_DB_DUMPS}" + : >"${LSS_DB_DUMPS_CONF}" + run cmd_run + [ "${status}" -eq 1 ] + [[ ${output} == *"the database dumps failed; the backup goes on with the files."* ]] + grep -q '^restic backup ' "${STUB_LOG}" +} + +@test "the restore test compares the canary of the latest snapshot and reads part of the data" { + load_backup + printf 'abc 123\n' >"${BATS_TEST_TMPDIR}/expected" + mkdir -p "${LSS_BACKUP_STATE}" + printf 'abc 123\n' >"${LSS_BACKUP_STATE}/canary-last" + RESTORED='abc 123' + cmd_restore_test + grep -q "^restic restore latest --host .* --tag lss --target .* --include ${LSS_BACKUP_STATE}/canary/canary$" "${STUB_LOG}" + grep -qxF 'restic check --read-data-subset=5%' "${STUB_LOG}" + [ "$(cat "${LSS_BACKUP_STATE}/restore-test-success")" = 1 ] + grep -qxF 'lss_backup_restore_test_success 1' "${LSS_TEXTFILE_DIR}/lss_backup.prom" + RESTORED='something else' + run cmd_restore_test + [ "${status}" -eq 1 ] + [[ ${output} == *"the restored canary file differs from the one backed up last."* ]] + [ "$(cat "${LSS_BACKUP_STATE}/restore-test-success")" = 0 ] + RESTORED='abc 123' + RESTIC_FAIL=restore run cmd_restore_test + [[ ${output} == *"restic restore of the latest snapshot failed."* ]] + RESTIC_FAIL=check run cmd_restore_test + [[ ${output} == *"restic check --read-data-subset=5% reports errors."* ]] +} + +@test "the probe backs up the canary alone, restores it and keeps one probe snapshot" { + load_backup + # shellcheck disable=SC2329 # called by the functions under test + canary() { + mkdir -p "${CANARY_DIR}" + printf 'probe 1\n' >"${CANARY}" + printf 'probe 1' + } + RESTORED='probe 1' + run cmd_probe + [ "${status}" -eq 0 ] + grep -qxF "restic backup --tag lss-probe ${LSS_BACKUP_STATE}/canary" "${STUB_LOG}" + grep -q '^restic restore latest .* --tag lss-probe ' "${STUB_LOG}" + grep -qxF 'restic forget --tag lss-probe --keep-last 1' "${STUB_LOG}" +} + +@test "the backup script needs root and a known command" { + load_backup + run main + if [[ ${EUID} -ne 0 ]]; then + [ "${status}" -eq 1 ] + [[ ${output} == *"must run as root."* ]] + fi + run dispatch unknown + [ "${status}" -eq 2 ] + [[ ${output} == *"usage: backup run|init|restore-test|probe|restic ARGS"* ]] +} + +@test "backup restic runs restic with the repository and its password, without --quiet" { + load_backup + # shellcheck disable=SC2329 # called by dispatch + env_stub() { + printf '%s|%s|%s\n' "${RESTIC_REPOSITORY:-}" "${RESTIC_PASSWORD_FILE:-}" "$*" + } + [ "$(RESTIC=env_stub dispatch restic cat config)" = "/srv/repo|${LSS_SECRETS_DIR}/restic-password|cat config" ] +} + +# --- db-dumps --------------------------------------------------------------------------------------------------- + +load_dumps() { + # shellcheck source=/dev/null + source "${REPO_ROOT}/libexec/lss-db-dumps" + CONF=${BATS_TEST_TMPDIR}/db-dumps.conf + printf 'DB_DUMPS_DIR=%s/dumps\nDB_POSTGRESQL_PORT=5433\n' "${BATS_TEST_TMPDIR}" >"${CONF}" + return +} + +@test "only the engines lss installed are dumped, each to a file of its own" { + load_dumps + state_set mariadb status installed + state_set postgresql status installed + state_set redis status failed + # shellcheck disable=SC2329 # called by the functions under test + dump() { + printf 'dump %s\n' "$1" >>"${STUB_LOG}" + printf 'DATA\n' >"$2" + } + run_dumps + grep -qxF 'dump mariadb' "${STUB_LOG}" + grep -qxF 'dump postgresql' "${STUB_LOG}" + run ! grep -q 'dump redis' "${STUB_LOG}" + [ -s "${BATS_TEST_TMPDIR}/dumps/mariadb.sql" ] + [ -s "${BATS_TEST_TMPDIR}/dumps/postgresql.sql" ] + grep -q '^mariadb ok [0-9]*$' "${LSS_DB_DUMPS_STATE}/status" + [ "$(stat -c '%a' "${BATS_TEST_TMPDIR}/dumps")" = 700 ] +} + +@test "a failed dump keeps the previous file and fails the run" { + load_dumps + state_set postgresql status installed + mkdir -p "${BATS_TEST_TMPDIR}/dumps" + printf 'OLD\n' >"${BATS_TEST_TMPDIR}/dumps/postgresql.sql" + # shellcheck disable=SC2329 # called by the functions under test + dump() { + printf 'PART' >"$2" + return 1 + } + run run_dumps + [ "${status}" -eq 1 ] + [[ ${output} == *"the dump of postgresql failed; ${BATS_TEST_TMPDIR}/dumps/postgresql.sql keeps the previous one."* ]] + [ "$(cat "${BATS_TEST_TMPDIR}/dumps/postgresql.sql")" = OLD ] + [ ! -e "${BATS_TEST_TMPDIR}/dumps/postgresql.sql.part" ] + grep -q '^postgresql failed ' "${LSS_DB_DUMPS_STATE}/status" +} + +@test "without databases nothing is dumped" { + load_dumps + run run_dumps + [ "${status}" -eq 0 ] + [[ ${output} == *"lss installed 0 databases here; nothing to dump."* ]] +} + +@test "each engine is dumped with its own tool and options" { + load_dumps + # shellcheck disable=SC2329 # called by the functions under test + first_program() { + printf '%s' "$1" + } + # env -i PATH=... COMMAND: the stub runs COMMAND, so the functions below stand in for the programs. + # shellcheck disable=SC2329 # called by the functions under test + env() { + [[ $1 == -i && $2 == "PATH=${CLEAN_PATH}" ]] || return 1 + shift 2 + "$@" + } + # shellcheck disable=SC2329 # called by the functions under test + mariadb-dump() { + printf 'mariadb-dump %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by the functions under test + runuser() { + printf 'runuser %s\n' "$*" >>"${STUB_LOG}" + } + dump mariadb "${BATS_TEST_TMPDIR}/m.sql" + grep -qxF 'mariadb-dump --user=root --all-databases --single-transaction --routines --events --triggers --hex-blob' "${STUB_LOG}" + dump postgresql "${BATS_TEST_TMPDIR}/p.sql" + grep -qxF "runuser -u postgres -- env -i PATH=${CLEAN_PATH} pg_dumpall -p 5433 --clean --if-exists" "${STUB_LOG}" + [ "$(file_of mongodb)" = mongodb.archive ] + [ "$(file_of valkey)" = valkey.rdb ] + [ "$(file_of mysql)" = mysql.sql ] + run dump other "${BATS_TEST_TMPDIR}/o" + [ "${status}" -eq 1 ] +} + +@test "Redis and Valkey are dumped once a fresh snapshot is saved, with the password in the environment" { + load_dumps + ETC_DIR=${BATS_TEST_TMPDIR}/etc + DATA_ROOT=${BATS_TEST_TMPDIR}/lib + mkdir -p "${ETC_DIR}" "${DATA_ROOT}/valkey" + printf 'pw\n' >"${ETC_DIR}/valkey-password" + printf 'RDB\n' >"${DATA_ROOT}/valkey/dump.rdb" + # shellcheck disable=SC2329 # called by the functions under test + first_program() { + printf 'cli_stub' + } + SAVES=0 + # shellcheck disable=SC2329 # called by the functions under test + cli_stub() { + printf 'cli %s auth=%s\n' "$*" "${VALKEYCLI_AUTH:-}" >>"${STUB_LOG}" + case ${*: -1} in + lastsave) printf '%s\n' "$((1000 + SAVES))" ;; + bgsave) SAVES=${TEST_SAVES:-1} ;; + *) ;; + esac + } + # shellcheck disable=SC2329 # called by the functions under test + sleep() { + : + } + dump_rdb valkey "${BATS_TEST_TMPDIR}/v.rdb" + grep -qxF 'cli -p 6379 --no-auth-warning bgsave auth=pw' "${STUB_LOG}" + [ "$(cat "${BATS_TEST_TMPDIR}/v.rdb")" = RDB ] + SAVES=0 + TEST_SAVES=0 + run dump_rdb valkey "${BATS_TEST_TMPDIR}/w.rdb" + [ "${status}" -eq 1 ] + [ ! -e "${BATS_TEST_TMPDIR}/w.rdb" ] +} diff --git a/tests/unit/core.bats b/tests/unit/core.bats index 1f69798..bc41a5c 100644 --- a/tests/unit/core.bats +++ b/tests/unit/core.bats @@ -202,6 +202,22 @@ write_release() { [ "$(cat "${dest}")" = keep ] } +@test "a removed file is no longer expected, also when it was gone already" { + local kept=${BATS_TEST_TMPDIR}/kept.conf gone=${BATS_TEST_TMPDIR}/gone.conf taken=${BATS_TEST_TMPDIR}/taken.conf + LSS_MODULE=demo + file_write 0644 "${OWNER}" "${kept}" <<<"x" + file_write 0644 "${OWNER}" "${gone}" <<<"y" + file_write 0644 "${OWNER}" "${taken}" <<<"z" + file_remove "${gone}" + rm "${taken}" + [ "$(files_drifted demo)" = "${taken} (missing)" ] + file_remove "${taken}" + [ -z "$(files_drifted demo)" ] + printf 'edited\n' >"${kept}" + [ "$(files_drifted demo)" = "${kept}" ] + LSS_STATE_DIR=${BATS_TEST_TMPDIR}/none files_forget_path "${kept}" +} + @test "files_forget drops the checksums of a module" { LSS_MODULE=demo file_write 0644 "${OWNER}" "${BATS_TEST_TMPDIR}/f.conf" <<<"x" diff --git a/tests/unit/guard.bats b/tests/unit/guard.bats index 176833a..63402f3 100644 --- a/tests/unit/guard.bats +++ b/tests/unit/guard.bats @@ -173,3 +173,72 @@ STUB [ "${#FINDINGS[@]}" -eq 0 ] [ "${#NOTES[@]}" -eq 0 ] } + +# --- backup and metrics ----------------------------------------------------------------------------------------- + +# A backup script that records its call and exits with $BACKUP_EXIT; stamps of the last run in $BACKUP_STATE. +fake_backup() { + BACKUP=${BATS_TEST_TMPDIR}/backup + BACKUP_STATE=${BATS_TEST_TMPDIR}/restic + mkdir -p "${BACKUP_STATE}" + printf '#!/bin/sh\necho "backup $*" >>"%s"\nexit ${BACKUP_EXIT:-0}\n' "${STUB_LOG}" >"${BACKUP}" + chmod +x "${BACKUP}" + return +} + +@test "without the restic module the backup is not checked" { + BACKUP=${BATS_TEST_TMPDIR}/missing + check_backup + [ "${#FINDINGS[@]}" -eq 0 ] +} + +@test "a backup older than GUARD_BACKUP_MAX_AGE is a finding, also when none ever succeeded" { + fake_backup + printf 'GUARD_BACKUP_MAX_AGE=24\n' >"${LSS_GUARD_CONF}" + date -d '-30 hours' +%s >"${BACKUP_STATE}/last-success" + date +%s >"${BACKUP_STATE}/restore-test" + check_backup + [ "${#FINDINGS[@]}" -eq 1 ] + [[ ${FINDINGS[0]} == "The last successful backup is 30 hours old, more than 24 (journalctl -u lss-backup)." ]] + FINDINGS=() + rm -f "${BACKUP_STATE}/last-success" + mkdir -p "${STATE_DIR}/modules" + printf 'status=installed\ninstalled_at=%s\n' "$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ)" >"${STATE_DIR}/modules/restic" + check_backup + [[ ${FINDINGS[0]} == "The last successful backup is 48 hours old, more than 24 (journalctl -u lss-backup)." ]] + run ! grep -qxF "backup restore-test" "${STUB_LOG}" +} + +@test "the restore test runs weekly; a failed one is a finding until the next one passes" { + fake_backup + date +%s >"${BACKUP_STATE}/last-success" + check_backup + grep -qxF 'backup restore-test' "${STUB_LOG}" + [ "${#FINDINGS[@]}" -eq 0 ] + : >"${STUB_LOG}" + date -d '-2 days' +%s >"${BACKUP_STATE}/restore-test" + printf '0\n' >"${BACKUP_STATE}/restore-test-success" + check_backup + run ! grep -qxF "backup restore-test" "${STUB_LOG}" + [[ ${FINDINGS[0]} == "The last restore test of the backup failed (${BACKUP} restore-test shows why)." ]] + FINDINGS=() + date -d '-8 days' +%s >"${BACKUP_STATE}/restore-test" + BACKUP_EXIT=1 check_backup + [[ ${FINDINGS[0]} == "The weekly restore test of the backup failed (${BACKUP} restore-test shows why)." ]] +} + +@test "the result of the run goes to the node exporter when it is installed" { + TEXTFILE_DIR=${BATS_TEST_TMPDIR}/textfile + export_metrics + [ ! -e "${TEXTFILE_DIR}" ] + mkdir -p "${TEXTFILE_DIR}" + FINDINGS=(one two) + FIXED=(ssh-hardening) + NOTES=() + export_metrics + grep -qxF 'lss_guard_findings 2' "${TEXTFILE_DIR}/lss_guard.prom" + grep -qxF 'lss_guard_fixed 1' "${TEXTFILE_DIR}/lss_guard.prom" + grep -qxF 'lss_guard_notes 0' "${TEXTFILE_DIR}/lss_guard.prom" + grep -q '^lss_guard_last_run_timestamp_seconds [0-9]*$' "${TEXTFILE_DIR}/lss_guard.prom" + [ "$(stat -c '%a' "${TEXTFILE_DIR}/lss_guard.prom")" = 644 ] +} diff --git a/tests/unit/helpers.bash b/tests/unit/helpers.bash index 2e3a849..8fdb399 100644 --- a/tests/unit/helpers.bash +++ b/tests/unit/helpers.bash @@ -22,7 +22,7 @@ lss_test_setup() { export OWNER : >"${STUB_LOG}" local library - for library in log os config files template secrets state pkg repo fetch service firewall container web module ui; do + for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring module ui; do # shellcheck source=/dev/null source "${REPO_ROOT}/lib/${library}.sh" done diff --git a/tests/unit/monitoring.bats b/tests/unit/monitoring.bats new file mode 100644 index 0000000..22c311c --- /dev/null +++ b/tests/unit/monitoring.bats @@ -0,0 +1,586 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# The monitoring layer (lib/monitoring.sh) and the WireGuard address of the server (lib/firewall.sh): addresses and +# access, the files Prometheus and Grafana read, drop-ins and units, upstream releases and the checks of the modules. + +setup() { + load helpers + lss_test_setup + use_os debian-13 + shopt -s lastpipe + export TMPDIR=${BATS_TEST_TMPDIR} + export LSS_MODULE=grafana + WRITES=${BATS_TEST_TMPDIR}/writes + : >"${WRITES}" + MON_TARGETS_DIR=${BATS_TEST_TMPDIR}/prometheus/targets + MON_ALERTMANAGERS_DIR=${BATS_TEST_TMPDIR}/prometheus/alertmanagers + MON_PROMETHEUS_DIR=${BATS_TEST_TMPDIR}/prometheus + MON_DATASOURCES_DIR=${BATS_TEST_TMPDIR}/grafana/datasources + MON_BIN_DIR=${BATS_TEST_TMPDIR}/bin + mkdir -p "${MON_BIN_DIR}" + MON_CACHE_DIR=${BATS_TEST_TMPDIR}/cache + MON_POLICY_RC=${BATS_TEST_TMPDIR}/policy-rc.d + # file_write records "== MODE OWNER DEST" and the content, and reports a change unless TEST_FILE_CHANGED=0. + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + printf '== %s %s %s\n' "$1" "$2" "$3" >>"${WRITES}" + cat >>"${WRITES}" + LSS_FILE_CHANGED=${TEST_FILE_CHANGED:-1} + if [[ ${LSS_FILE_CHANGED} == 1 ]]; then + files_note_change "wrote $3" + fi + } + # install without -o and -g, so it runs as the test user. + # shellcheck disable=SC2329 # called by the functions under test + install() { + local -a args=() + while [[ $# -gt 0 ]]; do + case $1 in + -o | -g) shift ;; + *) args+=("$1") ;; + esac + shift + done + command install "${args[@]}" + } +} + +# --- addresses and access --------------------------------------------------------------------------------------- + +@test "the server takes the first address of the WireGuard network" { + [ "$(fw_vpn_address)" = 10.66.0.1 ] + config_set LSS_WIREGUARD_NETWORK 172.20.5.7/16 + [ "$(fw_vpn_address)" = 172.20.0.1 ] + config_set LSS_WIREGUARD_NETWORK 192.168.10.8/30 + [ "$(fw_vpn_address)" = 192.168.10.9 ] +} + +@test "the WireGuard network must be IPv4 with a prefix from /8 to /30" { + config_set LSS_WIREGUARD_NETWORK fd00::/64 + run fw_vpn_address + [ "${status}" -eq 1 ] + [[ ${output} == *"must be an IPv4 network such as 10.66.0.0/24."* ]] + config_set LSS_WIREGUARD_NETWORK 10.66.0.0/31 + run fw_vpn_address + [ "${status}" -eq 1 ] + [[ ${output} == *"needs a prefix from /8 to /30, got /31."* ]] +} + +@test "a module listens on 127.0.0.1, or on the WireGuard address with access vpn" { + run ! mon_vpn grafana + [ "$(mon_address grafana)" = 127.0.0.1 ] + config_set LSS_GRAFANA_ACCESS vpn + mon_vpn grafana + [ "$(mon_address grafana)" = 10.66.0.1 ] +} + +@test "access vpn needs the WireGuard address on an interface" { + # shellcheck disable=SC2329 # called by the functions under test + ip() { + printf '%s\n' "${TEST_IP:-}" + } + mon_vpn_ready + config_set LSS_GRAFANA_ACCESS vpn + run mon_vpn_ready + [ "${status}" -eq 1 ] + [[ ${output} == *"LSS_GRAFANA_ACCESS=vpn needs the WireGuard address 10.66.0.1; install the wireguard module first"* ]] + TEST_IP='4: wg0 inet 10.66.0.1/24 scope global wg0' + mon_vpn_ready +} + +@test "with access vpn the unit starts after wg0; with access local the drop-in goes away" { + local dropin=${BATS_TEST_TMPDIR}/grafana-server.service.d/lss-vpn.conf + config_set LSS_GRAFANA_ACCESS vpn + mon_vpn_dropin grafana-server.service + [ "${LSS_FILE_CHANGED}" = 1 ] + grep -qxF '== 0644 root:root /etc/systemd/system/grafana-server.service.d/lss-vpn.conf' "${WRITES}" + grep -qxF 'After=wg-quick@wg0.service' "${WRITES}" + grep -qxF 'Wants=wg-quick@wg0.service' "${WRITES}" + grep -q 'systemctl daemon-reload' "${STUB_LOG}" + config_set LSS_GRAFANA_ACCESS local + mon_vpn_dropin grafana-server.service + [ "${LSS_FILE_CHANGED}" = 0 ] + # An existing drop-in is removed (the unit name leads into the test directory). + [ -d /etc/systemd/system ] || skip "no /etc/systemd/system on this machine" + mkdir -p "$(dirname "${dropin}")" + : >"${dropin}" + mon_vpn_dropin "../../..${BATS_TEST_TMPDIR}/grafana-server.service" + [ "${LSS_FILE_CHANGED}" = 1 ] + [ ! -e "${dropin}" ] +} + +@test "ports open to the WireGuard network with access vpn and close with access local" { + # shellcheck disable=SC2329 # called by the functions under test + fw_allow() { + printf 'allow %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by the functions under test + fw_remove_module() { + printf 'remove %s\n' "${LSS_MODULE}" >>"${STUB_LOG}" + } + config_set LSS_GRAFANA_ACCESS vpn + mon_firewall 3000/tcp 3001/udp + grep -qxF 'allow 3000/tcp:vpn' "${STUB_LOG}" + grep -qxF 'allow 3001/udp:vpn' "${STUB_LOG}" + config_set LSS_GRAFANA_ACCESS local + mon_firewall 3000/tcp + grep -qxF 'remove grafana' "${STUB_LOG}" +} + +# --- admin paths ------------------------------------------------------------------------------------------------ + +admin_stubs() { + # shellcheck disable=SC2329 # called by the functions under test + web_server() { + printf '%s' "${TEST_SERVER:-}" + } + # shellcheck disable=SC2329 # called by the functions under test + web_admin_write() { + printf 'admin_write %s\n' "$*" >>"${STUB_LOG}" + [[ ${TEST_FILE_CHANGED:-1} == 0 ]] || files_note_change "admin $1" + } + # shellcheck disable=SC2329 # called by the functions under test + web_admin_remove() { + printf 'admin_remove %s\n' "$*" >>"${STUB_LOG}" + [[ ${TEST_FILE_CHANGED:-1} == 0 ]] || files_note_change "removed admin $1" + } + # shellcheck disable=SC2329 # called by the functions under test + web_reload() { + printf 'reload\n' >>"${STUB_LOG}" + } +} + +@test "the admin path follows LSS__WEB_ADMIN and reloads the web server only on a change" { + admin_stubs + mon_web_admin proxy http://127.0.0.1:3000 + [ ! -s "${STUB_LOG}" ] + TEST_SERVER=nginx + config_set LSS_GRAFANA_WEB_ADMIN true + mon_web_admin proxy http://127.0.0.1:3000 + grep -qxF 'admin_write grafana grafana proxy http://127.0.0.1:3000' "${STUB_LOG}" + grep -qxF 'reload' "${STUB_LOG}" + : >"${STUB_LOG}" + TEST_FILE_CHANGED=0 mon_web_admin proxy http://127.0.0.1:3000 + run ! grep -q reload "${STUB_LOG}" + config_set LSS_GRAFANA_WEB_ADMIN false + mon_web_admin proxy http://127.0.0.1:3000 + grep -qxF 'admin_remove grafana' "${STUB_LOG}" + : >"${STUB_LOG}" + mon_web_admin_remove + grep -qxF 'admin_remove grafana' "${STUB_LOG}" + grep -qxF 'reload' "${STUB_LOG}" + TEST_SERVER='' + : >"${STUB_LOG}" + mon_web_admin_remove + [ ! -s "${STUB_LOG}" ] +} + +@test "an admin path the removed web server took along is no longer expected" { + local list=${LSS_STATE_DIR}/managed/grafana + mkdir -p "${LSS_STATE_DIR}/managed" + printf 'abc\t/etc/nginx/lss/admin/grafana.conf\nabc\t/etc/lss/grafana/admin-password\n' >"${list}" + admin_stubs + TEST_SERVER='' + config_set LSS_GRAFANA_WEB_ADMIN true + mon_web_admin proxy http://127.0.0.1:3000 + [ "$(cat "${list}")" = $'abc\t/etc/lss/grafana/admin-password' ] + [ ! -s "${STUB_LOG}" ] +} + +# --- Prometheus and Grafana ------------------------------------------------------------------------------------- + +@test "a scrape target names its job and metrics path for Prometheus" { + LSS_MODULE=prometheus + mon_target prometheus prometheus 127.0.0.1:9090 /prometheus/metrics + grep -qxF "== 0644 root:root ${MON_TARGETS_DIR}/prometheus.yml" "${WRITES}" + grep -qxF -e "- targets: ['127.0.0.1:9090']" "${WRITES}" + grep -qxF ' job: prometheus' "${WRITES}" + grep -qxF ' __metrics_path__: /prometheus/metrics' "${WRITES}" + mon_target node node 127.0.0.1:9100 + grep -qxF ' __metrics_path__: /metrics' "${WRITES}" + : >"${MON_TARGETS_DIR}/node.yml" + # shellcheck disable=SC2329 # called by the functions under test + file_remove() { + rm -f -- "$1" + } + mon_target_remove node + [ ! -e "${MON_TARGETS_DIR}" ] +} + +@test "a data source is provisioned for Grafana, which restarts only when it runs and the file changed" { + LSS_MODULE=prometheus + mon_datasource Prometheus prometheus http://127.0.0.1:9090/prometheus true + grep -qxF "== 0644 root:root ${MON_DATASOURCES_DIR}/prometheus.yaml" "${WRITES}" + grep -qxF ' - name: Prometheus' "${WRITES}" + grep -qxF ' uid: lss-prometheus' "${WRITES}" + grep -qxF ' url: http://127.0.0.1:9090/prometheus' "${WRITES}" + grep -qxF ' isDefault: true' "${WRITES}" + run ! grep -q 'systemctl restart' "${STUB_LOG}" + export STUB_ACTIVE_UNITS=grafana-server + LSS_MODULE=alertmanager + mon_datasource Alertmanager alertmanager http://127.0.0.1:9093/alertmanager + grep -qxF ' isDefault: false' "${WRITES}" + grep -qxF ' implementation: prometheus' "${WRITES}" + grep -q 'systemctl restart grafana-server' "${STUB_LOG}" + : >"${STUB_LOG}" + TEST_FILE_CHANGED=0 mon_datasource Alertmanager alertmanager http://127.0.0.1:9093/alertmanager + run ! grep -q 'systemctl restart' "${STUB_LOG}" + file_remove() { + printf 'remove %s\n' "$1" >>"${STUB_LOG}" + } + mon_datasource_remove + grep -qxF "remove ${MON_DATASOURCES_DIR}/alertmanager.yaml" "${STUB_LOG}" +} + +@test "a data source that Grafana does not take stops the run" { + export STUB_ACTIVE_UNITS=grafana-server STUB_EXIT_systemctl=1 + run mon_datasource Loki loki http://127.0.0.1:3100 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: Grafana does not start with the new data source."* ]] +} + +@test "units and drop-ins of the monitoring services" { + LSS_MODULE=prometheus + mon_unit prometheus prometheus '/usr/local/bin/prometheus --config.file=/x' 'Prometheus monitoring system' + grep -qxF '== 0644 root:root /etc/systemd/system/prometheus.service' "${WRITES}" + grep -qxF 'Description=Prometheus monitoring system' "${WRITES}" + grep -qxF 'User=prometheus' "${WRITES}" + grep -qxF 'ExecStart=/usr/local/bin/prometheus --config.file=/x' "${WRITES}" + grep -qxF "ExecReload=/bin/kill -HUP \$MAINPID" "${WRITES}" + grep -qxF 'WantedBy=multi-user.target' "${WRITES}" + : >"${WRITES}" + mon_exec prometheus '/usr/bin/prometheus --config.file=/x' + grep -qxF '== 0644 root:root /etc/systemd/system/prometheus.service.d/lss-config.conf' "${WRITES}" + [ "$(grep -c '^ExecStart=' "${WRITES}")" = 2 ] + grep -qxF 'EnvironmentFile=' "${WRITES}" + grep -qxF 'ExecStart=/usr/bin/prometheus --config.file=/x' "${WRITES}" +} + +@test "Grafana's repository goes with the last of grafana, loki and alloy" { + # shellcheck disable=SC2329 # called by the functions under test + repo_remove() { + printf 'repo_remove %s\n' "$1" >>"${STUB_LOG}" + } + state_set grafana status installed + state_set loki status installed + mon_grafana_repo_remove + [ ! -s "${STUB_LOG}" ] + state_set loki status removed + mon_grafana_repo_remove + grep -qxF 'repo_remove grafana' "${STUB_LOG}" +} + +# --- packages and upstream releases ----------------------------------------------------------------------------- + +@test "Debian packages install with service starts forbidden, and the policy goes away afterwards" { + export STUB_OUT_dpkg_query='deinstall ok config-files' + # shellcheck disable=SC2329 # called by the functions under test + pkg_install() { + printf 'pkg_install %s policy=%s\n' "$*" "$(tail -n 1 "${MON_POLICY_RC}" 2>/dev/null)" >>"${STUB_LOG}" + } + mon_install_stopped prometheus + grep -qxF 'pkg_install prometheus policy=exit 101' "${STUB_LOG}" + [ ! -e "${MON_POLICY_RC}" ] + # A policy of the system's own stays untouched. + printf '#!/bin/sh\nexit 0\n' >"${MON_POLICY_RC}" + mon_install_stopped prometheus + grep -qxF 'pkg_install prometheus policy=exit 0' "${STUB_LOG}" + [ -f "${MON_POLICY_RC}" ] + rm -f -- "${MON_POLICY_RC}" + use_os rocky-9 + : >"${STUB_LOG}" + mon_install_stopped grafana + grep -qxF 'pkg_install grafana policy=' "${STUB_LOG}" +} + +@test "the policy is removed when the installation fails" { + export STUB_OUT_dpkg_query='deinstall ok config-files' + # shellcheck disable=SC2329 # called by the functions under test + pkg_install() { + die "apt failed" + } + # With set -u, as lss runs: the trap must not depend on the locals of the dying function. + # shellcheck disable=SC2329 # called through run + nounset() { + set -u + "$@" + } + run nounset mon_install_stopped prometheus + [ "${status}" -eq 1 ] + [[ ${output} != *"unbound variable"* ]] + [ ! -e "${MON_POLICY_RC}" ] +} + +# A release archive with the programs NAME and NAME-tool in a top-level directory, as fetch_verified would download it. +fake_release() { + local name=$1 version=$2 dir + dir=${BATS_TEST_TMPDIR}/release/${name}-${version}.linux-amd64 + mkdir -p "${dir}" + printf '#!/bin/sh\necho "%s, version %s (branch: HEAD)"\n' "${name}" "${version}" >"${dir}/${name}" + printf '#!/bin/sh\n' >"${dir}/${name}tool" + chmod +x "${dir}/${name}" "${dir}/${name}tool" + tar -czf "${BATS_TEST_TMPDIR}/release.tar.gz" -C "${BATS_TEST_TMPDIR}/release" "${name}-${version}.linux-amd64" + # shellcheck disable=SC2329 # called by the functions under test + fetch_verified() { + printf 'fetch %s\n' "$1" >>"${STUB_LOG}" + mkdir -p "$(dirname "$2")" + cp "${BATS_TEST_TMPDIR}/release.tar.gz" "$2" + } +} + +@test "an upstream release is installed once per version and removed again" { + fake_release prom 3.13.4 + mon_upstream_install prom 3.13.4 prom promtool + grep -qxF 'fetch prom-3.13.4' "${STUB_LOG}" + [ -x "${MON_BIN_DIR}/prom" ] + [ -x "${MON_BIN_DIR}/promtool" ] + [ -z "$(ls -A "${MON_CACHE_DIR}")" ] + [ "$(files_changes_of grafana)" = 1 ] + : >"${STUB_LOG}" + mon_upstream_install prom 3.13.4 prom promtool + [ ! -s "${STUB_LOG}" ] + mon_upstream_remove prom promtool missing + [ ! -e "${MON_BIN_DIR}/prom" ] + [ ! -e "${MON_BIN_DIR}/promtool" ] + [ "$(files_changes_of grafana)" = 3 ] +} + +@test "an archive without the expected program stops the run" { + fake_release prom 3.13.4 + run mon_upstream_install prom 3.13.4 prom other + [ "${status}" -eq 1 ] + [[ ${output} == *"the prom 3.13.4 archive has no other; its layout changed."* ]] +} + +@test "service users are created once, without login" { + # shellcheck disable=SC2329 # called by the functions under test + useradd() { + printf 'useradd %s\n' "$*" >>"${STUB_LOG}" + } + export STUB_GETENT="" + mon_user prometheus /var/lib/prometheus + grep -qxF 'useradd --system --user-group --home-dir /var/lib/prometheus --no-create-home --shell /usr/sbin/nologin prometheus' "${STUB_LOG}" + export STUB_GETENT=passwd:prometheus + : >"${STUB_LOG}" + mon_user prometheus /var/lib/prometheus + run ! grep -q useradd "${STUB_LOG}" +} + +@test "data directories are created and their owner and mode kept, never through a link" { + local dir=${BATS_TEST_TMPDIR}/data + mon_dir "${dir}" "${OWNER}" 0750 + [ "$(stat -c '%a' "${dir}")" = 750 ] + chmod 0755 "${dir}" + mon_dir "${dir}" "${OWNER}" 0750 + [ "$(stat -c '%a' "${dir}")" = 750 ] + mon_dir "${dir}" "${OWNER}" 0750 + [ "$(files_changes_of grafana)" = 2 ] + ln -s "${dir}" "${BATS_TEST_TMPDIR}/link" + run mon_dir "${BATS_TEST_TMPDIR}/link" "${OWNER}" 0750 + [ "${status}" -eq 1 ] + [[ ${output} == *"is a symbolic link."* ]] +} + +@test "only data directories below /var/lib are deleted" { + run mon_delete /etc/grafana + [ "${status}" -eq 1 ] + [[ ${output} == *"refusing to delete /etc/grafana."* ]] + run mon_delete /var/lib/../etc + [ "${status}" -eq 1 ] + mon_delete /var/lib/lss-test-does-not-exist + [ "$(files_changes_of grafana)" = 0 ] +} + +# --- checks ----------------------------------------------------------------------------------------------------- + +@test "HTTP checks wait for 200 and report the last status" { + # shellcheck disable=SC2329 # called by the functions under test + sleep() { + : + } + # shellcheck disable=SC2329 # called by the functions under test + curl() { + printf '%s' "${TEST_HTTP_CODE}" + } + TEST_HTTP_CODE=200 + [ "$(mon_http http://127.0.0.1:9090/-/ready)" = 200 ] + mon_wait_http http://127.0.0.1:9090/-/ready + TEST_HTTP_CODE=503 + run mon_wait_http http://127.0.0.1:9090/-/ready 4 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: http://127.0.0.1:9090/-/ready does not answer (503)."* ]] +} + +@test "listening sockets must be on the expected addresses, of the named process" { + # shellcheck disable=SC2329 # called by the functions under test + ss() { + printf '%s\n' "${TEST_SS:-}" + } + TEST_SS=$'LISTEN 0 4096 127.0.0.1:3000 0.0.0.0:* users:(("grafana",pid=10,fd=8))\nLISTEN 0 4096 [::1]:3000 [::]:* users:(("grafana",pid=10,fd=9))' + mon_listen_check 3000 127.0.0.1 ::1 + run mon_listen_check 3000 127.0.0.1 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: port 3000 is open on ::1, expected 127.0.0.1 only."* ]] + TEST_SS='LISTEN 0 4096 0.0.0.0:53 0.0.0.0:* users:(("systemd-resolve",pid=5,fd=3))' + run mon_listen_check --process unbound 53 127.0.0.1 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: nothing listens on port 53."* ]] + TEST_SS=$'LISTEN 0 4096 10.66.0.1%wg0:53 0.0.0.0:* users:(("unbound",pid=5,fd=3))\nLISTEN 0 4096 0.0.0.0:53 0.0.0.0:* users:(("dnsmasq",pid=6,fd=3))' + mon_listen_check --process unbound 53 10.66.0.1 +} + +# --- the monitoring modules ------------------------------------------------------------------------------------- + +# load_module NAME: sources the module as lss does for a step. +load_module() { + local file + file=$(module_file "$1") + export LSS_MODULE=$1 + # shellcheck source=/dev/null + source "${file}" + return +} + +@test "every monitoring module checks its access setting" { + local module + for module in node-exporter prometheus alertmanager grafana loki netdata; do + load_module "${module}" + mod_check + config_set "$(config_key "${module}" access)" public + run mod_check + [ "${status}" -eq 1 ] + [[ ${output} == *"must be local or vpn."* ]] + done +} + +@test "the node exporter listens on its address and reads the textfile directory" { + load_module node-exporter + [ "$(_node_command)" = '/usr/bin/prometheus-node-exporter --web.listen-address=127.0.0.1:9100 --collector.textfile.directory=/var/lib/prometheus/node-exporter' ] + use_os rocky-9 + config_set LSS_NODE_EXPORTER_ACCESS vpn + [ "$(_node_command)" = '/usr/local/bin/node_exporter --web.listen-address=10.66.0.1:9100 --collector.textfile.directory=/var/lib/prometheus/node-exporter' ] + [ "$(_node_user)" = node_exporter ] +} + +@test "Prometheus finds targets and Alertmanagers in files and serves below /prometheus/" { + load_module prometheus + run _prom_config + [[ ${output} == *" - ${MON_PROMETHEUS_DIR}/rules/*.yml"* ]] + [[ ${output} == *" - path_prefix: /alertmanager/"* ]] + [[ ${output} == *" - files: ['${MON_ALERTMANAGERS_DIR}/*.yml']"* ]] + [[ ${output} == *" - files: ['${MON_TARGETS_DIR}/*.yml']"* ]] + config_set LSS_PROMETHEUS_RETENTION 30d + run _prom_command + [[ ${output} == '/usr/bin/prometheus --config.file='*' --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=30d --web.listen-address=127.0.0.1:9090 --web.external-url=http://127.0.0.1:9090/prometheus/' ]] + use_os almalinux-10 + run _prom_command + [[ ${output} == '/usr/local/bin/prometheus '*'--storage.tsdb.path=/var/lib/prometheus/data '* ]] + config_set LSS_PROMETHEUS_RETENTION 2weeks + run mod_check + [[ ${output} == *"LSS_PROMETHEUS_RETENTION must look like 15d"* ]] +} + +@test "the alert rules cover targets, disks, the guard and the backup" { + local rules=${REPO_ROOT}/templates/prometheus/rules.yml alert + for alert in LssTargetDown LssDiskAlmostFull LssMemoryLow LssGuardFindings LssGuardStale LssBackupFailed LssBackupTooOld LssRestoreTestFailed; do + grep -qxF " - alert: ${alert}" "${rules}" + done + # The template engine leaves the placeholders of Prometheus itself alone. + run template_render "${rules}" + [ "${status}" -eq 0 ] + [[ ${output} == *'{{ $labels.instance }}'* ]] +} + +@test "Alertmanager sends to e-mail and webhook when set and keeps its cluster port closed" { + load_module alertmanager + run _am_config + [[ ${output} == *$'receivers:\n - name: lss'* ]] + [[ ${output} != *email_configs* && ${output} != *webhook_configs* ]] + config_set LSS_ALERTMANAGER_EMAIL ops@example.com + config_set LSS_ALERTMANAGER_WEBHOOK https://hooks.example.com/abc + run _am_config + [[ ${output} == *" - to: 'ops@example.com'"* ]] + [[ ${output} == *" smarthost: 'localhost:25'"* ]] + [[ ${output} == *" - url: 'https://hooks.example.com/abc'"* ]] + run _am_command + [[ ${output} == *' --web.external-url=http://127.0.0.1:9093/alertmanager/ --cluster.listen-address=' ]] + config_set LSS_ALERTMANAGER_WEBHOOK 'http://hooks.example.com/x' + run mod_check + [[ ${output} == *"LSS_ALERTMANAGER_WEBHOOK must be an https:// URL."* ]] + config_set LSS_ALERTMANAGER_WEBHOOK '' + config_set LSS_ALERTMANAGER_EMAIL 'not mail' + run mod_check + [[ ${output} == *"is not an e-mail address."* ]] +} + +@test "Grafana runs below /grafana/ with its password from a file and nothing sent home" { + load_module grafana + run _grafana_environment + [[ ${output} == *'Environment=GF_SERVER_HTTP_ADDR=127.0.0.1'* ]] + [[ ${output} == *'Environment=GF_SERVER_ROOT_URL=http://127.0.0.1:3000/grafana/'* ]] + [[ ${output} == *'Environment=GF_SERVER_SERVE_FROM_SUB_PATH=true'* ]] + [[ ${output} == *"Environment=GF_SECURITY_ADMIN_PASSWORD=\$__file{${MON_GRAFANA_DIR}/admin-password}"* ]] + [[ ${output} == *'Environment=GF_ANALYTICS_REPORTING_ENABLED=false'* ]] + # The password itself never goes into the unit. + run ! grep -E '^Environment=GF_SECURITY_ADMIN_PASSWORD=[^$]' <<<"${output}" +} + +@test "Loki keeps every component on this server with retention" { + load_module loki + config_set LSS_LOKI_RETENTION_DAYS 14 + run _loki_config + [[ ${output} == *$' http_listen_address: 127.0.0.1\n http_listen_port: 3100\n grpc_listen_address: 127.0.0.1'* ]] + [[ ${output} == *' store: inmemory'* ]] + [[ ${output} == *' retention_period: 336h'* ]] + [[ ${output} == *' reporting_enabled: false'* ]] + config_set LSS_LOKI_RETENTION_DAYS 0 + run mod_check + [[ ${output} == *"LSS_LOKI_RETENTION_DAYS must be at least 1."* ]] +} + +@test "Alloy ships the journal to the local Loki" { + load_module alloy + run _alloy_config + [[ ${output} == *'loki.source.journal "journal" {'* ]] + [[ ${output} == *' max_age = "12h"'* ]] + [[ ${output} == *'url = "http://127.0.0.1:3100/loki/api/v1/push"'* ]] + config_set LSS_LOKI_ACCESS vpn + run _alloy_config + [[ ${output} == *'url = "http://10.66.0.1:3100/loki/api/v1/push"'* ]] + config_set LSS_ALLOY_MAX_AGE 2d + run mod_check + [[ ${output} == *"LSS_ALLOY_MAX_AGE must look like 12h or 30m."* ]] +} + +@test "Netdata binds to 127.0.0.1 and the WireGuard address, without registry or statistics" { + load_module netdata + # shellcheck disable=SC2329 # called by the functions under test + install() { + : + } + _netdata_files + grep -qxF $'\tbind to = 127.0.0.1' "${WRITES}" + grep -qxF '[registry]' "${WRITES}" + grep -qxF "== 0644 root:root ${NETDATA_OPT_OUT}" "${WRITES}" + config_set LSS_NETDATA_ACCESS vpn + [ "$(_netdata_bind)" = '127.0.0.1 10.66.0.1' ] +} + +@test "smartd finds its unit and configuration on each family and checks the temperatures" { + load_module smartd + # shellcheck disable=SC2329 # called by the functions under test + systemctl() { + printf 'smartmontools.service\n' + } + [ "$(_smartd_unit)" = smartmontools.service ] + [ "$(_smartd_conf)" = /etc/smartd.conf ] + use_os rocky-9 + [ "$(_smartd_conf)" = /etc/smartmontools/smartd.conf ] + mod_check + config_set LSS_SMARTD_TEMPERATURE 50 + run mod_check + [[ ${output} == *"LSS_SMARTD_TEMPERATURE must look like 45,55."* ]] +} diff --git a/tests/unit/network.bats b/tests/unit/network.bats new file mode 100644 index 0000000..adf7544 --- /dev/null +++ b/tests/unit/network.bats @@ -0,0 +1,292 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# The network modules: the checks and the configuration they write for WireGuard, Unbound and chrony, and the DNS +# messages dns-resolver verifies with. + +setup() { + load helpers + lss_test_setup + use_os debian-13 + export TMPDIR=${BATS_TEST_TMPDIR} + WRITES=${BATS_TEST_TMPDIR}/writes + : >"${WRITES}" + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + printf '== %s %s %s\n' "$1" "$2" "$3" >>"${WRITES}" + cat >>"${WRITES}" + LSS_FILE_CHANGED=${TEST_FILE_CHANGED:-1} + } +} + +# load_module NAME: sources the module as lss does for a step. +load_module() { + local file + file=$(module_file "$1") + export LSS_MODULE=$1 + # shellcheck source=/dev/null + source "${file}" + return +} + +KEY_A=xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg= +KEY_B=TrMvSoP4jYQlY6RIzBgbssQqY3vxI2Pi+y71lOWWXX0= + +# --- wireguard -------------------------------------------------------------------------------------------------- + +@test "wireguard accepts peers with a key and an address of their own in the network" { + load_module wireguard + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.2 phone:${KEY_B}:10.66.0.3/32" + mod_check +} + +@test "wireguard refuses bad names, keys, addresses and duplicates" { + load_module wireguard + config_set LSS_WIREGUARD_PEERS "Laptop:${KEY_A}:10.66.0.2" + run mod_check + [[ ${output} == *"'Laptop' is not a peer name"* ]] + config_set LSS_WIREGUARD_PEERS "laptop:abc=:10.66.0.2" + run mod_check + [[ ${output} == *"the key of 'laptop' is not a WireGuard public key."* ]] + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.67.0.2" + run mod_check + [[ ${output} == *"the address of 'laptop' must be an IPv4 address in 10.66.0.0/24."* ]] + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.1" + run mod_check + [[ ${output} == *"10.66.0.1 is the address of the server."* ]] + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.2 phone:${KEY_A}:10.66.0.3" + run mod_check + [[ ${output} == *"every peer needs its own name, key and address ('phone')."* ]] + config_set LSS_WIREGUARD_PEERS '' + config_set LSS_WIREGUARD_PORT 70000 + run mod_check + [[ ${output} == *"LSS_WIREGUARD_PORT must be between 1 and 65535."* ]] + config_set LSS_WIREGUARD_PORT 51820 + config_set LSS_WIREGUARD_ENDPOINT 'vpn.example.com;reboot' + run mod_check + [[ ${output} == *"is not a host name."* ]] +} + +@test "the WireGuard configuration holds no key: PostUp loads them from the secrets" { + load_module wireguard + config_set LSS_WIREGUARD_NETWORK 10.70.0.0/16 + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.70.3.4" + run _wg_config + [[ ${output} == *$'\nAddress = 10.70.0.1/16\nListenPort = 51820\n'* ]] + [[ ${output} == *"PostUp = wg set %i private-key ${LSS_SECRETS_DIR}/wireguard-private-key"* ]] + [[ ${output} == *"PostUp = wg set %i peer ${KEY_A} preshared-key ${LSS_SECRETS_DIR}/wireguard-psk-laptop"* ]] + [[ ${output} == *$'[Peer]\n# laptop\nPublicKey = '"${KEY_A}"$'\nAllowedIPs = 10.70.3.4/32'* ]] + [[ ${output} != *PrivateKey* && ${output} != *PresharedKey* ]] +} + +@test "the server key and one preshared key per peer are generated once" { + load_module wireguard + # shellcheck disable=SC2329 # called by the functions under test + wg() { + printf 'wg %s\n' "$*" >>"${STUB_LOG}" + printf 'GENERATED%s=\n' "${RANDOM}" + } + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.2" + _wg_keys + [ -s "${LSS_SECRETS_DIR}/wireguard-private-key" ] + [ -s "${LSS_SECRETS_DIR}/wireguard-psk-laptop" ] + [ "$(grep -c '^wg' "${STUB_LOG}")" = 2 ] + _wg_keys + [ "$(grep -c '^wg' "${STUB_LOG}")" = 2 ] +} + +# --- dns-resolver ----------------------------------------------------------------------------------------------- + +@test "dns-resolver checks access and the forward resolvers" { + load_module dns-resolver + mod_check + config_set LSS_DNS_RESOLVER_FORWARD '1.1.1.1 2606:4700:4700::1111@53' + mod_check + config_set LSS_DNS_RESOLVER_FORWARD 'dns.example.com' + run mod_check + [[ ${output} == *"'dns.example.com' is not ADDRESS, ADDRESS@PORT or ADDRESS@PORT#NAME."* ]] + config_set LSS_DNS_RESOLVER_FORWARD '1.1.1.1@853' + config_set LSS_DNS_RESOLVER_FORWARD_TLS true + run mod_check + [[ ${output} == *"needs #NAME (the name in its certificate) with forward_tls."* ]] + config_set LSS_DNS_RESOLVER_FORWARD '' + run mod_check + [[ ${output} == *"LSS_DNS_RESOLVER_FORWARD_TLS=true needs resolvers in LSS_DNS_RESOLVER_FORWARD."* ]] + config_set LSS_DNS_RESOLVER_FORWARD_TLS false + config_set LSS_DNS_RESOLVER_ACCESS public + run mod_check + [[ ${output} == *"LSS_DNS_RESOLVER_ACCESS must be local or vpn."* ]] +} + +@test "Unbound validates on 127.0.0.1, refuses everyone else and forwards over TLS when asked" { + load_module dns-resolver + # shellcheck disable=SC2329 # called by the functions under test + _dns_ipv6() { + return 1 + } + run _dns_config + [[ ${output} == *$'\tinterface: 127.0.0.1\n'* ]] + [[ ${output} != *'interface: ::1'* && ${output} != *ip-freebind* ]] + [[ ${output} == *$'\tdo-ip6: no\n'* ]] + [[ ${output} == *$'\taccess-control: 0.0.0.0/0 refuse\n'* ]] + [[ ${output} == *'auto-trust-anchor-file: "/var/lib/unbound/root.key"'* ]] + [[ ${output} == *'control-interface: /run/unbound.ctl'* ]] + [[ ${output} != *forward-zone* ]] + config_set LSS_DNS_RESOLVER_ACCESS vpn + config_set LSS_DNS_RESOLVER_FORWARD '9.9.9.9@853#dns.quad9.net' + config_set LSS_DNS_RESOLVER_FORWARD_TLS true + use_os rocky-9 + run _dns_config + [[ ${output} == *$'\tinterface: 10.66.0.1\n\tip-freebind: yes\n'* ]] + [[ ${output} == *$'\taccess-control: 10.66.0.0/24 allow\n'* ]] + [[ ${output} == *'tls-cert-bundle: "/etc/pki/tls/certs/ca-bundle.crt"'* ]] + [[ ${output} == *'control-interface: /run/unbound/unbound.ctl'* ]] + [[ ${output} == *$'forward-zone:\n\tname: "."\n\tforward-tls-upstream: yes\n\tforward-addr: 9.9.9.9@853#dns.quad9.net'* ]] +} + +@test "the resolv.conf for Unbound keeps the search domains of the original" { + load_module dns-resolver + DNS_SAVED=${BATS_TEST_TMPDIR}/resolv.conf.orig + printf 'nameserver 192.0.2.53\nsearch corp.example\noptions rotate\n' >"${DNS_SAVED}" + run _dns_resolv_conf + [[ ${output} == *$'\nnameserver 127.0.0.1\nsearch corp.example\noptions edns0 trust-ad'* ]] + [[ ${output} != *192.0.2.53* ]] +} + +@test "the server resolves through systemd-resolved, NetworkManager or resolv.conf, and not inside a container" { + load_module dns-resolver + export STUB_EXIT_systemd_detect_virt=1 + [ "$(_dns_system_mode)" = file ] + export STUB_ACTIVE_UNITS=NetworkManager + [ "$(_dns_system_mode)" = networkmanager ] + export STUB_ACTIVE_UNITS='NetworkManager systemd-resolved' + [ "$(_dns_system_mode)" = resolved ] + export STUB_EXIT_systemd_detect_virt=0 + [ "$(_dns_system_mode)" = none ] + export STUB_EXIT_systemd_detect_virt=1 + config_set LSS_DNS_RESOLVER_SYSTEM false + [ "$(_dns_system_mode)" = none ] +} + +@test "with systemd-resolved, its stub forwards every name to Unbound" { + load_module dns-resolver + DNS_RESOLVED_DROPIN=${BATS_TEST_TMPDIR}/resolved.conf + _dns_system_apply resolved + grep -qxF "== 0644 root:root ${DNS_RESOLVED_DROPIN}" "${WRITES}" + grep -qxF 'DNS=127.0.0.1' "${WRITES}" + grep -qxF 'Domains=~.' "${WRITES}" + grep -q 'systemctl restart systemd-resolved' "${STUB_LOG}" +} + +@test "DNS queries are built label by label with the AD bit, and replies are read back" { + load_module dns-resolver + [ "$(_dns_packet example.com 1)" = '\x4c\x53\x01\x20\x00\x01\x00\x00\x00\x00\x00\x00\x07example\x03com\x00\x00\x01\x00\x01' ] + [ "$(_dns_packet . 6)" = '\x4c\x53\x01\x20\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x01' ] + local reply='4c 53 81 a0 00 01 00 02 00 00 00 01' + [ "$(_dns_reply_rcode "${reply}")" = 0 ] + [ "$(_dns_reply_answers "${reply}")" = 2 ] + _dns_reply_ad "${reply}" + reply='4c 53 81 83 00 01 00 00 00 00 00 00' + [ "$(_dns_reply_rcode "${reply}")" = 3 ] + run ! _dns_reply_ad "${reply}" + run _dns_reply_rcode '12 34 81 80 00 01 00 01 00 00 00 00' + [ "${status}" -eq 1 ] + run _dns_reply_rcode '' + [ "${status}" -eq 1 ] + run _dns_reply_answers '4c 53' + [ "${status}" -eq 1 ] +} + +@test "dns-resolver counts Unbound as online only when the root zone resolves" { + load_module dns-resolver + # shellcheck disable=SC2329 # called by the function under test + _dns_query() { + echo "query" >>"${BATS_TEST_TMPDIR}/queries" + printf '%s' "${STUB_REPLY}" + } + STUB_REPLY='4c 53 81 a0 00 01 00 01 00 00 00 00' + _dns_online + STUB_REPLY='4c 53 81 82 00 01 00 00 00 00 00 00' + run ! _dns_online + # Silence (blocked outgoing DNS) is retried, three times in all. + rm -f "${BATS_TEST_TMPDIR}/queries" + STUB_REPLY='' + run ! _dns_online + [ "$(grep -c . "${BATS_TEST_TMPDIR}/queries")" = 3 ] +} + +# --- ntp-server ------------------------------------------------------------------------------------------------- + +@test "ntp-server opens port 123 to the WireGuard network or to everyone, rate-limited" { + load_module ntp-server + mod_check + run _ntp_config + [[ ${output} == *$'port 123\nallow 10.66.0.0/24\nallow 127.0.0.1\nallow ::1\nratelimit interval 1 burst 16 leak 2'* ]] + config_set LSS_NTP_SERVER_ACCESS public + config_set LSS_NTP_SERVER_RATELIMIT false + run _ntp_config + [[ ${output} == *$'allow all\n'* && ${output} != *ratelimit* ]] + config_set LSS_NTP_SERVER_ACCESS local + run mod_check + [[ ${output} == *"LSS_NTP_SERVER_ACCESS must be vpn or public."* ]] + export SVC_APPARMOR_ENABLED=${BATS_TEST_TMPDIR}/apparmor + echo N >"${SVC_APPARMOR_ENABLED}" + [ "$(_ntp_conf)" = /etc/chrony/conf.d/lss-ntp-server.conf ] + use_os rocky-9 + [ "$(_ntp_conf)" = /etc/chrony.d/lss-ntp-server.conf ] + # A RHEL container on a Debian or Ubuntu host: the host's AppArmor profile lets chronyd read /etc/chrony/ only. + echo Y >"${SVC_APPARMOR_ENABLED}" + [ "$(_ntp_conf)" = /etc/chrony/conf.d/lss-ntp-server.conf ] + [ "$(_ntp_unit)" = chronyd ] +} + +@test "inside a container chronyd serves the time without setting the clock" { + load_module ntp-server + export STUB_EXIT_systemd_detect_virt=0 + # The command line comes from the distribution's unit, which systemctl names. + local unit_file=${BATS_TEST_TMPDIR}/chrony.service + printf '[Service]\nUser=_chrony\nExecStart=!/usr/sbin/chronyd -n $DAEMON_OPTS\n' >"${unit_file}" + # shellcheck disable=SC2329 # called by the functions under test + systemctl() { + printf '%s\n' "${unit_file}" + } + _ntp_container_dropin + grep -qxF '== 0644 root:root /etc/systemd/system/chrony.service.d/lss-ntp-server.conf' "${WRITES}" + grep -qxF 'ConditionCapability=' "${WRITES}" + grep -qxF 'ConditionVirtualization=' "${WRITES}" + grep -qxF "ExecStart=!/usr/sbin/chronyd -n \$DAEMON_OPTS -x" "${WRITES}" + grep -qxF "Type=simple" "${WRITES}" + use_os rocky-9 + printf '[Service]\nExecStart=/usr/sbin/chronyd $OPTIONS\n' >"${unit_file}" + _ntp_container_dropin + grep -qxF "ExecStart=/usr/sbin/chronyd \$OPTIONS -x" "${WRITES}" + [ "$(grep -c "^Type=simple$" "${WRITES}")" = 1 ] + : >"${unit_file}" + run _ntp_container_dropin + [ "${status}" -eq 1 ] + [[ ${output} == *"chronyd.service has no command line in ${unit_file}."* ]] + export STUB_EXIT_systemd_detect_virt=1 + _ntp_container_dropin + [ "${LSS_FILE_CHANGED}" = 0 ] +} + +@test "the time module's chrony configuration reads the confdir of ntp-server last" { + load_module time + export SVC_APPARMOR_ENABLED=${BATS_TEST_TMPDIR}/apparmor + echo N >"${SVC_APPARMOR_ENABLED}" + # shellcheck disable=SC2329 # called by the functions under test + install() { + printf 'install %s\n' "$*" >>"${STUB_LOG}" + } + _time_chrony_config + grep -qxF '== 0644 root:root /etc/chrony/chrony.conf' "${WRITES}" + grep -qxF 'server time.cloudflare.com iburst nts' "${WRITES}" + grep -qxF 'port 0' "${WRITES}" + [ "$(tail -n 1 "${WRITES}")" = 'confdir /etc/chrony/conf.d' ] + grep -qxF 'install -d -m 0755 /etc/chrony/conf.d' "${STUB_LOG}" + use_os rocky-9 + _time_chrony_config + grep -qxF 'confdir /etc/chrony.d' "${WRITES}" +} diff --git a/tests/unit/system.bats b/tests/unit/system.bats index b8bf7e4..30baf05 100644 --- a/tests/unit/system.bats +++ b/tests/unit/system.bats @@ -35,7 +35,7 @@ calls() { @test "apt output drops the notices that report nothing to do and keeps the exit status" { use_os ubuntu-24.04 - export STUB_OUT_apt_get=$'No services need to be restarted.\nSetting up nginx (1.24.0)\nNo containers need to be restarted.\nNo user sessions are running outdated binaries.\nNo VM guests are running outdated hypervisor (qemu) binaries on this host.\nNot building database; man-db/auto-update is not \'true\'.' + export STUB_OUT_apt_get=$'No services need to be restarted.\nSetting up nginx (1.24.0)\nNo containers need to be restarted.\nNo user sessions are running outdated binaries.\nNo VM guests are running outdated hypervisor (qemu) binaries on this host.\nNot building database; man-db/auto-update is not \'true\'.\nNot creating home directory `/usr/share/grafana\'.' export STUB_EXIT_apt_get=100 run _pkg_apt install nginx [ "${status}" -eq 100 ] @@ -782,3 +782,14 @@ EOF [ "${status}" -eq 1 ] [[ ${output} == *"Repository demo checks no signature."* ]] } + +@test "repo_add writes a flat repository without components" { + repo_root + printf 'KEY=demo.asc\nFINGERPRINTS=AAAA1111\nDEB_URL=https://repo.example.invalid/{{id}}/\nDEB_SUITE={{codename}}/\nDEB_COMPONENTS=\n' \ + >"${LSS_ROOT}/data/repos/flat.conf" + use_os debian-13 + repo_add flat + grep -qxF 'URIs: https://repo.example.invalid/debian/' "${BATS_TEST_TMPDIR}/writes" + grep -qxF 'Suites: trixie/' "${BATS_TEST_TMPDIR}/writes" + run ! grep -q '^Components:' "${BATS_TEST_TMPDIR}/writes" +} diff --git a/tests/unit/web.bats b/tests/unit/web.bats index e7375f8..95eb9e2 100644 --- a/tests/unit/web.bats +++ b/tests/unit/web.bats @@ -748,6 +748,28 @@ EOF [ -f "${conf}/adminer.caddy" ] } +@test "strip admin paths hand the upstream its root, on every web server" { + as_test_user + use_server nginx + web_admin_write netdata netdata strip http://127.0.0.1:19999 + local conf=${BATS_TEST_TMPDIR}/conf + grep -qxF 'location = /netdata {' "${conf}/nginx/admin/netdata.conf" + grep -qxF $'\treturn 301 /netdata/;' "${conf}/nginx/admin/netdata.conf" + grep -qxF $'\tproxy_pass http://127.0.0.1:19999/;' "${conf}/nginx/admin/netdata.conf" + state_set nginx status removed + use_server apache + web_admin_write netdata netdata strip http://127.0.0.1:19999/ + grep -qxF 'RedirectMatch 301 ^/netdata$ /netdata/' "${conf}/apache/admin/netdata.conf" + grep -qxF $'\tProxyPass http://127.0.0.1:19999/ upgrade=websocket' "${conf}/apache/admin/netdata.conf" + grep -qxF $'\tProxyPassReverse http://127.0.0.1:19999/' "${conf}/apache/admin/netdata.conf" + state_set apache status removed + use_server caddy + web_admin_write netdata netdata strip http://127.0.0.1:19999 + grep -qxF 'redir /netdata /netdata/' "${conf}/caddy/admin/netdata.caddy" + grep -qxF 'handle_path /netdata/* {' "${conf}/caddy/admin/netdata.caddy" + grep -qxF $'\treverse_proxy http://127.0.0.1:19999' "${conf}/caddy/admin/netdata.caddy" +} + @test "admin paths need a valid name, a plain path and an installed web server" { as_test_user run web_admin_write pgadmin ../pgadmin proxy http://127.0.0.1:5050