From 12f2de8a40b09f287ddf1f6efb2efc90d573ca7f Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:07:38 +0200 Subject: [PATCH 01/32] feat(lib): helpers for the monitoring, backup and network modules lib/monitoring.sh gives the monitoring modules one way to bind to 127.0.0.1 or the WireGuard address, open ports to the VPN only, publish below the admin port, register Prometheus targets and Grafana data sources, install checksum-verified upstream releases and check listening sockets. fw_vpn_address names the server's WireGuard address, flat apt repositories need no component, and the web servers can proxy with the prefix stripped. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/firewall.sh | 15 ++ lib/module.sh | 3 +- lib/monitoring.sh | 359 ++++++++++++++++++++++++++++++++++++++++ lib/repo.sh | 15 +- lib/step.sh | 2 +- lib/web.sh | 28 +++- lss | 2 +- tests/unit/helpers.bash | 2 +- tests/unit/system.bats | 11 ++ tests/unit/web.bats | 22 +++ 10 files changed, 442 insertions(+), 17 deletions(-) create mode 100644 lib/monitoring.sh diff --git a/lib/firewall.sh b/lib/firewall.sh index 80098c0..37d9cd0 100644 --- a/lib/firewall.sh +++ b/lib/firewall.sh @@ -25,6 +25,21 @@ fw_vpn_network() { return } +# fw_vpn_address: the address of this server in the WireGuard network, the first host of it (10.66.0.1 for +# 10.66.0.0/24). The wireguard module gives it to wg0; admin interfaces with access vpn listen on it. +fw_vpn_address() { + local network prefix base + network=$(fw_vpn_network) + [[ ${network} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]{1,2})$ ]] || + die "LSS_WIREGUARD_NETWORK '${network}' must be an IPv4 network such as 10.66.0.0/24." + prefix=$((10#${BASH_REMATCH[2]})) + ((prefix >= 8 && prefix <= 30)) || die "LSS_WIREGUARD_NETWORK needs a prefix from /8 to /30, got /${prefix}." + base=$(_fw_ipv4_int "${network%/*}") + base=$(((base >> (32 - prefix) << (32 - prefix)) + 1)) + printf '%s.%s.%s.%s' $((base >> 24 & 255)) $((base >> 16 & 255)) $((base >> 8 & 255)) $((base & 255)) + return +} + fw_is_enabled() { if [[ $(fw_backend) == ufw ]]; then command -v ufw >/dev/null 2>&1 && ufw status 2>/dev/null | grep -q '^Status: active' diff --git a/lib/module.sh b/lib/module.sh index a2b9cfc..d174373 100644 --- a/lib/module.sh +++ b/lib/module.sh @@ -231,8 +231,7 @@ module_remove() { # config_unknown_keys: prints the given settings that no module reads, usually typos, one per line. config_unknown_keys() { - # LSS_WIREGUARD_NETWORK is read by the firewall for the vpn scope before the wireguard module exists. - local -A known=([LSS_MODULES]=1 [LSS_MODE]=1 [LSS_WIREGUARD_NETWORK]=1) + local -A known=([LSS_MODULES]=1 [LSS_MODE]=1) local name key _default _description while IFS= read -r name; do known[$(config_key "${name}" mode)]=1 diff --git a/lib/monitoring.sh b/lib/monitoring.sh new file mode 100644 index 0000000..615ba2a --- /dev/null +++ b/lib/monitoring.sh @@ -0,0 +1,359 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Monitoring layer shared by node-exporter, prometheus, alertmanager, grafana, loki, alloy and netdata, and the +# metric files of the guard and the backup. +# +# Conventions: +# MON_TEXTFILE_DIR node-exporter's textfile collector; the guard and the backup write their *.prom files there +# MON_TARGETS_DIR one file per module with its scrape target, which Prometheus reads by itself (file_sd), so +# the modules can come and go in any order +# MON_ALERTMANAGERS_DIR the same for the Alertmanager that Prometheus sends its alerts to +# MON_DATASOURCES_DIR one Grafana data source per module, read by Grafana when it starts +# +# Every interface with a UI or an API for people serves below a path of its own name (/prometheus/, /alertmanager/, +# /grafana/), on 127.0.0.1 or, with LSS__ACCESS=vpn, on the WireGuard address of the server. The web server +# can then publish it below the same path on its admin port (LSS__WEB_ADMIN), and none of the other modules +# needs to know whether it does. + +# shellcheck disable=SC2034 # read by node-exporter; libexec/lss-guard and libexec/lss-backup use the same path +MON_TEXTFILE_DIR=/var/lib/prometheus/node-exporter +MON_PROMETHEUS_DIR=/etc/lss/prometheus +MON_TARGETS_DIR=${MON_PROMETHEUS_DIR}/targets +MON_ALERTMANAGERS_DIR=${MON_PROMETHEUS_DIR}/alertmanagers +MON_GRAFANA_DIR=/etc/lss/grafana +MON_DATASOURCES_DIR=${MON_GRAFANA_DIR}/provisioning/datasources +MON_GRAFANA_UNIT=grafana-server +# shellcheck disable=SC2034 # read by the monitoring modules +MON_PROMETHEUS_PORT=9090 +# shellcheck disable=SC2034 # read by the monitoring modules +MON_ALERTMANAGER_PORT=9093 +# The unit of the WireGuard interface (modules/network/wireguard.sh). +MON_VPN_UNIT=wg-quick@wg0.service +MON_MANAGED_BY='# Managed by lss (module' +# Where the programs of upstream releases go (the RHEL family), and the policy file of Debian's package scripts. +MON_BIN_DIR=/usr/local/bin +MON_POLICY_RC=/usr/sbin/policy-rc.d +MON_CACHE_DIR=/var/cache/lss + +# mon_vpn MODULE: true when MODULE listens on the WireGuard network (LSS__ACCESS=vpn). +mon_vpn() { + [[ $(config_option "$1" access local) == vpn ]] + return +} + +# mon_address MODULE: the address MODULE listens on, 127.0.0.1 or the WireGuard address of the server. +mon_address() { + if mon_vpn "$1"; then + fw_vpn_address + else + printf '127.0.0.1' + fi + return +} + +# mon_vpn_ready: the WireGuard address must exist before a service of the current module can listen on it. Checked in +# mod_configure, because wireguard may be installed earlier in the same run. +mon_vpn_ready() { + local address + mon_vpn "${LSS_MODULE}" || return 0 + address=$(fw_vpn_address) + if ! ip -4 -o addr show 2>/dev/null | grep -qF " ${address}/"; then + die "$(config_key "${LSS_MODULE}" access)=vpn needs the WireGuard address ${address}; install the wireguard module first (lss install wireguard)." + fi + return +} + +# mon_vpn_dropin UNIT: with access vpn, a drop-in (lss-vpn.conf) starts UNIT after the WireGuard interface, whose +# address it listens on; with access local the drop-in goes away. Sets LSS_FILE_CHANGED. +mon_vpn_dropin() { + local unit=$1 + if mon_vpn "${LSS_MODULE}"; then + svc_dropin "${unit}" lss-vpn <_WEB_ADMIN=true and a web server, publishes the current module below +# /MODULE/ on the admin port (web_admin_write proxy or strip); otherwise removes that path. Reloads the web server +# only when something changed. +mon_web_admin() { + local kind=$1 url=$2 before + [[ -n $(web_server) ]] || return 0 + before=$(files_changes_of "${LSS_MODULE}") + if config_bool "$(config_key "${LSS_MODULE}" web_admin)"; then + web_admin_write "${LSS_MODULE}" "${LSS_MODULE}" "${kind}" "${url}" + else + web_admin_remove "${LSS_MODULE}" + fi + if [[ $(files_changes_of "${LSS_MODULE}") != "${before}" ]]; then + web_reload + fi + return +} + +# mon_web_admin_remove: drops the admin path of the current module (mod_remove). +mon_web_admin_remove() { + local before + [[ -n $(web_server) ]] || return 0 + before=$(files_changes_of "${LSS_MODULE}") + web_admin_remove "${LSS_MODULE}" + if [[ $(files_changes_of "${LSS_MODULE}") != "${before}" ]]; then + web_reload + fi + return +} + +# mon_target NAME JOB HOST:PORT [METRICS_PATH]: the scrape target of the current module for Prometheus. +mon_target() { + local name=$1 job=$2 target=$3 path=${4:-/metrics} + install -d -m 0755 "${MON_TARGETS_DIR}" + file_write 0644 root:root "${MON_TARGETS_DIR}/${name}.yml" </dev/null || true + return +} + +# mon_datasource NAME TYPE URL [DEFAULT]: the Grafana data source of the current module. Grafana reads the files when it +# starts, so a running Grafana is restarted after a change; without Grafana the file waits for it. +mon_datasource() { + local name=$1 type=$2 url=$3 default=${4:-false} file + file="${MON_DATASOURCES_DIR}/${LSS_MODULE}.yaml" + install -d -m 0755 "${MON_DATASOURCES_DIR}" + file_write 0644 root:root "${file}" < <( + printf '%s %s). Grafana reads it when it starts.\napiVersion: 1\ndatasources:\n' "${MON_MANAGED_BY}" "${LSS_MODULE}" + printf ' - name: %s\n uid: lss-%s\n type: %s\n access: proxy\n url: %s\n' "${name}" "${LSS_MODULE}" "${type}" "${url}" + printf ' isDefault: %s\n editable: false\n' "${default}" + if [[ ${type} == alertmanager ]]; then + printf ' jsonData:\n implementation: prometheus\n handleGrafanaManagedAlerts: false\n' + fi + ) + if [[ ${LSS_FILE_CHANGED} == 1 ]] && svc_is_active "${MON_GRAFANA_UNIT}"; then + svc_restart "${MON_GRAFANA_UNIT}" || die "${LSS_MODULE}: Grafana does not start with the new data source." + fi + return +} + +# mon_datasource_remove: removes the data source file of the current module. Grafana keeps the data source until it is +# deleted in Grafana. +mon_datasource_remove() { + file_remove "${MON_DATASOURCES_DIR}/${LSS_MODULE}.yaml" + return +} + +# mon_install_stopped PACKAGE...: installs packages whose Debian scripts would start the service right away, listening on +# every address with the package defaults, before lss has written its configuration. A temporary policy-rc.d forbids +# those starts (exit 101: action not allowed); mod_configure starts the service with the lss settings. The RHEL family +# starts no service on installation. +mon_install_stopped() { + local policy=${MON_POLICY_RC} + if ! os_is_debian || [[ -e ${policy} || -L ${policy} ]] || [[ -z $(pkg_missing "$@") ]]; then + pkg_install "$@" + return + fi + ( + trap 'rm -f -- "${policy}"' EXIT + printf '#!/bin/sh\n# Written by lss for one package installation.\nexit 101\n' >"${policy}" + chmod 0755 "${policy}" + pkg_install "$@" + ) + return +} + +# mon_grafana_repo_remove: grafana, loki and alloy share Grafana's repository; it goes with the last of them. +mon_grafana_repo_remove() { + local module + for module in grafana loki alloy; do + [[ ${module} != "${LSS_MODULE}" ]] || continue + state_installed "${module}" && return 0 + done + repo_remove grafana + return +} + +# mon_unit UNIT USER COMMAND DESCRIPTION: the unit of a program lss installs itself (the RHEL family, from the upstream +# release), running COMMAND as USER. Sets LSS_FILE_CHANGED. +mon_unit() { + local unit=$1 user=$2 command=$3 description=$4 + svc_unit "${unit}.service" <&1 || true) + [[ ${current} != *"version ${version} "* ]] || return 0 + archive=${MON_CACHE_DIR}/${name}-${version}.tar.gz + fetch_verified "${name}-${version}" "${archive}" + dir=$(mktemp -d "${MON_CACHE_DIR}/${name}.XXXXXX") + log_command tar -xzf "${archive}" -C "${dir}" --strip-components=1 --no-same-owner + for program in "$@"; do + [[ -f ${dir}/${program} && ! -L ${dir}/${program} ]] || + die "${LSS_MODULE}: the ${name} ${version} archive has no ${program}; its layout changed." + install -m 0755 -o root -g root "${dir}/${program}" "${MON_BIN_DIR}/${program}" + if command -v restorecon >/dev/null 2>&1; then + restorecon -F "${MON_BIN_DIR}/${program}" 2>/dev/null || true + fi + done + rm -rf -- "${dir}" "${archive}" + files_note_change "installed ${name} ${version} to ${MON_BIN_DIR}" + return +} + +# mon_upstream_remove PROGRAM... +mon_upstream_remove() { + local program + for program in "$@"; do + [[ -e ${MON_BIN_DIR}/${program} ]] || continue + rm -f -- "${MON_BIN_DIR}/${program}" + files_note_change "deleted ${MON_BIN_DIR}/${program}" + done + return +} + +# mon_user NAME HOME: a system user without login for a service (the RHEL family, where no package creates it). +mon_user() { + local name=$1 home=$2 + getent passwd "${name}" >/dev/null && return 0 + useradd --system --user-group --home-dir "${home}" --no-create-home --shell /usr/sbin/nologin "${name}" + files_note_change "created user ${name}" + return +} + +# mon_dir PATH OWNER MODE: creates a data directory, or fixes its owner and mode. +mon_dir() { + local path=$1 owner=$2 mode=$3 + [[ ! -L ${path} ]] || die "${LSS_MODULE}: ${path} is a symbolic link." + if [[ ! -d ${path} ]]; then + install -d -m "${mode}" -o "${owner%%:*}" -g "${owner#*:}" "${path}" + files_note_change "created ${path}" + elif [[ $(stat -c '%U:%G %a' "${path}") != "${owner} ${mode#0}" ]]; then + chown "${owner}" "${path}" + chmod "${mode}" "${path}" + files_note_change "owner and mode of ${path}" + fi + return +} + +# mon_delete PATH...: deletes data directories of the current module (mod_remove with --purge). Each PATH must be a +# directory below /var/lib, and never a symbolic link. +mon_delete() { + local path + for path in "$@"; do + [[ ${path} == /var/lib/?* && ${path} != *..* ]] || die "${LSS_MODULE}: refusing to delete ${path}." + [[ ! -L ${path} ]] || die "${LSS_MODULE}: refusing to delete ${path}: it is a symbolic link." + [[ -d ${path} ]] || continue + rm -rf -- "${path}" + files_note_change "deleted ${path}" + done + return +} + +# mon_http URL: the HTTP status code of URL, 000 while nothing answers. +mon_http() { + curl -s --max-time 10 -o /dev/null -w '%{http_code}' "$1" 2>/dev/null || true + return +} + +# mon_wait_http URL [SECONDS]: waits until URL answers with 200. +mon_wait_http() { + local url=$1 seconds=${2:-60} i status='' + for ((i = 0; i < seconds; i += 2)); do + status=$(mon_http "${url}") + [[ ${status} != 200 ]] || return 0 + sleep 2 + done + die "${LSS_MODULE}: ${url} does not answer (${status})." +} + +# mon_listen_check [--process NAME] PORT ADDRESS...: every TCP socket listening on PORT (of the process NAME only) is on +# one of the ADDRESSes. +mon_listen_check() { + local process='' port listeners address + if [[ $1 == --process ]]; then + process=$2 + shift 2 + fi + port=$1 + shift + listeners=$(ss -Hltnp "sport = :${port}" | awk -v process="${process}" ' + process == "" || index($0, "((\"" process "\"") { address = $4; sub(/:[0-9]+$/, "", address) + sub(/%.*/, "", address); gsub(/[][]/, "", address); print address }') + [[ -n ${listeners} ]] || die "${LSS_MODULE}: nothing listens on port ${port}." + while IFS= read -r address; do + [[ " $* " == *" ${address} "* ]] || die "${LSS_MODULE}: port ${port} is open on ${address}, expected $* only." + done <<<"${listeners}" + return +} diff --git a/lib/repo.sh b/lib/repo.sh index 8cf3445..4be8d6e 100644 --- a/lib/repo.sh +++ b/lib/repo.sh @@ -81,15 +81,12 @@ repo_add() { mkdir -p /etc/apt/keyrings file_write 0644 root:root "${keyring}" < <(gpg --batch --dearmor <"${key}") local changed=${LSS_FILE_CHANGED} - file_write 0644 root:root "/etc/apt/sources.list.d/lss-${name}.sources" < +EOF + elif [[ ${kind} == strip ]]; then + cat < + ProxyPass ${target%/}/ upgrade=websocket + ProxyPassReverse ${target%/}/ + EOF else cat <"${STUB_LOG}" local library - for library in log os config files template secrets state pkg repo fetch service firewall container web module ui; do + for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring module ui; do # shellcheck source=/dev/null source "${REPO_ROOT}/lib/${library}.sh" done diff --git a/tests/unit/system.bats b/tests/unit/system.bats index 9e9b9e1..8f23088 100644 --- a/tests/unit/system.bats +++ b/tests/unit/system.bats @@ -701,3 +701,14 @@ EOF [ "${status}" -eq 1 ] [[ ${output} == *"Repository demo checks no signature."* ]] } + +@test "repo_add writes a flat repository without components" { + repo_root + printf 'KEY=demo.asc\nFINGERPRINTS=AAAA1111\nDEB_URL=https://repo.example.invalid/{{id}}/\nDEB_SUITE={{codename}}/\nDEB_COMPONENTS=\n' \ + >"${LSS_ROOT}/data/repos/flat.conf" + use_os debian-13 + repo_add flat + grep -qxF 'URIs: https://repo.example.invalid/debian/' "${BATS_TEST_TMPDIR}/writes" + grep -qxF 'Suites: trixie/' "${BATS_TEST_TMPDIR}/writes" + run ! grep -q '^Components:' "${BATS_TEST_TMPDIR}/writes" +} diff --git a/tests/unit/web.bats b/tests/unit/web.bats index af0a26a..bf8e840 100644 --- a/tests/unit/web.bats +++ b/tests/unit/web.bats @@ -686,6 +686,28 @@ EOF [ -f "${conf}/adminer.caddy" ] } +@test "strip admin paths hand the upstream its root, on every web server" { + as_test_user + use_server nginx + web_admin_write netdata netdata strip http://127.0.0.1:19999 + local conf=${BATS_TEST_TMPDIR}/conf + grep -qxF 'location = /netdata {' "${conf}/nginx/admin/netdata.conf" + grep -qxF $'\treturn 301 /netdata/;' "${conf}/nginx/admin/netdata.conf" + grep -qxF $'\tproxy_pass http://127.0.0.1:19999/;' "${conf}/nginx/admin/netdata.conf" + state_set nginx status removed + use_server apache + web_admin_write netdata netdata strip http://127.0.0.1:19999/ + grep -qxF 'RedirectMatch 301 ^/netdata$ /netdata/' "${conf}/apache/admin/netdata.conf" + grep -qxF $'\tProxyPass http://127.0.0.1:19999/ upgrade=websocket' "${conf}/apache/admin/netdata.conf" + grep -qxF $'\tProxyPassReverse http://127.0.0.1:19999/' "${conf}/apache/admin/netdata.conf" + state_set apache status removed + use_server caddy + web_admin_write netdata netdata strip http://127.0.0.1:19999 + grep -qxF 'redir /netdata /netdata/' "${conf}/caddy/admin/netdata.caddy" + grep -qxF 'handle_path /netdata/* {' "${conf}/caddy/admin/netdata.caddy" + grep -qxF $'\treverse_proxy http://127.0.0.1:19999' "${conf}/caddy/admin/netdata.caddy" +} + @test "admin paths need a valid name, a plain path and an installed web server" { as_test_user run web_admin_write pgadmin ../pgadmin proxy http://127.0.0.1:5050 From 78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:08:20 +0200 Subject: [PATCH 02/32] feat(network): wireguard, dns-resolver and ntp-server modules wireguard sets up wg0 with keys in the lss secrets and peers from the settings; dns-resolver runs a validating Unbound on localhost and points the server at it, restoring the previous resolver on removal; ntp-server opens the chrony of the time module to the WireGuard network or everyone, rate-limited. time now writes a confdir line so ntp-server can add its own file without owning chrony.conf. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/base/time.sh | 35 ++- modules/network/dns-resolver.sh | 386 ++++++++++++++++++++++++++++++++ modules/network/ntp-server.sh | 157 +++++++++++++ modules/network/wireguard.sh | 193 ++++++++++++++++ templates/time/chrony.conf | 5 +- tests/unit/network.bats | 252 +++++++++++++++++++++ 6 files changed, 1018 insertions(+), 10 deletions(-) create mode 100644 modules/network/dns-resolver.sh create mode 100644 modules/network/ntp-server.sh create mode 100644 modules/network/wireguard.sh create mode 100644 tests/unit/network.bats diff --git a/modules/base/time.sh b/modules/base/time.sh index b8c1676..7e35b52 100644 --- a/modules/base/time.sh +++ b/modules/base/time.sh @@ -41,8 +41,32 @@ mod_install() { return } +# The directory chrony reads further settings from, where ntp-server puts its own: inside chrony's directory of /etc, +# which the AppArmor profile of chronyd on Ubuntu lets it read. +_time_confdir() { + if os_is_debian; then printf '/etc/chrony/conf.d'; else printf '/etc/chrony.d'; fi + return +} + +# _time_chrony_config: writes chrony's configuration (sets LSS_FILE_CHANGED). ntp-server calls it as well, inside a +# container, where this module leaves chrony out. +_time_chrony_config() { + local server suffix='' + opt_bool nts && suffix=' nts' + TIME_SOURCES='' + for server in $(opt servers); do + TIME_SOURCES+="server ${server} iburst${suffix}"$'\n' + done + TIME_SOURCES=${TIME_SOURCES%$'\n'} + if os_is_debian; then TIME_DRIFTFILE=/var/lib/chrony/chrony.drift; else TIME_DRIFTFILE=/var/lib/chrony/drift; fi + TIME_CONFDIR=$(_time_confdir) + install -d -m 0755 "${TIME_CONFDIR}" + template_install time chrony.conf 0644 root:root "$(_time_config)" + return +} + mod_configure() { - local timezone zone_file server suffix='' + local timezone zone_file timezone=$(opt timezone) zone_file="/usr/share/zoneinfo/${timezone}" [[ -f ${zone_file} ]] || die "Unknown time zone '${timezone}' (no ${zone_file})." @@ -60,14 +84,7 @@ mod_configure() { return 0 fi - opt_bool nts && suffix=' nts' - TIME_SOURCES='' - for server in $(opt servers); do - TIME_SOURCES+="server ${server} iburst${suffix}"$'\n' - done - TIME_SOURCES=${TIME_SOURCES%$'\n'} - if os_is_debian; then TIME_DRIFTFILE=/var/lib/chrony/chrony.drift; else TIME_DRIFTFILE=/var/lib/chrony/drift; fi - template_install time chrony.conf 0644 root:root "$(_time_config)" + _time_chrony_config local changed=${LSS_FILE_CHANGED} # systemd-timesyncd must not fight chrony for the clock. svc_disable systemd-timesyncd diff --git a/modules/network/dns-resolver.sh b/modules/network/dns-resolver.sh new file mode 100644 index 0000000..6e9a0ba --- /dev/null +++ b/modules/network/dns-resolver.sh @@ -0,0 +1,386 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="dns-resolver" +MOD_GROUP="network" +MOD_DESCRIPTION="Unbound as a validating (DNSSEC) caching resolver on 127.0.0.1, optionally for the WireGuard network; the server itself resolves through it" +MOD_REQUIRES="system" +MOD_OPTIONS="access|local|Who may ask: local (this server only) or vpn (also the WireGuard network, on the server's WireGuard address) +forward||Resolvers to forward every query to instead of asking the root servers, separated by spaces (1.1.1.1, or 1.1.1.1@853#cloudflare-dns.com with forward_tls); empty: Unbound resolves from the root servers itself +forward_tls|false|Forward over DNS over TLS; every forward entry then needs @853#NAME, the name its certificate must carry +system|true|Make this server resolve through Unbound (systemd-resolved, NetworkManager or /etc/resolv.conf); inside a container the engine owns /etc/resolv.conf, so it stays as it is" +MOD_PORTS="53/udp and 53/tcp from the WireGuard network with access vpn; none with access local" + +DNS_CONF=/etc/unbound/unbound.conf +DNS_ANCHOR=/var/lib/unbound/root.key +DNS_RESOLV=/etc/resolv.conf +# The resolv.conf found before lss changed it, put back when the module is removed. +DNS_SAVED_DIR=${LSS_STATE_DIR:-/var/lib/lss}/dns-resolver +DNS_SAVED=${DNS_SAVED_DIR}/resolv.conf +DNS_RESOLVED_DROPIN=/etc/systemd/resolved.conf.d/lss-dns-resolver.conf +DNS_NM_CONF=/etc/NetworkManager/conf.d/lss-dns-resolver.conf +# dhclient rewrites /etc/resolv.conf on every lease renewal unless a hook replaces its make_resolv_conf. +DNS_DHCLIENT_HOOK=/etc/dhcp/dhclient-enter-hooks.d/lss-dns-resolver +DNS_UNIT=unbound +DNS_LOOPBACK=127.0.0.1 + +mod_check() { + local entry + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_DNS_RESOLVER_ACCESS must be local or vpn." + opt_bool system || true + for entry in $(opt forward); do + [[ ${entry} =~ ^[0-9A-Fa-f.:]+(@[0-9]{1,5})?(#[A-Za-z0-9.-]+)?$ ]] || + die "LSS_DNS_RESOLVER_FORWARD: '${entry}' is not ADDRESS, ADDRESS@PORT or ADDRESS@PORT#NAME." + if opt_bool forward_tls && [[ ${entry} != *'#'* ]]; then + die "LSS_DNS_RESOLVER_FORWARD: '${entry}' needs #NAME (the name in its certificate) with forward_tls." + fi + done + if opt_bool forward_tls && [[ -z $(opt forward) ]]; then + die "LSS_DNS_RESOLVER_FORWARD_TLS=true needs resolvers in LSS_DNS_RESOLVER_FORWARD." + fi + return +} + +mod_install() { + # unbound-anchor creates and refreshes the DNSSEC trust anchor of the root zone; Debian ships it separately. + pkg_install_family "unbound unbound-anchor dns-root-data" unbound + return +} + +# IPv6 is used where the server has a default route for it; otherwise Unbound would try IPv6 root servers in vain. +_dns_ipv6() { + [[ -n $(ip -6 route show default 2>/dev/null) ]] && grep -q '^0\{31\}1 ' /proc/net/if_inet6 2>/dev/null + return +} + +# The control socket: in /run on Debian and Ubuntu, in Unbound's own directory under SELinux on the RHEL family. +_dns_control() { + if os_is_debian; then printf '/run/unbound.ctl'; else printf '/run/unbound/unbound.ctl'; fi + return +} + +_dns_ca_bundle() { + if os_is_debian; then printf '/etc/ssl/certs/ca-certificates.crt'; else printf '/etc/pki/tls/certs/ca-bundle.crt'; fi + return +} + +_dns_config() { + local ip6=no entry network + _dns_ipv6 && ip6=yes + network=$(fw_vpn_network) + printf '# Managed by lss (module dns-resolver). Change LSS_DNS_RESOLVER_* in /etc/lss/lss.conf and run lss configure dns-resolver.\n' + printf 'server:\n\tverbosity: 0\n\tuse-syslog: yes\n\tport: 53\n\tinterface: %s\n' "${DNS_LOOPBACK}" + [[ ${ip6} == no ]] || printf '\tinterface: ::1\n' + if [[ $(opt access) == vpn ]]; then + # The WireGuard address may come up after Unbound; freebind lets Unbound listen on it anyway. + printf '\tinterface: %s\n\tip-freebind: yes\n' "$(fw_vpn_address)" + fi + printf '\tdo-ip4: yes\n\tdo-ip6: %s\n\tdo-udp: yes\n\tdo-tcp: yes\n' "${ip6}" + printf '\taccess-control: 0.0.0.0/0 refuse\n\taccess-control: ::/0 refuse\n' + printf '\taccess-control: 127.0.0.0/8 allow\n\taccess-control: ::1/128 allow\n' + [[ $(opt access) != vpn ]] || printf '\taccess-control: %s allow\n' "${network}" + cat </dev/null 2>&1 || true + if [[ ! -s ${DNS_ANCHOR} && -s /usr/share/dns/root.key ]]; then + cp /usr/share/dns/root.key "${DNS_ANCHOR}" + fi + [[ -s ${DNS_ANCHOR} ]] || die "dns-resolver: unbound-anchor could not create ${DNS_ANCHOR}." + files_note_change "created the DNSSEC trust anchor" + fi + if [[ $(stat -c '%U' "${DNS_ANCHOR}") != unbound ]]; then + chown unbound:unbound "${DNS_ANCHOR}" + fi + if command -v restorecon >/dev/null 2>&1; then + restorecon -F "${DNS_ANCHOR}" 2>/dev/null || true + fi + return +} + +# How the server picks its resolver: systemd-resolved, NetworkManager, a plain /etc/resolv.conf, or nothing to change +# (inside a container, or with system=false). +_dns_system_mode() { + if ! opt_bool system || os_in_container; then + printf 'none' + elif svc_is_active systemd-resolved; then + printf 'resolved' + elif svc_is_active NetworkManager; then + printf 'networkmanager' + else + printf 'file' + fi + return +} + +# _dns_resolv_conf: /etc/resolv.conf pointing at Unbound, with the search domains of the original file. +_dns_resolv_conf() { + printf '# Managed by lss (module dns-resolver): this server resolves through Unbound.\nnameserver %s\n' "${DNS_LOOPBACK}" + if [[ -f ${DNS_SAVED} && ! -L ${DNS_SAVED} ]]; then + grep -E '^(search|domain)[[:space:]]' "${DNS_SAVED}" || true + fi + printf 'options edns0 trust-ad\n' + return +} + +_dns_save_original() { + [[ ! -e ${DNS_SAVED} && ! -L ${DNS_SAVED} ]] || return 0 + install -d -m 0700 "${DNS_SAVED_DIR}" + cp -a "${DNS_RESOLV}" "${DNS_SAVED}" + return +} + +_dns_system_apply() { + local mode=$1 changed=0 content + case ${mode} in + resolved) + # resolved keeps its stub on 127.0.0.53 and forwards everything to Unbound, which does the DNSSEC validation. + file_write 0644 root:root "${DNS_RESOLVED_DROPIN}" </dev/null || true + files_note_change "restored ${DNS_RESOLV}" + fi + if [[ -e ${DNS_NM_CONF} ]]; then + file_remove "${DNS_NM_CONF}" + systemctl reload NetworkManager || true + fi + [[ $(files_changes_of dns-resolver) == "${before}" ]] || log_info "dns-resolver: the server uses its previous resolvers again." + return +} + +mod_configure() { + local before config mode + before=$(files_changes_of dns-resolver) + mon_vpn_ready + _dns_anchor + config=$(_dns_config) + file_write 0644 root:root "${DNS_CONF}" <<<"${config}" + unbound-checkconf "${DNS_CONF}" >/dev/null || die "dns-resolver: unbound-checkconf rejects ${DNS_CONF}." + svc_sandbox "${DNS_UNIT}.service" ReadWritePaths=/var/lib/unbound + mon_vpn_dropin "${DNS_UNIT}.service" + svc_enable "${DNS_UNIT}" + if [[ $(files_changes_of dns-resolver) != "${before}" ]]; then + svc_restart "${DNS_UNIT}" || die "dns-resolver: Unbound does not start with the new configuration." + fi + mon_firewall 53/udp 53/tcp + # The server switches to Unbound only once it answers. + _dns_wait + mode=$(_dns_system_mode) + if [[ ${mode} == none ]]; then + _dns_system_undo + else + _dns_system_apply "${mode}" + fi + return +} + +# _dns_packet NAME TYPE: a DNS query (recursion desired, AD bit set to ask for the validation result) as printf %b +# escapes. NAME is a host name or "." for the root. +_dns_packet() { + local name=$1 type=$2 label packet='\x4c\x53\x01\x20\x00\x01\x00\x00\x00\x00\x00\x00' + local -a labels=() + IFS=. read -ra labels <<<"${name%.}" + for label in "${labels[@]}"; do + [[ -n ${label} ]] || continue + packet+=$(printf '\\x%02x%s' "${#label}" "${label}") + done + packet+=$(printf '\\x00\\x%02x\\x%02x\\x00\\x01' $((type >> 8)) $((type & 255))) + printf '%s' "${packet}" + return +} + +# _dns_query SERVER NAME TYPE: sends one query over UDP and prints the reply as hexadecimal bytes separated by +# spaces; nothing when no reply came within 5 seconds. +_dns_query() { + local server=$1 name=$2 type=$3 fd reply + exec {fd}<>"/dev/udp/${server}/53" || return 0 + printf '%b' "$(_dns_packet "${name}" "${type}")" >&"${fd}" + reply=$(timeout 5 dd bs=4096 count=1 status=none <&"${fd}" | od -An -v -tx1 | tr -s ' \n' ' ' || true) + exec {fd}>&- + printf '%s' "${reply# }" + return +} + +# _dns_reply_rcode REPLY / _dns_reply_ad REPLY / _dns_reply_answers REPLY: fields of a reply from _dns_query. +_dns_reply_rcode() { + local -a bytes=() + read -ra bytes <<<"$1" + [[ ${#bytes[@]} -ge 12 && ${bytes[0]}${bytes[1]} == 4c53 ]] || return 1 + printf '%s' $((0x${bytes[3]} & 15)) + return +} + +_dns_reply_ad() { + local -a bytes=() + read -ra bytes <<<"$1" + [[ ${#bytes[@]} -ge 12 ]] && (((0x${bytes[3]} & 0x20) != 0)) + return +} + +_dns_reply_answers() { + local -a bytes=() + read -ra bytes <<<"$1" + [[ ${#bytes[@]} -ge 12 ]] || return 1 + printf '%s' $((0x${bytes[6]} * 256 + 0x${bytes[7]})) + return +} + +# Waits until Unbound answers a local name (no network needed): the built-in zone localhost. +_dns_wait() { + local i reply='' + for ((i = 0; i < 15; i++)); do + reply=$(_dns_query "${DNS_LOOPBACK}" localhost 1) + if [[ $(_dns_reply_rcode "${reply}") == 0 && $(_dns_reply_answers "${reply}") -ge 1 ]]; then + return 0 + fi + sleep 1 + done + die "dns-resolver: Unbound does not answer on ${DNS_LOOPBACK}:53." +} + +mod_verify() { + local reply rcode address + local -a addresses=("${DNS_LOOPBACK}") + svc_is_active "${DNS_UNIT}" || die "dns-resolver: ${DNS_UNIT} is not running." + _dns_wait + unbound-control -c "${DNS_CONF}" status >/dev/null 2>&1 || die "dns-resolver: unbound-control gets no answer." + [[ $(unbound-control -c "${DNS_CONF}" get_option module-config 2>/dev/null) == *validator* ]] || + die "dns-resolver: Unbound does not validate DNSSEC." + [[ -s ${DNS_ANCHOR} ]] || die "dns-resolver: the DNSSEC trust anchor ${DNS_ANCHOR} is missing." + # The signed root zone must come back validated (AD). Without a reachable resolver upstream only the offline checks + # above apply. + reply=$(_dns_query "${DNS_LOOPBACK}" . 6) + rcode=$(_dns_reply_rcode "${reply}" || true) + if [[ ${rcode} == 0 ]]; then + _dns_reply_ad "${reply}" || die "dns-resolver: the answer for the root zone is not DNSSEC-validated." + else + log_warn "dns-resolver: the root zone did not resolve (rcode ${rcode:-none}); DNSSEC was checked offline only." + fi + _dns_ipv6 && addresses+=(::1) + [[ $(opt access) != vpn ]] || addresses+=("$(fw_vpn_address)") + mon_listen_check --process unbound 53 "${addresses[@]}" + case $(_dns_system_mode) in + resolved) + resolvectl dns 2>/dev/null | grep -qE "^Global: .*${DNS_LOOPBACK}" || + die "dns-resolver: systemd-resolved does not forward to Unbound." + ;; + networkmanager | file) + address=$(awk '$1 == "nameserver" { print $2; exit }' "${DNS_RESOLV}") + [[ ${address} == "${DNS_LOOPBACK}" ]] || die "dns-resolver: ${DNS_RESOLV} does not point at Unbound." + ;; + *) ;; + esac + return +} + +mod_remove() { + _dns_system_undo + svc_disable "${DNS_UNIT}" + svc_dropin_remove "${DNS_UNIT}.service" lss-sandbox + svc_dropin_remove "${DNS_UNIT}.service" lss-vpn + file_remove "${DNS_CONF}" + pkg_remove_family "unbound unbound-anchor" unbound + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete /var/lib/unbound + return +} diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh new file mode 100644 index 0000000..d837f75 --- /dev/null +++ b/modules/network/ntp-server.sh @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="ntp-server" +MOD_GROUP="network" +MOD_DESCRIPTION="chrony of the time module also serves NTP on port 123, to the WireGuard network or to everyone, rate-limited" +MOD_REQUIRES="time" +MOD_OPTIONS="access|vpn|Who may ask for the time: vpn (the WireGuard network) or public (everyone) +ratelimit|true|Answer each client at most about once a second, with short bursts, so the server cannot amplify floods" +MOD_PORTS="123/udp from the WireGuard network (access vpn) or from everyone (access public)" + +# chrony keeps the time module's configuration, which reads the files of this directory last; the file here turns the +# NTP port on and names who may ask. Inside a container the time module leaves chrony out (the host owns the clock), so +# this module sets chrony up there to serve the time without ever setting the clock (chronyd -x). +NTP_FILE_NAME=lss-ntp-server.conf +NTP_PORT=123 + +_ntp_unit() { + if os_is_debian; then printf 'chrony'; else printf 'chronyd'; fi + return +} + +_ntp_conf() { + if os_is_debian; then printf '/etc/chrony/conf.d/%s' "${NTP_FILE_NAME}"; else printf '/etc/chrony.d/%s' "${NTP_FILE_NAME}"; fi + return +} + +mod_check() { + [[ $(opt access) =~ ^(vpn|public)$ ]] || die "LSS_NTP_SERVER_ACCESS must be vpn or public." + opt_bool ratelimit || true + return +} + +mod_install() { + pkg_install chrony + return +} + +# _ntp_time_config: chrony's configuration as the time module writes it, from the time module's options. Inside a +# container only; elsewhere the time module has written it already. +_ntp_time_config() { + ( + # shellcheck source=/dev/null + source "$(module_file time)" + LSS_MODULE="time" + _time_chrony_config + ) + return +} + +_ntp_config() { + printf '# Managed by lss (module ntp-server). Change LSS_NTP_SERVER_* in /etc/lss/lss.conf and run lss configure ntp-server.\n' + printf 'port %s\n' "${NTP_PORT}" + if [[ $(opt access) == public ]]; then + printf 'allow all\n' + else + printf 'allow %s\n' "$(fw_vpn_network)" + fi + # lss verify asks over the loopback interface. + printf 'allow 127.0.0.1\nallow ::1\n' + if opt_bool ratelimit; then + printf 'ratelimit interval 1 burst 16 leak 2\n' + fi + return +} + +# _ntp_container_dropin: inside a container chronyd serves the time but must not set the clock it shares with the host +# (-x), and the unit's condition on CAP_SYS_TIME no longer applies. +_ntp_container_dropin() { + local unit options=OPTIONS + unit="$(_ntp_unit).service" + os_is_debian && options=DAEMON_OPTS + if os_in_container; then + svc_dropin "${unit}" lss-ntp-server <"/dev/udp/${address}/${NTP_PORT}" || return 0 + # 48 bytes: LI 0, VN 4, mode 3, then zeros. + printf '\x23%047d' 0 | tr 0 '\000' >&"${fd}" + reply=$(timeout 5 dd bs=512 count=1 status=none <&"${fd}" | od -An -N1 -tu1 | tr -d ' \n' || true) + exec {fd}>&- + printf '%s' "${reply}" + return +} + +mod_verify() { + local unit i first='' + unit=$(_ntp_unit) + svc_is_active "${unit}" || die "ntp-server: ${unit} is not running." + # chronyd answers clients also before it is synchronised itself (with the leap indicator "unsynchronised"). + for ((i = 0; i < 10; i++)); do + first=$(_ntp_probe 127.0.0.1) + [[ -z ${first} ]] || break + sleep 1 + done + [[ -n ${first} ]] || die "ntp-server: chronyd does not answer NTP requests on port ${NTP_PORT}." + (((first & 7) == 4)) || die "ntp-server: the answer on port ${NTP_PORT} is no NTP server reply (mode $((first & 7)))." + ss -Hlun "sport = :${NTP_PORT}" | grep -q . || die "ntp-server: nothing listens on ${NTP_PORT}/udp." + return +} + +mod_remove() { + local unit + unit=$(_ntp_unit) + file_remove "$(_ntp_conf)" + svc_dropin_remove "${unit}.service" lss-sandbox + if os_in_container; then + svc_dropin_remove "${unit}.service" lss-ntp-server + svc_disable "${unit}" + pkg_remove chrony + elif svc_is_active "${unit}"; then + # The time module's chrony keeps the clock, with the NTP port closed again. + svc_restart "${unit}" || die "ntp-server: chronyd does not start without the NTP port." + fi + return +} diff --git a/modules/network/wireguard.sh b/modules/network/wireguard.sh new file mode 100644 index 0000000..4bd3c2a --- /dev/null +++ b/modules/network/wireguard.sh @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="wireguard" +MOD_GROUP="network" +MOD_DESCRIPTION="WireGuard VPN server on wg0 with the peers from the settings and its keys in the lss secrets; admin interfaces with access vpn listen on it" +MOD_REQUIRES="system" +MOD_OPTIONS="network|10.66.0.0/24|VPN network (IPv4, /8 to /30); the server takes its first address, the peers others; the vpn scope of the firewall is this network +port|51820|UDP port WireGuard listens on, open to everyone +peers||Peers as NAME:PUBLIC_KEY:ADDRESS, separated by spaces (laptop::10.66.0.2); each also gets a preshared key (lss secrets show wireguard-psk-NAME) +endpoint||Host name or address the peers connect to, for the peer settings lss prints; empty: the address of the default route" +MOD_PORTS="51820/udp from everyone" +MOD_SECRETS="wireguard-private-key wireguard-psk-" + +# Peers reach the server itself (its admin interfaces, SSH, the DNS resolver), not the networks behind it: lss does not +# turn on IP forwarding, which kernel-hardening keeps off. +WG_INTERFACE=wg0 +WG_CONF=/etc/wireguard/${WG_INTERFACE}.conf +WG_UNIT=wg-quick@${WG_INTERFACE} +WG_KEY_SECRET=wireguard-private-key +# The public key is no secret; peers need it. +WG_PUBLIC_KEY=/etc/lss/wireguard-public-key + +_wg_server_address() { + fw_vpn_address + return +} + +_wg_prefix() { + local network + network=$(opt network) + printf '%s' "${network##*/}" + return +} + +# _wg_in_network ADDRESS: ADDRESS lies in the VPN network. +_wg_in_network() { + local address=$1 network prefix + network=$(opt network) + prefix=$((10#${network##*/})) + (($(_fw_ipv4_int "${address}") >> (32 - prefix) == $(_fw_ipv4_int "${network%/*}") >> (32 - prefix))) + return +} + +mod_check() { + local port peer name key address server endpoint + local -A names=() keys=() addresses=() + server=$(_wg_server_address) + port=$(opt_int port) + ((port >= 1 && port <= 65535)) || die "LSS_WIREGUARD_PORT must be between 1 and 65535." + for peer in $(opt peers); do + IFS=: read -r name key address <<<"${peer}" + address=${address%/32} + [[ ${name} =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]] || + die "LSS_WIREGUARD_PEERS: '${name}' is not a peer name (a-z, 0-9 and -, up to 32 characters)." + # A Curve25519 key in base64: 43 characters and "=", the last one from the set a 256-bit value allows. + [[ ${key} =~ ^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$ ]] || + die "LSS_WIREGUARD_PEERS: the key of '${name}' is not a WireGuard public key." + if [[ ! ${address} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || ! _wg_in_network "${address}"; then + die "LSS_WIREGUARD_PEERS: the address of '${name}' must be an IPv4 address in $(opt network)." + fi + [[ ${address} != "${server}" ]] || die "LSS_WIREGUARD_PEERS: ${address} is the address of the server." + [[ -z ${names[${name}]+x} && -z ${keys[${key}]+x} && -z ${addresses[${address}]+x} ]] || + die "LSS_WIREGUARD_PEERS: every peer needs its own name, key and address ('${name}')." + names[${name}]=1 + keys[${key}]=1 + addresses[${address}]=1 + done + endpoint=$(opt endpoint) + [[ -z ${endpoint} || ${endpoint} =~ ^[A-Za-z0-9.:-]+$ ]] || die "LSS_WIREGUARD_ENDPOINT '${endpoint}' is not a host name." + return +} + +mod_install() { + pkg_install wireguard-tools + return +} + +# _wg_keys: the private key of the server and a preshared key per peer, made once by wg itself and kept in the lss +# secrets. +_wg_keys() { + local peer name _key _address value + if ! secret_exists "${WG_KEY_SECRET}"; then + value=$(wg genkey) + secret_set "${WG_KEY_SECRET}" "${value}" + log_info "Generated secret ${WG_KEY_SECRET}" + files_note_change "generated the WireGuard key" + fi + for peer in $(opt peers); do + IFS=: read -r name _key _address <<<"${peer}" + secret_exists "wireguard-psk-${name}" && continue + value=$(wg genpsk) + secret_set "wireguard-psk-${name}" "${value}" + log_info "Generated secret wireguard-psk-${name} (lss secrets show wireguard-psk-${name})" + files_note_change "generated the preshared key of ${name}" + done + return +} + +# _wg_config: wg-quick configuration without keys; PostUp hands the private and preshared keys to the interface from +# the secrets, so they exist in one place only. +_wg_config() { + local peer name key address + printf '# Managed by lss (module wireguard). Change LSS_WIREGUARD_* in /etc/lss/lss.conf and run lss configure wireguard.\n' + printf '[Interface]\nAddress = %s/%s\nListenPort = %s\n' "$(_wg_server_address)" "$(_wg_prefix)" "$(opt_int port)" + printf 'PostUp = wg set %%i private-key %s\n' "$(secret_path "${WG_KEY_SECRET}")" + for peer in $(opt peers); do + IFS=: read -r name key address <<<"${peer}" + printf 'PostUp = wg set %%i peer %s preshared-key %s\n' "${key}" "$(secret_path "wireguard-psk-${name}")" + done + for peer in $(opt peers); do + IFS=: read -r name key address <<<"${peer}" + printf '\n[Peer]\n# %s\nPublicKey = %s\nAllowedIPs = %s/32\n' "${name}" "${key}" "${address%/32}" + done + return +} + +# The settings a peer needs, printed after a change (no secret: the preshared key is named, not shown). +_wg_hints() { + local peer name _key address endpoint public + endpoint=$(opt endpoint) + if [[ -z ${endpoint} ]]; then + endpoint=$(_fw_host_addresses) + endpoint=${endpoint%%$'\n'*} + fi + public=$(<"${WG_PUBLIC_KEY}") + for peer in $(opt peers); do + IFS=: read -r name _key address <<<"${peer}" + log_info "wireguard: peer ${name}: Address = ${address%/32}/32; [Peer] PublicKey = ${public}, Endpoint = ${endpoint:-SERVER}:$(opt_int port), AllowedIPs = $(_wg_server_address)/32, PresharedKey from lss secrets show wireguard-psk-${name}" + done + return +} + +mod_configure() { + local before public config + before=$(files_changes_of wireguard) + _wg_keys + public=$(wg pubkey <"$(secret_path "${WG_KEY_SECRET}")") + file_write 0644 root:root "${WG_PUBLIC_KEY}" <<<"${public}" + install -d -m 0700 /etc/wireguard + config=$(_wg_config) + file_write 0600 root:root "${WG_CONF}" <<<"${config}" + svc_sandbox "${WG_UNIT}.service" + fw_allow "$(opt_int port)/udp" + svc_enable "${WG_UNIT}" + if [[ $(files_changes_of wireguard) != "${before}" ]]; then + svc_restart "${WG_UNIT}" || die "wireguard: ${WG_INTERFACE} does not come up with the new configuration." + _wg_hints + fi + return +} + +mod_verify() { + local peer _name key _address expected actual without + svc_is_active "${WG_UNIT}" || die "wireguard: ${WG_UNIT} is not running." + [[ $(wg show "${WG_INTERFACE}" listen-port) == "$(opt_int port)" ]] || + die "wireguard: ${WG_INTERFACE} does not listen on port $(opt_int port)." + [[ $(wg show "${WG_INTERFACE}" public-key) == "$(<"${WG_PUBLIC_KEY}")" ]] || + die "wireguard: ${WG_INTERFACE} does not use the key in the lss secrets." + ip -4 -o addr show dev "${WG_INTERFACE}" | grep -qF " $(_wg_server_address)/$(_wg_prefix) " || + die "wireguard: ${WG_INTERFACE} does not have the address $(_wg_server_address)/$(_wg_prefix)." + expected='' + for peer in $(opt peers); do + IFS=: read -r _name key _address <<<"${peer}" + expected+="${key}"$'\n' + done + expected=$(sort <<<"${expected%$'\n'}") + actual=$(wg show "${WG_INTERFACE}" peers | sort) + [[ ${actual} == "${expected}" ]] || die "wireguard: the peers of ${WG_INTERFACE} differ from LSS_WIREGUARD_PEERS." + # Only the peers without one are printed; the preshared keys themselves never leave wg. + without=$(wg show "${WG_INTERFACE}" preshared-keys | awk '$2 == "(none)" { print $1 }') + [[ -z ${without} ]] || die "wireguard: peers without a preshared key: ${without//$'\n'/ }." + return +} + +mod_remove() { + local secret + if svc_is_enabled "${WG_UNIT}" || svc_is_active "${WG_UNIT}"; then + systemctl disable --now "${WG_UNIT}" >/dev/null 2>&1 || die "wireguard: ${WG_UNIT} does not stop." + files_note_change "disabled ${WG_UNIT}" + fi + svc_dropin_remove "${WG_UNIT}.service" lss-sandbox + file_remove "${WG_CONF}" + file_remove "${WG_PUBLIC_KEY}" + pkg_remove wireguard-tools + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + for secret in "${LSS_SECRETS_DIR}"/wireguard-*; do + [[ -f ${secret} ]] || continue + secret_remove "${secret##*/}" + done + return +} diff --git a/templates/time/chrony.conf b/templates/time/chrony.conf index f871278..b46b2b0 100644 --- a/templates/time/chrony.conf +++ b/templates/time/chrony.conf @@ -7,6 +7,9 @@ makestep 1.0 3 rtcsync leapsectz right/UTC logdir /var/log/chrony -# No NTP service for other hosts and no remote command port (chronyc uses the local socket). +# No NTP service for other hosts (unless the ntp-server module opens one in the confdir below) and no remote command +# port (chronyc uses the local socket). port 0 cmdport 0 +# Last, so that the files there win: the ntp-server module serves time to a network from there. +confdir {{TIME_CONFDIR}} diff --git a/tests/unit/network.bats b/tests/unit/network.bats new file mode 100644 index 0000000..1589607 --- /dev/null +++ b/tests/unit/network.bats @@ -0,0 +1,252 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# The network modules: the checks and the configuration they write for WireGuard, Unbound and chrony, and the DNS +# messages dns-resolver verifies with. + +setup() { + load helpers + lss_test_setup + use_os debian-13 + export TMPDIR=${BATS_TEST_TMPDIR} + WRITES=${BATS_TEST_TMPDIR}/writes + : >"${WRITES}" + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + printf '== %s %s %s\n' "$1" "$2" "$3" >>"${WRITES}" + cat >>"${WRITES}" + LSS_FILE_CHANGED=${TEST_FILE_CHANGED:-1} + } +} + +# load_module NAME: sources the module as lss does for a step. +load_module() { + local file + file=$(module_file "$1") + export LSS_MODULE=$1 + # shellcheck source=/dev/null + source "${file}" + return +} + +KEY_A=xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg= +KEY_B=TrMvSoP4jYQlY6RIzBgbssQqY3vxI2Pi+y71lOWWXX0= + +# --- wireguard -------------------------------------------------------------------------------------------------- + +@test "wireguard accepts peers with a key and an address of their own in the network" { + load_module wireguard + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.2 phone:${KEY_B}:10.66.0.3/32" + mod_check +} + +@test "wireguard refuses bad names, keys, addresses and duplicates" { + load_module wireguard + config_set LSS_WIREGUARD_PEERS "Laptop:${KEY_A}:10.66.0.2" + run mod_check + [[ ${output} == *"'Laptop' is not a peer name"* ]] + config_set LSS_WIREGUARD_PEERS "laptop:abc=:10.66.0.2" + run mod_check + [[ ${output} == *"the key of 'laptop' is not a WireGuard public key."* ]] + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.67.0.2" + run mod_check + [[ ${output} == *"the address of 'laptop' must be an IPv4 address in 10.66.0.0/24."* ]] + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.1" + run mod_check + [[ ${output} == *"10.66.0.1 is the address of the server."* ]] + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.2 phone:${KEY_A}:10.66.0.3" + run mod_check + [[ ${output} == *"every peer needs its own name, key and address ('phone')."* ]] + config_set LSS_WIREGUARD_PEERS '' + config_set LSS_WIREGUARD_PORT 70000 + run mod_check + [[ ${output} == *"LSS_WIREGUARD_PORT must be between 1 and 65535."* ]] + config_set LSS_WIREGUARD_PORT 51820 + config_set LSS_WIREGUARD_ENDPOINT 'vpn.example.com;reboot' + run mod_check + [[ ${output} == *"is not a host name."* ]] +} + +@test "the WireGuard configuration holds no key: PostUp loads them from the secrets" { + load_module wireguard + config_set LSS_WIREGUARD_NETWORK 10.70.0.0/16 + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.70.3.4" + run _wg_config + [[ ${output} == *$'\nAddress = 10.70.0.1/16\nListenPort = 51820\n'* ]] + [[ ${output} == *"PostUp = wg set %i private-key ${LSS_SECRETS_DIR}/wireguard-private-key"* ]] + [[ ${output} == *"PostUp = wg set %i peer ${KEY_A} preshared-key ${LSS_SECRETS_DIR}/wireguard-psk-laptop"* ]] + [[ ${output} == *$'[Peer]\n# laptop\nPublicKey = '"${KEY_A}"$'\nAllowedIPs = 10.70.3.4/32'* ]] + [[ ${output} != *PrivateKey* && ${output} != *PresharedKey* ]] +} + +@test "the server key and one preshared key per peer are generated once" { + load_module wireguard + # shellcheck disable=SC2329 # called by the functions under test + wg() { + printf 'wg %s\n' "$*" >>"${STUB_LOG}" + printf 'GENERATED%s=\n' "${RANDOM}" + } + config_set LSS_WIREGUARD_PEERS "laptop:${KEY_A}:10.66.0.2" + _wg_keys + [ -s "${LSS_SECRETS_DIR}/wireguard-private-key" ] + [ -s "${LSS_SECRETS_DIR}/wireguard-psk-laptop" ] + [ "$(grep -c '^wg' "${STUB_LOG}")" = 2 ] + _wg_keys + [ "$(grep -c '^wg' "${STUB_LOG}")" = 2 ] +} + +# --- dns-resolver ----------------------------------------------------------------------------------------------- + +@test "dns-resolver checks access and the forward resolvers" { + load_module dns-resolver + mod_check + config_set LSS_DNS_RESOLVER_FORWARD '1.1.1.1 2606:4700:4700::1111@53' + mod_check + config_set LSS_DNS_RESOLVER_FORWARD 'dns.example.com' + run mod_check + [[ ${output} == *"'dns.example.com' is not ADDRESS, ADDRESS@PORT or ADDRESS@PORT#NAME."* ]] + config_set LSS_DNS_RESOLVER_FORWARD '1.1.1.1@853' + config_set LSS_DNS_RESOLVER_FORWARD_TLS true + run mod_check + [[ ${output} == *"needs #NAME (the name in its certificate) with forward_tls."* ]] + config_set LSS_DNS_RESOLVER_FORWARD '' + run mod_check + [[ ${output} == *"LSS_DNS_RESOLVER_FORWARD_TLS=true needs resolvers in LSS_DNS_RESOLVER_FORWARD."* ]] + config_set LSS_DNS_RESOLVER_FORWARD_TLS false + config_set LSS_DNS_RESOLVER_ACCESS public + run mod_check + [[ ${output} == *"LSS_DNS_RESOLVER_ACCESS must be local or vpn."* ]] +} + +@test "Unbound validates on 127.0.0.1, refuses everyone else and forwards over TLS when asked" { + load_module dns-resolver + # shellcheck disable=SC2329 # called by the functions under test + _dns_ipv6() { + return 1 + } + run _dns_config + [[ ${output} == *$'\tinterface: 127.0.0.1\n'* ]] + [[ ${output} != *'interface: ::1'* && ${output} != *ip-freebind* ]] + [[ ${output} == *$'\tdo-ip6: no\n'* ]] + [[ ${output} == *$'\taccess-control: 0.0.0.0/0 refuse\n'* ]] + [[ ${output} == *'auto-trust-anchor-file: "/var/lib/unbound/root.key"'* ]] + [[ ${output} == *'control-interface: /run/unbound.ctl'* ]] + [[ ${output} != *forward-zone* ]] + config_set LSS_DNS_RESOLVER_ACCESS vpn + config_set LSS_DNS_RESOLVER_FORWARD '9.9.9.9@853#dns.quad9.net' + config_set LSS_DNS_RESOLVER_FORWARD_TLS true + use_os rocky-9 + run _dns_config + [[ ${output} == *$'\tinterface: 10.66.0.1\n\tip-freebind: yes\n'* ]] + [[ ${output} == *$'\taccess-control: 10.66.0.0/24 allow\n'* ]] + [[ ${output} == *'tls-cert-bundle: "/etc/pki/tls/certs/ca-bundle.crt"'* ]] + [[ ${output} == *'control-interface: /run/unbound/unbound.ctl'* ]] + [[ ${output} == *$'forward-zone:\n\tname: "."\n\tforward-tls-upstream: yes\n\tforward-addr: 9.9.9.9@853#dns.quad9.net'* ]] +} + +@test "the resolv.conf for Unbound keeps the search domains of the original" { + load_module dns-resolver + DNS_SAVED=${BATS_TEST_TMPDIR}/resolv.conf.orig + printf 'nameserver 192.0.2.53\nsearch corp.example\noptions rotate\n' >"${DNS_SAVED}" + run _dns_resolv_conf + [[ ${output} == *$'\nnameserver 127.0.0.1\nsearch corp.example\noptions edns0 trust-ad'* ]] + [[ ${output} != *192.0.2.53* ]] +} + +@test "the server resolves through systemd-resolved, NetworkManager or resolv.conf, and not inside a container" { + load_module dns-resolver + export STUB_EXIT_systemd_detect_virt=1 + [ "$(_dns_system_mode)" = file ] + export STUB_ACTIVE_UNITS=NetworkManager + [ "$(_dns_system_mode)" = networkmanager ] + export STUB_ACTIVE_UNITS='NetworkManager systemd-resolved' + [ "$(_dns_system_mode)" = resolved ] + export STUB_EXIT_systemd_detect_virt=0 + [ "$(_dns_system_mode)" = none ] + export STUB_EXIT_systemd_detect_virt=1 + config_set LSS_DNS_RESOLVER_SYSTEM false + [ "$(_dns_system_mode)" = none ] +} + +@test "with systemd-resolved, its stub forwards every name to Unbound" { + load_module dns-resolver + DNS_RESOLVED_DROPIN=${BATS_TEST_TMPDIR}/resolved.conf + _dns_system_apply resolved + grep -qxF "== 0644 root:root ${DNS_RESOLVED_DROPIN}" "${WRITES}" + grep -qxF 'DNS=127.0.0.1' "${WRITES}" + grep -qxF 'Domains=~.' "${WRITES}" + grep -q 'systemctl restart systemd-resolved' "${STUB_LOG}" +} + +@test "DNS queries are built label by label with the AD bit, and replies are read back" { + load_module dns-resolver + [ "$(_dns_packet example.com 1)" = '\x4c\x53\x01\x20\x00\x01\x00\x00\x00\x00\x00\x00\x07example\x03com\x00\x00\x01\x00\x01' ] + [ "$(_dns_packet . 6)" = '\x4c\x53\x01\x20\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x01' ] + local reply='4c 53 81 a0 00 01 00 02 00 00 00 01' + [ "$(_dns_reply_rcode "${reply}")" = 0 ] + [ "$(_dns_reply_answers "${reply}")" = 2 ] + _dns_reply_ad "${reply}" + reply='4c 53 81 83 00 01 00 00 00 00 00 00' + [ "$(_dns_reply_rcode "${reply}")" = 3 ] + run ! _dns_reply_ad "${reply}" + run _dns_reply_rcode '12 34 81 80 00 01 00 01 00 00 00 00' + [ "${status}" -eq 1 ] + run _dns_reply_rcode '' + [ "${status}" -eq 1 ] + run _dns_reply_answers '4c 53' + [ "${status}" -eq 1 ] +} + +# --- ntp-server ------------------------------------------------------------------------------------------------- + +@test "ntp-server opens port 123 to the WireGuard network or to everyone, rate-limited" { + load_module ntp-server + mod_check + run _ntp_config + [[ ${output} == *$'port 123\nallow 10.66.0.0/24\nallow 127.0.0.1\nallow ::1\nratelimit interval 1 burst 16 leak 2'* ]] + config_set LSS_NTP_SERVER_ACCESS public + config_set LSS_NTP_SERVER_RATELIMIT false + run _ntp_config + [[ ${output} == *$'allow all\n'* && ${output} != *ratelimit* ]] + config_set LSS_NTP_SERVER_ACCESS local + run mod_check + [[ ${output} == *"LSS_NTP_SERVER_ACCESS must be vpn or public."* ]] + [ "$(_ntp_conf)" = /etc/chrony/conf.d/lss-ntp-server.conf ] + use_os rocky-9 + [ "$(_ntp_conf)" = /etc/chrony.d/lss-ntp-server.conf ] + [ "$(_ntp_unit)" = chronyd ] +} + +@test "inside a container chronyd serves the time without setting the clock" { + load_module ntp-server + export STUB_EXIT_systemd_detect_virt=0 + _ntp_container_dropin + grep -qxF '== 0644 root:root /etc/systemd/system/chrony.service.d/lss-ntp-server.conf' "${WRITES}" + grep -qxF 'ConditionCapability=' "${WRITES}" + grep -qxF "ExecStart=/usr/sbin/chronyd \$DAEMON_OPTS -x" "${WRITES}" + use_os rocky-9 + _ntp_container_dropin + grep -qxF "ExecStart=/usr/sbin/chronyd \$OPTIONS -x" "${WRITES}" + export STUB_EXIT_systemd_detect_virt=1 + _ntp_container_dropin + [ "${LSS_FILE_CHANGED}" = 0 ] +} + +@test "the time module's chrony configuration reads the confdir of ntp-server last" { + load_module time + # shellcheck disable=SC2329 # called by the functions under test + install() { + printf 'install %s\n' "$*" >>"${STUB_LOG}" + } + _time_chrony_config + grep -qxF '== 0644 root:root /etc/chrony/chrony.conf' "${WRITES}" + grep -qxF 'server time.cloudflare.com iburst nts' "${WRITES}" + grep -qxF 'port 0' "${WRITES}" + [ "$(tail -n 1 "${WRITES}")" = 'confdir /etc/chrony/conf.d' ] + grep -qxF 'install -d -m 0755 /etc/chrony/conf.d' "${STUB_LOG}" + use_os rocky-9 + _time_chrony_config + grep -qxF 'confdir /etc/chrony.d' "${WRITES}" +} From c5cf12e3656a20a1efb44e28361ad6accc5e94bd Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:08:31 +0200 Subject: [PATCH 03/32] feat(monitoring): Prometheus, Grafana, Loki, Alloy, Netdata and smartd modules Debian and Ubuntu get Prometheus, Alertmanager and the node exporter from the distribution, the RHEL family the upstream releases pinned by sha256. Grafana, Loki and Alloy come from Grafana's repository, Netdata from its own, both with pinned key fingerprints. Alloy replaces Promtail, which reached its end of life on 2026-03-02. Every interface listens on 127.0.0.1 or the WireGuard address and runs with a sandbox drop-in. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- data/checksums.txt | 10 + data/keys/grafana.asc | 41 ++ data/keys/netdata.asc | 93 +++++ data/repos/grafana.conf | 17 + data/repos/netdata.conf | 18 + modules/monitoring/alertmanager.sh | 178 +++++++++ modules/monitoring/alloy.sh | 119 ++++++ modules/monitoring/grafana.sh | 130 +++++++ modules/monitoring/loki.sh | 162 ++++++++ modules/monitoring/netdata.sh | 124 ++++++ modules/monitoring/node-exporter.sh | 103 +++++ modules/monitoring/prometheus.sh | 165 ++++++++ modules/monitoring/smartd.sh | 92 +++++ templates/prometheus/rules.yml | 58 +++ tests/unit/monitoring.bats | 567 ++++++++++++++++++++++++++++ 15 files changed, 1877 insertions(+) create mode 100644 data/keys/grafana.asc create mode 100644 data/keys/netdata.asc create mode 100644 data/repos/grafana.conf create mode 100644 data/repos/netdata.conf create mode 100644 modules/monitoring/alertmanager.sh create mode 100644 modules/monitoring/alloy.sh create mode 100644 modules/monitoring/grafana.sh create mode 100644 modules/monitoring/loki.sh create mode 100644 modules/monitoring/netdata.sh create mode 100644 modules/monitoring/node-exporter.sh create mode 100644 modules/monitoring/prometheus.sh create mode 100644 modules/monitoring/smartd.sh create mode 100644 templates/prometheus/rules.yml create mode 100644 tests/unit/monitoring.bats diff --git a/data/checksums.txt b/data/checksums.txt index b34dd0b..f8ecf5e 100644 --- a/data/checksums.txt +++ b/data/checksums.txt @@ -91,3 +91,13 @@ k3s-selinux-el10 any 7ab503adacf07c3235ba543b94b6a109ec7b192596b887c200c90ab7a8b # as the asset digests GitHub reports (2026-10-13). 0.75.0 was out for less than two weeks then. trivy-0.74.0 x86_64 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-64bit.tar.gz trivy-0.74.0 aarch64 b94ce1976bbf3c15b514b605ee88be7c6d94a29be2302847ff01cb794d47aad5 https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-ARM64.tar.gz + +# Prometheus 3.13.4 (the 3.13 long-term support release; 3.13.4 is younger than two weeks but fixes two security +# issues), Alertmanager 0.34.1 and node_exporter 1.12.1 for the RHEL family, which has no packages for them: SHA-256 from +# sha256sums.txt of each GitHub release, the same as the asset digests GitHub reports (2026-10-14). +prometheus-3.13.4 x86_64 87f21a66f96c597a189cef8d640e8921b621fc17a9feff707c332c4f3b3ddd56 https://github.com/prometheus/prometheus/releases/download/v3.13.4/prometheus-3.13.4.linux-amd64.tar.gz +prometheus-3.13.4 aarch64 ffa86c7d6e7f7e7dc8a79c81c5c2a6c9164464b79335f25613f4327c863b8489 https://github.com/prometheus/prometheus/releases/download/v3.13.4/prometheus-3.13.4.linux-arm64.tar.gz +alertmanager-0.34.1 x86_64 265b9d1e55ef0d5306a436018af6d2b686c2ce051f03d968f7464ecb1372a7e8 https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-amd64.tar.gz +alertmanager-0.34.1 aarch64 d98d6cbaf52151c7e76e24355fec88b11cebcb9875d4cdd8b76ddce7a7e5535c https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-arm64.tar.gz +node_exporter-1.12.1 x86_64 b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz +node_exporter-1.12.1 aarch64 ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-arm64.tar.gz diff --git a/data/keys/grafana.asc b/data/keys/grafana.asc new file mode 100644 index 0000000..2c6e93d --- /dev/null +++ b/data/keys/grafana.asc @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGTnhmkBDADUE+SzjRRyitIm1siGxiHlIlnn6KO4C4GfEuV+PNzqxvwYO+1r +mcKlGDU0ugo8ohXruAOC77Kwc4keVGNU89BeHvrYbIftz/yxEneuPsCbGnbDMIyC +k44UOetRtV9/59Gj5YjNqnsZCr+e5D/JfrHUJTTwKLv88A9eHKxskrlZr7Un7j3i +Ef3NChlOh2Zk9Wfk8IhAqMMTferU4iTIhQk+5fanShtXIuzBaxU3lkzFSG7VuAH4 +CBLPWitKRMn5oqXUE0FZbRYL/6Qz0Gt6YCJsZbaQ3Am7FCwWCp9+ZHbR9yU+bkK0 +Dts4PNx4Wr9CktHIvbypT4Lk2oJEPWjcCJQHqpPQZXbnclXRlK5Ea0NVpaQdGK+v +JS4HGxFFjSkvTKAZYgwOk93qlpFeDML3TuSgWxuw4NIDitvewudnaWzfl9tDIoVS +Bb16nwJ8bMDzovC/RBE14rRKYtMLmBsRzGYHWd0NnX+FitAS9uURHuFxghv9GFPh +eTaXvc4glM94HBUAEQEAAbQmR3JhZmFuYSBMYWJzIDxlbmdpbmVlcmluZ0BncmFm +YW5hLmNvbT6JAdQEEwEKAD4CGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQS1 +Oud7rbYwpoMEYAWWP6J3EEWFRQUCaKhvPQUJB4NP1AAKCRCWP6J3EEWFRUjOC/9Y +dWOWJLJVKzLx8uv5YVzebyw15HevhKahbznJX5fHnE8irjkiPFltVEZ4T37s5afR +GBEJnR1UFd80s7jzwbuoZh/zEB3jN8q50g64AznuzDa0PWKzaY7Tgkssx3+hs6TS +vIwV4z8T7f56lDudeHxHXx+htRnZ3ebKNPCJS7+G12GF6W3C3znpdjgvhVUB0uxd ++42V0fRqk2GLNZeKS9988fi5dYRAy9Ozwced7ByCFjde9FBgUtrH3mG1/ibzLEh0 +4k02nYjc8mrH32t4UCWpxQEJ1vZA2vT2HN3/cH/4uyFdyU6OHkMyMbz6lmeXe71d +F5hOB4+/RP6Ndyj7ViRNDbm70NRBaFne/+YOJvmMfJTCh7YbF5qEn1ihGkJJ0ohE +u2IB+EGEhyiDm8SIsj1uMw7n17iIPNtbsU5GgnmLtfguP/WbwKV2UeuxTpiOeYb6 +blDwRlh48uHMlA5HBW+487Jktw3iPj1IKhdtAC9CU3xAvzDcseMbgmM6Xj2bSQG5 +AY0EZOeGaQEMALNIFUricEIwtZiX7vSDjwxobbqPKqzdek8x3ud0CyYlrbGHy0k+ +FDEXstjJQQ1s9rjJSu3sv5wyg9GDAUH3nzO976n/ZZvKPti3p2XU2UFx5gYkaaFV +D56yYxqGY0YU5ft6BG+RUz3iEPg3UBUzt0sCIYnG9+CsDqGOnRYIIa46fu2/H9Vu +8JvvSq9xbsK9CfoQDkIcoQOixPuI4P7eHtswCeYR/1LUTWEnYQWsBCf57cEpzR6t +7mlQnzQo9z4i/kp4S0ybDB77wnn+isMADOS+/VpXO+M7Zj5tpfJ6PkKch3SGXdUy +3zht8luFOYpJr2lVzp7n3NwB4zW08RptTzTgFAaW/NH2JjYI+rDvQm4jNs08Dtsp +nm4OQvBA9Df/6qwMEOZ9i10ixqk+55UpQFJ3nf4uKlSUM7bKXXVcD/odq804Y/K4 +y3csE059YVIyaPexEvYSYlHE2odJWRg2Q1VehmrOSC8Qps3xpU7dTHXD74ZpaYbr +haViRS5v/lCsiwARAQABiQG8BBgBCgAmAhsMFiEEtTrne622MKaDBGAFlj+idxBF +hUUFAmiobzkFCQeDT9AACgkQlj+idxBFhUVsmQwA0PA/zd7NqtnZ/Z8857gp2Wq2 +/e4EX8nRjsW2ZlrZfbU5oMQv9OZZ4z1UjIKEUV+TnCwXEKXTMJomdekQSSayVVx/ +u5w+0YM8gRuQGrG8hW0GRR8sHIeuwBFlyQrlwxUwXvDOPDYyieETjaQqMucupIKo +IPm3CjFySvfizvSWUVSWBnGmQfpv6OiGYawvwfewcQHUdLMgWN3lYlzGQJL4+OMm +7XcB8VNTa586Q00fmjDfktHYvGpmhqr3gsd4gS3AjTk0zI65qXBRJkdqVnwUrMUD +8TcxXYNXf90mhR0NWkLmp6kBYiW8+QY6ndMmRVpodg1A87qgMYaZUAAlxCS4XKTU +r+/YMDYOWgLN6i4UeYG/3/hsnAEHm5ITojfh6cLfdlhjohFTnD0IYw3AsNJXRzKB +1g5FTBKLLLIdXgS/3rWV1qjAd3drQVIMCku6HKl/vT4ftrBHeSyV7eLwOYbe3/bw +8VMx+lmMheD8/qJMia1om0iBBRSXRjY//f+Lllqm +=TH3J +-----END PGP PUBLIC KEY BLOCK----- diff --git a/data/keys/netdata.asc b/data/keys/netdata.asc new file mode 100644 index 0000000..53e5093 --- /dev/null +++ b/data/keys/netdata.asc @@ -0,0 +1,93 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGfW4AkBEACc94RKVQuMuw/T52gD5g527H5A97wIUMIARDKswAAHhLWlGtgl +0JrmS6mzBjWGW80aAe1sfsxJE6fntqwri8u4U/kOY3X7e6yk+Gverzkeu1pcgKL8 +eD9o9U4l3l6/CtPjYH/1HNyTED8VN9iJzzn0Snkq0GDJHd4pyevlE5r3lHdYY3PX +1aaIQAZSWby1eBONKKJzNj/pvSza5OWuGhWy3kHNaWypuwqhMczQhMNhLNUjQBUi +ci7vRVAiFp/DvJcMmNW/ZBbZZaVsgW8SxfMFBzKiGfINYzn3dEFL3H6w7KDs3EWu ++RaAXyBjgNqGZrDJMNWE4NP4rqgIwCgTuphJ95R9fEz+Gvv6uh5BjoAOUNJ0hE/K +t+XBit/u0ravcTuHzHt/1bN27wzWgBVixoNGu7jmVE+t5sU07IUQoj+q0i6byBmN +xgZUftGFwcHRraVViAX4BXNuBs+7b1liS4opuhAxDWhW3dHZqQBZCHDWm7ewIRVd +piEzowILaOCMmZiiJINKrSOEvs073sMcPMvY+RDxE+9kHcj92KO1bPDZHrnMKbLf +4IB8ZPMDtfhp/aX00dbAR3BCq5fcjqRMGgfLREer3FekMbusjbwAnlS8uA748HFu +uPWqbqYUo8cWJkmV2+Dd0eUtKRnmxFJ65r6lHvR0NDHi13m7v3RugL0y3wARAQAB +tD9OZXRkYXRhYm90IChOZXRkYXRhIFJlcG9zaXRvcnkgU2lnbmluZyBLZXkpIDxi +b3RAbmV0ZGF0YS5jbG91ZD6JAlQEEwEKAD4WIQRuFV3BU5Brc3ZadKmd1KdM7PqP +TwUCZ9bgCQIbAwUJEswDAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCd1KdM +7PqPT3ArEACBMqt3lnRJNnzwOaQomzVkooj4+tBt0/5noyQMGOfJlEQU8scMquqp +kMRuq+5q15ZWQh5K9vmdLfukw4fzwgdjYmQ9e924q7K1/8EvEYmPe4MAd+D4rMoC +0eTioSCUZx5H/8bkKrpx39w5gvQzonhxNjfmY1dYrcVgCFXRXweO6aq06Gw5YWS+ +EAuSTC0G3UN2mHdpJ37AyvkY5Blv36l3S7uSfGO68bQsoKiK493ysLMdFuWD2lu1 +Y18LB8Y0mxsdynVbaknhtXtS6iPtKr5yeIGd3KpdcW3ftLgJEnkj4o/Bhl6doeXx +dOyRsyb7H9YVNqstOZHlbjr+mOmFR/2NU+cO57wgfaGtfu6FyI2E/Fg7YBShQ37j +BBaZ3jKF3i/PpApO2mhoHiNs9VSKBmLuptL0w0Xdk4OhnSlAalTWHYX02eBYHMSH +a9YSupGSgiBYLA8S/1IJ1nJ8NGDeLKQpvphjjlP0/lFCEcEquqbW8WHGwEpUECQ6 +TLPOseUR4Xy+ji4wXVRB6S18m+j6NwAt0fp8sVOex3j4BGbvTAnC+9wefjQR+j3H +ZzDVCf/NehLHarIVAWN01lnXSgEUG9SgTQ6rqFGaxvX4k/zV4juhN0jhodL362DK +y/TGwSySlKhyNl1uP3D3xdTs+yd1fqEe+MpPbM6dfscqXnRTBZVOmbkCDQRn1uAJ +ARAAvpTd4NkjSUUHUlMEhOiZpAfEgBo/H5jS7Pj12weG5dFQ6Or/A1aNH0/NXUSs +lU/9IJwSegLTwze3/mXY7t0ix1UcPW5OLltmiOlj7arnFBe/wkxZzR+it18/EKkG +KbXSw410a79j8Xbz9hLfvIAb6CVebr93yljyFAYZENTM6DAa1UkdjeGno3afxeay +HLAk42mKYbHyHe77hhx0YaMhjvmqIyDITH/A+21UWaLbdCw1EzChID5MQWnifMqM +M/zIOciHRYSuey3OZ5h3UATZftkUCbK/p2hN94oHO48jRQZ+ATClG1owQTj7faU4 +cjC0RH4cPZ0EjoLnFq7dE41c+ANu+ky1i+OLaL0uwog1UdDsQjTnWS/ec2EBcQuv +WRllxRNK88x9TMr2ntfiMZumULumtlv8dgsSyu7S/94im4cNg0WkeKvKDv8tuMMN +/xUwTQ0N8NVzrGfHAljDwdgd0gEyiiz5f9LF/pwdl3Kt9HS/gOGSBkEJsTpSCNdU +lcE9k+wCYqFXuZXn+Ll/Ju4FkvXkZPGpN9hWtCVf4tDOu5FJPpo8+S8RbvxGbS7k +nfeyeGXjyuSiqvvm/2nAG0tfFeAZLt9WeyHaZRFiZmHkZc69cHudqcDpHcjUe82k +G22TuBie89mFOD13m6ItQ930xosyFmaAVEiHwobbZYPzBc0AEQEAAYkCPAQYAQoA +JhYhBG4VXcFTkGtzdlp0qZ3Up0zs+o9PBQJn1uAJAhsMBQkSzAMAAAoJEJ3Up0zs ++o9P8RYP/R81xloQl8YJ19d2DnaHCej10SXUBTeCY7u960B61UjAzCPu1SSY5/rr +EkWy/uqstpztDeXrzeSGFKyiZOahwoCjkf/d3nV9eagFugC2Za6Mdes1ekToU134 +28rwDLybUZSIFXlM210MDUlNVPxwCXxuPPt+I1KfHkVVH8K6XOnTngqYyaO89VLK +iZM4dI1yV34CZ4DYfzer2o2LQcTYilXrEdmuPoj9RSj75TOUtJUS21bQpvqtwL0o +ImTqiV2ZhhoBE60h8Cz2r0ALO1oTuo7hs2LaZpd9icwoP/ZlYobKmFRZ03fnnz/q +drYlGJt6uPNdfjGOrhdh0SSjoShInAIULbyxYmEuJEVx8CbTjd2NLH57TmlVN9P0 +u1PfeTWNnRxVeyxk8xov/JU/zyzDsKdOp/8GokAADfaBpbVPbT0/X30IetpO6O5q +ExGSvwhmUXmYYxbawigtHL6SFAPAbIJLhEguGrXm6mOmQh5LiRV3qoCUmoghUzk2 +yTidLDuQOCRN8bQ1me6Pt/VYT9TUbdUwEvs2zyChmVOfOzQloTGvybJohCp+uHYU +bshAmsA26Iw3y2ZRSVWF4YJyjLMyi/VV7qVYMTLdusgGJsOdlfGOUbZWGInX6o6S +9RApebBUDUvjM3Cz+iSof9/OBP7w/oEF1LmLUtoDKZZZPotHK7mk +=1wXu +-----END PGP PUBLIC KEY BLOCK----- +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGK+ooUBDACpmop8b+UQurHIEVOZ6YXyzcjC08LoPh4+PBxjHE+TuxFQkFXS +PRGp4Vi97NEMcEJbM+Hg/lqp1VNKBD2djHyQhmL6SHpM2ecCEjj2XUXlB9LDX5MR +ml+K4mhxKplmJ1a6Mr/sgdrG9dEMjnSEWM4Li0EWlv/9FXgujZDDnw5fG6ZHn6sj +NZgQRaskpbcoNK6btP3uFUF301w5ypoS9LtWVH49ixN3lHcmvJdtazpIcgmus65G +vGREYrqkVwFGGdbrcE5qYulfNIH1QK2a1PtCbJuqXA5pgz7FypmP+h2BOLfJf4G2 +04CNyrXCbyBzzK09s2QzJNL5kjsOzb31amGfUFfgjt+GYLMdZabnZfBBsY7Ujpkl +msymJXPR5TjHiSCKNBoN5ENJ3QjYlb61NcQ86v7eSoDOG+0io6zoTp+9cLgOq20Y +PpPNVHDm7tG95S3RChYUxAHmcgp/t7r4MbFgXdH68WN1PjDBAxUbskDtUVMBlL5d +DzHJZ6Psj8BvbTkAEQEAAbQeTmV0ZGF0YWJvdCA8Ym90QG5ldGRhdGEuY2xvdWQ+ +iQHOBBMBCgA4AhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAFiEEZYj917FHIf58 +MRXm+Rd7UmX1Y0YFAmK+tOUACgkQ+Rd7UmX1Y0Y2iwwAgAQE0FxqUVelGLfgAVPX +vG9yr13MWoQ6s13LX1BaxCpGajgcLBrrTjaJo3VDxrI2I8KN3o+8QQhnbtzHMLcG +1Hv3RAcBBVDDs7iPnj5MaucR8ZXPbeJVglO6Du3cLwXV19rQxJC1DUTL2IDezcWy +eaZOgK/PybHcbUZ5pfZcIP1vbpATGyHKD4sIVofkIS0/aaeNwyb+OnORF6C9XkoO +mMzbx5dX6hB2kOKiZeF2ayQq5Lk5grT0wS1L/TeG/SonFORUzjzbHe6mUHfSTt2m +dIBTdRyglv8paCW5TAbFMPxnUcDeCFlROZIE+TTFueizJOJ8czcGujbXiU30JgoF +lNNM2nJvts/C/mMj+A0YxddGvgy/K1vdBAFFod3FVH090Xi1YFw7ZDWGFXnW8gqa +HpVFg4ibtbH5HR22MoQYWWxaWl0sus8TlYIMlRvMSbVQhvTRBJ/JAaet6+9BEXwm +1WKQ8fkQP7GpXOssNIxf4co0r+rboIWIIGshcQ6sevzfuQGNBGK+ooUBDADIlkQ7 +PNVbQHP5rDAAYPFVUiJFlXbR6SeF2JqDk0QkG2+UlAGFulb/MUKwNW8SwTPxdaeP +ysEZ4GskK1OB/uN/5Bbyh0iI4Hwmbl91aWz/EZD8IE+zYmwVgHypQEBVjibeZWhy +dNMa7ZLYW7HVIG/OZOZqCPWSqLRKApHUIuCX82BVnT65+MpCicMfv9kXhzImQPyD +3Sxx1CC9t2FmRTtI+22tYcrayQLFj0GBEkYVD43X6eMdSWkHPv5kHiX0S8VKrZkk +dQbsm0Z3KPwgs8cXEDi0vnfHIVXbj5u3dR2m0L8eIXuBLSZEvsbLoG/beI6poELP +rhFLdk06XVGNvf+TUqJe5t//bV4WvOvy7+l7odUWNk+X0s7mYQdJDYYO+ju1iJ84 +PmXJaJ7N+Cp5aYLsEjpKOrLWwDaN2ZlisZm68Q5mdtLcRxWm3BPe1TK1ojFenMrX +g/qK4bRBHJDBVvZYh5mZHoDYmUcrqQBl4NQbwmXCEWBHtsi1+73UJRniuWMAEQEA +AYkBtgQYAQoAIAIbDBYhBGWI/dexRyH+fDEV5vkXe1Jl9WNGBQJivrT4AAoJEPkX +e1Jl9WNGWsUL/AksktdmVxrBRjuhjysfynwwt78PlZSdPPQ9K5Gn68x9xovht7tq +5/f7S7nUNUA6xJoR+VJB+e8R4NnopvI+4O+yK45V2JAry9+xOXWHuLDNjf9TdprP +ICNawkFzAAGAlbUXce+QU70ei2azuqpT5pes47yAJ2xp8VsVK6MMHcFfVLw9mQ9Y +Y8Zz9Q4UoIrN80GWAR0+8mObwMgSzm5XjfJKbjXyRoai6JXAQe5WCrVgW7g0EPKY +4JU3mPcNOxhTALC8xYGJGQ4MnVdGQEkbVk6byvUBRc6QaDRUz642OlK/6MrMaSEu +cYTaIOiGJ9nomyMmYR8/RVx49ZUeAVeyLpZQj3R10pNp7ntqPN/tzVOKanNOQpGV +YsCNLExxwo39uCsd2hFBw3nDC6QhOy6T6RLrjTGgsoJkiVm9LcKVwzrgTU3U20/7 +92tacDWvfpg6k/alEO+sXKEcxbJNP0LTUixHydNcCZx2srh3CclfiQUumnkokdTI +KvWTMBz97LPIMA== +=OUNZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/data/repos/grafana.conf b/data/repos/grafana.conf new file mode 100644 index 0000000..3541f40 --- /dev/null +++ b/data/repos/grafana.conf @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Grafana, Loki and Alloy from Grafana Labs' repositories (https://apt.grafana.com, https://rpm.grafana.com): one +# suite for every Debian and Ubuntu release, one repository for the whole RHEL family, x86_64 and arm64. Key fetched on +# 2026-10-14 from https://apt.grafana.com/gpg.key; https://rpm.grafana.com/gpg.key and https://packages.grafana.com/gpg.key +# carry the same key, and it signs the apt InRelease file and the rpm metadata. It expires on 2027-08-22; Grafana +# extends it before then. +KEY=grafana.asc +FINGERPRINTS=B53AE77BADB630A683046005963FA27710458545 +DEB_URL=https://apt.grafana.com +DEB_SUITE=stable +DEB_COMPONENTS=main +RPM_BASEURL=https://rpm.grafana.com +RPM_GPGCHECK=1 +RPM_REPO_GPGCHECK=1 diff --git a/data/repos/netdata.conf b/data/repos/netdata.conf new file mode 100644 index 0000000..132ada1 --- /dev/null +++ b/data/repos/netdata.conf @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Netdata from its own stable repository (https://repository.netdata.cloud/repos/stable/): Debian 13, Ubuntu 26.04 and +# EPEL 10 have no Netdata package, so every system uses Netdata's, which gives all of them the same version. Flat apt +# repositories per release (the suite ends in a slash, no components), one rpm repository per RHEL major release, for +# x86_64 and arm64; checked on 2026-10-14 for all seven releases. Keys fetched the same day: the current signing key +# (https://repository.netdata.cloud/netdatabot.gpg.key, the fingerprint Netdata's install guide names) and the previous +# one (netdatabot.old.gpg.key), which still signs the apt InRelease files next to it. +KEY=netdata.asc +FINGERPRINTS=6E155DC153906B73765A74A99DD4A74CECFA8F4F 6588FDD7B14721FE7C3115E6F9177B5265F56346 +DEB_URL=https://repository.netdata.cloud/repos/stable/{{id}}/ +DEB_SUITE={{codename}}/ +DEB_COMPONENTS= +RPM_BASEURL=https://repository.netdata.cloud/repos/stable/el/{{major}}/$basearch/ +RPM_GPGCHECK=1 +RPM_REPO_GPGCHECK=1 diff --git a/modules/monitoring/alertmanager.sh b/modules/monitoring/alertmanager.sh new file mode 100644 index 0000000..631b911 --- /dev/null +++ b/modules/monitoring/alertmanager.sh @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="alertmanager" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Prometheus Alertmanager: sends the alerts of Prometheus by e-mail or webhook; UI and API below /alertmanager/ on 127.0.0.1:9093 or the WireGuard network" +MOD_REQUIRES="system" +MOD_OPTIONS="access|local|Who reaches http://SERVER:9093/alertmanager/: local (127.0.0.1 only; SSH tunnel) or vpn (WireGuard network only) +web_admin|false|Also publish Alertmanager below /alertmanager/ on the admin port of the web server (needs nginx, apache or caddy) +email||Send alerts to this address (needs the mail-relay module or another mail server on smarthost) +smarthost|localhost:25|Mail server Alertmanager hands e-mails to, HOST:PORT without authentication +webhook||Send alerts as JSON to this HTTPS URL (Alertmanager's webhook format; ntfy, n8n, a chat bridge, ...)" +MOD_PORTS="9093/tcp from the WireGuard network with access vpn; none with access local" + +# Debian and Ubuntu package Alertmanager (0.23 to 0.28); the RHEL family gets the checksum-verified upstream release +# with a unit of the same name. The configuration below works with all of these versions. +AM_VERSION=0.34.1 +AM_UNIT=prometheus-alertmanager +AM_CONF=/etc/lss/alertmanager/alertmanager.yml +AM_PATH=alertmanager + +_am_user() { + if os_is_debian; then printf 'prometheus'; else printf 'alertmanager'; fi + return +} + +_am_data() { + if os_is_debian; then printf '/var/lib/prometheus/alertmanager'; else printf '/var/lib/alertmanager'; fi + return +} + +_am_binary() { + if os_is_debian; then printf '/usr/bin/prometheus-alertmanager'; else printf '/usr/local/bin/alertmanager'; fi + return +} + +_am_url() { + printf 'http://%s:%s/%s' "$(mon_address alertmanager)" "${MON_ALERTMANAGER_PORT}" "${AM_PATH}" + return +} + +mod_check() { + local email webhook + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_ALERTMANAGER_ACCESS must be local or vpn." + opt_bool web_admin || true + email=$(opt email) + webhook=$(opt webhook) + [[ -z ${email} || ${email} =~ ^[^@[:space:]\'\"]+@[^@[:space:]\'\"]+\.[^@[:space:]\'\"]+$ ]] || + die "LSS_ALERTMANAGER_EMAIL '${email}' is not an e-mail address." + [[ $(opt smarthost) =~ ^[A-Za-z0-9.:-]+:[0-9]{1,5}$ ]] || die "LSS_ALERTMANAGER_SMARTHOST must look like HOST:PORT." + [[ -z ${webhook} || ${webhook} =~ ^https://[^[:space:]\'\"]+$ ]] || die "LSS_ALERTMANAGER_WEBHOOK must be an https:// URL." + return +} + +mod_install() { + if os_is_debian; then + mon_install_stopped prometheus-alertmanager + else + mon_user alertmanager "$(_am_data)" + mon_upstream_install alertmanager "${AM_VERSION}" alertmanager amtool + fi + return +} + +# _am_config: one receiver for every alert. Without e-mail and webhook the alerts are only shown (Alertmanager UI, +# Grafana). +_am_config() { + local email webhook host + email=$(opt email) + webhook=$(opt webhook) + host=$(hostname -f 2>/dev/null || hostname) + printf '%s alertmanager). Change LSS_ALERTMANAGER_* in /etc/lss/lss.conf and run lss configure alertmanager.\n' "${MON_MANAGED_BY}" + cat <<'EOF' +route: + receiver: lss + group_by: [alertname, instance] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h +receivers: + - name: lss +EOF + if [[ -n ${email} ]]; then + printf " email_configs:\n - to: '%s'\n from: 'alertmanager@%s'\n smarthost: '%s'\n require_tls: false\n send_resolved: true\n" \ + "${email}" "${host}" "$(opt smarthost)" + fi + if [[ -n ${webhook} ]]; then + printf " webhook_configs:\n - url: '%s'\n send_resolved: true\n" "${webhook}" + fi + return +} + +_am_command() { + # An empty cluster address turns the gossip port 9094 off: a single Alertmanager has no peers. + printf '%s --config.file=%s --storage.path=%s --web.listen-address=%s:%s --web.external-url=%s/ --cluster.listen-address=' \ + "$(_am_binary)" "${AM_CONF}" "$(_am_data)" "$(mon_address alertmanager)" "${MON_ALERTMANAGER_PORT}" "$(_am_url)" + return +} + +_am_amtool() { + if os_is_debian; then printf '/usr/bin/amtool'; else printf '/usr/local/bin/amtool'; fi + return +} + +mod_configure() { + local before changed config reload=0 + before=$(files_changes_of alertmanager) + mon_vpn_ready + mon_dir "$(_am_data)" "$(_am_user):$(_am_user)" 0750 + install -d -m 0755 "$(dirname "${AM_CONF}")" + config=$(_am_config) + # The webhook URL is a credential (whoever has it can post), so only Alertmanager reads the file. + file_write 0640 "root:$(_am_user)" "${AM_CONF}" <<<"${config}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || reload=1 + "$(_am_amtool)" check-config "${AM_CONF}" >/dev/null || die "alertmanager: amtool rejects ${AM_CONF}." + install -d -m 0755 "${MON_ALERTMANAGERS_DIR}" + file_write 0644 root:root "${MON_ALERTMANAGERS_DIR}/alertmanager.yml" </dev/null || true + svc_disable "${AM_UNIT}" + svc_dropin_remove "${AM_UNIT}.service" lss-sandbox + svc_dropin_remove "${AM_UNIT}.service" lss-vpn + file_remove "${AM_CONF}" + rmdir "$(dirname "${AM_CONF}")" 2>/dev/null || true + if os_is_debian; then + svc_dropin_remove "${AM_UNIT}.service" lss-config + pkg_remove prometheus-alertmanager + else + file_remove "/etc/systemd/system/${AM_UNIT}.service" + svc_daemon_reload + mon_upstream_remove alertmanager amtool + fi + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete "$(_am_data)" + return +} diff --git a/modules/monitoring/alloy.sh b/modules/monitoring/alloy.sh new file mode 100644 index 0000000..f648018 --- /dev/null +++ b/modules/monitoring/alloy.sh @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="alloy" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Grafana Alloy ships the systemd journal of the server to Loki (Promtail's successor); its UI on 127.0.0.1:12345" +MOD_REQUIRES="system loki" +MOD_OPTIONS="max_age|12h|How far back Alloy reads the journal when it starts without a saved position (Go duration: 12h, 30m)" +MOD_PORTS="none (UI on 127.0.0.1:12345)" + +# Promtail reached its end of life on 2 March 2026 (https://grafana.com/docs/loki/latest/send-data/promtail/, checked +# 2026-10-14); Grafana Alloy replaces it and comes from the same repository (data/repos/grafana.conf) for every +# supported system. The package starts nothing on installation. +ALLOY_UNIT=alloy +ALLOY_CONF=/etc/lss/alloy/config.alloy +ALLOY_DATA=/var/lib/alloy/data +ALLOY_UI=127.0.0.1:12345 + +mod_check() { + [[ $(opt max_age) =~ ^[1-9][0-9]*(h|m)$ ]] || die "LSS_ALLOY_MAX_AGE must look like 12h or 30m." + return +} + +mod_install() { + repo_add grafana + pkg_install alloy + return +} + +# _alloy_config: the journal with the unit, the priority and the host as labels, pushed to Loki. +_alloy_config() { + cat </dev/null || true + pkg_remove alloy + mon_grafana_repo_remove + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete /var/lib/alloy + return +} diff --git a/modules/monitoring/grafana.sh b/modules/monitoring/grafana.sh new file mode 100644 index 0000000..ee8634c --- /dev/null +++ b/modules/monitoring/grafana.sh @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="grafana" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Grafana OSS from Grafana's repository with the data sources of the monitoring modules; below /grafana/ on 127.0.0.1:3000 or the WireGuard network" +MOD_REQUIRES="system prometheus" +MOD_OPTIONS="access|local|Who reaches http://SERVER:3000/grafana/: local (127.0.0.1 only; SSH tunnel) or vpn (WireGuard network only) +web_admin|false|Also publish Grafana below /grafana/ on the admin port of the web server (needs nginx, apache or caddy)" +MOD_PORTS="3000/tcp from the WireGuard network with access vpn; none with access local" +MOD_SECRETS="grafana-admin-password" + +# Grafana's own repository (data/repos/grafana.conf, pinned key) serves the same release to every supported system. +# The settings go in as GF_* variables of a drop-in, which win over grafana.ini; the package keeps its own file. +GRAFANA_PORT=3000 +GRAFANA_PATH=grafana +# The admin password for Grafana's first start, readable by Grafana only (the secrets directory is root's). +GRAFANA_PASSWORD_FILE=${MON_GRAFANA_DIR}/admin-password + +_grafana_url() { + printf 'http://%s:%s/%s' "$(mon_address grafana)" "${GRAFANA_PORT}" "${GRAFANA_PATH}" + return +} + +mod_check() { + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_GRAFANA_ACCESS must be local or vpn." + opt_bool web_admin || true + return +} + +mod_install() { + repo_add grafana + pkg_install grafana + return +} + +_grafana_environment() { + cat <= 1)) || die "LSS_LOKI_RETENTION_DAYS must be at least 1." + return +} + +# _loki_config: one process with every component, an in-memory ring (no gossip port), the TSDB index and the chunks on +# the local disk; the compactor deletes what is older than the retention. +_loki_config() { + cat </dev/null 2>&1 || + die "loki: Loki rejects ${LOKI_CONF} (loki -config.file=${LOKI_CONF} -verify-config shows why)." + svc_enable "${LOKI_UNIT}" + if [[ $(files_changes_of loki) != "${before}" ]] || ! svc_is_active "${LOKI_UNIT}"; then + svc_restart "${LOKI_UNIT}" || die "loki: Loki does not start." + fi + mon_firewall "${LOKI_PORT}/tcp" + mon_target loki loki "$(mon_address loki):${LOKI_PORT}" + mon_datasource Loki loki "$(_loki_url)" + return +} + +# _loki_probe: pushes one line and waits until a query returns it. +_loki_probe() { + local url token now i result='' + url=$(_loki_url) + token="lss-verify-$(date +%s)-${RANDOM}" + now=$(date +%s%N) + curl -s --max-time 10 -o /dev/null -H 'Content-Type: application/json' --data-binary @- "${url}/loki/api/v1/push" </dev/null || true + pkg_remove loki + mon_grafana_repo_remove + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete "${LOKI_DATA}" + return +} diff --git a/modules/monitoring/netdata.sh b/modules/monitoring/netdata.sh new file mode 100644 index 0000000..c575676 --- /dev/null +++ b/modules/monitoring/netdata.sh @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="netdata" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="Netdata real-time dashboard without cloud, registry or telemetry; on 127.0.0.1:19999 or the WireGuard network, optionally below /netdata/ on the admin port" +MOD_REQUIRES="system" +MOD_OPTIONS="access|local|Who reaches http://SERVER:19999: local (127.0.0.1 only; SSH tunnel) or vpn (also the WireGuard network) +web_admin|false|Also publish Netdata below /netdata/ on the admin port of the web server (needs nginx, apache or caddy) +ml|false|Machine-learning anomaly detection (costs CPU and memory on small servers)" +MOD_PORTS="19999/tcp from the WireGuard network with access vpn; none with access local" + +# Netdata's own repository (data/repos/netdata.conf, pinned keys) for every supported system: Debian 13, Ubuntu 26.04 +# and EPEL 10 have no Netdata package. Its packages need EPEL on the RHEL family. +NETDATA_UNIT=netdata +NETDATA_PORT=19999 +NETDATA_CONF=/etc/netdata/netdata.conf +NETDATA_CLOUD_CONF=/var/lib/netdata/cloud.d/cloud.conf +NETDATA_OPT_OUT=/etc/netdata/.opt-out-from-anonymous-statistics + +mod_check() { + [[ $(opt access) =~ ^(local|vpn)$ ]] || die "LSS_NETDATA_ACCESS must be local or vpn." + opt_bool web_admin || true + opt_bool ml || true + return +} + +_netdata_bind() { + local bind=127.0.0.1 + mon_vpn netdata && bind+=" $(fw_vpn_address)" + printf '%s' "${bind}" + return +} + +# _netdata_files: the settings that keep Netdata on this server, written before the package as well, so its first +# start already uses them. +_netdata_files() { + local ml=no + opt_bool ml && ml=yes + install -d -m 0755 /etc/netdata + file_write 0644 root:root "${NETDATA_CONF}" </dev/null || die "prometheus: promtool rejects ${PROM_CONF}." + mon_target prometheus prometheus "$(mon_address prometheus):${MON_PROMETHEUS_PORT}" "/${PROM_PATH}/metrics" + changed=$(files_changes_of prometheus) + if os_is_debian; then + mon_exec "${PROM_UNIT}" "$(_prom_command)" + else + mon_unit "${PROM_UNIT}" prometheus "$(_prom_command)" "Prometheus monitoring system" + fi + svc_sandbox "${PROM_UNIT}.service" "ReadWritePaths=$(_prom_data)" + mon_vpn_dropin "${PROM_UNIT}.service" + svc_enable "${PROM_UNIT}" + if [[ $(files_changes_of prometheus) != "${changed}" ]]; then + svc_restart "${PROM_UNIT}" || die "prometheus: Prometheus does not start." + elif [[ $(files_changes_of prometheus) != "${before}" && ${reload} == 1 ]]; then + # A new configuration or new rules only need a reload, which keeps the data in memory. + svc_reload "${PROM_UNIT}" || die "prometheus: Prometheus does not reload its configuration." + fi + mon_firewall "${MON_PROMETHEUS_PORT}/tcp" + mon_web_admin proxy "http://$(mon_address prometheus):${MON_PROMETHEUS_PORT}" + mon_datasource Prometheus prometheus "$(_prom_url)" true + return +} + +mod_verify() { + local url rules + url=$(_prom_url) + svc_is_active "${PROM_UNIT}" || die "prometheus: ${PROM_UNIT} is not running." + mon_wait_http "${url}/-/ready" 120 + rules=$(curl -s --max-time 10 "${url}/api/v1/rules") + [[ ${rules} == *'"name":"LssBackupTooOld"'* ]] || die "prometheus: the lss alert rules are not loaded." + # Prometheus scrapes itself and the node exporter through the target files. + _prom_wait_target node + mon_listen_check --process prometheus "${MON_PROMETHEUS_PORT}" "$(mon_address prometheus)" + return +} + +# _prom_wait_target JOB: waits until Prometheus has scraped the target of JOB successfully. +_prom_wait_target() { + local job=$1 i result='' + for ((i = 0; i < 45; i++)); do + result=$(curl -s --max-time 10 -G "$(_prom_url)/api/v1/query" --data-urlencode "query=up{job=\"${job}\"}") + [[ ${result} != *'"value":['*',"1"]'* ]] || return 0 + sleep 2 + done + die "prometheus: the target ${job} is not up (${result:-no answer})." +} + +mod_remove() { + mon_web_admin_remove + mon_datasource_remove + mon_target_remove prometheus + svc_disable "${PROM_UNIT}" + svc_dropin_remove "${PROM_UNIT}.service" lss-sandbox + svc_dropin_remove "${PROM_UNIT}.service" lss-vpn + file_remove "${PROM_RULES_DIR}/lss.yml" + file_remove "${PROM_CONF}" + rmdir "${PROM_RULES_DIR}" 2>/dev/null || true + if os_is_debian; then + svc_dropin_remove "${PROM_UNIT}.service" lss-config + pkg_remove prometheus + else + file_remove "/etc/systemd/system/${PROM_UNIT}.service" + svc_daemon_reload + mon_upstream_remove prometheus promtool + fi + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + mon_delete "$(_prom_data)" + return +} diff --git a/modules/monitoring/smartd.sh b/modules/monitoring/smartd.sh new file mode 100644 index 0000000..7f6aa32 --- /dev/null +++ b/modules/monitoring/smartd.sh @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="smartd" +MOD_GROUP="monitoring" +MOD_DESCRIPTION="smartd watches the SMART health of every drive, runs short self-tests daily and long ones weekly, and logs failures and temperatures to the journal" +MOD_REQUIRES="system" +MOD_OPTIONS="temperature|45,55|Log a drive temperature above the first value (Celsius) and warn above the second" +MOD_PORTS="none" + +# Virtual machines and containers usually show no SMART drives. smartd then has nothing to watch; it keeps running +# (-q never) instead of failing, so the module behaves the same on every server and picks up drives that appear later +# on the next restart. +SMARTD_BINARY=/usr/sbin/smartd +# Self-tests: short ones daily at 02:00, long ones on Saturdays at 03:00 (smartd -s, a regular expression). +SMARTD_TESTS='(S/../.././02|L/../../6/03)' + +_smartd_conf() { + if os_is_debian; then printf '/etc/smartd.conf'; else printf '/etc/smartmontools/smartd.conf'; fi + return +} + +# The unit is smartmontools.service on Debian and Ubuntu (smartd.service is an alias there) and smartd.service on the +# RHEL family; drop-ins must go to the real name. +_smartd_unit() { + local unit + unit=$(systemctl show -p Id --value smartd.service 2>/dev/null || true) + printf '%s' "${unit:-smartd.service}" + return +} + +mod_check() { + [[ $(opt temperature) =~ ^[0-9]{2},[0-9]{2,3}$ ]] || die "LSS_SMARTD_TEMPERATURE must look like 45,55." + return +} + +mod_install() { + pkg_install smartmontools + return +} + +mod_configure() { + local before unit + before=$(files_changes_of smartd) + unit=$(_smartd_unit) + # Every drive that speaks SMART: health, attributes and error logs (-a), automatic offline tests and attribute + # autosave on, drives in standby are not woken up (and not counted as missed), self-tests by schedule, temperature. + file_write 0644 root:root "$(_smartd_conf)" </dev/null | grep -c '^/dev/' || true) + if ((devices == 0)); then + log_info "smartd: smartctl finds 0 drives here (virtual machine or container); smartd keeps running for drives added later." + fi + return +} + +mod_remove() { + local unit + unit=$(_smartd_unit) + svc_disable "${unit%.service}" + svc_dropin_remove "${unit}" lss-sandbox + svc_dropin_remove "${unit}" lss-config + file_remove "$(_smartd_conf)" + pkg_remove smartmontools + return +} diff --git a/templates/prometheus/rules.yml b/templates/prometheus/rules.yml new file mode 100644 index 0000000..f287cec --- /dev/null +++ b/templates/prometheus/rules.yml @@ -0,0 +1,58 @@ +# Managed by lss (module prometheus). Alert rules for the server lss set up; lss configure prometheus replaces local +# changes. Alertmanager (module alertmanager) sends them on; without it they show in Prometheus and Grafana. +groups: + - name: lss + rules: + - alert: LssTargetDown + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.job }} on {{ $labels.instance }} cannot be scraped" + - alert: LssDiskAlmostFull + expr: >- + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"} < 0.10 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.mountpoint }} has less than 10 % free space" + - alert: LssMemoryLow + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.05 + for: 15m + labels: + severity: warning + annotations: + summary: "less than 5 % of the memory is available" + - alert: LssGuardFindings + expr: lss_guard_findings > 0 + labels: + severity: warning + annotations: + summary: "the lss guard reports {{ $value }} finding(s); see journalctl -u lss-guard" + - alert: LssGuardStale + expr: time() - lss_guard_last_run_timestamp_seconds > 26 * 3600 + labels: + severity: warning + annotations: + summary: "the lss guard has not run for more than 26 hours" + - alert: LssBackupFailed + expr: lss_backup_last_run_success == 0 + labels: + severity: critical + annotations: + summary: "the last backup failed; see journalctl -u lss-backup" + - alert: LssBackupTooOld + expr: time() - lss_backup_last_success_timestamp_seconds > 30 * 3600 + labels: + severity: critical + annotations: + summary: "the last successful backup is more than 30 hours old" + - alert: LssRestoreTestFailed + expr: lss_backup_restore_test_success == 0 + labels: + severity: critical + annotations: + summary: "the last restore test of the backup failed" diff --git a/tests/unit/monitoring.bats b/tests/unit/monitoring.bats new file mode 100644 index 0000000..d8df208 --- /dev/null +++ b/tests/unit/monitoring.bats @@ -0,0 +1,567 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# The monitoring layer (lib/monitoring.sh) and the WireGuard address of the server (lib/firewall.sh): addresses and +# access, the files Prometheus and Grafana read, drop-ins and units, upstream releases and the checks of the modules. + +setup() { + load helpers + lss_test_setup + use_os debian-13 + shopt -s lastpipe + export TMPDIR=${BATS_TEST_TMPDIR} + export LSS_MODULE=grafana + WRITES=${BATS_TEST_TMPDIR}/writes + : >"${WRITES}" + MON_TARGETS_DIR=${BATS_TEST_TMPDIR}/prometheus/targets + MON_ALERTMANAGERS_DIR=${BATS_TEST_TMPDIR}/prometheus/alertmanagers + MON_PROMETHEUS_DIR=${BATS_TEST_TMPDIR}/prometheus + MON_DATASOURCES_DIR=${BATS_TEST_TMPDIR}/grafana/datasources + MON_BIN_DIR=${BATS_TEST_TMPDIR}/bin + mkdir -p "${MON_BIN_DIR}" + MON_CACHE_DIR=${BATS_TEST_TMPDIR}/cache + MON_POLICY_RC=${BATS_TEST_TMPDIR}/policy-rc.d + # file_write records "== MODE OWNER DEST" and the content, and reports a change unless TEST_FILE_CHANGED=0. + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + printf '== %s %s %s\n' "$1" "$2" "$3" >>"${WRITES}" + cat >>"${WRITES}" + LSS_FILE_CHANGED=${TEST_FILE_CHANGED:-1} + if [[ ${LSS_FILE_CHANGED} == 1 ]]; then + files_note_change "wrote $3" + fi + } + # install without -o and -g, so it runs as the test user. + # shellcheck disable=SC2329 # called by the functions under test + install() { + local -a args=() + while [[ $# -gt 0 ]]; do + case $1 in + -o | -g) shift ;; + *) args+=("$1") ;; + esac + shift + done + command install "${args[@]}" + } +} + +# --- addresses and access --------------------------------------------------------------------------------------- + +@test "the server takes the first address of the WireGuard network" { + [ "$(fw_vpn_address)" = 10.66.0.1 ] + config_set LSS_WIREGUARD_NETWORK 172.20.5.7/16 + [ "$(fw_vpn_address)" = 172.20.0.1 ] + config_set LSS_WIREGUARD_NETWORK 192.168.10.8/30 + [ "$(fw_vpn_address)" = 192.168.10.9 ] +} + +@test "the WireGuard network must be IPv4 with a prefix from /8 to /30" { + config_set LSS_WIREGUARD_NETWORK fd00::/64 + run fw_vpn_address + [ "${status}" -eq 1 ] + [[ ${output} == *"must be an IPv4 network such as 10.66.0.0/24."* ]] + config_set LSS_WIREGUARD_NETWORK 10.66.0.0/31 + run fw_vpn_address + [ "${status}" -eq 1 ] + [[ ${output} == *"needs a prefix from /8 to /30, got /31."* ]] +} + +@test "a module listens on 127.0.0.1, or on the WireGuard address with access vpn" { + run ! mon_vpn grafana + [ "$(mon_address grafana)" = 127.0.0.1 ] + config_set LSS_GRAFANA_ACCESS vpn + mon_vpn grafana + [ "$(mon_address grafana)" = 10.66.0.1 ] +} + +@test "access vpn needs the WireGuard address on an interface" { + # shellcheck disable=SC2329 # called by the functions under test + ip() { + printf '%s\n' "${TEST_IP:-}" + } + mon_vpn_ready + config_set LSS_GRAFANA_ACCESS vpn + run mon_vpn_ready + [ "${status}" -eq 1 ] + [[ ${output} == *"LSS_GRAFANA_ACCESS=vpn needs the WireGuard address 10.66.0.1; install the wireguard module first"* ]] + TEST_IP='4: wg0 inet 10.66.0.1/24 scope global wg0' + mon_vpn_ready +} + +@test "with access vpn the unit starts after wg0; with access local the drop-in goes away" { + local dropin=${BATS_TEST_TMPDIR}/grafana-server.service.d/lss-vpn.conf + config_set LSS_GRAFANA_ACCESS vpn + mon_vpn_dropin grafana-server.service + [ "${LSS_FILE_CHANGED}" = 1 ] + grep -qxF '== 0644 root:root /etc/systemd/system/grafana-server.service.d/lss-vpn.conf' "${WRITES}" + grep -qxF 'After=wg-quick@wg0.service' "${WRITES}" + grep -qxF 'Wants=wg-quick@wg0.service' "${WRITES}" + grep -q 'systemctl daemon-reload' "${STUB_LOG}" + config_set LSS_GRAFANA_ACCESS local + mon_vpn_dropin grafana-server.service + [ "${LSS_FILE_CHANGED}" = 0 ] + # An existing drop-in is removed (the unit name leads into the test directory). + [ -d /etc/systemd/system ] || skip "no /etc/systemd/system on this machine" + mkdir -p "$(dirname "${dropin}")" + : >"${dropin}" + mon_vpn_dropin "../../..${BATS_TEST_TMPDIR}/grafana-server.service" + [ "${LSS_FILE_CHANGED}" = 1 ] + [ ! -e "${dropin}" ] +} + +@test "ports open to the WireGuard network with access vpn and close with access local" { + # shellcheck disable=SC2329 # called by the functions under test + fw_allow() { + printf 'allow %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by the functions under test + fw_remove_module() { + printf 'remove %s\n' "${LSS_MODULE}" >>"${STUB_LOG}" + } + config_set LSS_GRAFANA_ACCESS vpn + mon_firewall 3000/tcp 3001/udp + grep -qxF 'allow 3000/tcp:vpn' "${STUB_LOG}" + grep -qxF 'allow 3001/udp:vpn' "${STUB_LOG}" + config_set LSS_GRAFANA_ACCESS local + mon_firewall 3000/tcp + grep -qxF 'remove grafana' "${STUB_LOG}" +} + +# --- admin paths ------------------------------------------------------------------------------------------------ + +admin_stubs() { + # shellcheck disable=SC2329 # called by the functions under test + web_server() { + printf '%s' "${TEST_SERVER:-}" + } + # shellcheck disable=SC2329 # called by the functions under test + web_admin_write() { + printf 'admin_write %s\n' "$*" >>"${STUB_LOG}" + [[ ${TEST_FILE_CHANGED:-1} == 0 ]] || files_note_change "admin $1" + } + # shellcheck disable=SC2329 # called by the functions under test + web_admin_remove() { + printf 'admin_remove %s\n' "$*" >>"${STUB_LOG}" + [[ ${TEST_FILE_CHANGED:-1} == 0 ]] || files_note_change "removed admin $1" + } + # shellcheck disable=SC2329 # called by the functions under test + web_reload() { + printf 'reload\n' >>"${STUB_LOG}" + } +} + +@test "the admin path follows LSS__WEB_ADMIN and reloads the web server only on a change" { + admin_stubs + mon_web_admin proxy http://127.0.0.1:3000 + [ ! -s "${STUB_LOG}" ] + TEST_SERVER=nginx + config_set LSS_GRAFANA_WEB_ADMIN true + mon_web_admin proxy http://127.0.0.1:3000 + grep -qxF 'admin_write grafana grafana proxy http://127.0.0.1:3000' "${STUB_LOG}" + grep -qxF 'reload' "${STUB_LOG}" + : >"${STUB_LOG}" + TEST_FILE_CHANGED=0 mon_web_admin proxy http://127.0.0.1:3000 + run ! grep -q reload "${STUB_LOG}" + config_set LSS_GRAFANA_WEB_ADMIN false + mon_web_admin proxy http://127.0.0.1:3000 + grep -qxF 'admin_remove grafana' "${STUB_LOG}" + : >"${STUB_LOG}" + mon_web_admin_remove + grep -qxF 'admin_remove grafana' "${STUB_LOG}" + grep -qxF 'reload' "${STUB_LOG}" + TEST_SERVER='' + : >"${STUB_LOG}" + mon_web_admin_remove + [ ! -s "${STUB_LOG}" ] +} + +# --- Prometheus and Grafana ------------------------------------------------------------------------------------- + +@test "a scrape target names its job and metrics path for Prometheus" { + LSS_MODULE=prometheus + mon_target prometheus prometheus 127.0.0.1:9090 /prometheus/metrics + grep -qxF "== 0644 root:root ${MON_TARGETS_DIR}/prometheus.yml" "${WRITES}" + grep -qxF -e "- targets: ['127.0.0.1:9090']" "${WRITES}" + grep -qxF ' job: prometheus' "${WRITES}" + grep -qxF ' __metrics_path__: /prometheus/metrics' "${WRITES}" + mon_target node node 127.0.0.1:9100 + grep -qxF ' __metrics_path__: /metrics' "${WRITES}" + : >"${MON_TARGETS_DIR}/node.yml" + # shellcheck disable=SC2329 # called by the functions under test + file_remove() { + rm -f -- "$1" + } + mon_target_remove node + [ ! -e "${MON_TARGETS_DIR}" ] +} + +@test "a data source is provisioned for Grafana, which restarts only when it runs and the file changed" { + LSS_MODULE=prometheus + mon_datasource Prometheus prometheus http://127.0.0.1:9090/prometheus true + grep -qxF "== 0644 root:root ${MON_DATASOURCES_DIR}/prometheus.yaml" "${WRITES}" + grep -qxF ' - name: Prometheus' "${WRITES}" + grep -qxF ' uid: lss-prometheus' "${WRITES}" + grep -qxF ' url: http://127.0.0.1:9090/prometheus' "${WRITES}" + grep -qxF ' isDefault: true' "${WRITES}" + run ! grep -q 'systemctl restart' "${STUB_LOG}" + export STUB_ACTIVE_UNITS=grafana-server + LSS_MODULE=alertmanager + mon_datasource Alertmanager alertmanager http://127.0.0.1:9093/alertmanager + grep -qxF ' isDefault: false' "${WRITES}" + grep -qxF ' implementation: prometheus' "${WRITES}" + grep -q 'systemctl restart grafana-server' "${STUB_LOG}" + : >"${STUB_LOG}" + TEST_FILE_CHANGED=0 mon_datasource Alertmanager alertmanager http://127.0.0.1:9093/alertmanager + run ! grep -q 'systemctl restart' "${STUB_LOG}" + file_remove() { + printf 'remove %s\n' "$1" >>"${STUB_LOG}" + } + mon_datasource_remove + grep -qxF "remove ${MON_DATASOURCES_DIR}/alertmanager.yaml" "${STUB_LOG}" +} + +@test "a data source that Grafana does not take stops the run" { + export STUB_ACTIVE_UNITS=grafana-server STUB_EXIT_systemctl=1 + run mon_datasource Loki loki http://127.0.0.1:3100 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: Grafana does not start with the new data source."* ]] +} + +@test "units and drop-ins of the monitoring services" { + LSS_MODULE=prometheus + mon_unit prometheus prometheus '/usr/local/bin/prometheus --config.file=/x' 'Prometheus monitoring system' + grep -qxF '== 0644 root:root /etc/systemd/system/prometheus.service' "${WRITES}" + grep -qxF 'Description=Prometheus monitoring system' "${WRITES}" + grep -qxF 'User=prometheus' "${WRITES}" + grep -qxF 'ExecStart=/usr/local/bin/prometheus --config.file=/x' "${WRITES}" + grep -qxF "ExecReload=/bin/kill -HUP \$MAINPID" "${WRITES}" + grep -qxF 'WantedBy=multi-user.target' "${WRITES}" + : >"${WRITES}" + mon_exec prometheus '/usr/bin/prometheus --config.file=/x' + grep -qxF '== 0644 root:root /etc/systemd/system/prometheus.service.d/lss-config.conf' "${WRITES}" + [ "$(grep -c '^ExecStart=' "${WRITES}")" = 2 ] + grep -qxF 'EnvironmentFile=' "${WRITES}" + grep -qxF 'ExecStart=/usr/bin/prometheus --config.file=/x' "${WRITES}" +} + +@test "Grafana's repository goes with the last of grafana, loki and alloy" { + # shellcheck disable=SC2329 # called by the functions under test + repo_remove() { + printf 'repo_remove %s\n' "$1" >>"${STUB_LOG}" + } + state_set grafana status installed + state_set loki status installed + mon_grafana_repo_remove + [ ! -s "${STUB_LOG}" ] + state_set loki status removed + mon_grafana_repo_remove + grep -qxF 'repo_remove grafana' "${STUB_LOG}" +} + +# --- packages and upstream releases ----------------------------------------------------------------------------- + +@test "Debian packages install with service starts forbidden, and the policy goes away afterwards" { + export STUB_OUT_dpkg_query='deinstall ok config-files' + # shellcheck disable=SC2329 # called by the functions under test + pkg_install() { + printf 'pkg_install %s policy=%s\n' "$*" "$(tail -n 1 "${MON_POLICY_RC}" 2>/dev/null)" >>"${STUB_LOG}" + } + mon_install_stopped prometheus + grep -qxF 'pkg_install prometheus policy=exit 101' "${STUB_LOG}" + [ ! -e "${MON_POLICY_RC}" ] + # A policy of the system's own stays untouched. + printf '#!/bin/sh\nexit 0\n' >"${MON_POLICY_RC}" + mon_install_stopped prometheus + grep -qxF 'pkg_install prometheus policy=exit 0' "${STUB_LOG}" + [ -f "${MON_POLICY_RC}" ] + rm -f -- "${MON_POLICY_RC}" + use_os rocky-9 + : >"${STUB_LOG}" + mon_install_stopped grafana + grep -qxF 'pkg_install grafana policy=' "${STUB_LOG}" +} + +@test "the policy is removed when the installation fails" { + export STUB_OUT_dpkg_query='deinstall ok config-files' + # shellcheck disable=SC2329 # called by the functions under test + pkg_install() { + die "apt failed" + } + run mon_install_stopped prometheus + [ "${status}" -eq 1 ] + [ ! -e "${MON_POLICY_RC}" ] +} + +# A release archive with the programs NAME and NAME-tool in a top-level directory, as fetch_verified would download it. +fake_release() { + local name=$1 version=$2 dir + dir=${BATS_TEST_TMPDIR}/release/${name}-${version}.linux-amd64 + mkdir -p "${dir}" + printf '#!/bin/sh\necho "%s, version %s (branch: HEAD)"\n' "${name}" "${version}" >"${dir}/${name}" + printf '#!/bin/sh\n' >"${dir}/${name}tool" + chmod +x "${dir}/${name}" "${dir}/${name}tool" + tar -czf "${BATS_TEST_TMPDIR}/release.tar.gz" -C "${BATS_TEST_TMPDIR}/release" "${name}-${version}.linux-amd64" + # shellcheck disable=SC2329 # called by the functions under test + fetch_verified() { + printf 'fetch %s\n' "$1" >>"${STUB_LOG}" + mkdir -p "$(dirname "$2")" + cp "${BATS_TEST_TMPDIR}/release.tar.gz" "$2" + } +} + +@test "an upstream release is installed once per version and removed again" { + fake_release prom 3.13.4 + mon_upstream_install prom 3.13.4 prom promtool + grep -qxF 'fetch prom-3.13.4' "${STUB_LOG}" + [ -x "${MON_BIN_DIR}/prom" ] + [ -x "${MON_BIN_DIR}/promtool" ] + [ -z "$(ls -A "${MON_CACHE_DIR}")" ] + [ "$(files_changes_of grafana)" = 1 ] + : >"${STUB_LOG}" + mon_upstream_install prom 3.13.4 prom promtool + [ ! -s "${STUB_LOG}" ] + mon_upstream_remove prom promtool missing + [ ! -e "${MON_BIN_DIR}/prom" ] + [ ! -e "${MON_BIN_DIR}/promtool" ] + [ "$(files_changes_of grafana)" = 3 ] +} + +@test "an archive without the expected program stops the run" { + fake_release prom 3.13.4 + run mon_upstream_install prom 3.13.4 prom other + [ "${status}" -eq 1 ] + [[ ${output} == *"the prom 3.13.4 archive has no other; its layout changed."* ]] +} + +@test "service users are created once, without login" { + # shellcheck disable=SC2329 # called by the functions under test + useradd() { + printf 'useradd %s\n' "$*" >>"${STUB_LOG}" + } + export STUB_GETENT="" + mon_user prometheus /var/lib/prometheus + grep -qxF 'useradd --system --user-group --home-dir /var/lib/prometheus --no-create-home --shell /usr/sbin/nologin prometheus' "${STUB_LOG}" + export STUB_GETENT=passwd:prometheus + : >"${STUB_LOG}" + mon_user prometheus /var/lib/prometheus + run ! grep -q useradd "${STUB_LOG}" +} + +@test "data directories are created and their owner and mode kept, never through a link" { + local dir=${BATS_TEST_TMPDIR}/data + mon_dir "${dir}" "${OWNER}" 0750 + [ "$(stat -c '%a' "${dir}")" = 750 ] + chmod 0755 "${dir}" + mon_dir "${dir}" "${OWNER}" 0750 + [ "$(stat -c '%a' "${dir}")" = 750 ] + mon_dir "${dir}" "${OWNER}" 0750 + [ "$(files_changes_of grafana)" = 2 ] + ln -s "${dir}" "${BATS_TEST_TMPDIR}/link" + run mon_dir "${BATS_TEST_TMPDIR}/link" "${OWNER}" 0750 + [ "${status}" -eq 1 ] + [[ ${output} == *"is a symbolic link."* ]] +} + +@test "only data directories below /var/lib are deleted" { + run mon_delete /etc/grafana + [ "${status}" -eq 1 ] + [[ ${output} == *"refusing to delete /etc/grafana."* ]] + run mon_delete /var/lib/../etc + [ "${status}" -eq 1 ] + mon_delete /var/lib/lss-test-does-not-exist + [ "$(files_changes_of grafana)" = 0 ] +} + +# --- checks ----------------------------------------------------------------------------------------------------- + +@test "HTTP checks wait for 200 and report the last status" { + # shellcheck disable=SC2329 # called by the functions under test + sleep() { + : + } + # shellcheck disable=SC2329 # called by the functions under test + curl() { + printf '%s' "${TEST_HTTP_CODE}" + } + TEST_HTTP_CODE=200 + [ "$(mon_http http://127.0.0.1:9090/-/ready)" = 200 ] + mon_wait_http http://127.0.0.1:9090/-/ready + TEST_HTTP_CODE=503 + run mon_wait_http http://127.0.0.1:9090/-/ready 4 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: http://127.0.0.1:9090/-/ready does not answer (503)."* ]] +} + +@test "listening sockets must be on the expected addresses, of the named process" { + # shellcheck disable=SC2329 # called by the functions under test + ss() { + printf '%s\n' "${TEST_SS:-}" + } + TEST_SS=$'LISTEN 0 4096 127.0.0.1:3000 0.0.0.0:* users:(("grafana",pid=10,fd=8))\nLISTEN 0 4096 [::1]:3000 [::]:* users:(("grafana",pid=10,fd=9))' + mon_listen_check 3000 127.0.0.1 ::1 + run mon_listen_check 3000 127.0.0.1 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: port 3000 is open on ::1, expected 127.0.0.1 only."* ]] + TEST_SS='LISTEN 0 4096 0.0.0.0:53 0.0.0.0:* users:(("systemd-resolve",pid=5,fd=3))' + run mon_listen_check --process unbound 53 127.0.0.1 + [ "${status}" -eq 1 ] + [[ ${output} == *"grafana: nothing listens on port 53."* ]] + TEST_SS=$'LISTEN 0 4096 10.66.0.1%wg0:53 0.0.0.0:* users:(("unbound",pid=5,fd=3))\nLISTEN 0 4096 0.0.0.0:53 0.0.0.0:* users:(("dnsmasq",pid=6,fd=3))' + mon_listen_check --process unbound 53 10.66.0.1 +} + +# --- the monitoring modules ------------------------------------------------------------------------------------- + +# load_module NAME: sources the module as lss does for a step. +load_module() { + local file + file=$(module_file "$1") + export LSS_MODULE=$1 + # shellcheck source=/dev/null + source "${file}" + return +} + +@test "every monitoring module checks its access setting" { + local module + for module in node-exporter prometheus alertmanager grafana loki netdata; do + load_module "${module}" + mod_check + config_set "$(config_key "${module}" access)" public + run mod_check + [ "${status}" -eq 1 ] + [[ ${output} == *"must be local or vpn."* ]] + done +} + +@test "the node exporter listens on its address and reads the textfile directory" { + load_module node-exporter + [ "$(_node_command)" = '/usr/bin/prometheus-node-exporter --web.listen-address=127.0.0.1:9100 --collector.textfile.directory=/var/lib/prometheus/node-exporter' ] + use_os rocky-9 + config_set LSS_NODE_EXPORTER_ACCESS vpn + [ "$(_node_command)" = '/usr/local/bin/node_exporter --web.listen-address=10.66.0.1:9100 --collector.textfile.directory=/var/lib/prometheus/node-exporter' ] + [ "$(_node_user)" = node_exporter ] +} + +@test "Prometheus finds targets and Alertmanagers in files and serves below /prometheus/" { + load_module prometheus + run _prom_config + [[ ${output} == *" - ${MON_PROMETHEUS_DIR}/rules/*.yml"* ]] + [[ ${output} == *" - path_prefix: /alertmanager/"* ]] + [[ ${output} == *" - files: ['${MON_ALERTMANAGERS_DIR}/*.yml']"* ]] + [[ ${output} == *" - files: ['${MON_TARGETS_DIR}/*.yml']"* ]] + config_set LSS_PROMETHEUS_RETENTION 30d + run _prom_command + [[ ${output} == '/usr/bin/prometheus --config.file='*' --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=30d --web.listen-address=127.0.0.1:9090 --web.external-url=http://127.0.0.1:9090/prometheus/' ]] + use_os almalinux-10 + run _prom_command + [[ ${output} == '/usr/local/bin/prometheus '*'--storage.tsdb.path=/var/lib/prometheus/data '* ]] + config_set LSS_PROMETHEUS_RETENTION 2weeks + run mod_check + [[ ${output} == *"LSS_PROMETHEUS_RETENTION must look like 15d"* ]] +} + +@test "the alert rules cover targets, disks, the guard and the backup" { + local rules=${REPO_ROOT}/templates/prometheus/rules.yml alert + for alert in LssTargetDown LssDiskAlmostFull LssMemoryLow LssGuardFindings LssGuardStale LssBackupFailed LssBackupTooOld LssRestoreTestFailed; do + grep -qxF " - alert: ${alert}" "${rules}" + done + # The template engine leaves the placeholders of Prometheus itself alone. + run template_render "${rules}" + [ "${status}" -eq 0 ] + [[ ${output} == *'{{ $labels.instance }}'* ]] +} + +@test "Alertmanager sends to e-mail and webhook when set and keeps its cluster port closed" { + load_module alertmanager + run _am_config + [[ ${output} == *$'receivers:\n - name: lss'* ]] + [[ ${output} != *email_configs* && ${output} != *webhook_configs* ]] + config_set LSS_ALERTMANAGER_EMAIL ops@example.com + config_set LSS_ALERTMANAGER_WEBHOOK https://hooks.example.com/abc + run _am_config + [[ ${output} == *" - to: 'ops@example.com'"* ]] + [[ ${output} == *" smarthost: 'localhost:25'"* ]] + [[ ${output} == *" - url: 'https://hooks.example.com/abc'"* ]] + run _am_command + [[ ${output} == *' --web.external-url=http://127.0.0.1:9093/alertmanager/ --cluster.listen-address=' ]] + config_set LSS_ALERTMANAGER_WEBHOOK 'http://hooks.example.com/x' + run mod_check + [[ ${output} == *"LSS_ALERTMANAGER_WEBHOOK must be an https:// URL."* ]] + config_set LSS_ALERTMANAGER_WEBHOOK '' + config_set LSS_ALERTMANAGER_EMAIL 'not mail' + run mod_check + [[ ${output} == *"is not an e-mail address."* ]] +} + +@test "Grafana runs below /grafana/ with its password from a file and nothing sent home" { + load_module grafana + run _grafana_environment + [[ ${output} == *'Environment=GF_SERVER_HTTP_ADDR=127.0.0.1'* ]] + [[ ${output} == *'Environment=GF_SERVER_ROOT_URL=http://127.0.0.1:3000/grafana/'* ]] + [[ ${output} == *'Environment=GF_SERVER_SERVE_FROM_SUB_PATH=true'* ]] + [[ ${output} == *"Environment=GF_SECURITY_ADMIN_PASSWORD__FILE=${MON_GRAFANA_DIR}/admin-password"* ]] + [[ ${output} == *'Environment=GF_ANALYTICS_REPORTING_ENABLED=false'* ]] + [[ ${output} != *GF_SECURITY_ADMIN_PASSWORD=* ]] +} + +@test "Loki keeps every component on this server with retention" { + load_module loki + config_set LSS_LOKI_RETENTION_DAYS 14 + run _loki_config + [[ ${output} == *$' http_listen_address: 127.0.0.1\n http_listen_port: 3100\n grpc_listen_address: 127.0.0.1'* ]] + [[ ${output} == *' store: inmemory'* ]] + [[ ${output} == *' retention_period: 336h'* ]] + [[ ${output} == *' reporting_enabled: false'* ]] + config_set LSS_LOKI_RETENTION_DAYS 0 + run mod_check + [[ ${output} == *"LSS_LOKI_RETENTION_DAYS must be at least 1."* ]] +} + +@test "Alloy ships the journal to the local Loki" { + load_module alloy + run _alloy_config + [[ ${output} == *'loki.source.journal "journal" {'* ]] + [[ ${output} == *' max_age = "12h"'* ]] + [[ ${output} == *'url = "http://127.0.0.1:3100/loki/api/v1/push"'* ]] + config_set LSS_LOKI_ACCESS vpn + run _alloy_config + [[ ${output} == *'url = "http://10.66.0.1:3100/loki/api/v1/push"'* ]] + config_set LSS_ALLOY_MAX_AGE 2d + run mod_check + [[ ${output} == *"LSS_ALLOY_MAX_AGE must look like 12h or 30m."* ]] +} + +@test "Netdata binds to 127.0.0.1 and the WireGuard address, without registry, cloud or statistics" { + load_module netdata + # shellcheck disable=SC2329 # called by the functions under test + install() { + : + } + _netdata_files + grep -qxF $'\tbind to = 127.0.0.1' "${WRITES}" + grep -qxF '[registry]' "${WRITES}" + grep -qxF "== 0644 root:root ${NETDATA_OPT_OUT}" "${WRITES}" + grep -qxF "== 0644 root:root ${NETDATA_CLOUD_CONF}" "${WRITES}" + config_set LSS_NETDATA_ACCESS vpn + [ "$(_netdata_bind)" = '127.0.0.1 10.66.0.1' ] +} + +@test "smartd finds its unit and configuration on each family and checks the temperatures" { + load_module smartd + # shellcheck disable=SC2329 # called by the functions under test + systemctl() { + printf 'smartmontools.service\n' + } + [ "$(_smartd_unit)" = smartmontools.service ] + [ "$(_smartd_conf)" = /etc/smartd.conf ] + use_os rocky-9 + [ "$(_smartd_conf)" = /etc/smartmontools/smartd.conf ] + mod_check + config_set LSS_SMARTD_TEMPERATURE 50 + run mod_check + [[ ${output} == *"LSS_SMARTD_TEMPERATURE must look like 45,55."* ]] +} From 1c0d8759ccc2343a06faebdfb3c6b4fe17876c40 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:09:03 +0200 Subject: [PATCH 04/32] feat(backup): restic and db-dumps modules restic backs up to a local path, SFTP or S3 on a systemd timer, applies the retention, checks the repository and exports its state for Prometheus; the password, the S3 keys and the SSH key live in the lss secrets. db-dumps dumps every database engine lss installed before each backup. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- libexec/lss-backup | 335 +++++++++++++++++++++++++++++++++++++ libexec/lss-db-dumps | 200 ++++++++++++++++++++++ modules/backup/db-dumps.sh | 84 ++++++++++ modules/backup/restic.sh | 180 ++++++++++++++++++++ tests/unit/backup.bats | 299 +++++++++++++++++++++++++++++++++ 5 files changed, 1098 insertions(+) create mode 100755 libexec/lss-backup create mode 100755 libexec/lss-db-dumps create mode 100644 modules/backup/db-dumps.sh create mode 100644 modules/backup/restic.sh create mode 100644 tests/unit/backup.bats diff --git a/libexec/lss-backup b/libexec/lss-backup new file mode 100755 index 0000000..a8172fa --- /dev/null +++ b/libexec/lss-backup @@ -0,0 +1,335 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lss backup: restic backups of the server. Installed by the restic module as /usr/local/lib/lss/backup and run daily +# by lss-backup.timer. Settings come from /etc/lss/backup.conf (KEY=value lines, parsed, never sourced); the +# repository password and the credentials stay in /etc/lss/secrets and reach restic through files and the environment, +# never through its command line. +# +# backup run database dumps (db-dumps module), backup, forget and prune by the retention, check +# backup init creates the repository unless it exists +# backup restore-test restores the canary file of the latest backup and compares it, then reads a part of the data +# backup probe a small backup of the canary file and its restore test (lss verify, before the first run) +# backup restic ARGS restic with the settings of this server, for example: backup restic snapshots +# +# The state (time and result of the last run and restore test) is kept in /var/lib/lss/restic and written as metrics +# for the node exporter, which the guard and the Prometheus alert rules read. +set -Eeuo pipefail +shopt -s inherit_errexit + +CONF=${LSS_BACKUP_CONF:-/etc/lss/backup.conf} +SECRETS_DIR=${LSS_SECRETS_DIR:-/etc/lss/secrets} +STATE=${LSS_BACKUP_STATE:-/var/lib/lss/restic} +TEXTFILE_DIR=${LSS_TEXTFILE_DIR:-/var/lib/prometheus/node-exporter} +DB_DUMPS=${LSS_DB_DUMPS:-/usr/local/lib/lss/db-dumps} +DB_DUMPS_CONF=${LSS_DB_DUMPS_CONF:-/etc/lss/db-dumps.conf} +CACHE=${LSS_BACKUP_CACHE:-/var/cache/restic} +RESTIC=${LSS_RESTIC:-restic} +CANARY_DIR=${STATE}/canary +CANARY=${CANARY_DIR}/canary +TAG=lss +PROBE_TAG=lss-probe +# Global also when the script is sourced inside a function (the unit tests). +declare -ga RESTIC_OPTIONS=() + +say() { + printf 'lss-backup: %s\n' "$*" + return +} + +setting() { + local key=$1 default=${2-} line + if [[ -r ${CONF} ]]; then + while IFS= read -r line || [[ -n ${line} ]]; do + if [[ ${line} == "${key}="* ]]; then + printf '%s' "${line#*=}" + return 0 + fi + done <"${CONF}" + fi + printf '%s' "${default}" +} + +secret_file() { + printf '%s/%s' "${SECRETS_DIR}" "$1" + return +} + +# sftp_target REPOSITORY: user@host (and -p PORT) of an sftp repository, as ssh arguments separated by spaces. +# Repositories look like sftp:user@host:/path or sftp://user@host:port//path. +sftp_target() { + local repository=$1 rest host port='' + if [[ ${repository} == sftp://* ]]; then + rest=${repository#sftp://} + host=${rest%%/*} + if [[ ${host} =~ ^(.+):([0-9]+)$ ]]; then + host=${BASH_REMATCH[1]} + port=${BASH_REMATCH[2]} + fi + else + rest=${repository#sftp:} + host=${rest%%:*} + fi + [[ ${host} =~ ^[A-Za-z0-9._@-]+$ ]] || return 1 + printf '%s' "${host}" + [[ -z ${port} ]] || printf ' -p %s' "${port}" + return +} + +# prepare: the environment and options restic needs for the configured repository. +prepare() { + local repository target key_id secret + repository=$(setting BACKUP_REPOSITORY) + [[ -n ${repository} ]] || { + say "BACKUP_REPOSITORY is empty in ${CONF}." >&2 + return 1 + } + export RESTIC_REPOSITORY=${repository} + export RESTIC_PASSWORD_FILE + RESTIC_PASSWORD_FILE=$(secret_file restic-password) + export RESTIC_CACHE_DIR=${CACHE} + export TMPDIR=${CACHE}/tmp + mkdir -p "${TMPDIR}" + chmod 0700 "${CACHE}" "${TMPDIR}" + RESTIC_OPTIONS=(--quiet) + case ${repository} in + s3:*) + key_id=$(<"$(secret_file restic-s3-key-id)") + secret=$(<"$(secret_file restic-s3-secret)") + export AWS_ACCESS_KEY_ID=${key_id} AWS_SECRET_ACCESS_KEY=${secret} + if [[ -n $(setting BACKUP_S3_REGION) ]]; then + export AWS_DEFAULT_REGION + AWS_DEFAULT_REGION=$(setting BACKUP_S3_REGION) + fi + ;; + sftp:*) + target=$(sftp_target "${repository}") || { + say "BACKUP_REPOSITORY '${repository}' names no host." >&2 + return 1 + } + # The key in the secrets, the host key pinned on first contact, never a password prompt. + RESTIC_OPTIONS+=(-o "sftp.command=ssh -i $(secret_file restic-ssh-key) -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=${STATE}/known_hosts ${target} -s sftp") + ;; + *) ;; + esac + return +} + +run_restic() { + "${RESTIC}" "${RESTIC_OPTIONS[@]}" "$@" + return +} + +stamp() { + local name=$1 value=$2 + mkdir -p "${STATE}" + printf '%s\n' "${value}" >"${STATE}/${name}.tmp" + mv -f -- "${STATE}/${name}.tmp" "${STATE}/${name}" + return +} + +stamp_get() { + local file=${STATE}/$1 + if [[ -f ${file} ]]; then + printf '%s' "$(<"${file}")" + fi + return +} + +# metrics: the stamps as Prometheus metrics for the node exporter's textfile collector (when it is installed). +metrics() { + local file name value + [[ -d ${TEXTFILE_DIR} ]] || return 0 + file=${TEXTFILE_DIR}/lss_backup.prom + for name in last_run_timestamp_seconds:last-run last_run_success:last-run-success \ + last_success_timestamp_seconds:last-success restore_test_timestamp_seconds:restore-test \ + restore_test_success:restore-test-success; do + value=$(stamp_get "${name#*:}") + [[ ${value} =~ ^[0-9]+$ ]] || continue + printf '# TYPE lss_backup_%s gauge\nlss_backup_%s %s\n' "${name%%:*}" "${name%%:*}" "${value}" + done >"${file}.tmp" + chmod 0644 "${file}.tmp" + mv -f -- "${file}.tmp" "${file}" + return +} + +# backup_paths: the configured paths that exist (restic counts a missing one as an error). +backup_paths() { + local path + local -a paths=() + read -ra paths <<<"$(setting BACKUP_PATHS /etc)" + for path in "${paths[@]}"; do + if [[ -e ${path} ]]; then + printf '%s\n' "${path}" + fi + done + printf '%s\n' "${CANARY_DIR}" + return +} + +# canary: a file with fresh content in every backup; the restore test compares it with what it gets back. +canary() { + local value + mkdir -p "${CANARY_DIR}" + chmod 0700 "${STATE}" "${CANARY_DIR}" + value="$(date +%s) $(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" + printf '%s\n' "${value}" >"${CANARY}" + printf '%s' "${value}" + return +} + +cmd_init() { + prepare + if run_restic cat config >/dev/null 2>&1; then + return 0 + fi + run_restic init >/dev/null + say "created the repository ${RESTIC_REPOSITORY%%\?*}." + return +} + +cmd_run() { + local status=0 code=0 value now pattern + local -a paths=() excludes=() patterns=() + now=$(date +%s) + stamp last-run "${now}" + prepare + if [[ -x ${DB_DUMPS} && -f ${DB_DUMPS_CONF} ]]; then + "${DB_DUMPS}" || { + status=1 + say "the database dumps failed; the backup goes on with the files." + } + fi + cmd_init + mapfile -t paths < <(backup_paths) + # Read as words, never expanded as globs here: the patterns are restic's. + read -ra patterns <<<"$(setting BACKUP_EXCLUDE)" + for pattern in "${patterns[@]}"; do + excludes+=(--exclude "${pattern}") + done + value=$(canary) + run_restic backup --tag "${TAG}" --exclude-caches "${excludes[@]}" "${paths[@]}" || code=$? + case ${code} in + 0) stamp canary-last "${value}" ;; + 3) + # Some files could not be read; the snapshot exists without them. + stamp canary-last "${value}" + status=1 + say "restic could not read some files; see the lines above." + ;; + *) + status=1 + say "restic backup failed (exit code ${code})." + ;; + esac + if ! run_restic forget --tag "${TAG}" --group-by host,tags --prune \ + --keep-daily "$(setting BACKUP_KEEP_DAILY 7)" --keep-weekly "$(setting BACKUP_KEEP_WEEKLY 4)" \ + --keep-monthly "$(setting BACKUP_KEEP_MONTHLY 6)"; then + status=1 + say "restic forget and prune failed." + fi + if ! run_restic check; then + status=1 + say "restic check reports errors in the repository." + fi + if ((status == 0)); then + stamp last-success "${now}" + stamp last-run-success 1 + say "backup complete." + else + stamp last-run-success 0 + fi + metrics + return "${status}" +} + +# restore_test [TAG]: restores the canary of the latest snapshot of this host and compares it. +restore_test() { + local tag=${1:-${TAG}} target expected actual status=0 subset + prepare + expected=$(stamp_get canary-last) + target=$(mktemp -d "${TMPDIR}/restore.XXXXXX") + if ! run_restic restore latest --host "$(hostname)" --tag "${tag}" --target "${target}" --include "${CANARY}"; then + say "restic restore of the latest snapshot failed." + status=1 + fi + actual='' + if [[ -f ${target}${CANARY} ]]; then + actual=$(<"${target}${CANARY}") + fi + rm -rf -- "${target}" + if ((status == 0)) && [[ -z ${actual} || ${actual} != "${expected}" ]]; then + say "the restored canary file differs from the one backed up last." + status=1 + fi + # Reads a share of the packs, so damage on the storage side shows up without downloading everything every week. + subset=$(setting BACKUP_CHECK_SUBSET 5%) + if ((status == 0)) && ! run_restic check --read-data-subset="${subset}"; then + say "restic check --read-data-subset=${subset} reports errors." + status=1 + fi + return "${status}" +} + +cmd_restore_test() { + local status=0 + restore_test || status=$? + stamp restore-test "$(date +%s)" + if ((status == 0)); then + stamp restore-test-success 1 + say "restore test passed." + else + stamp restore-test-success 0 + fi + metrics + return "${status}" +} + +# cmd_probe: proves that the repository takes a backup and gives it back, with the canary file only. +cmd_probe() { + local value + cmd_init + value=$(canary) + run_restic backup --tag "${PROBE_TAG}" "${CANARY_DIR}" + stamp canary-last "${value}" + restore_test "${PROBE_TAG}" + run_restic forget --tag "${PROBE_TAG}" --keep-last 1 >/dev/null + say "probe passed: the repository takes a backup and restores it." + return +} + +# dispatch COMMAND [ARGS]: runs one subcommand. +dispatch() { + local command=${1:-run} + shift || true + case ${command} in + run) cmd_run ;; + init) cmd_init ;; + restore-test) cmd_restore_test ;; + probe) cmd_probe ;; + restic) + prepare + "${RESTIC}" "${RESTIC_OPTIONS[@]:1}" "$@" + ;; + *) + say "usage: backup run|init|restore-test|probe|restic ARGS" >&2 + exit 2 + ;; + esac +} + +main() { + [[ ${EUID} -eq 0 ]] || { + say "must run as root." >&2 + exit 1 + } + umask 077 + dispatch "$@" +} + +# The unit tests source this file without running it. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + main "$@" +fi diff --git a/libexec/lss-db-dumps b/libexec/lss-db-dumps new file mode 100755 index 0000000..68f3505 --- /dev/null +++ b/libexec/lss-db-dumps @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lss db-dumps: consistent dumps of the databases lss installed, for the restic backup. Installed by the db-dumps +# module as /usr/local/lib/lss/db-dumps; lss-backup runs it before every backup. Settings come from +# /etc/lss/db-dumps.conf (KEY=value lines, parsed, never sourced). +# +# Which engines exist is read at run time from the lss module state (/var/lib/lss/modules), so a database installed +# later is dumped without touching this module: +# mariadb, mysql mariadb-dump / mysqldump of every database in one transaction, with routines, events and triggers +# postgresql pg_dumpall (roles, every database) as the postgres user +# mongodb mongodump of every database as an archive, with the administrator from the lss secrets +# redis, valkey BGSAVE, then a copy of the RDB snapshot +# One file per engine in /var/backups/lss-db (root only), replaced on every run; restic keeps the history. +set -Eeuo pipefail +shopt -s inherit_errexit + +CONF=${LSS_DB_DUMPS_CONF:-/etc/lss/db-dumps.conf} +SECRETS_DIR=${LSS_SECRETS_DIR:-/etc/lss/secrets} +MODULES_DIR=${LSS_STATE_DIR:-/var/lib/lss}/modules +STATUS_DIR=${LSS_DB_DUMPS_STATE:-${LSS_STATE_DIR:-/var/lib/lss}/db-dumps} +# The password files of Redis and Valkey (their modules write them) and the parent of their data directories. +ETC_DIR=${LSS_DB_DUMPS_ETC:-/etc/lss} +DATA_ROOT=${LSS_DB_DUMPS_DATA:-/var/lib} +ENGINES="mariadb mysql postgresql mongodb redis valkey" + +say() { + printf 'lss-db-dumps: %s\n' "$*" + return +} + +setting() { + local key=$1 default=${2-} line + if [[ -r ${CONF} ]]; then + while IFS= read -r line || [[ -n ${line} ]]; do + if [[ ${line} == "${key}="* ]]; then + printf '%s' "${line#*=}" + return 0 + fi + done <"${CONF}" + fi + printf '%s' "${default}" +} + +# installed ENGINE: lss installed the module ENGINE. +installed() { + local file=${MODULES_DIR}/$1 + [[ -f ${file} ]] && grep -qx 'status=installed' "${file}" + return +} + +dump_dir() { + setting DB_DUMPS_DIR /var/backups/lss-db + return +} + +# first_program NAME...: the first program that exists, by path or in the PATH. +first_program() { + local program + for program in "$@"; do + if [[ ${program} == /* && -x ${program} ]] || { [[ ${program} != /* ]] && command -v "${program}" >/dev/null 2>&1; }; then + printf '%s' "${program}" + return 0 + fi + done + return 1 +} + +# The SQL servers: root logs in through the Unix socket (MariaDB, MySQL on Debian and Ubuntu) or with the password in +# /root/.my.cnf (MySQL on the RHEL family), which a service does not find on its own (no HOME). +dump_mysql() { + local engine=$1 out=$2 dump socket='' + local -a options=() + if [[ ${engine} == mariadb ]]; then + dump=$(first_program mariadb-dump mysqldump) || return 1 + else + dump=$(first_program /opt/mysql/bin/mysqldump mysqldump) || return 1 + # GTID metadata would make the dump fail to load on a server with its own GTIDs. + options+=(--set-gtid-purged=OFF) + if [[ -S /run/mysqld/mysqld.sock ]]; then socket=/run/mysqld/mysqld.sock; else socket=/var/lib/mysql/mysql.sock; fi + options+=(--socket="${socket}") + fi + if [[ -f /root/.my.cnf ]]; then + options=(--defaults-extra-file=/root/.my.cnf "${options[@]}") + fi + "${dump}" "${options[@]}" --user=root --all-databases --single-transaction --routines --events --triggers \ + --hex-blob >"${out}" + return +} + +dump_postgresql() { + local out=$1 dumpall port + dumpall=$(first_program pg_dumpall /usr/pgsql-*/bin/pg_dumpall) || return 1 + port=$(setting DB_POSTGRESQL_PORT 5432) + # As postgres from /, so the dump runs with the peer login and never inside a directory a user controls. + (cd / && runuser -u postgres -- "${dumpall}" -p "${port}" --clean --if-exists) >"${out}" + return +} + +# MongoDB: the password goes to mongodump in a configuration file of root's, never on the command line. +dump_mongodb() { + local out=$1 port password config status=0 + command -v mongodump >/dev/null 2>&1 || return 1 + port=$(setting DB_MONGODB_PORT 27017) + password=$(<"${SECRETS_DIR}/mongodb-admin") + config=$(mktemp) + printf 'password: "%s"\n' "${password}" >"${config}" + mongodump --quiet --config="${config}" --host=127.0.0.1 --port="${port}" --username=lss-admin \ + --authenticationDatabase=admin --archive="${out}" || status=$? + rm -f -- "${config}" + return "${status}" +} + +# Redis and Valkey: a fresh snapshot (BGSAVE), copied once LASTSAVE moves. The password goes through the environment. +dump_rdb() { + local engine=$1 out=$2 cli password port before i now + if [[ ${engine} == redis ]]; then + cli=$(first_program redis-cli) || return 1 + else + cli=$(first_program /opt/valkey/bin/valkey-cli valkey-cli) || return 1 + fi + password=$(<"${ETC_DIR}/${engine}-password") + port=$(setting "DB_${engine^^}_PORT" 6379) + before=$(REDISCLI_AUTH=${password} VALKEYCLI_AUTH=${password} "${cli}" -p "${port}" --no-auth-warning lastsave) + [[ ${before} =~ ^[0-9]+$ ]] || return 1 + REDISCLI_AUTH=${password} VALKEYCLI_AUTH=${password} "${cli}" -p "${port}" --no-auth-warning bgsave >/dev/null + for ((i = 0; i < 300; i++)); do + now=$(REDISCLI_AUTH=${password} VALKEYCLI_AUTH=${password} "${cli}" -p "${port}" --no-auth-warning lastsave) + [[ ${now} == "${before}" ]] || break + sleep 1 + done + [[ ${now} != "${before}" ]] || return 1 + cp -- "${DATA_ROOT}/${engine}/dump.rdb" "${out}" + return +} + +dump() { + local engine=$1 out=$2 + case ${engine} in + mariadb | mysql) dump_mysql "${engine}" "${out}" ;; + postgresql) dump_postgresql "${out}" ;; + mongodb) dump_mongodb "${out}" ;; + redis | valkey) dump_rdb "${engine}" "${out}" ;; + *) return 1 ;; + esac + return +} + +file_of() { + case $1 in + mariadb | mysql | postgresql) printf '%s.sql' "$1" ;; + mongodb) printf 'mongodb.archive' ;; + *) printf '%s.rdb' "$1" ;; + esac + return +} + +# run_dumps: one dump per installed engine; fails when one of them failed. +run_dumps() { + local dir engine out failed=0 count=0 + dir=$(dump_dir) + mkdir -p "${dir}" "${STATUS_DIR}" + chmod 0700 "${dir}" "${STATUS_DIR}" + : >"${STATUS_DIR}/status.tmp" + for engine in ${ENGINES}; do + installed "${engine}" || continue + count=$((count + 1)) + out=${dir}/$(file_of "${engine}") + # The dump goes to a temporary file first, so a failed run never replaces the last good dump. + if dump "${engine}" "${out}.part" && [[ -s ${out}.part ]]; then + mv -f -- "${out}.part" "${out}" + printf '%s ok %s\n' "${engine}" "$(date +%s)" >>"${STATUS_DIR}/status.tmp" + say "dumped ${engine} to ${out}." + else + rm -f -- "${out}.part" + printf '%s failed %s\n' "${engine}" "$(date +%s)" >>"${STATUS_DIR}/status.tmp" + say "the dump of ${engine} failed; ${out} keeps the previous one." + failed=1 + fi + done + mv -f -- "${STATUS_DIR}/status.tmp" "${STATUS_DIR}/status" + ((count > 0)) || say "lss installed 0 databases here; nothing to dump." + return "${failed}" +} + +main() { + [[ ${EUID} -eq 0 ]] || { + say "must run as root." >&2 + exit 1 + } + umask 077 + run_dumps +} + +# The unit tests source this file without running it. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + main "$@" +fi diff --git a/modules/backup/db-dumps.sh b/modules/backup/db-dumps.sh new file mode 100644 index 0000000..ee1c104 --- /dev/null +++ b/modules/backup/db-dumps.sh @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="db-dumps" +MOD_GROUP="backup" +MOD_DESCRIPTION="Consistent dumps of the MariaDB, MySQL, PostgreSQL, MongoDB, Redis and Valkey servers lss installed, before every restic backup" +MOD_REQUIRES="restic" +MOD_OPTIONS="dir|/var/backups/lss-db|Directory of the dumps (root only); restic backs it up when it is in LSS_RESTIC_PATHS" +MOD_PORTS="none" + +# A copy of a database's files taken while it runs may not load again; a dump does. libexec/lss-db-dumps finds the +# installed engines at run time, so databases added later are dumped without lss configure db-dumps. +DUMPS_SCRIPT=/usr/local/lib/lss/db-dumps +DUMPS_CONF=/etc/lss/db-dumps.conf +DUMPS_STATUS=/var/lib/lss/db-dumps/status + +mod_check() { + [[ $(opt dir) =~ ^/var/backups/[A-Za-z0-9._-]+$ ]] || die "LSS_DB_DUMPS_DIR must be a directory below /var/backups." + if [[ " $(config_option restic paths '') " != *" $(opt dir) "* && -n $(config_option restic paths '') ]]; then + log_warn "db-dumps: $(opt dir) is not in LSS_RESTIC_PATHS, so the dumps stay on this server only." + fi + return +} + +# mongodump is a separate package from the MongoDB repository the mongodb module set up; configure installs it too, +# for a mongodb module that came later. +mod_install() { + if state_installed mongodb; then + pkg_install mongodb-database-tools + fi + return +} + +mod_configure() { + mod_install + install -d -m 0700 "$(opt dir)" + file_write 0755 root:root "${DUMPS_SCRIPT}" <"${LSS_ROOT}/libexec/lss-db-dumps" + file_write 0600 root:root "${DUMPS_CONF}" </dev/null 2>&1; then + die "db-dumps: mongodump is missing; run lss configure db-dumps." + fi + # Dumps run with the backup; here only for engines that have never been dumped, so a second verify stays quiet. + if [[ -n $(_dumps_missing) ]]; then + "${DUMPS_SCRIPT}" || die "db-dumps: the dump of $(_dumps_missing)failed; see above." + fi + return +} + +mod_remove() { + file_remove "${DUMPS_SCRIPT}" + file_remove "${DUMPS_CONF}" + pkg_remove mongodb-database-tools + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + rm -rf -- "$(opt dir)" + mon_delete /var/lib/lss/db-dumps + files_note_change "deleted the database dumps" + return +} diff --git a/modules/backup/restic.sh b/modules/backup/restic.sh new file mode 100644 index 0000000..14f9261 --- /dev/null +++ b/modules/backup/restic.sh @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="restic" +MOD_GROUP="backup" +MOD_DESCRIPTION="Encrypted, deduplicated daily backups with restic to a local path, SFTP or S3, with retention, repository checks and a weekly restore test (guard)" +MOD_REQUIRES="system" +MOD_OPTIONS="repository|/var/backups/lss-restic|Where the backups go: a local path (better another disk), sftp:USER@HOST:/PATH or s3:https://HOST/BUCKET/PATH +paths|/etc /root /home /srv /opt /usr/local /var/www /var/spool/cron /var/lib/lss /var/backups/lss-db|What to back up, separated by spaces; paths that do not exist are skipped +exclude|/var/lib/lss/runs|Paths or patterns restic leaves out, separated by spaces (restic --exclude) +time|03:00|Time of the daily backup (HH:MM) +keep_daily|7|Daily snapshots to keep +keep_weekly|4|Weekly snapshots to keep +keep_monthly|6|Monthly snapshots to keep +check_subset|5%|Share of the data the weekly restore test reads back (restic check --read-data-subset) +s3_key_id||S3 access key ID, stored in the lss secrets (restic-s3-key-id); s3 repositories only +s3_secret||S3 secret access key, stored in the lss secrets (restic-s3-secret); s3 repositories only +s3_region||S3 region, when the provider needs one" +MOD_SECRETS="restic-password restic-ssh-key restic-s3-key-id restic-s3-secret" +MOD_PORTS="none (outgoing SSH or HTTPS to the repository)" + +# Every supported release packages restic (0.12 on Ubuntu 22.04 up to 0.19 in EPEL); the commands lss uses work with +# all of them. libexec/lss-backup does the work; the unit runs it daily. +BACKUP_SCRIPT=/usr/local/lib/lss/backup +BACKUP_CONF=/etc/lss/backup.conf +BACKUP_UNIT=lss-backup +BACKUP_STATE=/var/lib/lss/restic +BACKUP_CACHE=/var/cache/restic +BACKUP_PUBLIC_KEY=/etc/lss/restic-ssh-key.pub + +_restic_kind() { + case $(opt repository) in + sftp:*) printf 'sftp' ;; + s3:*) printf 's3' ;; + *) printf 'local' ;; + esac + return +} + +mod_check() { + local repository number + repository=$(opt repository) + case ${repository} in + sftp:*) [[ ${repository} =~ ^sftp:(//)?[A-Za-z0-9._-]+@[A-Za-z0-9.-]+(:[0-9]+)?[:/].+$ ]] || + die "LSS_RESTIC_REPOSITORY: an sftp repository looks like sftp:USER@HOST:/PATH." ;; + s3:*) [[ ${repository} =~ ^s3:https://[A-Za-z0-9.:-]+/[A-Za-z0-9._/-]+$ ]] || + die "LSS_RESTIC_REPOSITORY: an s3 repository looks like s3:https://HOST/BUCKET/PATH." ;; + /*) [[ ${repository} =~ ^/[A-Za-z0-9._/-]+$ && ${repository} != *..* ]] || + die "LSS_RESTIC_REPOSITORY: '${repository}' is not a plain absolute path." ;; + *) die "LSS_RESTIC_REPOSITORY must be an absolute path, sftp:USER@HOST:/PATH or s3:https://HOST/BUCKET/PATH." ;; + esac + [[ $(opt paths) =~ ^/[A-Za-z0-9._/[:space:]-]*$ ]] || die "LSS_RESTIC_PATHS must be absolute paths separated by spaces." + [[ $(opt exclude) =~ ^[A-Za-z0-9._/*?[:space:]-]*$ ]] || die "LSS_RESTIC_EXCLUDE holds characters restic patterns do not need." + [[ $(opt time) =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || die "LSS_RESTIC_TIME must look like 03:00." + for number in keep_daily keep_weekly keep_monthly; do + opt_int "${number}" >/dev/null + done + [[ $(opt check_subset) =~ ^([1-9][0-9]?|100)%$ ]] || die "LSS_RESTIC_CHECK_SUBSET must be a percentage such as 5%." + if [[ $(_restic_kind) == s3 ]] && ! secret_exists restic-s3-key-id && [[ -z $(opt s3_key_id) || -z $(opt s3_secret) ]]; then + die "LSS_RESTIC_REPOSITORY is an s3 repository: set LSS_RESTIC_S3_KEY_ID and LSS_RESTIC_S3_SECRET." + fi + [[ $(opt s3_region) =~ ^[a-z0-9-]*$ ]] || die "LSS_RESTIC_S3_REGION must look like eu-central-1." + return +} + +mod_install() { + if os_is_rhel; then + pkg_enable_epel + fi + pkg_install restic + return +} + +# _restic_secrets: the repository password (generated once), the S3 keys from the settings, the SSH key for sftp. +_restic_secrets() { + local value key + if ! secret_exists restic-password; then + secret_get restic-password 48 >/dev/null + log_warn "restic: generated the repository password. Keep a copy outside this server (lss secrets show restic-password); without it no backup can be restored." + files_note_change "generated the restic repository password" + fi + for key in s3_key_id:restic-s3-key-id s3_secret:restic-s3-secret; do + value=$(opt "${key%%:*}") + [[ -n ${value} ]] || continue + log_mask "${value}" + if ! secret_exists "${key#*:}" || [[ $(secret_get "${key#*:}") != "${value}" ]]; then + secret_set "${key#*:}" "${value}" + files_note_change "stored ${key#*:}" + fi + done + if [[ $(_restic_kind) == sftp && ! -s $(secret_path restic-ssh-key) ]]; then + ssh-keygen -q -t ed25519 -N '' -C "lss-restic@$(hostname)" -f "$(secret_path restic-ssh-key)" + mv -f -- "$(secret_path restic-ssh-key).pub" "${BACKUP_PUBLIC_KEY}" + chmod 0644 "${BACKUP_PUBLIC_KEY}" + files_note_change "generated the SSH key for the sftp repository" + fi + if [[ -s ${BACKUP_PUBLIC_KEY} ]]; then + log_info "restic: the sftp server must accept this key for the repository user: $(<"${BACKUP_PUBLIC_KEY}")" + fi + return +} + +_restic_conf() { + printf '# Managed by lss (module restic). Change LSS_RESTIC_* in /etc/lss/lss.conf and run lss configure restic.\n' + printf 'BACKUP_REPOSITORY=%s\nBACKUP_PATHS=%s\nBACKUP_EXCLUDE=%s\n' "$(opt repository)" "$(opt paths)" "$(opt exclude)" + printf 'BACKUP_KEEP_DAILY=%s\nBACKUP_KEEP_WEEKLY=%s\nBACKUP_KEEP_MONTHLY=%s\n' "$(opt_int keep_daily)" \ + "$(opt_int keep_weekly)" "$(opt_int keep_monthly)" + printf 'BACKUP_CHECK_SUBSET=%s\nBACKUP_S3_REGION=%s\n' "$(opt check_subset)" "$(opt s3_region)" + return +} + +mod_configure() { + local repository writable='' + repository=$(opt repository) + _restic_secrets + install -d -m 0700 "${BACKUP_STATE}" "${BACKUP_CACHE}" + if [[ $(_restic_kind) == local ]]; then + install -d -m 0700 "${repository}" + writable=${repository} + fi + file_write 0755 root:root "${BACKUP_SCRIPT}" <"${LSS_ROOT}/libexec/lss-backup" + _restic_conf | file_write 0600 root:root "${BACKUP_CONF}" + # Reads everything, writes only its state, its cache, the dumps, the metrics and a local repository. + svc_unit "${BACKUP_UNIT}.service" < <( + printf '# Managed by lss (module restic).\n[Unit]\nDescription=lss backup with restic\n' + printf 'After=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\n' + printf 'ExecStart=%s run\nNice=10\nIOSchedulingClass=idle\n' "${BACKUP_SCRIPT}" + svc_scanner_sandbox strict "${BACKUP_STATE}" "${BACKUP_CACHE}" /var/backups/lss-db "${MON_TEXTFILE_DIR}" \ + ${writable:+"${writable}"} + ) + svc_unit "${BACKUP_UNIT}.timer" </dev/null 2>&1 || + die "restic: the repository $(opt repository) does not open with the password from the lss secrets." + # Before the first scheduled backup a small probe proves that a backup goes in and comes back out. + if [[ ! -s ${BACKUP_STATE}/last-success && ! -s ${BACKUP_STATE}/probe ]]; then + "${BACKUP_SCRIPT}" probe || die "restic: the probe backup and its restore test fail." + date +%s >"${BACKUP_STATE}/probe" + fi + return +} + +mod_remove() { + svc_disable "${BACKUP_UNIT}.timer" + file_remove "/etc/systemd/system/${BACKUP_UNIT}.timer" + file_remove "/etc/systemd/system/${BACKUP_UNIT}.service" + svc_daemon_reload + file_remove "${BACKUP_SCRIPT}" + file_remove "${BACKUP_CONF}" + file_remove "${BACKUP_PUBLIC_KEY}" + rm -f -- "${MON_TEXTFILE_DIR}/lss_backup.prom" + pkg_remove restic + # The repository holds the backups and is never deleted by lss, not even with --purge. + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + rm -rf -- "${BACKUP_CACHE}" + mon_delete "${BACKUP_STATE}" + log_info "restic: the repository $(opt repository) and the password (lss secrets show restic-password) stay; delete them yourself when the backups are no longer needed." + return +} diff --git a/tests/unit/backup.bats b/tests/unit/backup.bats new file mode 100644 index 0000000..40c81ca --- /dev/null +++ b/tests/unit/backup.bats @@ -0,0 +1,299 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# The backup scripts (libexec/lss-backup, libexec/lss-db-dumps) against a restic stub, and the checks of the restic +# module. + +setup() { + load helpers + lss_test_setup + use_os debian-13 + export TMPDIR=${BATS_TEST_TMPDIR} + export LSS_BACKUP_CONF=${BATS_TEST_TMPDIR}/backup.conf + export LSS_BACKUP_STATE=${BATS_TEST_TMPDIR}/restic-state + export LSS_BACKUP_CACHE=${BATS_TEST_TMPDIR}/cache + export LSS_TEXTFILE_DIR=${BATS_TEST_TMPDIR}/textfile + export LSS_DB_DUMPS=${BATS_TEST_TMPDIR}/db-dumps + export LSS_DB_DUMPS_CONF=${BATS_TEST_TMPDIR}/db-dumps.conf + export LSS_DB_DUMPS_STATE=${BATS_TEST_TMPDIR}/db-dumps-state + export LSS_RESTIC=restic_stub + mkdir -p "${LSS_TEXTFILE_DIR}" "${LSS_SECRETS_DIR}" + printf 'BACKUP_REPOSITORY=/srv/repo\nBACKUP_PATHS=/etc /does-not-exist\nBACKUP_EXCLUDE=*.tmp\n' >"${LSS_BACKUP_CONF}" + printf 'password' >"${LSS_SECRETS_DIR}/restic-password" + RESTORED='' +} + +# restic stub: records the call (without --quiet), fails the subcommand named in $RESTIC_FAIL with $RESTIC_CODE, and +# "restore" writes $RESTORED as the canary below --target. +restic_stub() { + local args=" $* " target='' previous='' arg line + line=${args/ --quiet / } + printf 'restic%s\n' "${line% }" >>"${STUB_LOG}" + for arg in "$@"; do + [[ ${previous} != --target ]] || target=${arg} + previous=${arg} + done + if [[ -n ${RESTIC_FAIL:-} && ${args} == *" ${RESTIC_FAIL} "* ]]; then + return "${RESTIC_CODE:-1}" + fi + if [[ ${args} == *' restore '* && -n ${RESTORED} ]]; then + mkdir -p "${target}${LSS_BACKUP_STATE}/canary" + printf '%s\n' "${RESTORED}" >"${target}${LSS_BACKUP_STATE}/canary/canary" + fi + return 0 +} + +load_backup() { + # shellcheck source=/dev/null + source "${REPO_ROOT}/libexec/lss-backup" + export -f restic_stub + return +} + +@test "sftp repositories give restic an ssh command with the key and the pinned host" { + load_backup + [ "$(sftp_target sftp:backup@nas.example.com:/srv/restic)" = backup@nas.example.com ] + [ "$(sftp_target sftp://backup@nas.example.com:2222//srv/restic)" = 'backup@nas.example.com -p 2222' ] + run sftp_target 'sftp:;rm -rf /:/x' + [ "${status}" -eq 1 ] + printf 'BACKUP_REPOSITORY=sftp:backup@nas.example.com:/srv/restic\n' >"${LSS_BACKUP_CONF}" + prepare + [ "${RESTIC_REPOSITORY}" = sftp:backup@nas.example.com:/srv/restic ] + [ "${RESTIC_PASSWORD_FILE}" = "${LSS_SECRETS_DIR}/restic-password" ] + [[ ${RESTIC_OPTIONS[2]} == "sftp.command=ssh -i ${LSS_SECRETS_DIR}/restic-ssh-key "*"-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=${LSS_BACKUP_STATE}/known_hosts backup@nas.example.com -s sftp" ]] +} + +@test "s3 repositories take their keys from the secrets, through the environment" { + load_backup + printf 'BACKUP_REPOSITORY=s3:https://s3.example.com/bucket/host\nBACKUP_S3_REGION=eu-central-1\n' >"${LSS_BACKUP_CONF}" + printf 'KEYID' >"${LSS_SECRETS_DIR}/restic-s3-key-id" + printf 'SECRET' >"${LSS_SECRETS_DIR}/restic-s3-secret" + prepare + [ "${AWS_ACCESS_KEY_ID}" = KEYID ] + [ "${AWS_SECRET_ACCESS_KEY}" = SECRET ] + [ "${AWS_DEFAULT_REGION}" = eu-central-1 ] + [ "${RESTIC_OPTIONS[*]}" = --quiet ] + printf 'BACKUP_REPOSITORY=\n' >"${LSS_BACKUP_CONF}" + run prepare + [ "${status}" -eq 1 ] + [[ ${output} == *"BACKUP_REPOSITORY is empty"* ]] +} + +@test "a run dumps the databases, backs up the existing paths, applies the retention, checks and records success" { + load_backup + printf '#!/bin/sh\necho dumps >>"%s"\n' "${STUB_LOG}" >"${LSS_DB_DUMPS}" + chmod +x "${LSS_DB_DUMPS}" + : >"${LSS_DB_DUMPS_CONF}" + cmd_run + grep -qxF 'dumps' "${STUB_LOG}" + grep -qxF 'restic cat config' "${STUB_LOG}" + grep -qxF "restic backup --tag lss --exclude-caches --exclude *.tmp /etc ${LSS_BACKUP_STATE}/canary" "${STUB_LOG}" + grep -qxF 'restic forget --tag lss --group-by host,tags --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6' "${STUB_LOG}" + grep -qxF 'restic check' "${STUB_LOG}" + [ "$(cat "${LSS_BACKUP_STATE}/last-run-success")" = 1 ] + [ "$(cat "${LSS_BACKUP_STATE}/canary-last")" = "$(cat "${LSS_BACKUP_STATE}/canary/canary")" ] + grep -q '^lss_backup_last_success_timestamp_seconds [0-9]*$' "${LSS_TEXTFILE_DIR}/lss_backup.prom" + grep -qxF 'lss_backup_last_run_success 1' "${LSS_TEXTFILE_DIR}/lss_backup.prom" +} + +@test "a repository that does not exist yet is created" { + load_backup + RESTIC_FAIL='cat' cmd_init + grep -qxF 'restic init' "${STUB_LOG}" +} + +@test "a backup with unreadable files or a failed check counts as failed and keeps the last success" { + load_backup + RESTIC_FAIL=backup RESTIC_CODE=3 run cmd_run + [ "${status}" -eq 1 ] + [[ ${output} == *"restic could not read some files"* ]] + [ "$(cat "${LSS_BACKUP_STATE}/last-run-success")" = 0 ] + [ ! -e "${LSS_BACKUP_STATE}/last-success" ] + [ -s "${LSS_BACKUP_STATE}/canary-last" ] + RESTIC_FAIL=backup RESTIC_CODE=1 run cmd_run + [[ ${output} == *"restic backup failed (exit code 1)."* ]] + RESTIC_FAIL=check run cmd_run + [[ ${output} == *"restic check reports errors in the repository."* ]] + RESTIC_FAIL=forget run cmd_run + [[ ${output} == *"restic forget and prune failed."* ]] + grep -qxF 'lss_backup_last_run_success 0' "${LSS_TEXTFILE_DIR}/lss_backup.prom" +} + +@test "failed database dumps fail the run but the files are still backed up" { + load_backup + printf '#!/bin/sh\nexit 1\n' >"${LSS_DB_DUMPS}" + chmod +x "${LSS_DB_DUMPS}" + : >"${LSS_DB_DUMPS_CONF}" + run cmd_run + [ "${status}" -eq 1 ] + [[ ${output} == *"the database dumps failed; the backup goes on with the files."* ]] + grep -q '^restic backup ' "${STUB_LOG}" +} + +@test "the restore test compares the canary of the latest snapshot and reads part of the data" { + load_backup + printf 'abc 123\n' >"${BATS_TEST_TMPDIR}/expected" + mkdir -p "${LSS_BACKUP_STATE}" + printf 'abc 123\n' >"${LSS_BACKUP_STATE}/canary-last" + RESTORED='abc 123' + cmd_restore_test + grep -q "^restic restore latest --host .* --tag lss --target .* --include ${LSS_BACKUP_STATE}/canary/canary$" "${STUB_LOG}" + grep -qxF 'restic check --read-data-subset=5%' "${STUB_LOG}" + [ "$(cat "${LSS_BACKUP_STATE}/restore-test-success")" = 1 ] + grep -qxF 'lss_backup_restore_test_success 1' "${LSS_TEXTFILE_DIR}/lss_backup.prom" + RESTORED='something else' + run cmd_restore_test + [ "${status}" -eq 1 ] + [[ ${output} == *"the restored canary file differs from the one backed up last."* ]] + [ "$(cat "${LSS_BACKUP_STATE}/restore-test-success")" = 0 ] + RESTORED='abc 123' + RESTIC_FAIL=restore run cmd_restore_test + [[ ${output} == *"restic restore of the latest snapshot failed."* ]] + RESTIC_FAIL=check run cmd_restore_test + [[ ${output} == *"restic check --read-data-subset=5% reports errors."* ]] +} + +@test "the probe backs up the canary alone, restores it and keeps one probe snapshot" { + load_backup + # shellcheck disable=SC2329 # called by the functions under test + canary() { + mkdir -p "${CANARY_DIR}" + printf 'probe 1\n' >"${CANARY}" + printf 'probe 1' + } + RESTORED='probe 1' + run cmd_probe + [ "${status}" -eq 0 ] + grep -qxF "restic backup --tag lss-probe ${LSS_BACKUP_STATE}/canary" "${STUB_LOG}" + grep -q '^restic restore latest .* --tag lss-probe ' "${STUB_LOG}" + grep -qxF 'restic forget --tag lss-probe --keep-last 1' "${STUB_LOG}" +} + +@test "the backup script needs root and a known command" { + load_backup + run main + if [[ ${EUID} -ne 0 ]]; then + [ "${status}" -eq 1 ] + [[ ${output} == *"must run as root."* ]] + fi + run dispatch unknown + [ "${status}" -eq 2 ] + [[ ${output} == *"usage: backup run|init|restore-test|probe|restic ARGS"* ]] +} + +# --- db-dumps --------------------------------------------------------------------------------------------------- + +load_dumps() { + # shellcheck source=/dev/null + source "${REPO_ROOT}/libexec/lss-db-dumps" + CONF=${BATS_TEST_TMPDIR}/db-dumps.conf + printf 'DB_DUMPS_DIR=%s/dumps\nDB_POSTGRESQL_PORT=5433\n' "${BATS_TEST_TMPDIR}" >"${CONF}" + return +} + +@test "only the engines lss installed are dumped, each to a file of its own" { + load_dumps + state_set mariadb status installed + state_set postgresql status installed + state_set redis status failed + # shellcheck disable=SC2329 # called by the functions under test + dump() { + printf 'dump %s\n' "$1" >>"${STUB_LOG}" + printf 'DATA\n' >"$2" + } + run_dumps + grep -qxF 'dump mariadb' "${STUB_LOG}" + grep -qxF 'dump postgresql' "${STUB_LOG}" + run ! grep -q 'dump redis' "${STUB_LOG}" + [ -s "${BATS_TEST_TMPDIR}/dumps/mariadb.sql" ] + [ -s "${BATS_TEST_TMPDIR}/dumps/postgresql.sql" ] + grep -q '^mariadb ok [0-9]*$' "${LSS_DB_DUMPS_STATE}/status" + [ "$(stat -c '%a' "${BATS_TEST_TMPDIR}/dumps")" = 700 ] +} + +@test "a failed dump keeps the previous file and fails the run" { + load_dumps + state_set postgresql status installed + mkdir -p "${BATS_TEST_TMPDIR}/dumps" + printf 'OLD\n' >"${BATS_TEST_TMPDIR}/dumps/postgresql.sql" + # shellcheck disable=SC2329 # called by the functions under test + dump() { + printf 'PART' >"$2" + return 1 + } + run run_dumps + [ "${status}" -eq 1 ] + [[ ${output} == *"the dump of postgresql failed; ${BATS_TEST_TMPDIR}/dumps/postgresql.sql keeps the previous one."* ]] + [ "$(cat "${BATS_TEST_TMPDIR}/dumps/postgresql.sql")" = OLD ] + [ ! -e "${BATS_TEST_TMPDIR}/dumps/postgresql.sql.part" ] + grep -q '^postgresql failed ' "${LSS_DB_DUMPS_STATE}/status" +} + +@test "without databases nothing is dumped" { + load_dumps + run run_dumps + [ "${status}" -eq 0 ] + [[ ${output} == *"lss installed 0 databases here; nothing to dump."* ]] +} + +@test "each engine is dumped with its own tool and options" { + load_dumps + # shellcheck disable=SC2329 # called by the functions under test + first_program() { + printf '%s' "$1" + } + # shellcheck disable=SC2329 # called by the functions under test + mariadb-dump() { + printf 'mariadb-dump %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by the functions under test + runuser() { + printf 'runuser %s\n' "$*" >>"${STUB_LOG}" + } + dump mariadb "${BATS_TEST_TMPDIR}/m.sql" + grep -qxF 'mariadb-dump --user=root --all-databases --single-transaction --routines --events --triggers --hex-blob' "${STUB_LOG}" + dump postgresql "${BATS_TEST_TMPDIR}/p.sql" + grep -qxF 'runuser -u postgres -- pg_dumpall -p 5433 --clean --if-exists' "${STUB_LOG}" + [ "$(file_of mongodb)" = mongodb.archive ] + [ "$(file_of valkey)" = valkey.rdb ] + [ "$(file_of mysql)" = mysql.sql ] + run dump other "${BATS_TEST_TMPDIR}/o" + [ "${status}" -eq 1 ] +} + +@test "Redis and Valkey are dumped once a fresh snapshot is saved, with the password in the environment" { + load_dumps + ETC_DIR=${BATS_TEST_TMPDIR}/etc + DATA_ROOT=${BATS_TEST_TMPDIR}/lib + mkdir -p "${ETC_DIR}" "${DATA_ROOT}/valkey" + printf 'pw\n' >"${ETC_DIR}/valkey-password" + printf 'RDB\n' >"${DATA_ROOT}/valkey/dump.rdb" + # shellcheck disable=SC2329 # called by the functions under test + first_program() { + printf 'cli_stub' + } + SAVES=0 + # shellcheck disable=SC2329 # called by the functions under test + cli_stub() { + printf 'cli %s auth=%s\n' "$*" "${VALKEYCLI_AUTH:-}" >>"${STUB_LOG}" + case ${*: -1} in + lastsave) printf '%s\n' "$((1000 + SAVES))" ;; + bgsave) SAVES=${TEST_SAVES:-1} ;; + *) ;; + esac + } + # shellcheck disable=SC2329 # called by the functions under test + sleep() { + : + } + dump_rdb valkey "${BATS_TEST_TMPDIR}/v.rdb" + grep -qxF 'cli -p 6379 --no-auth-warning bgsave auth=pw' "${STUB_LOG}" + [ "$(cat "${BATS_TEST_TMPDIR}/v.rdb")" = RDB ] + SAVES=0 + TEST_SAVES=0 + run dump_rdb valkey "${BATS_TEST_TMPDIR}/w.rdb" + [ "${status}" -eq 1 ] + [ ! -e "${BATS_TEST_TMPDIR}/w.rdb" ] +} From 4864b91bcc835452305166272695cec6ada5d98c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:09:07 +0200 Subject: [PATCH 05/32] feat(guard): backup age, weekly restore test and Prometheus metrics With restic installed, guard reports a last successful backup older than LSS_GUARD_BACKUP_MAX_AGE hours and runs the restore test once a week. Its findings go to the node exporter's textfile directory, where an alert rule picks them up. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- libexec/lss-guard | 70 ++++++++++++++++++++++++++++++++++++++- modules/security/guard.sh | 4 ++- tests/unit/guard.bats | 69 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/libexec/lss-guard b/libexec/lss-guard index 274e055..3a8f825 100755 --- a/libexec/lss-guard +++ b/libexec/lss-guard @@ -10,7 +10,8 @@ # when GUARD_AUTOFIX=true), failed systemd units, disk space, pending security updates, vulnerable packages (debsecan # on Debian, dnf updateinfo on the RHEL family), failed SSH logins and active bans, the results of the daily AIDE, # rkhunter and ClamAV runs, the Lynis hardening index, certificates that expire soon, HIGH and CRITICAL vulnerabilities -# in the container images of Docker and Podman (Trivy). +# in the container images of Docker and Podman (Trivy), the age of the last backup and a weekly restore test (restic). +# The numbers of findings, fixes and notes also go to the node exporter for Prometheus (lss_guard_*). # Every finding goes to the journal (identifier lss-guard); a summary goes to the webhook and the e-mail address. # Notes (logins, bans) are part of the summary but do not make the run fail. set -Eeuo pipefail @@ -24,6 +25,13 @@ TLS_DIR=${LSS_TLS_DIR:-/etc/lss/tls} # Installed by the docker and podman modules (lib/container.sh). TRIVY=${LSS_GUARD_TRIVY:-/usr/local/bin/trivy} TRIVY_CACHE=${LSS_GUARD_TRIVY_CACHE:-/var/cache/lss-trivy} +# Installed by the restic module (libexec/lss-backup), with its state next to it. +BACKUP=${LSS_GUARD_BACKUP:-/usr/local/lib/lss/backup} +BACKUP_STATE=${LSS_BACKUP_STATE:-${STATE_DIR}/restic} +# The node exporter's textfile collector (lib/monitoring.sh); the guard's metrics go there when it exists. +TEXTFILE_DIR=${LSS_TEXTFILE_DIR:-/var/lib/prometheus/node-exporter} +# A restore test every week, a little earlier so a daily run at the same time does not slip to the next day. +RESTORE_TEST_INTERVAL=$((7 * 86400 - 3600)) # Global also when the script is sourced inside a function (the unit tests). declare -ga FINDINGS=() FIXED=() NOTES=() @@ -326,6 +334,64 @@ check_containers() { return } +# backup_stamp NAME: a time or result the backup script recorded (libexec/lss-backup); nothing when it never did. +backup_stamp() { + local file=${BACKUP_STATE}/$1 value='' + [[ -f ${file} ]] && value=$(<"${file}") + [[ ${value} =~ ^[0-9]+$ ]] && printf '%s' "${value}" + return 0 +} + +# The restic backup (when the restic module is installed): the last successful backup must be younger than +# GUARD_BACKUP_MAX_AGE hours (counted from the installation until the first one), and once a week a restore test +# proves that the latest backup comes back. +check_backup() { + local max now last installed age tested + [[ -x ${BACKUP} ]] || return 0 + max=$(setting GUARD_BACKUP_MAX_AGE 30) + now=$(date +%s) + last=$(backup_stamp last-success) + if [[ -z ${last} ]]; then + installed=$(sed -n 's/^installed_at=//p' "${STATE_DIR}/modules/restic" 2>/dev/null || true) + last=$(date -d "${installed:-now}" +%s 2>/dev/null || printf '%s' "${now}") + fi + age=$(((now - last) / 3600)) + if ((age >= max)); then + finding "The last successful backup is ${age} hours old, more than ${max} (journalctl -u lss-backup)." + fi + [[ -n $(backup_stamp last-success) ]] || return 0 + tested=$(backup_stamp restore-test) + if [[ -z ${tested} ]] || ((now - tested >= RESTORE_TEST_INTERVAL)); then + if "${BACKUP}" restore-test >/dev/null 2>&1; then + logger -t lss-guard -p auth.info -- "Backup restore test passed" + else + finding "The weekly restore test of the backup failed (${BACKUP} restore-test shows why)." + fi + elif [[ $(backup_stamp restore-test-success) == 0 ]]; then + finding "The last restore test of the backup failed (${BACKUP} restore-test shows why)." + fi + return +} + +# export_metrics: the result of this run for Prometheus, through the node exporter's textfile collector. +export_metrics() { + local file=${TEXTFILE_DIR}/lss_guard.prom + [[ -d ${TEXTFILE_DIR} ]] || return 0 + { + printf '# HELP lss_guard_findings Findings of the last lss guard run.\n# TYPE lss_guard_findings gauge\n' + printf 'lss_guard_findings %s\n' "${#FINDINGS[@]}" + printf '# HELP lss_guard_fixed Modules the last lss guard run restored.\n# TYPE lss_guard_fixed gauge\n' + printf 'lss_guard_fixed %s\n' "${#FIXED[@]}" + printf '# HELP lss_guard_notes Notes of the last lss guard run.\n# TYPE lss_guard_notes gauge\n' + printf 'lss_guard_notes %s\n' "${#NOTES[@]}" + printf '# HELP lss_guard_last_run_timestamp_seconds End of the last lss guard run.\n' + printf '# TYPE lss_guard_last_run_timestamp_seconds gauge\nlss_guard_last_run_timestamp_seconds %s\n' "$(date +%s)" + } >"${file}.tmp" + chmod 0644 "${file}.tmp" + mv -f -- "${file}.tmp" "${file}" + return +} + json_escape() { local text=$1 text=${text//\\/\\\\} @@ -375,6 +441,8 @@ main() { check_lynis check_certificates check_containers + check_backup + export_metrics local summary host host=$(hostname -f 2>/dev/null || hostname) if [[ ${#FINDINGS[@]} -eq 0 && ${#FIXED[@]} -eq 0 ]]; then diff --git a/modules/security/guard.sh b/modules/security/guard.sh index 307c78f..c4bcb81 100644 --- a/modules/security/guard.sh +++ b/modules/security/guard.sh @@ -13,6 +13,7 @@ webhook||Send findings to this HTTPS webhook (Slack, Teams, Mattermost, Discord disk_threshold|90|Report file systems that are fuller than this (percent) lynis_min|0|Report a Lynis hardening index below this (needs the lynis module) login_threshold|50|Report more failed SSH logins than this within 24 hours +backup_max_age|30|Report a last successful backup older than this (hours; needs the restic module, which the guard also restore-tests weekly) daily_summary|false|Also send a message when everything is fine" GUARD_SCRIPT=/usr/local/lib/lss/guard @@ -28,6 +29,7 @@ mod_check() { (($(opt_int disk_threshold) <= 100)) || die "LSS_GUARD_DISK_THRESHOLD must be at most 100." opt_int lynis_min >/dev/null opt_int login_threshold >/dev/null + (($(opt_int backup_max_age) >= 1)) || die "LSS_GUARD_BACKUP_MAX_AGE must be at least 1." email=$(opt email) webhook=$(opt webhook) [[ -z ${email} || ${email} =~ ^[^@[:space:]]+@[^@[:space:]]+\.[^@[:space:]]+$ ]] || die "LSS_GUARD_EMAIL '${email}' is not an e-mail address." @@ -59,7 +61,7 @@ mod_configure() { printf 'GUARD_DAILY_SUMMARY=%s\n' "${bool}" printf 'GUARD_EMAIL=%s\nGUARD_WEBHOOK=%s\n' "$(opt email)" "$(opt webhook)" printf 'GUARD_DISK_THRESHOLD=%s\nGUARD_LYNIS_MIN=%s\n' "$(opt_int disk_threshold)" "$(opt_int lynis_min)" - printf 'GUARD_LOGIN_THRESHOLD=%s\n' "$(opt_int login_threshold)" + printf 'GUARD_LOGIN_THRESHOLD=%s\nGUARD_BACKUP_MAX_AGE=%s\n' "$(opt_int login_threshold)" "$(opt_int backup_max_age)" ) svc_unit "${GUARD_UNIT}.service" <>"%s"\nexit ${BACKUP_EXIT:-0}\n' "${STUB_LOG}" >"${BACKUP}" + chmod +x "${BACKUP}" + return +} + +@test "without the restic module the backup is not checked" { + BACKUP=${BATS_TEST_TMPDIR}/missing + check_backup + [ "${#FINDINGS[@]}" -eq 0 ] +} + +@test "a backup older than GUARD_BACKUP_MAX_AGE is a finding, also when none ever succeeded" { + fake_backup + printf 'GUARD_BACKUP_MAX_AGE=24\n' >"${LSS_GUARD_CONF}" + date -d '-30 hours' +%s >"${BACKUP_STATE}/last-success" + date +%s >"${BACKUP_STATE}/restore-test" + check_backup + [ "${#FINDINGS[@]}" -eq 1 ] + [[ ${FINDINGS[0]} == "The last successful backup is 30 hours old, more than 24 (journalctl -u lss-backup)." ]] + FINDINGS=() + rm -f "${BACKUP_STATE}/last-success" + mkdir -p "${STATE_DIR}/modules" + printf 'status=installed\ninstalled_at=%s\n' "$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ)" >"${STATE_DIR}/modules/restic" + check_backup + [[ ${FINDINGS[0]} == "The last successful backup is 48 hours old, more than 24 (journalctl -u lss-backup)." ]] + run ! grep -q restore-test "${STUB_LOG}" +} + +@test "the restore test runs weekly; a failed one is a finding until the next one passes" { + fake_backup + date +%s >"${BACKUP_STATE}/last-success" + check_backup + grep -qxF 'backup restore-test' "${STUB_LOG}" + [ "${#FINDINGS[@]}" -eq 0 ] + : >"${STUB_LOG}" + date -d '-2 days' +%s >"${BACKUP_STATE}/restore-test" + printf '0\n' >"${BACKUP_STATE}/restore-test-success" + check_backup + run ! grep -q restore-test "${STUB_LOG}" + [[ ${FINDINGS[0]} == "The last restore test of the backup failed (${BACKUP} restore-test shows why)." ]] + FINDINGS=() + date -d '-8 days' +%s >"${BACKUP_STATE}/restore-test" + BACKUP_EXIT=1 check_backup + [[ ${FINDINGS[0]} == "The weekly restore test of the backup failed (${BACKUP} restore-test shows why)." ]] +} + +@test "the result of the run goes to the node exporter when it is installed" { + TEXTFILE_DIR=${BATS_TEST_TMPDIR}/textfile + export_metrics + [ ! -e "${TEXTFILE_DIR}" ] + mkdir -p "${TEXTFILE_DIR}" + FINDINGS=(one two) + FIXED=(ssh-hardening) + NOTES=() + export_metrics + grep -qxF 'lss_guard_findings 2' "${TEXTFILE_DIR}/lss_guard.prom" + grep -qxF 'lss_guard_fixed 1' "${TEXTFILE_DIR}/lss_guard.prom" + grep -qxF 'lss_guard_notes 0' "${TEXTFILE_DIR}/lss_guard.prom" + grep -q '^lss_guard_last_run_timestamp_seconds [0-9]*$' "${TEXTFILE_DIR}/lss_guard.prom" + [ "$(stat -c '%a' "${TEXTFILE_DIR}/lss_guard.prom")" = 644 ] +} From 37cbfc631472534be6b68b65e39c7f3643515c73 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:09:41 +0200 Subject: [PATCH 06/32] feat(profiles): devops profile; webhosting and all get the new modules devops is minimal-secure, Docker and the monitoring stack. A comment marks where Forgejo joins once its module exists. The integration matrix tests the backup, monitoring and network modules and the devops profile; the haproxy test port moves off 9100, which the node exporter takes. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- .github/workflows/integration.yml | 23 ++++++++++++++++++++--- profiles/all.conf | 2 +- profiles/devops.conf | 10 ++++++++++ profiles/webhosting.conf | 6 +++--- tests/integration/settings.conf | 12 +++++++++++- 5 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 profiles/devops.conf diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5bde302..706bf2e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] - group: [base, security, scanners, web, runtime, database, database-alt, container] + group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network] arch: [x64, arm64] include: - group: base @@ -54,6 +54,18 @@ jobs: # k3s need a kernel of their own (bridges, iptables, cgroups) and run in the VM jobs below. - group: container modules: podman + # restic alone (local repository), then the database dumps of every engine; the conflicting engines in a job + # of their own, as above. + - group: backup + modules: restic mariadb,postgresql,redis,restic,db-dumps + - group: backup-alt + modules: mysql,valkey,mongodb,restic,db-dumps + # The Prometheus stack, Grafana below the admin port of nginx, logs from the journal to Loki, Netdata on the + # WireGuard address (LSS_NETDATA_ACCESS=vpn) and smartd without drives. + - group: monitoring + modules: node-exporter prometheus,alertmanager,grafana nginx,grafana loki,alloy wireguard,netdata smartd + - group: network + modules: wireguard dns-resolver ntp-server steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -85,7 +97,7 @@ jobs: with: persist-credentials: false - name: Install, verify, reinstall, remove - run: tests/integration/run.sh host time swap firewall kernel-hardening mac auditd usbguard + run: tests/integration/run.sh host time swap firewall kernel-hardening mac auditd usbguard wireguard profile: name: ${{ matrix.runner }} VM / profile minimal-secure @@ -146,6 +158,11 @@ jobs: run: profile k8s-node - system: almalinux-10 run: profile k8s-node + # Docker with the monitoring stack, as on a DevOps server. + - system: debian-13 + run: profile devops + - system: almalinux-10 + run: profile devops steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -172,7 +189,7 @@ jobs: RUN: ${{ matrix.run }} run: | if [[ $RUN == kernel ]]; then - tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard + tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard wireguard elif [[ $RUN == containers ]]; then tests/integration/run.sh "$SYSTEM" firewall,docker,portainer firewall,k3s else diff --git a/profiles/all.conf b/profiles/all.conf index 3a84e6d..a6017fe 100644 --- a/profiles/all.conf +++ b/profiles/all.conf @@ -7,4 +7,4 @@ # For the nightly test and as a reference. ssh-2fa, clamav and usbguard stay opt-in as in minimal-secure. k3s # conflicts with docker (both bring their own containerd and claim the forwarding rules) and has its own profile, # k8s-node; podman is the alternative to docker. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server node-exporter prometheus alertmanager grafana loki alloy netdata smartd diff --git a/profiles/devops.conf b/profiles/devops.conf new file mode 100644 index 0000000..5a53f83 --- /dev/null +++ b/profiles/devops.conf @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Profile: DevOps server: minimal-secure plus Docker and the monitoring stack (Prometheus, Alertmanager, Grafana, Loki) +# +# Every interface stays on 127.0.0.1 (SSH tunnel) until LSS__ACCESS=vpn with the wireguard module or +# LSS__WEB_ADMIN=true with a web server says otherwise. netdata and restic are opt-in. A Git forge (Forgejo) +# and CI runners join this profile with their modules in a later stage. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard docker node-exporter prometheus alertmanager loki alloy grafana smartd diff --git a/profiles/webhosting.conf b/profiles/webhosting.conf index 116b30f..ac9e69b 100644 --- a/profiles/webhosting.conf +++ b/profiles/webhosting.conf @@ -4,6 +4,6 @@ # Profile: Web hosting: minimal-secure plus nginx, certificates, PHP, MariaDB, phpMyAdmin and sites # -# Sites go into LSS_VHOST_SITES, the host names of the server into LSS_TLS_DOMAINS. Backups (restic) join the profile -# with the backup modules. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mariadb phpmyadmin vhost modsecurity +# Sites go into LSS_VHOST_SITES, the host names of the server into LSS_TLS_DOMAINS. restic backs up the sites and +# the dumps of MariaDB daily; LSS_RESTIC_REPOSITORY should point to another disk or host. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mariadb phpmyadmin vhost modsecurity restic db-dumps diff --git a/tests/integration/settings.conf b/tests/integration/settings.conf index d8041aa..3c86208 100644 --- a/tests/integration/settings.conf +++ b/tests/integration/settings.conf @@ -15,7 +15,7 @@ LSS_TLS_SERVER=https://localhost:14000/dir LSS_TLS_CA_BUNDLE=/etc/lss-pebble/pebble.minica.pem LSS_TLS_DOMAINS=lss-test.example.com LSS_VHOST_SITES=demo:static:demo.example.com,www.demo.example.com app:proxy:app.example.com:http://127.0.0.1:9 -LSS_HAPROXY_SERVICES=echo:9100/local:127.0.0.1:9,127.0.0.1:10 +LSS_HAPROXY_SERVICES=echo:9190/local:127.0.0.1:9,127.0.0.1:10 # Database modules: one application user each, so verify also covers creating users, databases and their grants. # A small memory limit keeps redis and valkey from growing on the shared runner. @@ -33,3 +33,13 @@ LSS_MODSECURITY_EXCLUSIONS=wordpress nextcloud # Container modules: k3s without Traefik (fewer image pulls from Docker Hub on the shared runner). LSS_K3S_TRAEFIK=false + +# Monitoring: Prometheus, Grafana and Netdata also below the admin port when a web server is installed (nginx,grafana +# and the profile all). Netdata listens on the WireGuard address too, so its job installs wireguard first. +LSS_PROMETHEUS_WEB_ADMIN=true +LSS_GRAFANA_WEB_ADMIN=true +LSS_NETDATA_WEB_ADMIN=true +LSS_NETDATA_ACCESS=vpn + +# Network modules: one peer with a throwaway public key (any 32 bytes are a valid key; nobody holds a private half). +LSS_WIREGUARD_PEERS=ci:xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg=:10.66.0.2 From 5bf759c3eaf68436471cad151e97e0c27b023b20 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:09:47 +0200 Subject: [PATCH 07/32] docs: backup, monitoring and network in the README and changelog Module groups, addresses, settings and the guard's backup checks. The resource figures follow from the integration runs. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- CHANGELOG.md | 16 +++++++++ README.md | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff08c50..a2065b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,22 @@ now or later. - Supported systems: Debian 12 and 13, Ubuntu 22.04, 24.04 and 26.04, Rocky Linux 9, AlmaLinux 10. CI installs, verifies, reinstalls and removes every module on each of them. - OpenSSF Scorecard, signed-off commits (DCO), REUSE licensing, security policy and governance. +- Backup modules: `restic` (encrypted daily backups to a local path, SFTP or S3, retention, repository checks, the + password and keys in the lss secrets) and `db-dumps` (consistent dumps of every database engine lss installed, + taken before each backup). +- Monitoring modules: `node-exporter`, `prometheus` (with alert rules for down targets, full disks, guard findings and + old or failed backups), `alertmanager` (e-mail and webhook), `grafana`, `loki`, `alloy` (Promtail's successor; it + ships the journal to Loki), `netdata` (no cloud, registry or telemetry) and `smartd`. Every interface listens on + 127.0.0.1 or, with `access=vpn`, on the WireGuard address, and can be published below the web server's admin port. +- Network modules: `wireguard` (server with peers from the settings, keys in the lss secrets), `dns-resolver` (Unbound + with DNSSEC on localhost, optionally for the WireGuard network, used by the server itself) and `ntp-server` (chrony of + the time module serves the WireGuard network or everyone, rate-limited). +- Profile `devops` (minimal-secure, Docker and the monitoring stack); `webhosting` adds restic and db-dumps, `all` + every new module. +- guard checks the age of the last backup (`LSS_GUARD_BACKUP_MAX_AGE`), runs a restore test every week and exports + its results for Prometheus through the node exporter. +- The services of the backup, monitoring and network modules run with a systemd sandbox drop-in; a service that + listens on the WireGuard address starts after the interface. ### Removed diff --git a/README.md b/README.md index 53b5324..3a315aa 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,19 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `podman` | not measured yet | not measured yet | not measured yet | – (no daemon) | | `portainer` | not measured yet | not measured yet | not measured yet | Portainer container, up to 512 MiB | | `k3s` | not measured yet | not measured yet | not measured yet | k3s server, about 1 GiB with Traefik | +| `restic` | not measured yet | not measured yet | not measured yet | daily backup | +| `db-dumps` | not measured yet | not measured yet | not measured yet | dumps before each backup | +| `node-exporter` | not measured yet | not measured yet | not measured yet | node exporter | +| `prometheus` | not measured yet | not measured yet | not measured yet | Prometheus and its metrics on disk | +| `alertmanager` | not measured yet | not measured yet | not measured yet | Alertmanager | +| `grafana` | not measured yet | not measured yet | not measured yet | Grafana | +| `loki` | not measured yet | not measured yet | not measured yet | Loki and the logs on disk | +| `alloy` | not measured yet | not measured yet | not measured yet | Alloy | +| `netdata` | not measured yet | not measured yet | not measured yet | Netdata and its plugins | +| `smartd` | not measured yet | not measured yet | not measured yet | smartd | +| `wireguard` | not measured yet | not measured yet | not measured yet | – (kernel) | +| `dns-resolver` | not measured yet | not measured yet | not measured yet | Unbound | +| `ntp-server` | not measured yet | not measured yet | not measured yet | – (the chronyd of `time`) | The integration jobs print these figures as `RESOURCES` lines, so every new module and every new system adds its own. @@ -128,6 +141,9 @@ LSS_DOCKER_USERNS_REMAP=true LSS_PORTAINER_ACCESS=vpn LSS_K3S_API_ACCESS=vpn LSS_K3S_TRAEFIK=false +LSS_RESTIC_REPOSITORY=sftp:backup@nas.example.com:/srv/restic/server1 +LSS_WIREGUARD_PEERS=laptop::10.66.0.2 +LSS_GRAFANA_ACCESS=vpn ``` `--set` wins over the environment, which wins over the file, which wins over the profile, which wins over the @@ -162,7 +178,16 @@ first password with `lss secrets show pgadmin-admin-password`), all on the admin together with `docker`). See [Containers](#containers). The [hardening guide](docs/hardening.md) lists what each security module changes, the CIS Benchmark areas it covers -and where lss deliberately deviates. Backup, monitoring and mail follow as further module groups. +and where lss deliberately deviates. Mail follows as a further module group. + +**Backup:** `restic` (daily, encrypted backups to a local path, SFTP or S3) and `db-dumps` (dumps of the databases +before each backup). See [Backup](#backup). + +**Monitoring:** `node-exporter`, `prometheus`, `alertmanager`, `grafana`, `loki`, `alloy`, `netdata`, `smartd`. See +[Monitoring](#monitoring). + +**Network:** `wireguard` (VPN server), `dns-resolver` (Unbound with DNSSEC), `ntp-server` (chrony serves the time). +See [WireGuard, DNS and NTP](#wireguard-dns-and-ntp). ### Web sites @@ -214,6 +239,72 @@ sudo lss install --profile k8s-node --set LSS_ADMIN_USER_NAME=alice --set LSS_AD With `docker` or `podman` installed, the guard scans every image daily with Trivy and reports HIGH and CRITICAL vulnerabilities per image. +### Backup + +`restic` backs up `LSS_RESTIC_PATHS` (configuration, home directories, web roots, the database dumps) every night at +`LSS_RESTIC_TIME`, keeps 7 daily, 4 weekly and 6 monthly snapshots and checks the repository after each run. The +repository is a local path (`/var/backups/lss-restic` by default; put it on another disk), `sftp:USER@HOST:/PATH` or +`s3:https://HOST/BUCKET/PATH`. lss generates the repository password; keep a copy outside the server, since no backup +can be restored without it: + +```bash +sudo lss secrets show restic-password +``` + +For SFTP lss creates an SSH key and prints its public half, which the backup host must accept. For S3, +`LSS_RESTIC_S3_KEY_ID` and `LSS_RESTIC_S3_SECRET` go into the lss secrets. `/usr/local/lib/lss/backup restic +snapshots` runs restic with the settings of the server, and `/usr/local/lib/lss/backup restore-test` restores the +latest backup's test file and reads back 5 % of the data. + +`db-dumps` dumps every database engine lss installed (MariaDB, MySQL, PostgreSQL, MongoDB, Redis, Valkey) to +`/var/backups/lss-db` before each backup. A database added later is dumped without further setup. + +### Monitoring + +Prometheus scrapes the node exporter, itself, Alertmanager and Loki, and alerts on unreachable targets, full disks, +low memory, guard findings and backups that failed, are older than 30 hours or did not pass the restore test. +Alertmanager sends the alerts to `LSS_ALERTMANAGER_EMAIL` or `LSS_ALERTMANAGER_WEBHOOK`. Grafana starts with +Prometheus, Alertmanager and Loki as data sources; the user is `admin`, the first password `lss secrets show +grafana-admin-password`. Alloy ships the systemd journal to Loki (Promtail reached its end of life in March 2026). + +| Module | Address | `LSS__WEB_ADMIN=true` | +| --------------- | ---------------------------------------- | ----------------------------- | +| `prometheus` | `http://127.0.0.1:9090/prometheus/` | `/prometheus/` on port 8090 | +| `alertmanager` | `http://127.0.0.1:9093/alertmanager/` | `/alertmanager/` on port 8090 | +| `grafana` | `http://127.0.0.1:3000/grafana/` | `/grafana/` on port 8090 | +| `netdata` | `http://127.0.0.1:19999/` | `/netdata/` on port 8090 | +| `node-exporter` | `http://127.0.0.1:9100/metrics` | – | +| `loki` | `http://127.0.0.1:3100` | – | +| `alloy` | `http://127.0.0.1:12345` (always local) | – | + +With `LSS__ACCESS=vpn` a module listens on the server's WireGuard address instead (the first address of +`LSS_WIREGUARD_NETWORK`, 10.66.0.1 by default) and the firewall opens its port to the WireGuard network only; Netdata +keeps 127.0.0.1 as well. That needs the `wireguard` module. Debian and Ubuntu get Prometheus, Alertmanager and the +node exporter from the distribution, the RHEL family the checksum-verified upstream releases; Grafana, Loki and Alloy +come from Grafana's repository and Netdata from Netdata's, both with pinned keys. `smartd` watches the drives, also in +virtual machines, where it usually finds none. + +The profile `devops` installs `minimal-secure`, `docker` and the monitoring stack without Netdata: + +```bash +sudo lss install --profile devops --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github +``` + +### WireGuard, DNS and NTP + +`wireguard` sets up `wg0` with the server's key in the lss secrets and one preshared key per peer. Each peer is +`NAME:PUBLIC_KEY:ADDRESS` in `LSS_WIREGUARD_PEERS`; lss prints the settings a peer needs (the server's public key, +endpoint and port) after every change, and `lss secrets show wireguard-psk-NAME` the peer's preshared key. Peers reach +the server, not the networks behind it: IP forwarding stays off. + +`dns-resolver` runs Unbound on 127.0.0.1 as a validating (DNSSEC) resolver and makes the server use it, through +systemd-resolved, NetworkManager or `/etc/resolv.conf`; removing the module restores the previous resolver. +`LSS_DNS_RESOLVER_ACCESS=vpn` serves the WireGuard network too, `LSS_DNS_RESOLVER_FORWARD` forwards to other +resolvers, over TLS with `LSS_DNS_RESOLVER_FORWARD_TLS=true`. + +`ntp-server` opens chrony of the `time` module to the WireGuard network (or to everyone with +`LSS_NTP_SERVER_ACCESS=public`), with rate limiting. + ## Guard The `guard` module checks the server every day: it runs `lss verify`, repairs modules that fail or whose files were @@ -224,6 +315,10 @@ what the daily AIDE, rkhunter and ClamAV runs found, and reports more failed SSH `LSS_GUARD_LOGIN_THRESHOLD` with their main sources. Successful SSH logins and active fail2ban or CrowdSec bans are listed as notes. Findings go to a webhook (Slack, Teams, Mattermost, Discord, ntfy) or by e-mail. +With `restic` installed, the guard reports a last successful backup older than `LSS_GUARD_BACKUP_MAX_AGE` hours (30) +and runs a restore test every week. With `node-exporter` installed, the number of findings goes to Prometheus as +well, where an alert rule picks it up. + ## Documentation - [Architecture](docs/architecture.md): how a run works, where lss keeps its state From 25a4183e5ad33cb3c1be07ba491adffca5f3bb30 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:12:59 +0200 Subject: [PATCH 08/32] docs(module-guide): the helpers of lib/monitoring.sh How a module with a web interface binds, opens its ports, publishes below the admin port and registers with Prometheus and Grafana. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- docs/module-guide.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/module-guide.md b/docs/module-guide.md index 009ec8a..d0b49e2 100644 --- a/docs/module-guide.md +++ b/docs/module-guide.md @@ -102,6 +102,22 @@ All of `lib/` is loaded in every step. The ones modules use most: - `fw_trust NETWORK INTERFACE` accepts all traffic from NETWORK arriving on INTERFACE, recorded and removed like a port. Only for networks of the server itself (a pod network on its bridge). +### Monitoring and admin interfaces + +`lib/monitoring.sh` is for services with a web interface or an API, so they all bind, open and publish the same way: + +- `mon_address MODULE` is 127.0.0.1, or the server's WireGuard address (`fw_vpn_address`) with + `LSS__ACCESS=vpn`; `mon_vpn_dropin UNIT` starts the unit after `wg0` in that case, and `mon_firewall + PORT/PROTO...` opens the ports to the WireGuard network only. +- `mon_web_admin KIND URL` publishes the interface below the web server's admin port when `LSS__WEB_ADMIN` + is true (`proxy` keeps the path, `strip` removes the prefix); `mon_web_admin_remove` takes it away. +- `mon_target NAME JOB HOST:PORT [PATH]` adds a Prometheus scrape target, `mon_datasource NAME TYPE URL` a Grafana data + source; both have a `_remove` counterpart. +- `mon_upstream_install NAME VERSION PROGRAM...` installs programs from a release archive listed in + `data/checksums.txt`; `mon_install_stopped PKG...` installs Debian packages without letting them start their + service before lss configured it. +- `mon_wait_http URL [SECONDS]` and `mon_listen_check [--process NAME] PORT ADDRESS...` are for `mod_verify`. + ### Containers - `ctr_run NAME IMAGE [OPTION...] [-- COMMAND...]` runs a container with the engine of the server (docker or podman) From 0f463489214bc9aef8699756d2f4f34b0792f850 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:17:50 +0200 Subject: [PATCH 09/32] test(guard): look for the restore-test call, not the word in the finding The finding names the restore-test command, and finding also logs it, so the check matched its own message. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- tests/unit/guard.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/guard.bats b/tests/unit/guard.bats index 4e07c9a..63402f3 100644 --- a/tests/unit/guard.bats +++ b/tests/unit/guard.bats @@ -206,7 +206,7 @@ fake_backup() { printf 'status=installed\ninstalled_at=%s\n' "$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ)" >"${STATE_DIR}/modules/restic" check_backup [[ ${FINDINGS[0]} == "The last successful backup is 48 hours old, more than 24 (journalctl -u lss-backup)." ]] - run ! grep -q restore-test "${STUB_LOG}" + run ! grep -qxF "backup restore-test" "${STUB_LOG}" } @test "the restore test runs weekly; a failed one is a finding until the next one passes" { @@ -219,7 +219,7 @@ fake_backup() { date -d '-2 days' +%s >"${BACKUP_STATE}/restore-test" printf '0\n' >"${BACKUP_STATE}/restore-test-success" check_backup - run ! grep -q restore-test "${STUB_LOG}" + run ! grep -qxF "backup restore-test" "${STUB_LOG}" [[ ${FINDINGS[0]} == "The last restore test of the backup failed (${BACKUP} restore-test shows why)." ]] FINDINGS=() date -d '-8 days' +%s >"${BACKUP_STATE}/restore-test" From b4166f05fdce05e7b96236a99c283d4539aac170 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:17:52 +0200 Subject: [PATCH 10/32] ci(gitleaks): the WireGuard example public keys of the tests are no secret The network tests use the public keys of the wg(8) examples as peers. .gitleaksignore lists these two findings by fingerprint. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- .gitleaksignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .gitleaksignore diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..aab7c3e --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Findings of gitleaks that are no secrets, by fingerprint (commit:file:rule:line). +# WireGuard public keys from the wg(8) examples, used as peers in the unit tests; public keys are meant to be shared. +78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371:tests/unit/network.bats:generic-api-key:34 +78ea7af4af2a90de1e21a4a07aec3f6b0f8c5371:tests/unit/network.bats:generic-api-key:35 From 1beb39803d95c11013d13ad0e362c7566de8f91a Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:26:57 +0200 Subject: [PATCH 11/32] refactor(monitoring): one place builds the interface URLs; resolve Sonar findings mon_url builds the URL of a module's interface and states once why plain HTTP is safe there (127.0.0.1 or the encrypted WireGuard tunnel). Positional parameters go to named locals, and Loki's retention hours are computed before the heredoc. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/monitoring.sh | 26 ++++++++++++++++++++------ modules/monitoring/alertmanager.sh | 4 ++-- modules/monitoring/alloy.sh | 4 ++-- modules/monitoring/grafana.sh | 4 ++-- modules/monitoring/loki.sh | 7 +++++-- modules/monitoring/node-exporter.sh | 2 +- modules/monitoring/prometheus.sh | 4 ++-- 7 files changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/monitoring.sh b/lib/monitoring.sh index 615ba2a..7c11a24 100644 --- a/lib/monitoring.sh +++ b/lib/monitoring.sh @@ -36,16 +36,20 @@ MON_MANAGED_BY='# Managed by lss (module' MON_BIN_DIR=/usr/local/bin MON_POLICY_RC=/usr/sbin/policy-rc.d MON_CACHE_DIR=/var/cache/lss +# The interfaces speak plain HTTP; see mon_url. +MON_SCHEME=http # mon_vpn MODULE: true when MODULE listens on the WireGuard network (LSS__ACCESS=vpn). mon_vpn() { - [[ $(config_option "$1" access local) == vpn ]] + local module=$1 + [[ $(config_option "${module}" access local) == vpn ]] return } # mon_address MODULE: the address MODULE listens on, 127.0.0.1 or the WireGuard address of the server. mon_address() { - if mon_vpn "$1"; then + local module=$1 + if mon_vpn "${module}"; then fw_vpn_address else printf '127.0.0.1' @@ -53,6 +57,14 @@ mon_address() { return } +# mon_url MODULE PORT [PATH]: the URL of MODULE's interface. Plain HTTP is safe here: the interface listens on +# 127.0.0.1 or on the WireGuard address, and the WireGuard tunnel encrypts everything that leaves the server. +mon_url() { + local module=$1 port=$2 path=${3:-} + printf '%s://%s:%s%s' "${MON_SCHEME}" "$(mon_address "${module}")" "${port}" "${path}" + return +} + # mon_vpn_ready: the WireGuard address must exist before a service of the current module can listen on it. Checked in # mod_configure, because wireguard may be installed earlier in the same run. mon_vpn_ready() { @@ -146,7 +158,8 @@ EOF # mon_target_remove NAME mon_target_remove() { - file_remove "${MON_TARGETS_DIR}/$1.yml" + local name=$1 + file_remove "${MON_TARGETS_DIR}/${name}.yml" rmdir "${MON_TARGETS_DIR}" "${MON_ALERTMANAGERS_DIR}" "${MON_PROMETHEUS_DIR}" 2>/dev/null || true return } @@ -323,7 +336,8 @@ mon_delete() { # mon_http URL: the HTTP status code of URL, 000 while nothing answers. mon_http() { - curl -s --max-time 10 -o /dev/null -w '%{http_code}' "$1" 2>/dev/null || true + local url=$1 + curl -s --max-time 10 -o /dev/null -w '%{http_code}' "${url}" 2>/dev/null || true return } @@ -341,8 +355,8 @@ mon_wait_http() { # mon_listen_check [--process NAME] PORT ADDRESS...: every TCP socket listening on PORT (of the process NAME only) is on # one of the ADDRESSes. mon_listen_check() { - local process='' port listeners address - if [[ $1 == --process ]]; then + local process='' port listeners address first=$1 + if [[ ${first} == --process ]]; then process=$2 shift 2 fi diff --git a/modules/monitoring/alertmanager.sh b/modules/monitoring/alertmanager.sh index 631b911..1224237 100644 --- a/modules/monitoring/alertmanager.sh +++ b/modules/monitoring/alertmanager.sh @@ -36,7 +36,7 @@ _am_binary() { } _am_url() { - printf 'http://%s:%s/%s' "$(mon_address alertmanager)" "${MON_ALERTMANAGER_PORT}" "${AM_PATH}" + mon_url alertmanager "${MON_ALERTMANAGER_PORT}" "/${AM_PATH}" return } @@ -135,7 +135,7 @@ EOF svc_reload "${AM_UNIT}" || die "alertmanager: Alertmanager does not reload its configuration." fi mon_firewall "${MON_ALERTMANAGER_PORT}/tcp" - mon_web_admin proxy "http://$(mon_address alertmanager):${MON_ALERTMANAGER_PORT}" + mon_web_admin proxy "$(mon_url alertmanager "${MON_ALERTMANAGER_PORT}")" mon_datasource Alertmanager alertmanager "$(_am_url)" return } diff --git a/modules/monitoring/alloy.sh b/modules/monitoring/alloy.sh index f648018..779a5af 100644 --- a/modules/monitoring/alloy.sh +++ b/modules/monitoring/alloy.sh @@ -58,7 +58,7 @@ loki.source.journal "journal" { loki.write "local" { endpoint { - url = "http://$(mon_address loki):3100/loki/api/v1/push" + url = "$(mon_url loki 3100 /loki/api/v1/push)" } } EOF @@ -85,7 +85,7 @@ mod_configure() { # _alloy_probe: writes a line to the journal and waits until Loki returns it. _alloy_probe() { local token i result='' url - url="http://$(mon_address loki):3100/loki/api/v1/query_range" + url="$(mon_url loki 3100 /loki/api/v1/query_range)" token="lss-alloy-probe-$(date +%s)-${RANDOM}" logger -t lss-alloy-probe "${token}" for ((i = 0; i < 45; i++)); do diff --git a/modules/monitoring/grafana.sh b/modules/monitoring/grafana.sh index ee8634c..2d35e28 100644 --- a/modules/monitoring/grafana.sh +++ b/modules/monitoring/grafana.sh @@ -19,7 +19,7 @@ GRAFANA_PATH=grafana GRAFANA_PASSWORD_FILE=${MON_GRAFANA_DIR}/admin-password _grafana_url() { - printf 'http://%s:%s/%s' "$(mon_address grafana)" "${GRAFANA_PORT}" "${GRAFANA_PATH}" + mon_url grafana "${GRAFANA_PORT}" "/${GRAFANA_PATH}" return } @@ -78,7 +78,7 @@ mod_configure() { log_info "grafana: user admin, password from lss secrets show grafana-admin-password (set when Grafana creates its database; later changes go through Grafana)." fi mon_firewall "${GRAFANA_PORT}/tcp" - mon_web_admin proxy "http://$(mon_address grafana):${GRAFANA_PORT}" + mon_web_admin proxy "$(mon_url grafana "${GRAFANA_PORT}")" return } diff --git a/modules/monitoring/loki.sh b/modules/monitoring/loki.sh index bde9045..1efb4f7 100644 --- a/modules/monitoring/loki.sh +++ b/modules/monitoring/loki.sh @@ -19,7 +19,7 @@ LOKI_CONF=/etc/lss/loki/config.yml LOKI_DATA=/var/lib/loki _loki_url() { - printf 'http://%s:%s' "$(mon_address loki)" "${LOKI_PORT}" + mon_url loki "${LOKI_PORT}" return } @@ -34,6 +34,9 @@ mod_check() { # _loki_config: one process with every component, an in-memory ring (no gossip port), the TSDB index and the chunks on # the local disk; the compactor deletes what is older than the retention. _loki_config() { + local days hours + days=$(opt_int retention_days) + hours=$((days * 24)) cat < Date: Wed, 14 Oct 2026 16:55:58 +0200 Subject: [PATCH 12/32] fix(grafana): read the admin password through Grafana's $__file{} expansion GF_*__FILE variables are resolved by the entry script of Grafana's container image only; the packaged grafana-server ignored the variable and created the admin with the default password, so the login check failed. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/monitoring/grafana.sh | 4 +++- tests/unit/monitoring.bats | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/monitoring/grafana.sh b/modules/monitoring/grafana.sh index 2d35e28..ba113fa 100644 --- a/modules/monitoring/grafana.sh +++ b/modules/monitoring/grafana.sh @@ -45,7 +45,9 @@ Environment=GF_SERVER_ROOT_URL=$(_grafana_url)/ Environment=GF_SERVER_SERVE_FROM_SUB_PATH=true Environment=GF_PATHS_PROVISIONING=${MON_GRAFANA_DIR}/provisioning Environment=GF_SECURITY_ADMIN_USER=admin -Environment=GF_SECURITY_ADMIN_PASSWORD__FILE=${GRAFANA_PASSWORD_FILE} +# Grafana reads the password from the file itself (its \$__file{} expansion); the GF_*__FILE variables exist only in +# Grafana's container image, whose entry script resolves them. +Environment=GF_SECURITY_ADMIN_PASSWORD=\$__file{${GRAFANA_PASSWORD_FILE}} Environment=GF_SECURITY_DISABLE_GRAVATAR=true Environment=GF_USERS_ALLOW_SIGN_UP=false # The admin port of the web server may ask for its own basic auth; Grafana must not take that header for a login. diff --git a/tests/unit/monitoring.bats b/tests/unit/monitoring.bats index d8df208..443afb4 100644 --- a/tests/unit/monitoring.bats +++ b/tests/unit/monitoring.bats @@ -503,9 +503,10 @@ load_module() { [[ ${output} == *'Environment=GF_SERVER_HTTP_ADDR=127.0.0.1'* ]] [[ ${output} == *'Environment=GF_SERVER_ROOT_URL=http://127.0.0.1:3000/grafana/'* ]] [[ ${output} == *'Environment=GF_SERVER_SERVE_FROM_SUB_PATH=true'* ]] - [[ ${output} == *"Environment=GF_SECURITY_ADMIN_PASSWORD__FILE=${MON_GRAFANA_DIR}/admin-password"* ]] + [[ ${output} == *"Environment=GF_SECURITY_ADMIN_PASSWORD=\$__file{${MON_GRAFANA_DIR}/admin-password}"* ]] [[ ${output} == *'Environment=GF_ANALYTICS_REPORTING_ENABLED=false'* ]] - [[ ${output} != *GF_SECURITY_ADMIN_PASSWORD=* ]] + # The password itself never goes into the unit. + run ! grep -E '^Environment=GF_SECURITY_ADMIN_PASSWORD=[^$]' <<<"${output}" } @test "Loki keeps every component on this server with retention" { From 2125e1daadd5608507549583ea31ccf88feb836c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:56:00 +0200 Subject: [PATCH 13/32] fix(wireguard): let wg-quick run ip under SELinux NoNewPrivileges forbids the switch to ifconfig_t when wg-quick runs ip, so wg0 never came up on AlmaLinux 10 with SELinux enforcing. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/wireguard.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/network/wireguard.sh b/modules/network/wireguard.sh index 4bd3c2a..def326b 100644 --- a/modules/network/wireguard.sh +++ b/modules/network/wireguard.sh @@ -141,7 +141,9 @@ mod_configure() { install -d -m 0700 /etc/wireguard config=$(_wg_config) file_write 0600 root:root "${WG_CONF}" <<<"${config}" - svc_sandbox "${WG_UNIT}.service" + # wg-quick is a root script that runs ip and wg; under SELinux these switch to their own domains (ifconfig_t), + # which NoNewPrivileges forbids, so the interface would never come up on the RHEL family. + svc_sandbox "${WG_UNIT}.service" NoNewPrivileges=no fw_allow "$(opt_int port)/udp" svc_enable "${WG_UNIT}" if [[ $(files_changes_of wireguard) != "${before}" ]]; then From d1b44d9ce6c7043be861f325fbd33d0ba1ae5ea4 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:56:05 +0200 Subject: [PATCH 14/32] fix(ntp-server): keep the distribution's chronyd command line in containers Debian starts chronyd with '!' as root although the unit says User=_chrony; the replacement command line lost the prefix and chronyd stopped with 'Not superuser'. The drop-in now takes the vendor line and adds -x, and also lifts the unit's condition on running outside a container. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/ntp-server.sh | 21 +++++++++++++++++---- tests/unit/network.bats | 15 ++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh index d837f75..bfc53ad 100644 --- a/modules/network/ntp-server.sh +++ b/modules/network/ntp-server.sh @@ -65,20 +65,33 @@ _ntp_config() { return } +# _ntp_exec UNIT: the command line of the distribution's chronyd unit with -x added. It is taken over rather than +# written anew: Debian starts chronyd with "!" (as root although the unit says User=_chrony) and -n (Type=notify), +# the RHEL family lets it fork. +_ntp_exec() { + local unit=$1 fragment line + fragment=$(systemctl show -p FragmentPath --value "${unit}") + line=$(sed -n 's/^ExecStart=\(..*\)$/\1/p' "${fragment}" 2>/dev/null | tail -n 1) + [[ -n ${line} ]] || die "ntp-server: ${unit} has no command line in ${fragment:-its unit file}." + printf '%s -x' "${line}" + return +} + # _ntp_container_dropin: inside a container chronyd serves the time but must not set the clock it shares with the host -# (-x), and the unit's condition on CAP_SYS_TIME no longer applies. +# (-x), and the unit's conditions on CAP_SYS_TIME and on running outside a container no longer apply. _ntp_container_dropin() { - local unit options=OPTIONS + local unit exec unit="$(_ntp_unit).service" - os_is_debian && options=DAEMON_OPTS if os_in_container; then + exec=$(_ntp_exec "${unit}") || return 1 svc_dropin "${unit}" lss-ntp-server <"${unit_file}" + # shellcheck disable=SC2329 # called by the functions under test + systemctl() { + printf '%s\n' "${unit_file}" + } _ntp_container_dropin grep -qxF '== 0644 root:root /etc/systemd/system/chrony.service.d/lss-ntp-server.conf' "${WRITES}" grep -qxF 'ConditionCapability=' "${WRITES}" - grep -qxF "ExecStart=/usr/sbin/chronyd \$DAEMON_OPTS -x" "${WRITES}" + grep -qxF 'ConditionVirtualization=' "${WRITES}" + grep -qxF "ExecStart=!/usr/sbin/chronyd -n \$DAEMON_OPTS -x" "${WRITES}" use_os rocky-9 + printf '[Service]\nExecStart=/usr/sbin/chronyd $OPTIONS\n' >"${unit_file}" _ntp_container_dropin grep -qxF "ExecStart=/usr/sbin/chronyd \$OPTIONS -x" "${WRITES}" + : >"${unit_file}" + run _ntp_container_dropin + [ "${status}" -eq 1 ] + [[ ${output} == *"chronyd.service has no command line in ${unit_file}."* ]] export STUB_EXIT_systemd_detect_virt=1 _ntp_container_dropin [ "${LSS_FILE_CHANGED}" = 0 ] From 3561400b5f7f82c4852cb1b11786d75150befbdc Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:56:08 +0200 Subject: [PATCH 15/32] fix(dns-resolver): give a cold Unbound three tries for the root zone Priming the root servers and keys can take longer than one 5 s probe. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/dns-resolver.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/network/dns-resolver.sh b/modules/network/dns-resolver.sh index 6e9a0ba..d8cfa6f 100644 --- a/modules/network/dns-resolver.sh +++ b/modules/network/dns-resolver.sh @@ -348,9 +348,12 @@ mod_verify() { die "dns-resolver: Unbound does not validate DNSSEC." [[ -s ${DNS_ANCHOR} ]] || die "dns-resolver: the DNSSEC trust anchor ${DNS_ANCHOR} is missing." # The signed root zone must come back validated (AD). Without a reachable resolver upstream only the offline checks - # above apply. - reply=$(_dns_query "${DNS_LOOPBACK}" . 6) - rcode=$(_dns_reply_rcode "${reply}" || true) + # above apply. A cold Unbound first fetches the root servers and keys, so it gets three tries. + for _ in 1 2 3; do + reply=$(_dns_query "${DNS_LOOPBACK}" . 6) + rcode=$(_dns_reply_rcode "${reply}" || true) + [[ -z ${rcode} ]] || break + done if [[ ${rcode} == 0 ]]; then _dns_reply_ad "${reply}" || die "dns-resolver: the answer for the root zone is not DNSSEC-validated." else From 53c5ffe13c18dc5c741807df625efb1443077de7 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 16:56:11 +0200 Subject: [PATCH 16/32] fix(pkg): drop adduser's note on skipped home directories from the apt output Packages that add a system user (grafana) print 'Not creating home directory', a line that starts with Not. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/pkg.sh | 5 +++-- tests/unit/system.bats | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pkg.sh b/lib/pkg.sh index 23e9d73..b399070 100644 --- a/lib/pkg.sh +++ b/lib/pkg.sh @@ -10,11 +10,12 @@ LSS_PKG_UPDATED=${LSS_PKG_UPDATED:-0} # DPkg::Lock::Timeout waits for unattended-upgrades or apt-daily instead of failing on their lock; they start on # their own on a fresh server and as soon as the auto-updates module enables them. # The filter drops the notices that report nothing to do: the needrestart summaries and the man-db trigger on images -# that switched off man-db/auto-update. Errors and everything else stay in the log. +# that switched off man-db/auto-update, and adduser skipping the home directory of a package's system user. Errors and +# everything else stay in the log. _pkg_apt() { DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get -y -q -o DPkg::Lock::Timeout=600 \ -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold "$@" 2>&1 | - sed -E '/^(No (services|containers) need to be restarted\.|No user sessions are running outdated binaries\.|No VM guests are running outdated hypervisor|Not building database; man-db\/auto-update is not)/d' + sed -E '/^(No (services|containers) need to be restarted\.|No user sessions are running outdated binaries\.|No VM guests are running outdated hypervisor|Not building database; man-db\/auto-update is not|Not creating (home directory )?`)/d' return "${PIPESTATUS[0]}" } diff --git a/tests/unit/system.bats b/tests/unit/system.bats index 8f23088..8ac50b4 100644 --- a/tests/unit/system.bats +++ b/tests/unit/system.bats @@ -35,7 +35,7 @@ calls() { @test "apt output drops the notices that report nothing to do and keeps the exit status" { use_os ubuntu-24.04 - export STUB_OUT_apt_get=$'No services need to be restarted.\nSetting up nginx (1.24.0)\nNo containers need to be restarted.\nNo user sessions are running outdated binaries.\nNo VM guests are running outdated hypervisor (qemu) binaries on this host.\nNot building database; man-db/auto-update is not \'true\'.' + export STUB_OUT_apt_get=$'No services need to be restarted.\nSetting up nginx (1.24.0)\nNo containers need to be restarted.\nNo user sessions are running outdated binaries.\nNo VM guests are running outdated hypervisor (qemu) binaries on this host.\nNot building database; man-db/auto-update is not \'true\'.\nNot creating home directory `/usr/share/grafana\'.' export STUB_EXIT_apt_get=100 run _pkg_apt install nginx [ "${status}" -eq 100 ] From 59a8f30077fc6fd03218610c54230e04f849af6b Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 18:26:53 +0200 Subject: [PATCH 17/32] fix(wireguard): keep the host's /dev for wg-quick wg reads its settings from a pipe through /dev/fd; in the private /dev of PrivateDevices SELinux denied that on AlmaLinux 10 (fopen: Permission denied). Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/wireguard.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/network/wireguard.sh b/modules/network/wireguard.sh index def326b..b43e64d 100644 --- a/modules/network/wireguard.sh +++ b/modules/network/wireguard.sh @@ -142,8 +142,9 @@ mod_configure() { config=$(_wg_config) file_write 0600 root:root "${WG_CONF}" <<<"${config}" # wg-quick is a root script that runs ip and wg; under SELinux these switch to their own domains (ifconfig_t), - # which NoNewPrivileges forbids, so the interface would never come up on the RHEL family. - svc_sandbox "${WG_UNIT}.service" NoNewPrivileges=no + # which NoNewPrivileges forbids, so the interface would never come up on the RHEL family. wg reads its settings + # from a pipe through /dev/fd, which SELinux denies wg in the private /dev of PrivateDevices. + svc_sandbox "${WG_UNIT}.service" NoNewPrivileges=no PrivateDevices=no fw_allow "$(opt_int port)/udp" svc_enable "${WG_UNIT}" if [[ $(files_changes_of wireguard) != "${before}" ]]; then From 75ce862a24f09195c44f0cac414799f6969898bf Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 18:48:13 +0200 Subject: [PATCH 18/32] fix(netdata): check that the agent is not claimed instead of a Cloud switch Netdata 2 removed the setting that turned its Cloud off; /api/v1/info reports cloud-enabled as true on every agent. An agent connects only once claimed, so the verification now requires agent-claimed and aclk-available to be false, and lss no longer writes the ignored cloud.conf. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- CHANGELOG.md | 5 +++-- modules/monitoring/netdata.sh | 17 ++++++----------- tests/unit/monitoring.bats | 15 +++++++++++++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2065b6..d9765ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,8 +71,9 @@ now or later. taken before each backup). - Monitoring modules: `node-exporter`, `prometheus` (with alert rules for down targets, full disks, guard findings and old or failed backups), `alertmanager` (e-mail and webhook), `grafana`, `loki`, `alloy` (Promtail's successor; it - ships the journal to Loki), `netdata` (no cloud, registry or telemetry) and `smartd`. Every interface listens on - 127.0.0.1 or, with `access=vpn`, on the WireGuard address, and can be published below the web server's admin port. + ships the journal to Loki), `netdata` (never claimed to Netdata Cloud, no registry or telemetry) and `smartd`. + Every interface listens on 127.0.0.1 or, with `access=vpn`, on the WireGuard address, and can be published below + the web server's admin port. - Network modules: `wireguard` (server with peers from the settings, keys in the lss secrets), `dns-resolver` (Unbound with DNSSEC on localhost, optionally for the WireGuard network, used by the server itself) and `ntp-server` (chrony of the time module serves the WireGuard network or everyone, rate-limited). diff --git a/modules/monitoring/netdata.sh b/modules/monitoring/netdata.sh index c575676..0644cba 100644 --- a/modules/monitoring/netdata.sh +++ b/modules/monitoring/netdata.sh @@ -4,7 +4,7 @@ MOD_NAME="netdata" MOD_GROUP="monitoring" -MOD_DESCRIPTION="Netdata real-time dashboard without cloud, registry or telemetry; on 127.0.0.1:19999 or the WireGuard network, optionally below /netdata/ on the admin port" +MOD_DESCRIPTION="Netdata real-time dashboard, never claimed to Netdata Cloud, without registry or telemetry; on 127.0.0.1:19999 or the WireGuard network, optionally below /netdata/ on the admin port" MOD_REQUIRES="system" MOD_OPTIONS="access|local|Who reaches http://SERVER:19999: local (127.0.0.1 only; SSH tunnel) or vpn (also the WireGuard network) web_admin|false|Also publish Netdata below /netdata/ on the admin port of the web server (needs nginx, apache or caddy) @@ -16,7 +16,6 @@ MOD_PORTS="19999/tcp from the WireGuard network with access vpn; none with acces NETDATA_UNIT=netdata NETDATA_PORT=19999 NETDATA_CONF=/etc/netdata/netdata.conf -NETDATA_CLOUD_CONF=/var/lib/netdata/cloud.d/cloud.conf NETDATA_OPT_OUT=/etc/netdata/.opt-out-from-anonymous-statistics mod_check() { @@ -51,14 +50,10 @@ _netdata_files() { [statsd] enabled = no EOF - # No anonymous statistics (the file's existence is the opt-out) and no connection to Netdata Cloud. + # No anonymous statistics: the file's existence is the opt-out. Netdata 2 has no switch for its Cloud any more + # (the API reports cloud-enabled as true on every agent); an agent connects only once it is claimed, which lss + # never does, and mod_verify checks that it is not. file_write 0644 root:root "${NETDATA_OPT_OUT}" "${list}" + admin_stubs + TEST_SERVER='' + config_set LSS_GRAFANA_WEB_ADMIN true + mon_web_admin proxy http://127.0.0.1:3000 + [ "$(cat "${list}")" = $'abc\t/etc/lss/grafana/admin-password' ] + [ ! -s "${STUB_LOG}" ] +} + # --- Prometheus and Grafana ------------------------------------------------------------------------------------- @test "a scrape target names its job and metrics path for Prometheus" { @@ -536,7 +548,7 @@ load_module() { [[ ${output} == *"LSS_ALLOY_MAX_AGE must look like 12h or 30m."* ]] } -@test "Netdata binds to 127.0.0.1 and the WireGuard address, without registry, cloud or statistics" { +@test "Netdata binds to 127.0.0.1 and the WireGuard address, without registry or statistics" { load_module netdata # shellcheck disable=SC2329 # called by the functions under test install() { @@ -546,7 +558,6 @@ load_module() { grep -qxF $'\tbind to = 127.0.0.1' "${WRITES}" grep -qxF '[registry]' "${WRITES}" grep -qxF "== 0644 root:root ${NETDATA_OPT_OUT}" "${WRITES}" - grep -qxF "== 0644 root:root ${NETDATA_CLOUD_CONF}" "${WRITES}" config_set LSS_NETDATA_ACCESS vpn [ "$(_netdata_bind)" = '127.0.0.1 10.66.0.1' ] } From 7f358c02c6795839567dfcc8676d1ca802c9457c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 18:48:18 +0200 Subject: [PATCH 19/32] fix(files): a removed file is no longer expected by lss verify file_remove kept the checksum, so a drop-in a module stopped writing, or an admin path the removed web server took along, was reported as missing on every verify. mon_web_admin forgets the admin path when no web server is left; the integration test removes prometheus before nginx. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- .github/workflows/integration.yml | 2 +- lib/files.sh | 13 ++++++++++++- lib/monitoring.sh | 10 ++++++++-- tests/unit/core.bats | 16 ++++++++++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 706bf2e..b2c0c7a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,7 +63,7 @@ jobs: # The Prometheus stack, Grafana below the admin port of nginx, logs from the journal to Loki, Netdata on the # WireGuard address (LSS_NETDATA_ACCESS=vpn) and smartd without drives. - group: monitoring - modules: node-exporter prometheus,alertmanager,grafana nginx,grafana loki,alloy wireguard,netdata smartd + modules: node-exporter prometheus,alertmanager,grafana nginx,prometheus,grafana loki,alloy wireguard,netdata smartd - group: network modules: wireguard dns-resolver ntp-server steps: diff --git a/lib/files.sh b/lib/files.sh index 4dcc0c4..1995adc 100644 --- a/lib/files.sh +++ b/lib/files.sh @@ -124,15 +124,26 @@ file_write() { log_debug "Wrote ${dest}" } -# file_remove PATH: deletes a managed file with backup. +# file_remove PATH: deletes a managed file with backup. lss verify no longer expects it, also when it was gone already +# (a drop-in a module stops writing, a file the web server took with it). file_remove() { local path=$1 + files_forget_path "${path}" [[ -e ${path} || -L ${path} ]] || return 0 files_backup "${path}" rm -rf -- "${path}" files_note_change "removed ${path}" } +# files_forget_path PATH: drops PATH from the checksums of the current module. +files_forget_path() { + local path=$1 list="${LSS_STATE_DIR}/managed/${LSS_MODULE:-core}" + [[ -f ${list} ]] || return 0 + awk -F '\t' -v path="${path}" '$2 != path' "${list}" >"${list}.tmp" + mv -f -- "${list}.tmp" "${list}" + return +} + # Prints managed files of MODULE whose content no longer matches what lss wrote. files_drifted() { local list="${LSS_STATE_DIR}/managed/$1" sum path diff --git a/lib/monitoring.sh b/lib/monitoring.sh index 7c11a24..5d0b2bd 100644 --- a/lib/monitoring.sh +++ b/lib/monitoring.sh @@ -116,8 +116,14 @@ mon_firewall() { # /MODULE/ on the admin port (web_admin_write proxy or strip); otherwise removes that path. Reloads the web server # only when something changed. mon_web_admin() { - local kind=$1 url=$2 before - [[ -n $(web_server) ]] || return 0 + local kind=$1 url=$2 before server + if [[ -z $(web_server) ]]; then + # A web server removed since took the admin path with it; lss verify must not miss it. + for server in nginx apache caddy; do + files_forget_path "$(web_conf_dir "${server}")/admin/${LSS_MODULE}.$(_web_ext "${server}")" + done + return 0 + fi before=$(files_changes_of "${LSS_MODULE}") if config_bool "$(config_key "${LSS_MODULE}" web_admin)"; then web_admin_write "${LSS_MODULE}" "${LSS_MODULE}" "${kind}" "${url}" diff --git a/tests/unit/core.bats b/tests/unit/core.bats index 1f69798..bc41a5c 100644 --- a/tests/unit/core.bats +++ b/tests/unit/core.bats @@ -202,6 +202,22 @@ write_release() { [ "$(cat "${dest}")" = keep ] } +@test "a removed file is no longer expected, also when it was gone already" { + local kept=${BATS_TEST_TMPDIR}/kept.conf gone=${BATS_TEST_TMPDIR}/gone.conf taken=${BATS_TEST_TMPDIR}/taken.conf + LSS_MODULE=demo + file_write 0644 "${OWNER}" "${kept}" <<<"x" + file_write 0644 "${OWNER}" "${gone}" <<<"y" + file_write 0644 "${OWNER}" "${taken}" <<<"z" + file_remove "${gone}" + rm "${taken}" + [ "$(files_drifted demo)" = "${taken} (missing)" ] + file_remove "${taken}" + [ -z "$(files_drifted demo)" ] + printf 'edited\n' >"${kept}" + [ "$(files_drifted demo)" = "${kept}" ] + LSS_STATE_DIR=${BATS_TEST_TMPDIR}/none files_forget_path "${kept}" +} + @test "files_forget drops the checksums of a module" { LSS_MODULE=demo file_write 0644 "${OWNER}" "${BATS_TEST_TMPDIR}/f.conf" <<<"x" From 437b35c7c4080c2c18d84f63d76b21f709e5b72c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 19:12:49 +0200 Subject: [PATCH 20/32] fix(loki): let systemd create Loki's data directory on every start The package's installation script restarts Loki before mod_configure creates /var/lib/loki; with ReadWritePaths=- the directory stayed read-only in the sandbox and Loki hit its start limit (seen on the Debian 13 VM in the devops profile). StateDirectory creates it for the user loki, and a failed start is reset before lss enables the unit. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/monitoring/loki.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/monitoring/loki.sh b/modules/monitoring/loki.sh index 1efb4f7..bc1a50c 100644 --- a/modules/monitoring/loki.sh +++ b/modules/monitoring/loki.sh @@ -86,7 +86,9 @@ _loki_files() { config=$(_loki_config) file_write 0644 root:root "${LOKI_CONF}" <<<"${config}" mon_exec "${LOKI_UNIT}" "/usr/bin/loki -config.file=${LOKI_CONF}" - svc_sandbox "${LOKI_UNIT}.service" "ReadWritePaths=-${LOKI_DATA}" + # systemd creates the data directory for the user loki on every start (StateDirectory), so the start by the package's + # installation script, before mod_configure, finds it writable too and Loki does not end in a restart loop. + svc_sandbox "${LOKI_UNIT}.service" "StateDirectory=${LOKI_DATA#/var/lib/}" StateDirectoryMode=0750 mon_vpn_dropin "${LOKI_UNIT}.service" return } @@ -110,6 +112,8 @@ mod_configure() { _loki_files /usr/bin/loki -config.file="${LOKI_CONF}" -verify-config >/dev/null 2>&1 || die "loki: Loki rejects ${LOKI_CONF} (loki -config.file=${LOKI_CONF} -verify-config shows why)." + # A start by the package's installation script that failed must not leave the unit at its start limit. + systemctl reset-failed "${LOKI_UNIT}.service" >/dev/null 2>&1 || true svc_enable "${LOKI_UNIT}" if [[ $(files_changes_of loki) != "${before}" ]] || ! svc_is_active "${LOKI_UNIT}"; then svc_restart "${LOKI_UNIT}" || die "loki: Loki does not start." From 3b94d509658e13e6cf699be00875f13600ac764c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 19:41:11 +0200 Subject: [PATCH 21/32] fix(ntp-server): run chronyd without readiness notification in containers chrony 4.8 on Ubuntu 26.04 stops with 'Could not send notification to $NOTIFY_SOCKET' inside a container. Where the unit runs chronyd in the foreground, the container drop-in makes it Type=simple without a notification socket; the forking unit of the RHEL family stays as is. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/ntp-server.sh | 6 +++++- tests/unit/network.bats | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh index bfc53ad..07f7aba 100644 --- a/modules/network/ntp-server.sh +++ b/modules/network/ntp-server.sh @@ -80,16 +80,20 @@ _ntp_exec() { # _ntp_container_dropin: inside a container chronyd serves the time but must not set the clock it shares with the host # (-x), and the unit's conditions on CAP_SYS_TIME and on running outside a container no longer apply. _ntp_container_dropin() { - local unit exec + local unit exec notify='' unit="$(_ntp_unit).service" if os_in_container; then exec=$(_ntp_exec "${unit}") || return 1 + # chronyd in the foreground (-n) reports readiness to systemd; inside a container chrony 4.8 (Ubuntu 26.04) + # cannot reach the notification socket and stops. There systemd counts the started process as ready. + [[ " ${exec} " != *' -n '* ]] || notify=$'Type=simple\nNotifyAccess=none' svc_dropin "${unit}" lss-ntp-server <"${unit_file}" _ntp_container_dropin grep -qxF "ExecStart=/usr/sbin/chronyd \$OPTIONS -x" "${WRITES}" + [ "$(grep -c "^Type=simple$" "${WRITES}")" = 1 ] : >"${unit_file}" run _ntp_container_dropin [ "${status}" -eq 1 ] From c34032d4e9031dfd7501d75e41a5205c26ead5eb Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 20:07:49 +0200 Subject: [PATCH 22/32] fix(backup): the repository credentials reach restic alone prepare exported AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and RESTIC_PASSWORD_FILE before the database dumps ran, so the dump helper and its children (pg_dumpall as postgres) inherited them. The credentials now go into the environment of each restic process only (a subshell in run_restic), the dumps run before prepare, and the dump programs start with an empty environment; pg_dumpall as postgres gets nothing of root's. A unit test fails if the dump helper sees any AWS_ or RESTIC_ variable. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- libexec/lss-backup | 47 +++++++++++++++++++++++++----------------- libexec/lss-db-dumps | 9 +++++--- tests/unit/backup.bats | 42 ++++++++++++++++++++++++++++++------- 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/libexec/lss-backup b/libexec/lss-backup index a8172fa..8a21056 100755 --- a/libexec/lss-backup +++ b/libexec/lss-backup @@ -5,8 +5,9 @@ # lss backup: restic backups of the server. Installed by the restic module as /usr/local/lib/lss/backup and run daily # by lss-backup.timer. Settings come from /etc/lss/backup.conf (KEY=value lines, parsed, never sourced); the -# repository password and the credentials stay in /etc/lss/secrets and reach restic through files and the environment, -# never through its command line. +# repository password and the credentials stay in /etc/lss/secrets and reach restic through files and the environment +# of the restic process alone, never through its command line; the database dumps and every other child never see +# them. # # backup run database dumps (db-dumps module), backup, forget and prune by the retention, check # backup init creates the repository unless it exists @@ -33,6 +34,10 @@ TAG=lss PROBE_TAG=lss-probe # Global also when the script is sourced inside a function (the unit tests). declare -ga RESTIC_OPTIONS=() +# NAME=value pairs that only the restic process gets in its environment (run_restic); never exported here. +declare -ga RESTIC_ENV=() +REPOSITORY='' +BACKUP_TMP='' say() { printf 'lss-backup: %s\n' "$*" @@ -78,30 +83,29 @@ sftp_target() { return } -# prepare: the environment and options restic needs for the configured repository. +# prepare: the environment (RESTIC_ENV) and options restic needs for the configured repository. prepare() { - local repository target key_id secret + local repository target key_id secret region repository=$(setting BACKUP_REPOSITORY) [[ -n ${repository} ]] || { say "BACKUP_REPOSITORY is empty in ${CONF}." >&2 return 1 } - export RESTIC_REPOSITORY=${repository} - export RESTIC_PASSWORD_FILE - RESTIC_PASSWORD_FILE=$(secret_file restic-password) - export RESTIC_CACHE_DIR=${CACHE} - export TMPDIR=${CACHE}/tmp - mkdir -p "${TMPDIR}" - chmod 0700 "${CACHE}" "${TMPDIR}" + REPOSITORY=${repository} + BACKUP_TMP=${CACHE}/tmp + mkdir -p "${BACKUP_TMP}" + chmod 0700 "${CACHE}" "${BACKUP_TMP}" + RESTIC_ENV=("RESTIC_REPOSITORY=${repository}" "RESTIC_PASSWORD_FILE=$(secret_file restic-password)" + "RESTIC_CACHE_DIR=${CACHE}" "TMPDIR=${BACKUP_TMP}") RESTIC_OPTIONS=(--quiet) case ${repository} in s3:*) key_id=$(<"$(secret_file restic-s3-key-id)") secret=$(<"$(secret_file restic-s3-secret)") - export AWS_ACCESS_KEY_ID=${key_id} AWS_SECRET_ACCESS_KEY=${secret} - if [[ -n $(setting BACKUP_S3_REGION) ]]; then - export AWS_DEFAULT_REGION - AWS_DEFAULT_REGION=$(setting BACKUP_S3_REGION) + RESTIC_ENV+=("AWS_ACCESS_KEY_ID=${key_id}" "AWS_SECRET_ACCESS_KEY=${secret}") + region=$(setting BACKUP_S3_REGION) + if [[ -n ${region} ]]; then + RESTIC_ENV+=("AWS_DEFAULT_REGION=${region}") fi ;; sftp:*) @@ -117,8 +121,12 @@ prepare() { return } +# run_restic ARGS: restic with RESTIC_ENV exported in a subshell of its own, so no other process inherits it. run_restic() { - "${RESTIC}" "${RESTIC_OPTIONS[@]}" "$@" + ( + export "${RESTIC_ENV[@]}" + "${RESTIC}" "${RESTIC_OPTIONS[@]}" "$@" + ) return } @@ -186,7 +194,7 @@ cmd_init() { return 0 fi run_restic init >/dev/null - say "created the repository ${RESTIC_REPOSITORY%%\?*}." + say "created the repository ${REPOSITORY%%\?*}." return } @@ -195,13 +203,14 @@ cmd_run() { local -a paths=() excludes=() patterns=() now=$(date +%s) stamp last-run "${now}" - prepare + # The dumps run before prepare reads any credential. if [[ -x ${DB_DUMPS} && -f ${DB_DUMPS_CONF} ]]; then "${DB_DUMPS}" || { status=1 say "the database dumps failed; the backup goes on with the files." } fi + prepare cmd_init mapfile -t paths < <(backup_paths) # Read as words, never expanded as globs here: the patterns are restic's. @@ -250,7 +259,7 @@ restore_test() { local tag=${1:-${TAG}} target expected actual status=0 subset prepare expected=$(stamp_get canary-last) - target=$(mktemp -d "${TMPDIR}/restore.XXXXXX") + target=$(mktemp -d "${BACKUP_TMP}/restore.XXXXXX") if ! run_restic restore latest --host "$(hostname)" --tag "${tag}" --target "${target}" --include "${CANARY}"; then say "restic restore of the latest snapshot failed." status=1 diff --git a/libexec/lss-db-dumps b/libexec/lss-db-dumps index 68f3505..e6998d8 100755 --- a/libexec/lss-db-dumps +++ b/libexec/lss-db-dumps @@ -25,6 +25,9 @@ STATUS_DIR=${LSS_DB_DUMPS_STATE:-${LSS_STATE_DIR:-/var/lib/lss}/db-dumps} ETC_DIR=${LSS_DB_DUMPS_ETC:-/etc/lss} DATA_ROOT=${LSS_DB_DUMPS_DATA:-/var/lib} ENGINES="mariadb mysql postgresql mongodb redis valkey" +# The dump programs start with an empty environment and this PATH: nothing of the unit's or the caller's environment +# reaches them, and nothing at all a process of another user (postgres). +CLEAN_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin say() { printf 'lss-db-dumps: %s\n' "$*" @@ -85,7 +88,7 @@ dump_mysql() { if [[ -f /root/.my.cnf ]]; then options=(--defaults-extra-file=/root/.my.cnf "${options[@]}") fi - "${dump}" "${options[@]}" --user=root --all-databases --single-transaction --routines --events --triggers \ + env -i "PATH=${CLEAN_PATH}" "${dump}" "${options[@]}" --user=root --all-databases --single-transaction --routines --events --triggers \ --hex-blob >"${out}" return } @@ -95,7 +98,7 @@ dump_postgresql() { dumpall=$(first_program pg_dumpall /usr/pgsql-*/bin/pg_dumpall) || return 1 port=$(setting DB_POSTGRESQL_PORT 5432) # As postgres from /, so the dump runs with the peer login and never inside a directory a user controls. - (cd / && runuser -u postgres -- "${dumpall}" -p "${port}" --clean --if-exists) >"${out}" + (cd / && runuser -u postgres -- env -i "PATH=${CLEAN_PATH}" "${dumpall}" -p "${port}" --clean --if-exists) >"${out}" return } @@ -107,7 +110,7 @@ dump_mongodb() { password=$(<"${SECRETS_DIR}/mongodb-admin") config=$(mktemp) printf 'password: "%s"\n' "${password}" >"${config}" - mongodump --quiet --config="${config}" --host=127.0.0.1 --port="${port}" --username=lss-admin \ + env -i "PATH=${CLEAN_PATH}" mongodump --quiet --config="${config}" --host=127.0.0.1 --port="${port}" --username=lss-admin \ --authenticationDatabase=admin --archive="${out}" || status=$? rm -f -- "${config}" return "${status}" diff --git a/tests/unit/backup.bats b/tests/unit/backup.bats index 40c81ca..23434bf 100644 --- a/tests/unit/backup.bats +++ b/tests/unit/backup.bats @@ -60,21 +60,27 @@ load_backup() { [ "${status}" -eq 1 ] printf 'BACKUP_REPOSITORY=sftp:backup@nas.example.com:/srv/restic\n' >"${LSS_BACKUP_CONF}" prepare - [ "${RESTIC_REPOSITORY}" = sftp:backup@nas.example.com:/srv/restic ] - [ "${RESTIC_PASSWORD_FILE}" = "${LSS_SECRETS_DIR}/restic-password" ] + [ "${RESTIC_ENV[0]}" = RESTIC_REPOSITORY=sftp:backup@nas.example.com:/srv/restic ] + [ "${RESTIC_ENV[1]}" = "RESTIC_PASSWORD_FILE=${LSS_SECRETS_DIR}/restic-password" ] [[ ${RESTIC_OPTIONS[2]} == "sftp.command=ssh -i ${LSS_SECRETS_DIR}/restic-ssh-key "*"-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=${LSS_BACKUP_STATE}/known_hosts backup@nas.example.com -s sftp" ]] } -@test "s3 repositories take their keys from the secrets, through the environment" { +@test "s3 repositories take their keys from the secrets, through the environment of restic alone" { load_backup printf 'BACKUP_REPOSITORY=s3:https://s3.example.com/bucket/host\nBACKUP_S3_REGION=eu-central-1\n' >"${LSS_BACKUP_CONF}" printf 'KEYID' >"${LSS_SECRETS_DIR}/restic-s3-key-id" printf 'SECRET' >"${LSS_SECRETS_DIR}/restic-s3-secret" prepare - [ "${AWS_ACCESS_KEY_ID}" = KEYID ] - [ "${AWS_SECRET_ACCESS_KEY}" = SECRET ] - [ "${AWS_DEFAULT_REGION}" = eu-central-1 ] + [[ " ${RESTIC_ENV[*]} " == *' AWS_ACCESS_KEY_ID=KEYID AWS_SECRET_ACCESS_KEY=SECRET AWS_DEFAULT_REGION=eu-central-1 '* ]] [ "${RESTIC_OPTIONS[*]}" = --quiet ] + # Nothing of it is exported: restic gets it, nothing else does. + [ -z "$(printenv AWS_SECRET_ACCESS_KEY)" ] + [ -z "$(printenv RESTIC_PASSWORD_FILE)" ] + # shellcheck disable=SC2329 # called by run_restic + env_stub() { + printenv AWS_SECRET_ACCESS_KEY + } + [ "$(RESTIC=env_stub run_restic snapshots)" = SECRET ] printf 'BACKUP_REPOSITORY=\n' >"${LSS_BACKUP_CONF}" run prepare [ "${status}" -eq 1 ] @@ -121,6 +127,21 @@ load_backup() { grep -qxF 'lss_backup_last_run_success 0' "${LSS_TEXTFILE_DIR}/lss_backup.prom" } +@test "the database dumps never see the credentials of the repository" { + load_backup + printf 'BACKUP_REPOSITORY=s3:https://s3.example.com/bucket/host\nBACKUP_PATHS=/etc\n' >"${LSS_BACKUP_CONF}" + printf 'KEYID' >"${LSS_SECRETS_DIR}/restic-s3-key-id" + printf 'SECRET' >"${LSS_SECRETS_DIR}/restic-s3-secret" + printf '#!/bin/sh\nenv >"%s/dumps.env"\n' "${BATS_TEST_TMPDIR}" >"${LSS_DB_DUMPS}" + chmod +x "${LSS_DB_DUMPS}" + : >"${LSS_DB_DUMPS_CONF}" + cmd_run + [ -s "${BATS_TEST_TMPDIR}/dumps.env" ] + run grep -E '^(AWS_|RESTIC_)' "${BATS_TEST_TMPDIR}/dumps.env" + [ "${status}" -eq 1 ] + run ! grep -q '=SECRET$' "${BATS_TEST_TMPDIR}/dumps.env" +} + @test "failed database dumps fail the run but the files are still backed up" { load_backup printf '#!/bin/sh\nexit 1\n' >"${LSS_DB_DUMPS}" @@ -244,6 +265,13 @@ load_dumps() { first_program() { printf '%s' "$1" } + # env -i PATH=... COMMAND: the stub runs COMMAND, so the functions below stand in for the programs. + # shellcheck disable=SC2329 # called by the functions under test + env() { + [[ $1 == -i && $2 == "PATH=${CLEAN_PATH}" ]] || return 1 + shift 2 + "$@" + } # shellcheck disable=SC2329 # called by the functions under test mariadb-dump() { printf 'mariadb-dump %s\n' "$*" >>"${STUB_LOG}" @@ -255,7 +283,7 @@ load_dumps() { dump mariadb "${BATS_TEST_TMPDIR}/m.sql" grep -qxF 'mariadb-dump --user=root --all-databases --single-transaction --routines --events --triggers --hex-blob' "${STUB_LOG}" dump postgresql "${BATS_TEST_TMPDIR}/p.sql" - grep -qxF 'runuser -u postgres -- pg_dumpall -p 5433 --clean --if-exists' "${STUB_LOG}" + grep -qxF "runuser -u postgres -- env -i PATH=${CLEAN_PATH} pg_dumpall -p 5433 --clean --if-exists" "${STUB_LOG}" [ "$(file_of mongodb)" = mongodb.archive ] [ "$(file_of valkey)" = valkey.rdb ] [ "$(file_of mysql)" = mysql.sql ] From 66fabf0221b096fa67875b3d9f2cbee8c44bf445 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 21:18:34 +0200 Subject: [PATCH 23/32] fix(backup): give 'backup restic' the repository environment again The pass-through to restic still called it directly and lost the repository and password once they stopped being exported, so the restic module's verification failed. It now goes through restic_env like every other restic call. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- libexec/lss-backup | 14 ++++++++++---- tests/unit/backup.bats | 9 +++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/libexec/lss-backup b/libexec/lss-backup index 8a21056..42aa9db 100755 --- a/libexec/lss-backup +++ b/libexec/lss-backup @@ -121,15 +121,21 @@ prepare() { return } -# run_restic ARGS: restic with RESTIC_ENV exported in a subshell of its own, so no other process inherits it. -run_restic() { +# restic_env ARGS: restic with RESTIC_ENV exported in a subshell of its own, so no other process inherits it. +restic_env() { ( export "${RESTIC_ENV[@]}" - "${RESTIC}" "${RESTIC_OPTIONS[@]}" "$@" + "${RESTIC}" "$@" ) return } +# run_restic ARGS: restic_env with the options of the repository, quiet. +run_restic() { + restic_env "${RESTIC_OPTIONS[@]}" "$@" + return +} + stamp() { local name=$1 value=$2 mkdir -p "${STATE}" @@ -320,7 +326,7 @@ dispatch() { probe) cmd_probe ;; restic) prepare - "${RESTIC}" "${RESTIC_OPTIONS[@]:1}" "$@" + restic_env "${RESTIC_OPTIONS[@]:1}" "$@" ;; *) say "usage: backup run|init|restore-test|probe|restic ARGS" >&2 diff --git a/tests/unit/backup.bats b/tests/unit/backup.bats index 23434bf..472da90 100644 --- a/tests/unit/backup.bats +++ b/tests/unit/backup.bats @@ -204,6 +204,15 @@ load_backup() { [[ ${output} == *"usage: backup run|init|restore-test|probe|restic ARGS"* ]] } +@test "backup restic runs restic with the repository and its password, without --quiet" { + load_backup + # shellcheck disable=SC2329 # called by dispatch + env_stub() { + printf '%s|%s|%s\n' "${RESTIC_REPOSITORY:-}" "${RESTIC_PASSWORD_FILE:-}" "$*" + } + [ "$(RESTIC=env_stub dispatch restic cat config)" = "/srv/repo|${LSS_SECRETS_DIR}/restic-password|cat config" ] +} + # --- db-dumps --------------------------------------------------------------------------------------------------- load_dumps() { From b866935c40e57e6babd5aa63ae4987b85c59d9c2 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 22:34:20 +0200 Subject: [PATCH 24/32] fix(dns-resolver): skip the RHEL unit's unbound-anchor run before each start unbound.service of the RHEL family runs unbound-anchor as ExecStartPre and fails when the root servers are out of reach (Rocky 9 in CI). Unbound keeps the anchor current itself; DISABLE_UNBOUND_ANCHOR=yes is the unit's own switch. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/dns-resolver.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/network/dns-resolver.sh b/modules/network/dns-resolver.sh index d8cfa6f..c045272 100644 --- a/modules/network/dns-resolver.sh +++ b/modules/network/dns-resolver.sh @@ -257,6 +257,15 @@ mod_configure() { file_write 0644 root:root "${DNS_CONF}" <<<"${config}" unbound-checkconf "${DNS_CONF}" >/dev/null || die "dns-resolver: unbound-checkconf rejects ${DNS_CONF}." svc_sandbox "${DNS_UNIT}.service" ReadWritePaths=/var/lib/unbound + if os_is_rhel; then + # The unit of the RHEL family runs unbound-anchor before every start and fails when it cannot reach the root + # servers. Unbound keeps the anchor current itself (RFC 5011, _dns_anchor), so the unit's own update stays off. + svc_dropin "${DNS_UNIT}.service" lss-config <<'EOF' +# Managed by lss (module dns-resolver): Unbound updates its trust anchor itself. +[Service] +Environment=DISABLE_UNBOUND_ANCHOR=yes +EOF + fi mon_vpn_dropin "${DNS_UNIT}.service" svc_enable "${DNS_UNIT}" if [[ $(files_changes_of dns-resolver) != "${before}" ]]; then @@ -381,6 +390,7 @@ mod_remove() { svc_disable "${DNS_UNIT}" svc_dropin_remove "${DNS_UNIT}.service" lss-sandbox svc_dropin_remove "${DNS_UNIT}.service" lss-vpn + svc_dropin_remove "${DNS_UNIT}.service" lss-config file_remove "${DNS_CONF}" pkg_remove_family "unbound unbound-anchor" unbound [[ ${LSS_PURGE:-0} == 1 ]] || return 0 From 051cfd65a8f902816765a49afbade7ad4e55adf5 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 22:34:24 +0200 Subject: [PATCH 25/32] chore(ntp-server): log chronyd's view when it does not answer The parsed port, allow and confdir lines and chronyd's sockets go to the log before the verification fails, to find why AlmaLinux 10 does not serve on port 123. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/ntp-server.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh index 07f7aba..4dfe2b0 100644 --- a/modules/network/ntp-server.sh +++ b/modules/network/ntp-server.sh @@ -151,7 +151,12 @@ mod_verify() { [[ -z ${first} ]] || break sleep 1 done - [[ -n ${first} ]] || die "ntp-server: chronyd does not answer NTP requests on port ${NTP_PORT}." + if [[ -z ${first} ]]; then + # What chronyd made of its configuration and which sockets it has, for the log. + chronyd -p 2>&1 | grep -E '^(port|allow|confdir|cmdport)' | sed 's/^/ntp-server: chronyd -p: /' >&2 || true + ss -Hlunp 2>/dev/null | grep chronyd | sed 's/^/ntp-server: socket: /' >&2 || true + die "ntp-server: chronyd does not answer NTP requests on port ${NTP_PORT}." + fi (((first & 7) == 4)) || die "ntp-server: the answer on port ${NTP_PORT} is no NTP server reply (mode $((first & 7)))." ss -Hlun "sport = :${NTP_PORT}" | grep -q . || die "ntp-server: nothing listens on ${NTP_PORT}/udp." return From a6d0299870475905d76c8339710b9ef64daa0096 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Wed, 14 Oct 2026 23:13:57 +0200 Subject: [PATCH 26/32] docs(readme): resource figures of the stage-4 modules Taken from the RESOURCES lines of the integration run on all systems: the largest value, rounded up, and the resident size of each service. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3a315aa..bd3c9c6 100644 --- a/README.md +++ b/README.md @@ -72,19 +72,22 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `podman` | not measured yet | not measured yet | not measured yet | – (no daemon) | | `portainer` | not measured yet | not measured yet | not measured yet | Portainer container, up to 512 MiB | | `k3s` | not measured yet | not measured yet | not measured yet | k3s server, about 1 GiB with Traefik | -| `restic` | not measured yet | not measured yet | not measured yet | daily backup | -| `db-dumps` | not measured yet | not measured yet | not measured yet | dumps before each backup | -| `node-exporter` | not measured yet | not measured yet | not measured yet | node exporter | -| `prometheus` | not measured yet | not measured yet | not measured yet | Prometheus and its metrics on disk | -| `alertmanager` | not measured yet | not measured yet | not measured yet | Alertmanager | -| `grafana` | not measured yet | not measured yet | not measured yet | Grafana | -| `loki` | not measured yet | not measured yet | not measured yet | Loki and the logs on disk | -| `alloy` | not measured yet | not measured yet | not measured yet | Alloy | -| `netdata` | not measured yet | not measured yet | not measured yet | Netdata and its plugins | -| `smartd` | not measured yet | not measured yet | not measured yet | smartd | -| `wireguard` | not measured yet | not measured yet | not measured yet | – (kernel) | -| `dns-resolver` | not measured yet | not measured yet | not measured yet | Unbound | -| `ntp-server` | not measured yet | not measured yet | not measured yet | – (the chronyd of `time`) | +| `restic` | 35 s | 700 MiB (dnf), else 220 MiB | 260 MiB (dnf), else 115 MiB | daily backup, weekly restore test | +| `db-dumps` | measured with the databases | – | – | dumps before each backup | +| `node-exporter` | 20 s | 255 MiB | 110 MiB | node exporter, about 20 MiB | +| `prometheus` | 100 s with alertmanager and grafana | 860 MiB together | 2.2 GiB together | Prometheus about 75 MiB, and its metrics on disk | +| `alertmanager` | see `prometheus` | see `prometheus` | see `prometheus` | Alertmanager about 40 MiB | +| `grafana` | see `prometheus` | see `prometheus` | see `prometheus` | Grafana about 350 MiB, its plugins about 30 MiB each | +| `loki` | 50 s with alloy | 545 MiB together | 700 MiB together | Loki about 155 MiB, and the logs on disk | +| `alloy` | see `loki` | see `loki` | see `loki` | Alloy about 230 MiB | +| `netdata` | 45 s | 440 MiB | 680 MiB | Netdata and its plugins, about 220 MiB | +| `smartd` | 15 s | 140 MiB | 5 MiB | smartd about 5 MiB | +| `wireguard` | 15 s | 260 MiB | 100 MiB | – (kernel) | +| `dns-resolver` | 45 s | 180 MiB | 40 MiB | Unbound about 20 MiB | +| `ntp-server` | 15 s | 195 MiB | 10 MiB | – (the chronyd of `time`, about 10 MiB) | + +Modules the tests install together are measured together. The Rocky 9 VM in CI is slow throughout (eight minutes even +for `wireguard`); those outliers are left out. The integration jobs print these figures as `RESOURCES` lines, so every new module and every new system adds its own. From 279a293cefb1ca0ff80bc69a13e3b93d44aab11f Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 00:35:22 +0200 Subject: [PATCH 27/32] chore(ntp-server): show chrony's whole configuration when it does not answer chronyd -p listed port 0 from the main file but none of the lines of the confdir file on Rocky 9 and AlmaLinux 10; the full output, the confdir line, the directory and the running command line tell why. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/ntp-server.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh index 4dfe2b0..c387e01 100644 --- a/modules/network/ntp-server.sh +++ b/modules/network/ntp-server.sh @@ -26,6 +26,11 @@ _ntp_conf() { return } +_ntp_main_conf() { + if os_is_debian; then printf '/etc/chrony/chrony.conf'; else printf '/etc/chrony.conf'; fi + return +} + mod_check() { [[ $(opt access) =~ ^(vpn|public)$ ]] || die "LSS_NTP_SERVER_ACCESS must be vpn or public." opt_bool ratelimit || true @@ -153,7 +158,11 @@ mod_verify() { done if [[ -z ${first} ]]; then # What chronyd made of its configuration and which sockets it has, for the log. - chronyd -p 2>&1 | grep -E '^(port|allow|confdir|cmdport)' | sed 's/^/ntp-server: chronyd -p: /' >&2 || true + # chrony's configuration holds no secrets (keys live in a separate key file). + chronyd -p 2>&1 | grep -v '^server ' | sed 's/^/ntp-server: chronyd -p: /' >&2 || true + grep -n '^confdir' "$(_ntp_main_conf)" 2>&1 | sed 's/^/ntp-server: main file: /' >&2 || true + find "$(dirname "$(_ntp_conf)")" -maxdepth 1 -printf '%M %u %s %f\n' 2>&1 | sed 's/^/ntp-server: confdir: /' >&2 || true + ps -o args= -C chronyd 2>&1 | sed 's/^/ntp-server: process: /' >&2 || true ss -Hlunp 2>/dev/null | grep chronyd | sed 's/^/ntp-server: socket: /' >&2 || true die "ntp-server: chronyd does not answer NTP requests on port ${NTP_PORT}." fi From ecfce0ea1739e9404014b3215ee409bdf1a76bff Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 00:56:38 +0200 Subject: [PATCH 28/32] fix(dns-resolver): keep the server's resolvers where Unbound reaches no DNS On the CI machines outgoing DNS to the root servers is blocked. In the profile all job dns-resolver still pointed systemd-resolved at Unbound, and every later package download failed on name resolution. The server now switches to Unbound only after the signed root zone resolved; otherwise it keeps its previous resolvers and lss warns, naming LSS_DNS_RESOLVER_FORWARD. Verify skips the system check in that case. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- README.md | 4 +++- modules/network/dns-resolver.sh | 40 +++++++++++++++++++++++++-------- tests/unit/network.bats | 18 +++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bd3c9c6..b4d4da2 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,9 @@ endpoint and port) after every change, and `lss secrets show wireguard-psk-NAME` the server, not the networks behind it: IP forwarding stays off. `dns-resolver` runs Unbound on 127.0.0.1 as a validating (DNSSEC) resolver and makes the server use it, through -systemd-resolved, NetworkManager or `/etc/resolv.conf`; removing the module restores the previous resolver. +systemd-resolved, NetworkManager or `/etc/resolv.conf`; removing the module restores the previous resolver. Where +Unbound reaches no DNS server on the Internet (outgoing DNS blocked), the server keeps its previous resolver and lss +warns; `LSS_DNS_RESOLVER_FORWARD` then names resolvers Unbound can reach. `LSS_DNS_RESOLVER_ACCESS=vpn` serves the WireGuard network too, `LSS_DNS_RESOLVER_FORWARD` forwards to other resolvers, over TLS with `LSS_DNS_RESOLVER_FORWARD_TLS=true`. diff --git a/modules/network/dns-resolver.sh b/modules/network/dns-resolver.sh index c045272..321c268 100644 --- a/modules/network/dns-resolver.sh +++ b/modules/network/dns-resolver.sh @@ -272,9 +272,14 @@ EOF svc_restart "${DNS_UNIT}" || die "dns-resolver: Unbound does not start with the new configuration." fi mon_firewall 53/udp 53/tcp - # The server switches to Unbound only once it answers. + # The server switches to Unbound only once it answers, and only when it reaches the DNS of the Internet: where + # outgoing DNS is blocked, the switch would leave the server without any name resolution. _dns_wait mode=$(_dns_system_mode) + if [[ ${mode} != none ]] && ! _dns_online; then + log_warn "dns-resolver: Unbound reaches no DNS server on the Internet, so this server keeps its previous resolvers; LSS_DNS_RESOLVER_FORWARD names resolvers that are reachable." + mode=none + fi if [[ ${mode} == none ]]; then _dns_system_undo else @@ -347,8 +352,26 @@ _dns_wait() { die "dns-resolver: Unbound does not answer on ${DNS_LOOPBACK}:53." } +# _dns_root_reply: Unbound's answer for the SOA of the signed root zone. A cold Unbound first fetches the root servers +# and keys, so it gets three tries. +_dns_root_reply() { + local reply='' + for _ in 1 2 3; do + reply=$(_dns_query "${DNS_LOOPBACK}" . 6) + [[ -z $(_dns_reply_rcode "${reply}" || true) ]] || break + done + printf '%s' "${reply}" + return +} + +# _dns_online: whether Unbound resolves the root zone, which needs the root servers or the forward resolvers. +_dns_online() { + [[ $(_dns_reply_rcode "$(_dns_root_reply)" || true) == 0 ]] + return +} + mod_verify() { - local reply rcode address + local reply rcode address mode local -a addresses=("${DNS_LOOPBACK}") svc_is_active "${DNS_UNIT}" || die "dns-resolver: ${DNS_UNIT} is not running." _dns_wait @@ -357,21 +380,20 @@ mod_verify() { die "dns-resolver: Unbound does not validate DNSSEC." [[ -s ${DNS_ANCHOR} ]] || die "dns-resolver: the DNSSEC trust anchor ${DNS_ANCHOR} is missing." # The signed root zone must come back validated (AD). Without a reachable resolver upstream only the offline checks - # above apply. A cold Unbound first fetches the root servers and keys, so it gets three tries. - for _ in 1 2 3; do - reply=$(_dns_query "${DNS_LOOPBACK}" . 6) - rcode=$(_dns_reply_rcode "${reply}" || true) - [[ -z ${rcode} ]] || break - done + # above apply, and the server keeps its previous resolvers (mod_configure). + reply=$(_dns_root_reply) + rcode=$(_dns_reply_rcode "${reply}" || true) + mode=$(_dns_system_mode) if [[ ${rcode} == 0 ]]; then _dns_reply_ad "${reply}" || die "dns-resolver: the answer for the root zone is not DNSSEC-validated." else log_warn "dns-resolver: the root zone did not resolve (rcode ${rcode:-none}); DNSSEC was checked offline only." + mode=none fi _dns_ipv6 && addresses+=(::1) [[ $(opt access) != vpn ]] || addresses+=("$(fw_vpn_address)") mon_listen_check --process unbound 53 "${addresses[@]}" - case $(_dns_system_mode) in + case ${mode} in resolved) resolvectl dns 2>/dev/null | grep -qE "^Global: .*${DNS_LOOPBACK}" || die "dns-resolver: systemd-resolved does not forward to Unbound." diff --git a/tests/unit/network.bats b/tests/unit/network.bats index fcaac03..3505616 100644 --- a/tests/unit/network.bats +++ b/tests/unit/network.bats @@ -199,6 +199,24 @@ KEY_B=TrMvSoP4jYQlY6RIzBgbssQqY3vxI2Pi+y71lOWWXX0= [ "${status}" -eq 1 ] } +@test "dns-resolver counts Unbound as online only when the root zone resolves" { + load_module dns-resolver + # shellcheck disable=SC2329 # called by the function under test + _dns_query() { + echo "query" >>"${BATS_TEST_TMPDIR}/queries" + printf '%s' "${STUB_REPLY}" + } + STUB_REPLY='4c 53 81 a0 00 01 00 01 00 00 00 00' + _dns_online + STUB_REPLY='4c 53 81 82 00 01 00 00 00 00 00 00' + run ! _dns_online + # Silence (blocked outgoing DNS) is retried, three times in all. + rm -f "${BATS_TEST_TMPDIR}/queries" + STUB_REPLY='' + run ! _dns_online + [ "$(grep -c . "${BATS_TEST_TMPDIR}/queries")" = 3 ] +} + # --- ntp-server ------------------------------------------------------------------------------------------------- @test "ntp-server opens port 123 to the WireGuard network or to everyone, rate-limited" { From 95d7a43b29c89fe1da4fbed78ed0966f5e38bf10 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 00:56:44 +0200 Subject: [PATCH 29/32] fix(monitoring): remove the temporary policy-rc.d also when apt fails The EXIT trap read the local variable of mon_install_stopped. When pkg_install died, bash on Debian 13 ran the trap with that variable gone ("policy: unbound variable"), so policy-rc.d stayed and forbade every later service start. The trap now uses the global MON_POLICY_RC. The test runs the failure path with set -u; Git Bash does not reproduce the unbound case, CI's bash did. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/monitoring.sh | 3 ++- tests/unit/monitoring.bats | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/monitoring.sh b/lib/monitoring.sh index 5d0b2bd..187d0ed 100644 --- a/lib/monitoring.sh +++ b/lib/monitoring.sh @@ -208,7 +208,8 @@ mon_install_stopped() { return fi ( - trap 'rm -f -- "${policy}"' EXIT + # The global name: when pkg_install dies, the trap runs after the locals of this function are gone. + trap 'rm -f -- "${MON_POLICY_RC}"' EXIT printf '#!/bin/sh\n# Written by lss for one package installation.\nexit 101\n' >"${policy}" chmod 0755 "${policy}" pkg_install "$@" diff --git a/tests/unit/monitoring.bats b/tests/unit/monitoring.bats index 3cfd704..22c311c 100644 --- a/tests/unit/monitoring.bats +++ b/tests/unit/monitoring.bats @@ -302,8 +302,15 @@ admin_stubs() { pkg_install() { die "apt failed" } - run mon_install_stopped prometheus + # With set -u, as lss runs: the trap must not depend on the locals of the dying function. + # shellcheck disable=SC2329 # called through run + nounset() { + set -u + "$@" + } + run nounset mon_install_stopped prometheus [ "${status}" -eq 1 ] + [[ ${output} != *"unbound variable"* ]] [ ! -e "${MON_POLICY_RC}" ] } From f20a6aefeb601e736475439977a3a1f06e2188e2 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 03:26:29 +0200 Subject: [PATCH 30/32] chore(ntp-server): test chrony's parsing of the confdir when it does not answer Rocky 9 and AlmaLinux 10 list the confdir line and the file in it, yet chronyd -p shows none of its lines. The version, the end of the main file and parse runs of the file and of the confdir alone narrow it down. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/network/ntp-server.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh index c387e01..7a73289 100644 --- a/modules/network/ntp-server.sh +++ b/modules/network/ntp-server.sh @@ -163,6 +163,10 @@ mod_verify() { grep -n '^confdir' "$(_ntp_main_conf)" 2>&1 | sed 's/^/ntp-server: main file: /' >&2 || true find "$(dirname "$(_ntp_conf)")" -maxdepth 1 -printf '%M %u %s %f\n' 2>&1 | sed 's/^/ntp-server: confdir: /' >&2 || true ps -o args= -C chronyd 2>&1 | sed 's/^/ntp-server: process: /' >&2 || true + chronyd -v 2>&1 | sed 's/^/ntp-server: version: /' >&2 || true + tail -c 40 "$(_ntp_main_conf)" | od -c | sed 's/^/ntp-server: main file end: /' >&2 || true + chronyd -p -f "$(_ntp_conf)" 2>&1 | sed 's/^/ntp-server: own file: /' >&2 || true + chronyd -p "confdir $(dirname "$(_ntp_conf)")" 2>&1 | sed 's/^/ntp-server: confdir alone: /' >&2 || true ss -Hlunp 2>/dev/null | grep chronyd | sed 's/^/ntp-server: socket: /' >&2 || true die "ntp-server: chronyd does not answer NTP requests on port ${NTP_PORT}." fi From 1437887b6bb4e44bd1b26cb25c68d95eaba7c02a Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 05:16:30 +0200 Subject: [PATCH 31/32] fix(ntp-server): read the confdir under /etc/chrony on an AppArmor kernel A RHEL-family container on a Debian or Ubuntu host runs chronyd under the host's AppArmor profile for /usr/sbin/chronyd, which allows /etc/chrony/** and /etc/chrony.* only. chronyd could not open /etc/chrony.d/lss-ntp-server.conf (Permission denied) and never opened port 123. The time module and ntp-server now share one rule for the directory. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/base/time.sh | 9 +++++++-- modules/network/ntp-server.sh | 9 ++++++++- tests/unit/network.bats | 7 +++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/modules/base/time.sh b/modules/base/time.sh index 7e35b52..29db971 100644 --- a/modules/base/time.sh +++ b/modules/base/time.sh @@ -42,9 +42,14 @@ mod_install() { } # The directory chrony reads further settings from, where ntp-server puts its own: inside chrony's directory of /etc, -# which the AppArmor profile of chronyd on Ubuntu lets it read. +# which the AppArmor profile of chronyd (Debian, Ubuntu) lets it read. The RHEL family uses /etc/chrony.d, except under +# an AppArmor kernel (a container on a Debian or Ubuntu host), where the host's profile for /usr/sbin/chronyd applies. _time_confdir() { - if os_is_debian; then printf '/etc/chrony/conf.d'; else printf '/etc/chrony.d'; fi + if os_is_debian || [[ $(cat "${SVC_APPARMOR_ENABLED:-/sys/module/apparmor/parameters/enabled}" 2>/dev/null) == Y ]]; then + printf '/etc/chrony/conf.d' + else + printf '/etc/chrony.d' + fi return } diff --git a/modules/network/ntp-server.sh b/modules/network/ntp-server.sh index 7a73289..730af9d 100644 --- a/modules/network/ntp-server.sh +++ b/modules/network/ntp-server.sh @@ -21,8 +21,15 @@ _ntp_unit() { return } +# _ntp_conf: the file in the directory the time module's chrony configuration reads last. _ntp_conf() { - if os_is_debian; then printf '/etc/chrony/conf.d/%s' "${NTP_FILE_NAME}"; else printf '/etc/chrony.d/%s' "${NTP_FILE_NAME}"; fi + local dir + dir=$( + # shellcheck source=/dev/null + source "$(module_file time)" + _time_confdir + ) + printf '%s/%s' "${dir}" "${NTP_FILE_NAME}" return } diff --git a/tests/unit/network.bats b/tests/unit/network.bats index 3505616..adf7544 100644 --- a/tests/unit/network.bats +++ b/tests/unit/network.bats @@ -231,9 +231,14 @@ KEY_B=TrMvSoP4jYQlY6RIzBgbssQqY3vxI2Pi+y71lOWWXX0= config_set LSS_NTP_SERVER_ACCESS local run mod_check [[ ${output} == *"LSS_NTP_SERVER_ACCESS must be vpn or public."* ]] + export SVC_APPARMOR_ENABLED=${BATS_TEST_TMPDIR}/apparmor + echo N >"${SVC_APPARMOR_ENABLED}" [ "$(_ntp_conf)" = /etc/chrony/conf.d/lss-ntp-server.conf ] use_os rocky-9 [ "$(_ntp_conf)" = /etc/chrony.d/lss-ntp-server.conf ] + # A RHEL container on a Debian or Ubuntu host: the host's AppArmor profile lets chronyd read /etc/chrony/ only. + echo Y >"${SVC_APPARMOR_ENABLED}" + [ "$(_ntp_conf)" = /etc/chrony/conf.d/lss-ntp-server.conf ] [ "$(_ntp_unit)" = chronyd ] } @@ -269,6 +274,8 @@ KEY_B=TrMvSoP4jYQlY6RIzBgbssQqY3vxI2Pi+y71lOWWXX0= @test "the time module's chrony configuration reads the confdir of ntp-server last" { load_module time + export SVC_APPARMOR_ENABLED=${BATS_TEST_TMPDIR}/apparmor + echo N >"${SVC_APPARMOR_ENABLED}" # shellcheck disable=SC2329 # called by the functions under test install() { printf 'install %s\n' "$*" >>"${STUB_LOG}" From 73a47746d8269808f4252ea1b4f50603d0759b77 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 05:16:33 +0200 Subject: [PATCH 32/32] fix(k3s): wait for the secrets encryption status The encryption controller can report later than the API on a busy host; the profile run on AlmaLinux 10 failed now and then right after start. The check now waits up to 60 seconds and logs the status (key names and hash state, no key material) when it still fails. Refs #6 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/container/k3s.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/container/k3s.sh b/modules/container/k3s.sh index 9c6de6e..1e7b285 100644 --- a/modules/container/k3s.sh +++ b/modules/container/k3s.sh @@ -252,14 +252,23 @@ _k3s_wait() { } mod_verify() { - local key value actual status + local key value actual status i svc_is_active "${K3S_UNIT}" || die "k3s: the server is not running; see journalctl -u k3s." _k3s_wait 180 "${K3S_BIN}" kubectl get --raw=/readyz || die "k3s: the Kubernetes API is not ready." _k3s_wait 180 "${K3S_BIN}" kubectl get nodes -o 'jsonpath={.items[0].status.conditions[?(@.type=="Ready")].status}' || die "k3s: the node does not become Ready; see k3s kubectl describe nodes." [[ $(stat -c '%a %U' "${K3S_KUBECONFIG}") == '600 root' ]] || die "k3s: ${K3S_KUBECONFIG} must be mode 600 and owned by root." - status=$("${K3S_BIN}" secrets-encrypt status 2>/dev/null || true) - [[ ${status} == *'Encryption Status: Enabled'* ]] || die "k3s: secrets are not encrypted at rest." + # The status comes from the server's encryption controller, which can report later than the API on a busy host. + for ((i = 0; i < 60; i += 2)); do + status=$("${K3S_BIN}" secrets-encrypt status 2>&1 || true) + [[ ${status} != *'Encryption Status: Enabled'* ]] || break + sleep 2 + done + if [[ ${status} != *'Encryption Status: Enabled'* ]]; then + # The status names the keys and whether their hashes match, never the key material. + printf 'k3s: secrets-encrypt status: %s\n' "${status//$'\n'/$'\n'k3s: secrets-encrypt status: }" >&2 + die "k3s: secrets are not encrypted at rest." + fi if ! os_in_container; then while IFS='=' read -r key value; do key=${key// /}