From e5202b70d4d262bc49fbddc954ec886988346a4b Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 19:07:13 +0200 Subject: [PATCH 01/28] feat(apps): app modules as hardened Docker Compose stacks behind the web server Stage 6 brings the applications a small company runs itself: Nextcloud with Collabora Online, Forgejo, Uptime Kuma, Vaultwarden, Keycloak, Matomo, WordPress, n8n, MinIO (as the maintained fork Silo, since MinIO archived its community edition) and Self-Updater. The apps share lib/apps.sh so that every stack gets the same guarantees without each module repeating them: images pinned by digest, no capabilities, no new privileges, read-only roots where the image allows, memory and process limits, ports on 127.0.0.1 only and an HTTPS site of the installed web server in front. Secrets are copied from /etc/lss/secrets as read-only files for the container user (mapped through Docker's user namespace), so no password lands in a Compose file, an environment variable or a command line; first administrators get generated passwords that lss only points to. mod_verify uses each app (WebDAV round trip, document conversion, API create/delete, tracking and reporting, S3 bucket and object) instead of only checking that a port answers. MinIO and Self-Updater also run natively, so they are covered on arm64 in the container jobs; the Docker stacks run in new apps VM jobs on all seven systems. The office and devops profiles gain the apps they were meant to have. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- .github/workflows/integration.yml | 74 +++- README.md | 65 +++- data/checksums.txt | 11 + lib/apps.sh | 569 ++++++++++++++++++++++++++++++ lib/container.sh | 20 +- lib/files.sh | 8 +- lib/step.sh | 2 +- lss | 2 +- modules/apps/collabora.sh | 143 ++++++++ modules/apps/forgejo.sh | 167 +++++++++ modules/apps/keycloak.sh | 143 ++++++++ modules/apps/matomo.sh | 223 ++++++++++++ modules/apps/minio.sh | 257 ++++++++++++++ modules/apps/n8n.sh | 197 +++++++++++ modules/apps/nextcloud.sh | 207 +++++++++++ modules/apps/self-updater.sh | 249 +++++++++++++ modules/apps/uptime-kuma.sh | 149 ++++++++ modules/apps/vaultwarden.sh | 143 ++++++++ modules/apps/wordpress.sh | 152 ++++++++ profiles/all.conf | 5 +- profiles/devops.conf | 8 +- profiles/office.conf | 10 +- tests/integration/run.sh | 40 ++- tests/integration/settings.conf | 17 + tests/integration/vm.sh | 9 +- tests/unit/apps.bats | 450 +++++++++++++++++++++++ tests/unit/helpers.bash | 2 +- 27 files changed, 3285 insertions(+), 37 deletions(-) create mode 100644 lib/apps.sh create mode 100644 modules/apps/collabora.sh create mode 100644 modules/apps/forgejo.sh create mode 100644 modules/apps/keycloak.sh create mode 100644 modules/apps/matomo.sh create mode 100644 modules/apps/minio.sh create mode 100644 modules/apps/n8n.sh create mode 100644 modules/apps/nextcloud.sh create mode 100644 modules/apps/self-updater.sh create mode 100644 modules/apps/uptime-kuma.sh create mode 100644 modules/apps/vaultwarden.sh create mode 100644 modules/apps/wordpress.sh create mode 100644 tests/unit/apps.bats diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 537ef9b..9c0f50b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] - group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network, mail] + group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network, mail, apps] arch: [x64, arm64] include: - group: base @@ -71,6 +71,10 @@ jobs: # filter without public DNS. - group: mail modules: mail-relay guard,mail-relay mailserver nginx,php,mailserver,webmail + # The apps that run without Docker: Self-Updater deploys a Git repository (its own, by tag) and MinIO (Silo) + # runs from its package behind nginx. The Docker stacks of the other apps run in the apps VM jobs below. + - group: apps + modules: node,self-updater nginx,minio steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -136,9 +140,13 @@ jobs: system: [debian-12, debian-13, rocky-9, almalinux-10] run: [kernel, profile minimal-secure] include: - # Every module at once (profiles/all.conf), on a clean system without the runner's preinstalled databases. + # Every module at once (profiles/all.conf), on a clean system without the runner's preinstalled databases; + # with the apps it needs most of the memory and processors of the runner. - system: debian-13 run: profile all + memory: 12288 + cpus: 4 + disk: 40G # Docker with Portainer, then k3s (after Docker is removed; they conflict) on every supported system. The # runner itself already has Docker, so these need a clean VM. - system: debian-12 @@ -163,11 +171,18 @@ jobs: run: profile k8s-node - system: almalinux-10 run: profile k8s-node - # Docker with the monitoring stack, as on a DevOps server. + # Docker with the monitoring stack and Forgejo, as on a DevOps server. - system: debian-13 run: profile devops - system: almalinux-10 run: profile devops + # Mail, Nextcloud with Collabora Online and Vaultwarden, as on an office server. + - system: debian-13 + run: profile office + memory: 8192 + - system: almalinux-10 + run: profile office + memory: 8192 steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -192,6 +207,9 @@ jobs: env: SYSTEM: vm-${{ matrix.system }} RUN: ${{ matrix.run }} + LSS_VM_MEMORY: ${{ matrix.memory || 4096 }} + LSS_VM_CPUS: ${{ matrix.cpus || 2 }} + LSS_VM_DISK: ${{ matrix.disk || '20G' }} run: | if [[ $RUN == kernel ]]; then tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard wireguard @@ -207,3 +225,53 @@ jobs: esac tests/integration/run.sh "$SYSTEM" --profile "${RUN#profile }" "$minimum" fi + + # The apps as Docker Compose stacks behind each web server, on every supported system (x86_64 VMs: Docker needs a + # kernel of its own). Docker and the web server are the base; each app is installed, used (mod_verify), installed + # again and removed on its own. Nextcloud alone, then with Collabora Online (Nextcloud Office). + apps: + name: ${{ matrix.system }} VM / apps ${{ matrix.set }} + runs-on: ubuntu-24.04 + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] + set: [office, dev, web] + include: + - set: office + base: docker,nginx + modules: nextcloud collabora,nextcloud vaultwarden + - set: dev + base: docker,apache + modules: forgejo uptime-kuma n8n + - set: web + base: docker,caddy,tls + modules: keycloak matomo wordpress minio + settings: LSS_MINIO_MODE=docker + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install QEMU and allow access to KVM + run: | + sudo apt-get -qq update + sudo NEEDRESTART_SUSPEND=1 apt-get -qq install -y --no-install-recommends qemu-system-x86 qemu-utils cloud-image-utils >/dev/null + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Install, verify, reinstall, remove + env: + SYSTEM: vm-${{ matrix.system }} + BASE: ${{ matrix.base }} + MODULES: ${{ matrix.modules }} + LSS_TEST_SETTINGS: ${{ matrix.settings }} + LSS_VM_MEMORY: 8192 + LSS_VM_CPUS: 4 + LSS_VM_DISK: 40G + run: | + read -ra modules <<<"$MODULES" + tests/integration/run.sh "$SYSTEM" --base "$BASE" "${modules[@]}" diff --git a/README.md b/README.md index 0213b3a..940c7ad 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ every module on each tested system. | Profile `minimal-secure` | 1 GB; 2 GB on the RHEL family | 4 GB | 3 to 5 min plus AIDE | | `minimal-secure` with `crowdsec` | 1.5 GB; 2 GB on the RHEL family | 4.5 GB | about 30 s more | | `minimal-secure` with `clamav` | 2.5 GB | 4.5 GB | about 1 min more | +| Profile `office` | not measured yet | not measured yet | not measured yet | +| Profile `devops` | not measured yet | not measured yet | not measured yet | One CPU core is enough; more cores mainly shorten the first AIDE database build. The disk figure includes the swap file (`LSS_SWAP_SIZE`, 1 GB in the measurement), the journal (up to `LSS_LOGGING_MAX_USE`, 1 GB) and the audit logs @@ -88,6 +90,17 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `mail-relay` | 35 s | 250 MiB | 180 MiB (dnf), else 135 MiB | Postfix (master, queue manager, pickup) | | `mailserver` | 90 s | 800 MiB (dnf), else 225 MiB | 235 MiB (dnf), else 120 MiB | Postfix, Dovecot and Rspamd, up to 180 MiB | | `webmail` | 135 s with nginx, php and mailserver | 390 MiB together | 265 MiB together | Roundcube in PHP-FPM; with nginx and the mail server up to 290 MiB | +| `nextcloud` | not measured yet | not measured yet | not measured yet | Nextcloud (Apache) and PostgreSQL, up to 1.5 GiB | +| `collabora` | not measured yet | not measured yet | not measured yet | Collabora Online, up to 2 GiB | +| `forgejo` | not measured yet | not measured yet | not measured yet | Forgejo, up to 1 GiB | +| `uptime-kuma` | not measured yet | not measured yet | not measured yet | Uptime Kuma, up to 512 MiB | +| `vaultwarden` | not measured yet | not measured yet | not measured yet | Vaultwarden, up to 512 MiB | +| `keycloak` | not measured yet | not measured yet | not measured yet | Keycloak and PostgreSQL, up to 2 GiB | +| `matomo` | not measured yet | not measured yet | not measured yet | Matomo (Apache) and MariaDB, up to 1.5 GiB | +| `wordpress` | not measured yet | not measured yet | not measured yet | WordPress (Apache) and MariaDB, up to 1.3 GiB | +| `n8n` | not measured yet | not measured yet | not measured yet | n8n and PostgreSQL, up to 1.5 GiB | +| `minio` | not measured yet | not measured yet | not measured yet | Silo, up to 1 GiB in a container | +| `self-updater` | not measured yet | not measured yet | not measured yet | Self-Updater, about 60 MiB | Modules the tests install together are measured together. The Rocky 9 VM in CI is slow throughout (eight minutes even for `wireguard`); those outliers are left out, like one Rocky 9 container run of `mail-relay` that took six minutes. @@ -198,6 +211,11 @@ See [WireGuard, DNS and NTP](#wireguard-dns-and-ntp). **Mail:** `mail-relay` (the server sends its own mail, through a smarthost or directly) or `mailserver` (Postfix, Dovecot and Rspamd for your domains), `webmail` (Roundcube on the web server). See [Mail](#mail). +**Apps:** `nextcloud`, `collabora` (Collabora Online for Nextcloud Office), `forgejo` (Git hosting), `uptime-kuma`, +`vaultwarden` (password manager), `keycloak` (single sign-on), `matomo` (web analytics), `wordpress`, `n8n` (workflow +automation), `minio` (S3 object storage) and `self-updater` (deploys a Git repository when it changes). See +[Apps](#apps). + ### Web sites One web server owns ports 80 and 443. It answers only for the sites you set up; requests for any other host name get @@ -248,6 +266,47 @@ sudo lss install --profile k8s-node --set LSS_ADMIN_USER_NAME=alice --set LSS_AD With `docker` or `podman` installed, the guard scans every image daily with Trivy and reports HIGH and CRITICAL vulnerabilities per image. +### Apps + +The apps run as Docker Compose stacks behind the web server (nginx, Apache or Caddy), each on host names of its own: +`LSS__DOMAINS`, by default the module name in front of the server's name (`nextcloud.example.com`). Those names +must point to the server; with the `tls` module every app gets its own certificate. `minio` and `self-updater` also +run without Docker: Silo from its package, Self-Updater from its npm release. + +Every stack follows the container rules above: images pinned by digest, no capabilities, no new privileges, a +read-only root file system where the image allows it (Collabora Online and Keycloak write below their program +directory and are marked as exceptions), memory and process limits. The containers run as unprivileged users, mapped +to unused IDs on the host by Docker's user namespace. Ports bind to 127.0.0.1 only (`LSS__PORT`, 8081 to 8093); +the web server publishes them with TLS and the security headers of every lss site. Databases run in the same stack on +an internal network without a route out. + +Passwords never appear in a Compose file, an environment variable or a command line. lss creates them in +`/etc/lss/secrets` and hands each container a read-only copy as a file under `/run/secrets`. The first administrator +gets a generated password that lss does not print: `lss secrets show nextcloud-admin-password` and so on (for +Vaultwarden, `vaultwarden-admin-token` opens `/admin`, where you invite users; open registration is off everywhere). +Matomo stores its database password in its own `config.ini.php`, which lss cannot avoid. + +`lss verify` uses every app, not just its start page: it stores and reads a file in Nextcloud over WebDAV, converts a +document with Collabora Online, creates and deletes a repository in Forgejo, a monitor in Uptime Kuma, a user in +Keycloak and Vaultwarden and a workflow in n8n, tracks a page view in Matomo and reads it back, logs in to WordPress +and writes a bucket and an object to MinIO. Nextcloud and Collabora Online link themselves when both are installed, +in either order. + +Data lives in Docker volumes named after the stack (`lss-nextcloud_html`, `lss-nextcloud_db`, ...). `lss remove` +keeps them; `--purge` deletes them with the secrets. Back them up with `docker run` or a restic path of your own; +the `restic` module does not include them yet. + +`minio` is the MinIO server code, maintained as [PGSTY Silo](https://github.com/pgsty/silo) since MinIO archived its +community edition in 2026. The S3 API answers on `LSS_MINIO_DOMAINS`, the web console on +`LSS_MINIO_CONSOLE_DOMAINS`. `LSS_MINIO_MODE=docker` runs it as a container instead of the package. + +`self-updater` installs [Self-Updater](https://github.com/VectoDE/Self-Updater), which keeps a checkout of +`LSS_SELF_UPDATER_REPO` in `LSS_SELF_UPDATER_PATH` up to date (every new commit of a branch, or every release tag with +`LSS_SELF_UPDATER_TRACK=tag`), restarts `LSS_SELF_UPDATER_SERVICE` and rolls back when the health check fails. + +The profile `office` installs `minimal-secure`, mail with webmail, Nextcloud with Collabora Online, Vaultwarden and +restic; `devops` adds Forgejo to Docker and the monitoring stack. + ### Backup `restic` backs up `LSS_RESTIC_PATHS` (configuration, home directories, web roots, the database dumps) every night at @@ -293,7 +352,8 @@ node exporter from the distribution, the RHEL family the checksum-verified upstr come from Grafana's repository and Netdata from Netdata's, both with pinned keys. `smartd` watches the drives, also in virtual machines, where it usually finds none. -The profile `devops` installs `minimal-secure`, `docker` and the monitoring stack without Netdata: +The profile `devops` installs `minimal-secure`, `docker`, nginx with Forgejo and the monitoring stack without +Netdata: ```bash sudo lss install --profile devops --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github @@ -357,7 +417,8 @@ Many providers block outgoing port 25; ask for it to be opened or relay through `webmail` runs Roundcube as a site of the web server (`LSS_WEBMAIL_DOMAINS`, by default the mail host), with the certificate of the `tls` module when it is installed. It talks to Dovecot and Postfix on 127.0.0.1 only. The profile -`office` installs `minimal-secure`, nginx with `tls` and `php`, `mailserver`, `webmail` and `restic`: +`office` installs `minimal-secure`, nginx with `tls` and `php`, `mailserver`, `webmail`, Docker with `nextcloud`, +`collabora` and `vaultwarden` (see [Apps](#apps)) and `restic`: ```bash sudo lss install --profile office --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github --set LSS_MAILSERVER_DOMAINS=example.com --set LSS_MAILSERVER_USERS=alice --set LSS_TLS_DOMAINS=mail.example.com diff --git a/data/checksums.txt b/data/checksums.txt index 5c5f5be..42a2a90 100644 --- a/data/checksums.txt +++ b/data/checksums.txt @@ -106,3 +106,14 @@ alertmanager-0.34.1 x86_64 265b9d1e55ef0d5306a436018af6d2b686c2ce051f03d968f7464 alertmanager-0.34.1 aarch64 d98d6cbaf52151c7e76e24355fec88b11cebcb9875d4cdd8b76ddce7a7e5535c https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-arm64.tar.gz node_exporter-1.12.1 x86_64 b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz node_exporter-1.12.1 aarch64 ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-arm64.tar.gz + +# Self-Updater 4.2.1 (VectoDE, npm package of the release): SHA-256 digest of the release asset on GitHub (gh api +# repos/VectoDE/Self-Updater/releases/tags/v4.2.1, checked 2026-10-15; the download matched it). +self-updater-4.2.1 any 2a864c6fcdc6325b7ef87916ee277202ff675937223fdd6d585b7d4fa313e74f https://github.com/VectoDE/Self-Updater/releases/download/v4.2.1/vectode-self-updater-4.2.1.tgz +# PGSTY Silo RELEASE.2026-09-16T00-00-00Z (the maintained fork of the MinIO server; module minio, native mode): the +# packages of the GitHub release, SHA-256 digests of the release assets (gh api repos/pgsty/silo/releases, checked +# 2026-10-15). +silo-deb x86_64 176aad3944d9e9f949baf47bcea86e62a3271094be075c6547c056f652ae7e5f https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo_20260916000000.0.0-1PGSTY_amd64.deb +silo-deb aarch64 32f7fab06f38a90920bf7295017b031da0eb03fcb1e54fd948f109fd30e3a711 https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo_20260916000000.0.0-1PGSTY_arm64.deb +silo-rpm x86_64 cb4d6690d386e43dca0b9c6ae4e3abb1fad49c75f9df7d36502c5f8742ef3a77 https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo-20260916000000.0.0-1PGSTY.x86_64.rpm +silo-rpm aarch64 e7fa4c2fcf49a9d7632e91b4aceef3225c5330571cb19116e5ba59c8d2dc031c https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo-20260916000000.0.0-1PGSTY.aarch64.rpm diff --git a/lib/apps.sh b/lib/apps.sh new file mode 100644 index 0000000..a4a59b5 --- /dev/null +++ b/lib/apps.sh @@ -0,0 +1,569 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Apps: the modules of the group apps run as Docker Compose stacks behind the installed web server. Each stack is the +# project lss-NAME with its files in CTR_CONFIG_DIR/lss-NAME (the one directory of /etc its containers may mount): +# compose.yaml the stack, checked against the container rules (lib/container.sh) before every start +# secrets/ copies of the secrets in /etc/lss/secrets that a container reads, mounted read-only as +# /run/secrets/NAME and owned by the user the container runs as (seen from the host) +# config/ other files a container reads +# Nothing secret is written into compose.yaml, an environment variable or a command line: the images read +# /run/secrets/NAME themselves (*_FILE variables) or a small wrapper reads it before the program starts. +# +# Every published port binds to 127.0.0.1; the web server publishes the app as an HTTPS site with the security headers +# of lib/web.sh. Data lives in named volumes (lss-NAME_*), which stay when the module is removed, unless --purge. + +APP_START_TIMEOUT=${APP_START_TIMEOUT:-600} + +# app_project: the Compose project of the module running this step. +app_project() { + printf 'lss-%s' "${LSS_MODULE}" + return +} + +# app_dir: the directory of the stack. +app_dir() { + printf '%s/%s' "${CTR_CONFIG_DIR}" "$(app_project)" + return +} + +app_compose_file() { + printf '%s/compose.yaml' "$(app_dir)" + return +} + +# app_valid_domain NAME: a host name like cloud.example.com (lowercase, at least two labels). +app_valid_domain() { + local name=$1 + [[ ${name} =~ ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,61}[a-z0-9]$ ]] + return +} + +# app_check_domains: checks the option domains of the module (host names separated by spaces). +app_check_domains() { + local domain + for domain in $(opt domains); do + app_valid_domain "${domain}" || + die "$(config_key "${LSS_MODULE}" domains): '${domain}' is not a host name like ${LSS_MODULE}.example.com." + done + return +} + +# app_check_port OPTION: the option is a port from 1024 to 65535 other than the admin port of the web server. +app_check_port() { + local option=$1 port + port=$(opt_int "${option}") + ((port >= 1024 && port <= 65535)) || die "$(config_key "${LSS_MODULE}" "${option}") must be a port from 1024 to 65535." + [[ ${port} != "${WEB_ADMIN_PORT}" ]] || + die "$(config_key "${LSS_MODULE}" "${option}") must differ from the admin port of the web server (${WEB_ADMIN_PORT})." + return +} + +# app_domains: the host names of the app: the option domains, or NAME.. +app_domains() { + local domains host + domains=$(opt domains) + if [[ -z ${domains} ]]; then + host=$(hostname -f 2>/dev/null || true) + [[ -n ${host} ]] || host=$(hostname) + domains=${LSS_MODULE}.${host,,} + fi + printf '%s' "${domains}" + return +} + +# app_domain: the first host name of the app, the one its links use. +app_domain() { + local -a domains=() + read -ra domains <<<"$(app_domains)" + printf '%s' "${domains[0]}" + return +} + +# app_compose_ready: docker compose is there (the Compose plugin of Docker CE). +app_compose_ready() { + docker compose version >/dev/null 2>&1 || + die "${LSS_MODULE}: docker compose is missing; the apps need the Compose plugin (LSS_DOCKER_SOURCE=auto installs it)." + return +} + +# _app_userns_base FILE: the first ID of the range Docker maps containers into (/etc/subuid or /etc/subgid), or 0 +# when the daemon runs without user namespace remapping. +_app_userns_base() { + local file=$1 user base + user=$(jq -r '."userns-remap" // empty' "${APP_DOCKER_CONFIG:-/etc/docker/daemon.json}" 2>/dev/null) || user='' + if [[ -z ${user} ]]; then + printf '0' + return 0 + fi + [[ ${user} != default ]] || user=dockremap + user=${user%%:*} + base=$(awk -F ':' -v user="${user}" '$1 == user { print $2; exit }' "${file}" 2>/dev/null) || base='' + [[ ${base} =~ ^[0-9]+$ ]] || die "${LSS_MODULE}: ${file} has no ID range for ${user}, which Docker maps containers into." + printf '%s' "${base}" + return +} + +# app_host_owner UID[:GID]: the owner, as the host sees it, of a file the container user UID:GID must read. +app_host_owner() { + local ids=$1 uid gid + uid=${ids%%:*} + gid=${ids#*:} + [[ ${ids} == *:* ]] || gid=${uid} + [[ ${uid} =~ ^[0-9]+$ && ${gid} =~ ^[0-9]+$ ]] || die "${LSS_MODULE}: '${ids}' is not UID or UID:GID." + printf '%s:%s' "$((uid + $(_app_userns_base "${APP_SUBUID:-/etc/subuid}")))" \ + "$((gid + $(_app_userns_base "${APP_SUBGID:-/etc/subgid}")))" + return +} + +# APP_CHANGED: a file the containers mount changed in this step, so they must be created again (a bind-mounted file +# keeps showing the old copy once file_write replaced it). APP_COMPOSE_CHANGED: compose.yaml changed. +APP_CHANGED=0 +APP_COMPOSE_CHANGED=0 + +# app_file PATH UID[:GID] MODE: writes stdin to $(app_dir)/PATH for the container user UID:GID. +app_file() { + local path=$1 ids=$2 mode=$3 + [[ ${path} =~ ^[a-z0-9][a-z0-9._/-]*$ && /${path}/ != */../* ]] || die "${LSS_MODULE}: '${path}' is not a valid file name." + install -d -m 0700 "$(app_dir)" + install -d -m 0700 "$(dirname "$(app_dir)/${path}")" + file_write "${mode}" "$(app_host_owner "${ids}")" "$(app_dir)/${path}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || APP_CHANGED=1 + return +} + +# app_secret SECRET UID[:GID] [LENGTH] [FILE]: the secret from /etc/lss/secrets (created on first use), copied to +# secrets/FILE (default: SECRET) for the container user UID:GID, mode 0400, without a final newline. The container +# mounts it with app_mount secrets/FILE /run/secrets/NAME. +app_secret() { + local secret=$1 ids=$2 length=${3:-32} file=${4:-$1} value + secret_load "${secret}" value "${length}" + printf '%s' "${value}" | app_file "secrets/${file}" "${ids}" 0400 + return +} + +# app_mount PATH TARGET: the volume line of a Compose service that mounts $(app_dir)/PATH read-only at TARGET (with +# the shared SELinux label on the RHEL family, so every container of the stack may read it). +app_mount() { + local path=$1 target=$2 + printf ' - %s/%s:%s:ro,z\n' "$(app_dir)" "${path}" "${target}" + return +} + +# app_hardening MEMORY PIDS [writable]: the lines every Compose service of lss gets (indented for a service): no +# capabilities, no new privileges, memory and process limits, a read-only root file system unless "writable", restart +# after a crash or reboot. Services add the capabilities they need with cap_add. +app_hardening() { + local memory=$1 pids=$2 writable=${3:-} + printf ' restart: unless-stopped\n cap_drop: [ALL]\n security_opt: ["no-new-privileges:true"]\n' + printf ' mem_limit: %s\n memswap_limit: %s\n pids_limit: %s\n' "${memory}" "${memory}" "${pids}" + if [[ ${writable} == writable ]]; then + printf ' labels:\n lss.managed: "true"\n lss.writable: "true"\n' + else + printf ' read_only: true\n labels:\n lss.managed: "true"\n' + fi + return +} + +# app_compose: writes the stack from stdin to compose.yaml (root only: it names the files with the secrets). +app_compose() { + install -d -m 0700 "$(app_dir)" + file_write 0600 root:root "$(app_compose_file)" + APP_COMPOSE_CHANGED=${LSS_FILE_CHANGED} + return +} + +# _app_compose COMMAND...: docker compose for the stack of the module. +_app_compose() { + docker compose --project-name "$(app_project)" -f "$(app_compose_file)" "$@" + return +} + +# app_running: every service of the stack has a running container (except "init", which runs once per start). +app_running() { + local services running + services=$(_app_compose config --services 2>/dev/null | grep -vx init | sort) || return 1 + running=$(_app_compose ps --status running --services 2>/dev/null | sort) || return 1 + [[ -n ${services} && ${services} == "${running}" ]] + return +} + +# app_up: starts the stack after compose.yaml or a mounted file changed, or when a service is not running. A changed +# mounted file creates every container again, because a bind mount keeps the replaced file. +app_up() { + local -a options=() + app_compose_ready + if [[ ${APP_CHANGED} == 1 ]]; then + options=(--force-recreate) + elif [[ ${APP_COMPOSE_CHANGED} == 0 ]] && app_running; then + return 0 + fi + log_info "Starting $(app_project) (images are pulled on the first start)" + ctr_compose_up "$(app_project)" "$(app_compose_file)" "${options[@]}" + files_note_change "started $(app_project)" + APP_CHANGED=0 + APP_COMPOSE_CHANGED=0 + return +} + +# app_exec [OPTION...] SERVICE COMMAND...: runs COMMAND in the running container of SERVICE (no terminal, stdin passed +# on); OPTIONs of "docker compose exec" such as --user or --env come first. +app_exec() { + _app_compose exec -T "$@" + return +} + +# Nextcloud Office: the collabora module and the nextcloud module link each other, whichever comes second. +APP_NEXTCLOUD_PROJECT=lss-nextcloud + +# app_domains_of MODULE: the host names of another app module (its option domains, or MODULE.). +app_domains_of() { + local module=$1 domains host + domains=$(config_option "${module}" domains '') + if [[ -z ${domains} ]]; then + host=$(hostname -f 2>/dev/null || true) + [[ -n ${host} ]] || host=$(hostname) + domains=${module}.${host,,} + fi + printf '%s' "${domains}" + return +} + +# _app_nc_exec COMMAND...: a command in the Nextcloud container (stdin passed on). +_app_nc_exec() { + docker compose --project-name "${APP_NEXTCLOUD_PROJECT}" -f "${CTR_CONFIG_DIR}/${APP_NEXTCLOUD_PROJECT}/compose.yaml" exec -T app "$@" + return +} + +# _app_nc_occ ARG...: occ in the Nextcloud container. +_app_nc_occ() { + _app_nc_exec php occ "$@" + return +} + +# _app_nc_trust: the certificate Nextcloud must trust to reach the Collabora site: the CA of a private ACME server +# (LSS_TLS_CA_BUNDLE) or, without the tls module, the self-signed default certificate of the web server; nothing for +# Let's Encrypt. Prints the file, or nothing. +_app_nc_trust() { + local bundle + if state_installed tls; then + bundle=$(config_option tls ca_bundle '') + [[ -z ${bundle} || ! -s ${bundle} ]] || printf '%s' "${bundle}" + elif [[ -s ${WEB_TLS_DIR}/default/fullchain.pem ]]; then + printf '%s/default/fullchain.pem' "${WEB_TLS_DIR}" + fi + return +} + +# app_nextcloud_office: with both modules installed (or the other one being installed), Nextcloud Office +# (richdocuments) points at the Collabora site and checks that it reaches it. +app_nextcloud_office() { + local url trust sum module first + for module in nextcloud collabora; do + state_installed "${module}" || [[ ${LSS_MODULE} == "${module}" ]] || return 0 + done + read -r first _ <<<"$(app_domains_of collabora)" + url=https://${first} + if ! _app_nc_occ app:list --output=json 2>/dev/null | jq -e '.enabled.richdocuments' >/dev/null 2>&1; then + log_command _app_nc_occ app:install richdocuments || log_command _app_nc_occ app:enable richdocuments || + die "${LSS_MODULE}: the Nextcloud app Nextcloud Office (richdocuments) cannot be installed." + files_note_change "installed Nextcloud Office" + fi + if [[ $(_app_nc_occ config:app:get richdocuments wopi_url 2>/dev/null) != "${url}" ]]; then + log_command _app_nc_occ config:app:set richdocuments wopi_url --value="${url}" || + die "${LSS_MODULE}: occ cannot set the address of Collabora Online." + files_note_change "linked Nextcloud Office to ${url}" + fi + trust=$(_app_nc_trust) + if [[ -n ${trust} ]]; then + sum=$(sha256sum "${trust}" | cut -d ' ' -f 1) + if [[ $(state_get nextcloud trusted_cert) != "${sum}" ]]; then + _app_nc_exec sh -c 'cat >/tmp/lss-web.pem' <"${trust}" || + die "${LSS_MODULE}: cannot copy the certificate of the web server into the Nextcloud container." + log_command _app_nc_occ security:certificates:import /tmp/lss-web.pem || + die "${LSS_MODULE}: occ cannot add the certificate of the web server to the trusted ones of Nextcloud." + state_set nextcloud trusted_cert "${sum}" + files_note_change "Nextcloud trusts the certificate of the web server" + fi + fi + log_command _app_nc_occ richdocuments:activate-config || + die "${LSS_MODULE}: Nextcloud Office cannot reach Collabora Online at ${url}; see the log." + return +} + +# app_timer NAME INTERVAL DESCRIPTION SERVICE COMMAND...: the systemd timer lss-NAME.timer, which runs COMMAND in the +# container of SERVICE every INTERVAL (a systemd time span such as 5min), first INTERVAL after boot. +app_timer() { + local name=$1 interval=$2 description=$3 service=$4 docker + shift 4 + docker=$(command -v docker) || die "${LSS_MODULE}: docker is missing." + svc_unit "lss-${name}.service" <&2 2>/dev/null || true + _app_compose logs --tail 30 >&2 2>/dev/null || true + return 1 +} + +# app_verify_stack: the stack runs, its compose.yaml still passes the container rules and the site serves security +# headers. +app_verify_stack() { + local page + [[ -s $(app_compose_file) ]] || die "${LSS_MODULE}: $(app_compose_file) is missing." + [[ $(stat -c '%a %U' "$(app_compose_file)") == '600 root' ]] || die "${LSS_MODULE}: $(app_compose_file) must be mode 600, owned by root." + ctr_compose_check "$(app_compose_file)" "$(app_project)" || die "${LSS_MODULE}: the stack breaks the container rules; see above." + app_running || die "${LSS_MODULE}: a container of $(app_project) is not running (docker compose -p $(app_project) ps)." + page=$(app_get '' --head) || true + grep -qi '^strict-transport-security:' <<<"${page}" || + die "${LSS_MODULE}: https://$(app_domain)/ sends no Strict-Transport-Security header; the web server site is missing." + return +} + +# Database containers for the stacks: official images, pinned by digest (multi-architecture index, amd64 and arm64), +# running as their own unprivileged user without capabilities, only on the internal network "backend" of the stack. +# PostgreSQL 18.6 (alpine; postgres is 70:70) and MariaDB 11.8.9 LTS (mysql is 999:999), checked 2026-10-15. +APP_POSTGRES_IMAGE=docker.io/library/postgres@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 +APP_POSTGRES_USER=70:70 +APP_MARIADB_IMAGE=docker.io/library/mariadb@sha256:6422478cb8e159f080fb1d8ccf65101e26fe51385787fde7d16c3b165a331f15 +APP_MARIADB_USER=999:999 + +# app_postgres NAME: the service "db" (PostgreSQL with database and user NAME); the password is the secret +# -db-password, mounted as /run/secrets/db-password. Call app_secret for it with APP_POSTGRES_USER and the file +# name db-password.db first. +app_postgres() { + local name=$1 + cat <-db-root-password (db-root-password.db). +app_mariadb() { + local name=$1 + cat <"${file}" + printf '%s' "${file}" + return +} + +# app_site_remove NAME: removes the site NAME of app_site with its certificates. +app_site_remove() { + local name=$1 + if [[ -n $(web_server) ]]; then + web_site_remove "${name}" + web_reload + fi + file_remove "${WEB_TLS_DIR}/${name}" + if [[ -d ${WEB_CERT_LIVE}/${name} ]] && command -v certbot >/dev/null 2>&1; then + log_command certbot delete --non-interactive --cert-name "${name}" || + log_warn "${name}: certbot could not delete the certificate ${name}." + fi + return +} + +# app_remove: stops the stack and removes its site and files; the named volumes (the data) and the secrets stay unless +# --purge. +app_remove() { + local name=${LSS_MODULE} secret + app_site_remove "${name}" + if [[ -s $(app_compose_file) ]] && command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + local -a options=(--remove-orphans) + [[ ${LSS_PURGE:-0} != 1 ]] || options+=(--volumes) + log_command _app_compose down "${options[@]}" || die "${name}: docker compose cannot stop $(app_project)." + files_note_change "stopped $(app_project)" + fi + file_remove "$(app_dir)" + rmdir "${CTR_CONFIG_DIR}" 2>/dev/null || true + if [[ ${LSS_PURGE:-0} == 1 ]]; then + for secret in ${MOD_SECRETS:-}; do + secret_remove "${secret}" + done + fi + return +} diff --git a/lib/container.sh b/lib/container.sh index b6cc622..61f3741 100644 --- a/lib/container.sh +++ b/lib/container.sh @@ -424,11 +424,12 @@ CTR_COMPOSE_RULES=' # shellcheck disable=SC2016 # jq variables, not shell variables CTR_COMPOSE_BINDS='.services // {} | to_entries[] | .key as $name | .value.volumes[]? | select(.type == "bind") | "\($name)\t\(.source // "")"' -# ctr_compose_check FILE: checks every service of a Compose file against the container rules. docker compose resolves -# the file first (extends, anchors, defaults), so the checks see what Compose would run. Prints each violation and fails -# when there is one; a file it cannot read or check fails as well. +# ctr_compose_check FILE [PROJECT]: checks every service of a Compose file against the container rules. docker +# compose resolves the file first (extends, anchors, defaults), so the checks see what Compose would run. Prints each +# violation and fails when there is one; a file it cannot read or check fails as well. Bind mounts from /etc may come +# from CTR_CONFIG_DIR/PROJECT (the stacks of the app modules), without PROJECT from CTR_CONFIG_DIR/compose-SERVICE. ctr_compose_check() { - local file=$1 json problems binds service source problem line + local file=$1 project=${2:-} json problems binds service source problem line owner [[ -r ${file} ]] || die "container: cannot read the Compose file ${file}." json=$(docker compose -f "${file}" config --format json 2>/dev/null) || die "container: docker compose cannot read ${file} (docker compose -f ${file} config shows why)." @@ -437,7 +438,8 @@ ctr_compose_check() { binds=$(jq -r "${CTR_COMPOSE_BINDS}" <<<"${json}") || die "container: jq cannot read the mounts of ${file}." while IFS=$'\t' read -r service source; do [[ -n ${service} ]] || continue - problem=$(_ctr_host_source_problem "compose-${service}" "${source}") || + owner=${project:-compose-${service}} + problem=$(_ctr_host_source_problem "${owner}" "${source}") || die "container: cannot resolve the host path ${source} of ${service} in ${file}." [[ -z ${problem} ]] || problems+=$'\n'"${service}: ${problem}" done <<<"${binds}" @@ -449,13 +451,15 @@ ctr_compose_check() { return 1 } -# ctr_compose_up PROJECT FILE: starts a Compose project after ctr_compose_check passed (Docker only). +# ctr_compose_up PROJECT FILE [OPTION...]: starts a Compose project after ctr_compose_check passed (Docker only). The +# OPTIONs go to "docker compose up" (--force-recreate after a mounted file changed). ctr_compose_up() { local project=$1 file=$2 + shift 2 [[ $(ctr_engine) == docker ]] || die "container: Compose projects need the docker module." [[ ${project} =~ ^[a-z0-9][a-z0-9_-]*$ ]] || die "container: '${project}' is not a valid Compose project name." - ctr_compose_check "${file}" || die "container: ${file} breaks the container rules; see above." - log_command docker compose --project-name "${project}" -f "${file}" up --detach --remove-orphans --quiet-pull || + ctr_compose_check "${file}" "${project}" || die "container: ${file} breaks the container rules; see above." + log_command docker compose --project-name "${project}" -f "${file}" up --detach --remove-orphans --quiet-pull "$@" || die "container: docker compose cannot start ${project}; see the log." return } diff --git a/lib/files.sh b/lib/files.sh index 1995adc..0271b46 100644 --- a/lib/files.sh +++ b/lib/files.sh @@ -94,16 +94,18 @@ _files_record_checksum() { return } -# file_write MODE OWNER:GROUP DEST, content on stdin. Sets LSS_FILE_CHANGED to 1 or 0. +# file_write MODE OWNER:GROUP DEST, content on stdin. Sets LSS_FILE_CHANGED to 1 or 0. OWNER:GROUP are names, or +# numbers (UID:GID) for IDs without an account, such as the users of a container in a user namespace. file_write() { - local mode=$1 owner=$2 dest=$3 tmp current_mode current_owner + local mode=$1 owner=$2 dest=$3 tmp current_mode current_owner format='%U:%G' LSS_FILE_CHANGED=0 + [[ ! ${owner} =~ ^[0-9]+:[0-9]+$ ]] || format='%u:%g' mkdir -p "$(dirname "${dest}")" tmp=$(mktemp "${dest}.lss.XXXXXX") cat >"${tmp}" if [[ -f ${dest} ]] && cmp -s "${tmp}" "${dest}"; then current_mode=$(stat -c '%a' "${dest}") - current_owner=$(stat -c '%U:%G' "${dest}") + current_owner=$(stat -c "${format}" "${dest}") if [[ ${current_mode} == "${mode#0}" && ${current_owner} == "${owner}" ]]; then rm -f -- "${tmp}" _files_record_checksum "${dest}" diff --git a/lib/step.sh b/lib/step.sh index 757da7a..be9f141 100755 --- a/lib/step.sh +++ b/lib/step.sh @@ -15,7 +15,7 @@ set -Eeuo pipefail # change notes it sets reach the caller. shopt -s inherit_errexit lastpipe -for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring module; do +for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring apps module; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/lss b/lss index 532763c..68652c2 100755 --- a/lss +++ b/lss @@ -14,7 +14,7 @@ LSS_ROOT=$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd) LSS_VERSION=$(<"${LSS_ROOT}/VERSION") export LSS_ROOT LSS_VERSION -for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring module ui; do +for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring apps module ui; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/modules/apps/collabora.sh b/modules/apps/collabora.sh new file mode 100644 index 0000000..2304bb9 --- /dev/null +++ b/modules/apps/collabora.sh @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="collabora" +MOD_GROUP="apps" +MOD_DESCRIPTION="Collabora Online (CODE 26.04) as a Docker Compose stack behind the web server, the office suite of Nextcloud Office; only the Nextcloud host names may open documents" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Collabora site, separated by spaces; empty: collabora. +port|8082|Port of the Collabora container on 127.0.0.1, which the web server forwards to +memory|2g|Memory limit of the Collabora container +dictionaries|de_DE en_GB en_US|Spell checking languages, separated by spaces" +MOD_SECRETS="" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Collabora Online Development Edition 26.04.5.1.1, the multi-architecture index (amd64 and arm64), checked 2026-10-15. +# The image is distroless (no shell) and runs as cool (1001). +COOL_IMAGE=docker.io/collabora/code@sha256:bb8f3c9c5c9016045f25fccb5f0649be94469ebc7c7cc379f3576e470cccf33d + +mod_check() { + local languages='^[a-zA-Z_ -]*$' + app_check_domains + app_check_port port + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_COLLABORA_MEMORY must be a size such as 2g or 1536m." + [[ $(opt dictionaries) =~ ${languages} ]] || die "LSS_COLLABORA_DICTIONARIES must be language codes such as de_DE en_US." + return +} + +mod_install() { + app_compose_ready + return +} + +# _cool_wopi_hosts: the sites allowed to open documents (WOPI hosts): the host names of Nextcloud, as an alias group +# (the first one is the host, the others its aliases). Nothing else may use this server. +_cool_wopi_hosts() { + local domain list='' + for domain in $(app_domains_of nextcloud); do + list+=${list:+,}https://${domain}:443 + done + printf '%s' "${list}" + return +} + +# _cool_verify_tls: Collabora checks the certificate of Nextcloud only when it comes from a public CA (the tls module +# without a private ACME server); the image has no way to add another CA, and the connection stays on this server. +_cool_verify_tls() { + if state_installed tls && [[ -z $(config_option tls ca_bundle '') ]]; then + printf 'true' + else + printf 'false' + fi + return +} + +# _cool_compose: the stack. Collabora runs without capabilities and without new privileges, so it cannot build its +# chroot jails; COOL_FORCE_LANDLOCK makes it isolate the document processes with Landlock instead. The root file +# system stays writable because the jails are created below /opt/cool. The admin console is off: it would need a +# password in the environment. +_cool_compose() { + local domain params + domain=$(app_domain) + params="--o:ssl.enable=false --o:ssl.termination=true --o:admin_console.enable=false" + params+=" --o:ssl.ssl_verification=$(_cool_verify_tls) --o:logging.level=warning" + cat </dev/null || die "collabora: https://$(app_domain)/ does not answer." + if state_installed nextcloud; then + app_nextcloud_office + fi + return +} + +mod_verify() { + local page probe pdf status + app_verify_stack + page=$(app_wait '' 200) || die "collabora: https://$(app_domain)/ does not answer." + [[ ${page%$'\n'*} == OK ]] || die "collabora: https://$(app_domain)/ answers '${page%$'\n'*}' instead of OK." + page=$(app_wait hosting/discovery 200 60) || die "collabora: the WOPI discovery does not answer." + grep -q 'urlsrc=' <<<"${page}" || die "collabora: the WOPI discovery lists no editors." + # Used, not only running: Collabora converts a text document to PDF. + probe=$(mktemp) + pdf=$(mktemp) + printf 'lss verify %s\n' "$(date +%s)" >"${probe}" + status=$(app_status "$(app_get cool/convert-to/pdf -F "data=@${probe};filename=lss-verify.txt" -o "${pdf}" --max-time 120)") + if [[ ${status} != 200 || $(head -c 4 "${pdf}") != %PDF ]]; then + rm -f -- "${probe}" "${pdf}" + die "collabora: converting a document to PDF failed (HTTP ${status})." + fi + rm -f -- "${probe}" "${pdf}" + if state_installed nextcloud; then + log_command _app_nc_occ richdocuments:activate-config || + die "collabora: Nextcloud Office cannot reach Collabora Online; see the log." + fi + return +} + +mod_remove() { + # Nextcloud stops offering Nextcloud Office; installing the module again turns it back on. + if state_installed nextcloud && command -v docker >/dev/null 2>&1; then + log_command _app_nc_occ app:disable richdocuments || + log_warn "collabora: Nextcloud Office could not be turned off in Nextcloud." + fi + app_remove + return +} diff --git a/modules/apps/forgejo.sh b/modules/apps/forgejo.sh new file mode 100644 index 0000000..f4fd116 --- /dev/null +++ b/modules/apps/forgejo.sh @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="forgejo" +MOD_GROUP="apps" +MOD_DESCRIPTION="Forgejo 16 (Git hosting) as a rootless Docker Compose stack behind the web server, with SQLite, Git over HTTPS only and no open registration" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Forgejo site, separated by spaces; empty: forgejo. +port|8083|Port of the Forgejo container on 127.0.0.1, which the web server forwards to +admin_user|lssadmin|Name of the first administrator (the password is the secret forgejo-admin-password) +admin_email||E-mail address of the first administrator; empty: ADMIN_USER@ +memory|1g|Memory limit of the Forgejo container" +MOD_SECRETS="forgejo-admin-password forgejo-secret-key forgejo-internal-token" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Forgejo 16.0.6 rootless (runs as git, 1000:1000), the multi-architecture index (amd64 and arm64), checked +# 2026-10-15. A new release means a new digest here; Forgejo migrates its database when it starts. +FJ_IMAGE=codeberg.org/forgejo/forgejo@sha256:51b47921660e1ec7127f094208fe06901ba57df4fe6d59fcccc3a2419514a77f +FJ_USER=1000:1000 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{1,38}$ && $(opt admin_user) != admin ]] || + die "LSS_FORGEJO_ADMIN_USER must be a lowercase login name other than admin (reserved by Forgejo)." + [[ -z $(opt admin_email) || $(opt admin_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_FORGEJO_ADMIN_EMAIL must be an e-mail address." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_FORGEJO_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +# _fj_email: the e-mail address of the first administrator. +_fj_email() { + local email + email=$(opt admin_email) + [[ -n ${email} ]] || email=$(opt admin_user)@$(app_domain) + printf '%s' "${email}" + return +} + +# _fj_compose: the stack. Forgejo reads its keys from files (the *_URI settings), so they are neither in app.ini nor +# in the environment. SSH stays off: Git works over HTTPS through the web server. +_fj_compose() { + local domain + domain=$(app_domain) + cat </dev/null | awk 'NR > 1 { print $2 }' | grep -qx -- "${user}"; then + secret_exists forgejo-admin-password || + log_warn "forgejo: the administrator ${user} exists, but the secret forgejo-admin-password is missing." + return 0 + fi + output=$(_fj admin user create --username "${user}" --email "$(_fj_email)" --admin --random-password \ + --random-password-length 32 --must-change-password=false 2>&1) || { + log_error "forgejo: $(grep -v 'generated random password' <<<"${output}")" + die "forgejo: the administrator ${user} cannot be created." + } + password=$(sed -n "s/^generated random password is '\(.*\)'$/\1/p" <<<"${output}") + [[ -n ${password} ]] || die "forgejo: Forgejo did not report the password of ${user}." + secret_set forgejo-admin-password "${password}" + log_info "forgejo: created the administrator ${user}; password in $(secret_path forgejo-admin-password)" + files_note_change "created the Forgejo administrator ${user}" + return +} + +mod_configure() { + app_secret forgejo-secret-key "${FJ_USER}" 64 secret-key + app_secret forgejo-internal-token "${FJ_USER}" 64 internal-token + _fj_compose | app_compose + app_up + app_site "$(opt_int port)" + app_wait api/healthz 200 >/dev/null || die "forgejo: https://$(app_domain)/api/healthz does not answer." + _fj_admin + return +} + +# _fj_api METHOD PATH [CURL OPTION...]: an API request as the administrator; prints the body and the status. +_fj_api() { + local method=$1 path=$2 netrc page + shift 2 + netrc=$(app_netrc "$(opt admin_user)" forgejo-admin-password) + page=$(app_get "api/v1/${path}" --netrc-file "${netrc}" -X "${method}" -H 'Content-Type: application/json' "$@") || true + rm -f -- "${netrc}" + printf '%s' "${page}" + return +} + +mod_verify() { + local page repo user + user=$(opt admin_user) + app_verify_stack + page=$(app_wait api/healthz 200) || die "forgejo: https://$(app_domain)/api/healthz does not answer." + [[ $(jq -r '.status' <<<"${page%$'\n'*}" 2>/dev/null) =~ ^(pass|warn)$ ]] || die "forgejo: the health check fails." + page=$(app_wait user/login 200 60) || die "forgejo: the login page does not answer." + grep -q 'name="password"' <<<"${page}" || die "forgejo: https://$(app_domain)/user/login shows no login." + # Registration is off: the sign-up page has no form to repeat a password. + page=$(app_get user/sign_up) || true + ! grep -q 'name="retype"' <<<"${page}" || die "forgejo: anybody can register at https://$(app_domain)/user/sign_up." + # Used, not only running: the administrator creates a repository through the API, reads it and deletes it. + repo="lss-verify-$(date +%s)" + page=$(_fj_api POST user/repos -d "{\"name\":\"${repo}\",\"private\":true,\"auto_init\":true}") + [[ $(app_status "${page}") == 201 ]] || die "forgejo: creating a repository as ${user} answers $(app_status "${page}")." + page=$(_fj_api GET "repos/${user}/${repo}/contents/README.md") + [[ $(app_status "${page}") == 200 ]] || die "forgejo: the new repository has no README.md ($(app_status "${page}"))." + page=$(_fj_api DELETE "repos/${user}/${repo}") + [[ $(app_status "${page}") == 204 ]] || die "forgejo: deleting the test repository answers $(app_status "${page}")." + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/keycloak.sh b/modules/apps/keycloak.sh new file mode 100644 index 0000000..a30af69 --- /dev/null +++ b/modules/apps/keycloak.sh @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="keycloak" +MOD_GROUP="apps" +MOD_DESCRIPTION="Keycloak 26 (single sign-on with OpenID Connect and SAML) with PostgreSQL as a Docker Compose stack behind the web server" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Keycloak site, separated by spaces; empty: keycloak. +port|8086|Port of the Keycloak container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the first administrator of the master realm (the password is the secret keycloak-admin-password) +memory|1536m|Memory limit of the Keycloak container" +MOD_SECRETS="keycloak-admin-password keycloak-db-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Keycloak 26.8.1 (runs as keycloak, 1000:0), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +KC_IMAGE=quay.io/keycloak/keycloak@sha256:79c623a22abacf3fe6c00a5fb30f5b7843d717b234033b80995891c06116d244 +KC_USER=1000:0 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{1,31}$ ]] || die "LSS_KEYCLOAK_ADMIN_USER must be a lowercase login name." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_KEYCLOAK_MEMORY must be a size such as 1536m or 2g." + return +} + +mod_install() { + app_compose_ready + return +} + +# _kc_compose: the stack. The passwords reach Keycloak in a configuration file (KC_CONFIG_FILE), not in the +# environment. "start" builds the server for PostgreSQL and the health checks on the first start and writes below +# /opt/keycloak, so the root file system stays writable. +_kc_compose() { + cat </dev/null || + die "keycloak: the master realm at https://$(app_domain)/realms/master does not answer." + log_info "keycloak: the admin console is https://$(app_domain)/admin/; user $(opt admin_user), password in $(secret_path keycloak-admin-password)" + return +} + +# _kc_token: an access token of the administrator (admin-cli, password grant); the password goes to curl as a file. +_kc_token() { + local page + page=$(app_get realms/master/protocol/openid-connect/token -X POST -d grant_type=password -d client_id=admin-cli \ + --data-urlencode "username=$(opt admin_user)" --data-urlencode "password@$(secret_path keycloak-admin-password)") || true + [[ $(app_status "${page}") == 200 ]] || return 1 + jq -r '.access_token // empty' <<<"${page%$'\n'*}" + return +} + +mod_verify() { + local page token headers user id status + app_verify_stack + page=$(app_wait realms/master/.well-known/openid-configuration 200) || die "keycloak: the master realm does not answer." + [[ $(jq -r '.issuer' <<<"${page%$'\n'*}" 2>/dev/null) == "https://$(app_domain)/realms/master" ]] || + die "keycloak: the issuer is not https://$(app_domain)/realms/master (KC_HOSTNAME or the proxy headers are wrong)." + # Used, not only running: the administrator logs in, creates a user through the admin API and deletes it. + token=$(_kc_token) || die "keycloak: $(opt admin_user) cannot log in to the master realm." + [[ -n ${token} ]] || die "keycloak: the login of $(opt admin_user) returned no token." + headers=$(mktemp) + printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "${token}" >"${headers}" + user="lss-verify-$(date +%s)" + status=$(app_status "$(app_get admin/realms/master/users -X POST -H "@${headers}" -d "{\"username\":\"${user}\",\"enabled\":false}")") + if [[ ${status} != 201 ]]; then + rm -f -- "${headers}" + die "keycloak: creating a user answers ${status}." + fi + page=$(app_get "admin/realms/master/users?exact=true&username=${user}" -H "@${headers}") + id=$(jq -r '.[0].id // empty' <<<"${page%$'\n'*}" 2>/dev/null) || id='' + if [[ -z ${id} ]]; then + rm -f -- "${headers}" + die "keycloak: the new user is missing." + fi + status=$(app_status "$(app_get "admin/realms/master/users/${id}" -X DELETE -H "@${headers}")") + rm -f -- "${headers}" + [[ ${status} == 204 ]] || die "keycloak: deleting the test user answers ${status}." + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/matomo.sh b/modules/apps/matomo.sh new file mode 100644 index 0000000..97af363 --- /dev/null +++ b/modules/apps/matomo.sh @@ -0,0 +1,223 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="matomo" +MOD_GROUP="apps" +MOD_DESCRIPTION="Matomo 5 (web analytics) with MariaDB as a Docker Compose stack behind the web server, installed with a super user and a first website, reports archived by a systemd timer" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Matomo site, separated by spaces; empty: matomo. +port|8087|Port of the Matomo container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the super user (the password is the secret matomo-admin-password) +admin_email||E-mail address of the super user; empty: ADMIN_USER@ +site_url||Address of the first website Matomo measures; empty: https:// +memory|1g|Memory limit of the Matomo container" +MOD_SECRETS="matomo-admin-password matomo-db-password matomo-db-root-password matomo-api-token" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Matomo 5.14.0 (apache), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +MT_IMAGE=docker.io/library/matomo@sha256:1e00192627025a9eb9108858a0d2fcedb495958adf8ee12cd2b2e4ec306efae4 +# www-data in the image; the container runs as this user from the start, so it needs no capabilities. +MT_USER=33:33 +# A browser user agent, so Matomo does not drop the test visit of mod_verify as a bot. +MT_AGENT='Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0' + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-zA-Z0-9._@-]{2,100}$ ]] || die "LSS_MATOMO_ADMIN_USER must be a login name." + [[ -z $(opt admin_email) || $(opt admin_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_MATOMO_ADMIN_EMAIL must be an e-mail address." + [[ -z $(opt site_url) || $(opt site_url) =~ ^https?://[A-Za-z0-9.-]+(:[0-9]+)?(/[^[:space:]]*)?$ ]] || + die "LSS_MATOMO_SITE_URL must be an address such as https://www.example.com." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_MATOMO_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +# _mt_compose: the stack. The installer of Matomo takes the database password from MATOMO_DATABASE_PASSWORD_FILE. +_mt_compose() { + cat </dev/null + return +} + +# _mt_step JAR STEP [CURL OPTION...]: one step of the web installer; prints the body and the status. +_mt_step() { + local jar=$1 step=$2 + shift 2 + app_get "index.php?module=Installation&action=${step}" -b "${jar}" -c "${jar}" "$@" + return +} + +# _mt_post JAR STEP FIELD...: posts the form of STEP; fails unless Matomo moves on (redirect) or shows a page without a +# form error. +_mt_post() { + local jar=$1 step=$2 page status + shift 2 + page=$(_mt_step "${jar}" "${step}" -X POST "$@") || true + status=$(app_status "${page}") + if [[ ${status} != 302 && ${status} != 200 ]] || grep -q 'class="[^"]*form-errors\|class="[^"]*alert-danger' <<<"${page}"; then + log_error "matomo: the installer step ${step} answers ${status}: $(sed -n 's/.*\(alert-danger\|form-errors\)[^>]*>\([^<]*\).*/\2/p' <<<"${page}" | head -3)" + return 1 + fi + return 0 +} + +# _mt_install: the web installer, step by step (Matomo has no installer on the command line). The passwords go to +# curl as files, so they never show up in the process list; "**********" makes the installer use the database +# password from MATOMO_DATABASE_PASSWORD_FILE. +_mt_install() { + local jar email site domain password + _mt_installed && return 0 + domain=$(app_domain) + email=$(opt admin_email) + [[ -n ${email} ]] || email=$(opt admin_user)@${domain} + site=$(opt site_url) + [[ -n ${site} ]] || site=https://${domain} + secret_load matomo-admin-password password 32 + log_info "matomo: installing Matomo (super user $(opt admin_user), password in $(secret_path matomo-admin-password))" + jar=$(mktemp) + if ! { + _mt_step "${jar}" '' >/dev/null && + _mt_step "${jar}" systemCheck >/dev/null && + _mt_post "${jar}" databaseSetup -d host=db -d username=matomo -d 'password=**********' -d dbname=matomo \ + -d tables_prefix=matomo_ --data-urlencode 'adapter=PDO\MYSQL' -d schema=Mariadb && + _mt_step "${jar}" tablesCreation >/dev/null && + _mt_post "${jar}" setupSuperUser --data-urlencode "login=$(opt admin_user)" \ + --data-urlencode "password@$(secret_path matomo-admin-password)" \ + --data-urlencode "password_bis@$(secret_path matomo-admin-password)" --data-urlencode "email=${email}" && + _mt_post "${jar}" firstWebsiteSetup --data-urlencode "siteName=${domain}" --data-urlencode "url=${site}" \ + -d timezone=UTC -d ecommerce=0 && + _mt_step "${jar}" trackingCode >/dev/null && + _mt_post "${jar}" finished -d anonymise_ip=1 + }; then + rm -f -- "${jar}" + die "matomo: the installation failed; see above." + fi + rm -f -- "${jar}" + _mt_installed || die "matomo: Matomo is not installed after the web installer." + files_note_change "installed Matomo" + return +} + +# _mt_set SECTION KEY VALUE: a setting of config.ini.php, changed only when it differs. +_mt_set() { + local section=$1 key=$2 value=$3 + if ! app_exec app grep -qx "${key} = ${value}" /var/www/html/config/config.ini.php 2>/dev/null; then + log_command _mt_console config:set --section="${section}" --key="${key}" --value="${value}" || + die "matomo: the console cannot set ${section}.${key}." + files_note_change "matomo setting ${section}.${key}" + fi + return +} + +# _mt_token: an app token of the super user for the reporting API (the secret matomo-api-token). +_mt_token() { + local page token + secret_exists matomo-api-token && return 0 + page=$(app_get index.php -X POST -d module=API -d method=UsersManager.createAppSpecificTokenAuth -d format=json \ + --data-urlencode "userLogin=$(opt admin_user)" --data-urlencode "passwordConfirmation@$(secret_path matomo-admin-password)" \ + -d description=lss) || true + token=$(jq -r '.value // empty' <<<"${page%$'\n'*}" 2>/dev/null) || token='' + [[ -n ${token} ]] || die "matomo: the reporting API gives the super user no token (HTTP $(app_status "${page}"))." + secret_set matomo-api-token "${token}" + files_note_change "created the Matomo API token" + return +} + +mod_configure() { + app_db_secrets mariadb "${MT_USER}" + _mt_compose | app_compose + app_up + app_site "$(opt_int port)" + app_wait matomo.php 200 >/dev/null || die "matomo: https://$(app_domain)/matomo.php does not answer." + _mt_install + _mt_set General force_ssl 1 + _mt_set General assume_secure_protocol 1 + _mt_set General enable_browser_archiving_triggering 0 + _mt_token + app_timer matomo-archive 1h "Matomo report archiving" app php /var/www/html/console core:archive --url="https://$(app_domain)/" + return +} + +# _mt_api METHOD [CURL OPTION...]: a call of the reporting API with the token (in the request body, not the URL). +_mt_api() { + local method=$1 file + shift + file=$(secret_path matomo-api-token) + app_get index.php -X POST -d module=API -d "method=${method}" -d format=json --data-urlencode "token_auth@${file}" "$@" + return +} + +mod_verify() { + local page action + app_verify_stack + page=$(app_wait 'index.php?module=Login' 200) || die "matomo: the login page does not answer." + grep -q 'form_password\|name="form_password"' <<<"${page}" || die "matomo: https://$(app_domain)/index.php?module=Login shows no login." + # Used, not only running: a page view goes in through the tracker and comes out of the reporting API. + action="lss-verify-$(date +%s)" + page=$(app_get "matomo.php?idsite=1&rec=1&send_image=0&action_name=${action}&url=https%3A%2F%2F$(app_domain)%2F${action}" -A "${MT_AGENT}") + [[ $(app_status "${page}") =~ ^(200|204)$ ]] || die "matomo: the tracker answers $(app_status "${page}")." + page=$(_mt_api Live.getLastVisitsDetails -d idSite=1 -d period=day -d date=today -d filter_limit=20) + [[ $(app_status "${page}") == 200 ]] || die "matomo: the reporting API answers $(app_status "${page}")." + grep -q "${action}" <<<"${page}" || die "matomo: the test page view does not show up in the reporting API." + svc_is_enabled lss-matomo-archive.timer || die "matomo: the timer lss-matomo-archive for the report archiving is off." + timeout 600 systemctl start lss-matomo-archive.service || die "matomo: the report archiving fails (journalctl -u lss-matomo-archive)." + return +} + +mod_remove() { + app_timer_remove matomo-archive + app_remove + return +} diff --git a/modules/apps/minio.sh b/modules/apps/minio.sh new file mode 100644 index 0000000..c4368f2 --- /dev/null +++ b/modules/apps/minio.sh @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="minio" +MOD_GROUP="apps" +MOD_DESCRIPTION="S3-compatible object storage with the MinIO server code (PGSTY Silo, the maintained fork, since MinIO archived its community edition) behind the web server: the S3 API and the web console as two HTTPS sites" +MOD_REQUIRES="system nginx|apache|caddy" +MOD_MODES="native docker" +MOD_OPTIONS="domains||Host names of the S3 API, separated by spaces; empty: minio. +console_domains||Host names of the web console, separated by spaces; empty: minio-console. +api_port|8091|Port of the S3 API on 127.0.0.1, which the web server forwards to +console_port|8092|Port of the web console on 127.0.0.1, which the web server forwards to +root_user|lssadmin|Name of the root user (the password is the secret minio-root-password) +memory|1g|Memory limit of the container (docker mode)" +MOD_SECRETS="minio-root-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Silo RELEASE.2026-09-16T00-00-00Z: the image (multi-architecture index, amd64 and arm64) for the docker mode, the +# packages of the same release (data/checksums.txt) for the native mode. Checked 2026-10-15. +SILO_IMAGE=docker.io/pgsty/silo@sha256:635197cb9f36d01bee221d34d1c7d7960f6a95c48b0b6c01d99cd13bdae51a46 +SILO_VERSION=20260916000000 +# Not the root user of the image: an unprivileged user that owns only the data volume. +SILO_USER=1000:1000 +SILO_ENV=/etc/default/silo +SILO_ETC=/etc/silo +SILO_DATA=/var/lib/silo +# Silo falls back to the well-known minioadmin:minioadmin when it finds no credentials; mod_verify makes sure it did not. +SILO_DEFAULT_LOGIN=minioadmin + +mod_check() { + local domain + app_check_domains + for domain in $(opt console_domains); do + app_valid_domain "${domain}" || die "LSS_MINIO_CONSOLE_DOMAINS: '${domain}' is not a host name like minio-console.example.com." + done + app_check_port api_port + app_check_port console_port + [[ $(opt_int api_port) != "$(opt_int console_port)" ]] || die "LSS_MINIO_API_PORT and LSS_MINIO_CONSOLE_PORT must differ." + [[ $(opt root_user) =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]{2,63}$ && $(opt root_user) != "${SILO_DEFAULT_LOGIN}" ]] || + die "LSS_MINIO_ROOT_USER must be 3 to 64 letters, digits, dots, dashes or underscores, other than ${SILO_DEFAULT_LOGIN}." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_MINIO_MEMORY must be a size such as 1g or 768m." + return +} + +_minio_docker() { + [[ $(module_mode minio) == docker ]] + return +} + +# _minio_console_domains: the host names of the web console. +_minio_console_domains() { + local domains host + domains=$(opt console_domains) + if [[ -z ${domains} ]]; then + host=$(hostname -f 2>/dev/null || true) + [[ -n ${host} ]] || host=$(hostname) + domains=minio-console.${host,,} + fi + printf '%s' "${domains}" + return +} + +_minio_console_domain() { + local first + read -r first _ <<<"$(_minio_console_domains)" + printf '%s' "${first}" + return +} + +# _minio_package_installed: the pinned Silo release is installed. +_minio_package_installed() { + if os_is_debian; then + [[ $(dpkg-query -W -f='${Status} ${Version}' silo 2>/dev/null) == "install ok installed ${SILO_VERSION}"* ]] + else + [[ $(rpm -q --qf '%{VERSION}' silo 2>/dev/null) == "${SILO_VERSION}"* ]] + fi + return +} + +mod_install() { + local tmp + if _minio_docker; then + app_compose_ready + return 0 + fi + _minio_package_installed && return 0 + tmp=$(mktemp -d) + if os_is_debian; then + fetch_verified silo-deb "${tmp}/silo.deb" + pkg_update_once + log_command _pkg_apt install --no-install-recommends "${tmp}/silo.deb" || die "minio: apt cannot install Silo; see the log." + else + fetch_verified silo-rpm "${tmp}/silo.rpm" + log_command _pkg_dnf install "${tmp}/silo.rpm" || die "minio: dnf cannot install Silo; see the log." + fi + rm -rf -- "${tmp}" + _minio_package_installed || die "minio: Silo ${SILO_VERSION} is not installed after the package installation." + files_note_change "installed Silo ${SILO_VERSION}" + return +} + +# _minio_compose: the stack of the docker mode. The image reads the root user and password from the files +# /run/secrets/access_key and /run/secrets/secret_key (MINIO_ROOT_USER_FILE and MINIO_ROOT_PASSWORD_FILE). +_minio_compose() { + cat </dev/null 2>&1 || die "minio: the package did not create the user silo." + install -d -m 0750 -o root -g silo "${SILO_ETC}" + install -d -m 0750 -o silo -g silo "${SILO_DATA}" + file_write 0440 root:silo "${SILO_ETC}/root-password" <<<"${password}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + file_write 0644 root:root "${SILO_ENV}" </dev/null || die "minio: https://$(app_domain)/minio/health/ready does not answer." + log_info "minio: S3 API https://$(app_domain), console https://$(_minio_console_domain); user $(opt root_user), password in $(secret_path minio-root-password)" + return +} + +# _minio_s3 USER SECRET METHOD PATH [CURL OPTION...]: an S3 request signed with AWS Signature V4 by curl; the login +# comes from a netrc file. SECRET empty: the default login of the server. +_minio_s3() { + local user=$1 secret=$2 method=$3 path=$4 netrc page + shift 4 + if [[ -n ${secret} ]]; then + netrc=$(app_netrc "${user}" "${secret}") + else + netrc=$(mktemp) + chmod 600 "${netrc}" + printf 'machine %s login %s password %s\n' "$(app_domain)" "${SILO_DEFAULT_LOGIN}" "${SILO_DEFAULT_LOGIN}" >"${netrc}" + fi + page=$(app_get "${path}" --netrc-file "${netrc}" --aws-sigv4 aws:amz:us-east-1:s3 -X "${method}" "$@") || true + rm -f -- "${netrc}" + printf '%s' "${page}" + return +} + +mod_verify() { + local page bucket probe user status + user=$(opt root_user) + if _minio_docker; then + app_verify_stack + else + svc_is_active silo.service || die "minio: silo.service is not running." + [[ $(stat -c '%a %U:%G' "${SILO_ETC}/root-password") == '440 root:silo' ]] || + die "minio: ${SILO_ETC}/root-password must be mode 440, owned by root:silo." + page=$(app_get '' --head) || true + grep -qi '^strict-transport-security:' <<<"${page}" || die "minio: https://$(app_domain)/ sends no Strict-Transport-Security header." + fi + page=$(app_wait minio/health/ready 200) || die "minio: the S3 API does not answer." + page=$(APP_GET_DOMAIN=$(_minio_console_domain) app_wait '' 200 60) || die "minio: the web console does not answer." + grep -qi '"${probe}" + status=$(app_status "$(_minio_s3 "${user}" minio-root-password PUT "${bucket}")") + [[ ${status} == 200 ]] || die "minio: creating the bucket ${bucket} answers ${status}." + status=$(app_status "$(_minio_s3 "${user}" minio-root-password PUT "${bucket}/probe.txt" --data-binary "@${probe}")") + rm -f -- "${probe}" + [[ ${status} == 200 ]] || die "minio: storing an object answers ${status}." + page=$(_minio_s3 "${user}" minio-root-password GET "${bucket}/probe.txt") + [[ ${page%$'\n'*} == "${bucket}" ]] || die "minio: the stored object does not come back." + status=$(app_status "$(_minio_s3 "${user}" minio-root-password DELETE "${bucket}/probe.txt")") + [[ ${status} == 204 ]] || die "minio: deleting the object answers ${status}." + status=$(app_status "$(_minio_s3 "${user}" minio-root-password DELETE "${bucket}")") + [[ ${status} == 204 ]] || die "minio: deleting the bucket answers ${status}." + return +} + +mod_remove() { + app_site_remove minio-console + if _minio_docker; then + app_remove + return 0 + fi + app_site_remove minio + svc_disable silo.service + svc_dropin_remove silo.service lss-sandbox + file_remove "${SILO_ETC}" + file_remove "${SILO_ENV}" + pkg_remove silo + if [[ ${LSS_PURGE:-0} == 1 ]]; then + file_remove "${SILO_DATA}" + secret_remove minio-root-password + fi + return +} diff --git a/modules/apps/n8n.sh b/modules/apps/n8n.sh new file mode 100644 index 0000000..8affceb --- /dev/null +++ b/modules/apps/n8n.sh @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="n8n" +MOD_GROUP="apps" +MOD_DESCRIPTION="n8n 2 (workflow automation) with PostgreSQL as a Docker Compose stack behind the web server, with the owner account already created and telemetry off" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the n8n site, separated by spaces; empty: n8n. +port|8089|Port of the n8n container on 127.0.0.1, which the web server forwards to +owner_email||E-mail address of the owner account (its login); empty: admin@ +memory|1g|Memory limit of the n8n container" +MOD_SECRETS="n8n-owner-password n8n-encryption-key n8n-db-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# n8n 2.44.2 (runs as node, 1000:1000), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +N8N_IMAGE=docker.io/n8nio/n8n@sha256:b57e7a9c8e5ab1a8fe9b03fff3158052aec65f571ed1f759897a20f306ad329d +N8N_USER=1000:1000 +# n8n binds a browser session to this header; the requests of mod_verify send the same one throughout. +N8N_BROWSER_ID=lss-verify + +mod_check() { + app_check_domains + app_check_port port + [[ -z $(opt owner_email) || $(opt owner_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_N8N_OWNER_EMAIL must be an e-mail address." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_N8N_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +_n8n_email() { + local email + email=$(opt owner_email) + [[ -n ${email} ]] || email=admin@$(app_domain) + printf '%s' "${email}" + return +} + +# _n8n_compose: the stack. Every secret reaches n8n as a file (*_FILE); the frontend cache and /tmp are the only +# writable paths besides the volume. +_n8n_compose() { + local domain + domain=$(app_domain) + cat <"${file}" + return +} + +# _n8n_owner: the owner account (POST /rest/owner/setup, possible once, before anybody logs in). +_n8n_owner() { + local body page status + _n8n_password + body=$(mktemp) + chmod 600 "${body}" + # shellcheck disable=SC2016 # jq variables + _n8n_body "${body}" '{email: $email, firstName: "Server", lastName: "Admin", password: $password}' + page=$(app_get rest/owner/setup -X POST -H 'Content-Type: application/json' --data-binary "@${body}") || true + rm -f -- "${body}" + status=$(app_status "${page}") + if [[ ${status} == 200 ]]; then + log_info "n8n: created the owner $(_n8n_email); password in $(secret_path n8n-owner-password)" + files_note_change "created the n8n owner" + elif [[ ${status} != 400 ]] || ! grep -qi 'already setup' <<<"${page}"; then + die "n8n: the owner cannot be created (HTTP ${status})." + fi + return +} + +mod_configure() { + app_db_secrets postgres "${N8N_USER}" + app_secret n8n-encryption-key "${N8N_USER}" 48 encryption-key + _n8n_compose | app_compose + app_up + app_site "$(opt_int port)" + app_wait healthz/readiness 200 >/dev/null || die "n8n: https://$(app_domain)/healthz/readiness does not answer." + _n8n_owner + return +} + +# _n8n_rest JAR PATH [CURL OPTION...]: a request to the REST API of the editor in the session of JAR. +_n8n_rest() { + local jar=$1 path=$2 + shift 2 + app_get "rest/${path}" -b "${jar}" -c "${jar}" -H "browser-id: ${N8N_BROWSER_ID}" -H 'Content-Type: application/json' "$@" + return +} + +mod_verify() { + local page jar body id name + app_verify_stack + page=$(app_wait healthz/readiness 200) || die "n8n: https://$(app_domain)/healthz/readiness does not answer." + page=$(app_wait signin 200 60) || die "n8n: the sign-in page does not answer." + grep -qi 'n8n' <<<"${page}" || die "n8n: https://$(app_domain)/signin is not n8n." + # Used, not only running: the owner logs in, creates a workflow, reads it back and deletes it. + jar=$(mktemp) + body=$(mktemp) + chmod 600 "${body}" + # shellcheck disable=SC2016 # jq variables + _n8n_body "${body}" '{emailOrLdapLoginId: $email, email: $email, password: $password}' + page=$(_n8n_rest "${jar}" login -X POST --data-binary "@${body}") || true + rm -f -- "${body}" + if [[ $(app_status "${page}") != 200 ]]; then + rm -f -- "${jar}" + die "n8n: the owner $(_n8n_email) cannot log in (HTTP $(app_status "${page}"))." + fi + name="lss-verify-$(date +%s)" + page=$(_n8n_rest "${jar}" workflows -X POST -d "{\"name\":\"${name}\",\"nodes\":[],\"connections\":{},\"settings\":{}}") + id=$(jq -r '.data.id // empty' <<<"${page%$'\n'*}" 2>/dev/null) || id='' + if [[ -z ${id} ]]; then + rm -f -- "${jar}" + die "n8n: creating a workflow answers $(app_status "${page}")." + fi + page=$(_n8n_rest "${jar}" "workflows/${id}") + if [[ $(jq -r '.data.name // empty' <<<"${page%$'\n'*}" 2>/dev/null) != "${name}" ]]; then + rm -f -- "${jar}" + die "n8n: the new workflow does not come back." + fi + # n8n 2 deletes only archived workflows. + _n8n_rest "${jar}" "workflows/${id}/archive" -X POST -d '{}' >/dev/null || true + page=$(_n8n_rest "${jar}" "workflows/${id}" -X DELETE) + rm -f -- "${jar}" + [[ $(app_status "${page}") == 200 ]] || die "n8n: deleting the test workflow answers $(app_status "${page}")." + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/nextcloud.sh b/modules/apps/nextcloud.sh new file mode 100644 index 0000000..63f7c23 --- /dev/null +++ b/modules/apps/nextcloud.sh @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="nextcloud" +MOD_GROUP="apps" +MOD_DESCRIPTION="Nextcloud Hub 34 with PostgreSQL as a Docker Compose stack behind the web server, background jobs by a systemd timer, linked to Collabora Online when the collabora module is installed" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Nextcloud site, separated by spaces; empty: nextcloud. +port|8081|Port of the Nextcloud container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the first administrator (the password is the secret nextcloud-admin-password) +memory|1g|Memory limit of the Nextcloud container" +MOD_SECRETS="nextcloud-admin-password nextcloud-db-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Nextcloud 34.0.4 (apache), the multi-architecture index (amd64 and arm64), checked 2026-10-15. A new release means a +# new digest here; the container upgrades the installation when it starts. +NC_IMAGE=docker.io/library/nextcloud@sha256:0e5688ba0988b9112cf03f9c6bab53eccb466d278435f6596f16c1f25acacf74 +# www-data in the image; the container runs as this user from the start, so it needs no capabilities. +NC_USER=33:33 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{1,31}$ ]] || die "LSS_NEXTCLOUD_ADMIN_USER must be a lowercase login name." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_NEXTCLOUD_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +# _nc_compose: the stack. Nextcloud runs as www-data on a read-only root file system; its code, configuration and +# files are in the volume "html". The secrets reach it only as files; occ reads the passwords for the installation +# from stdin (_nc_install). +_nc_compose() { + local domain + domain=$(app_domain) + cat </dev/null | jq -r '.installed // false' 2>/dev/null) == true ]] + return +} + +# _nc_install: the first installation. occ asks for the database and the admin password when they are not on its +# command line; both come from the mounted secret files through a pipe, so they never appear in a process list. +_nc_install() { + local i + for ((i = 0; i < 100; i++)); do + _nc_occ status >/dev/null 2>&1 && break + sleep 3 + done + _nc_installed && return 0 + log_info "nextcloud: installing Nextcloud (admin user $(opt admin_user), password in $(secret_path nextcloud-admin-password))" + # shellcheck disable=SC2016 # expanded by the shell in the container + log_command app_exec -e SHELL_INTERACTIVE=1 app sh -c 'printf "%s\n%s\n" "$(cat /run/secrets/db-password)" "$(cat /run/secrets/admin-password)" | + php occ maintenance:install --database pgsql --database-host db --database-name nextcloud --database-user nextcloud \ + --admin-user "$1" --data-dir /var/www/html/data' sh "$(opt admin_user)" || + die "nextcloud: the installation failed; see the log." + _nc_installed || die "nextcloud: Nextcloud is not installed after occ maintenance:install." + files_note_change "installed Nextcloud" + return +} + +# _nc_set KEY VALUE: a system setting, changed only when it differs. +_nc_set() { + local key=$1 value=$2 index=${3:-} + local -a keys=("${key}") + [[ -z ${index} ]] || keys+=("${index}") + if [[ $(_nc_occ config:system:get "${keys[@]}" 2>/dev/null) != "${value}" ]]; then + log_command _nc_occ config:system:set "${keys[@]}" --value="${value}" || die "nextcloud: occ cannot set ${key}." + files_note_change "nextcloud setting ${key}" + fi + return +} + +# _nc_settings: host names, URLs and defaults after the installation. +_nc_settings() { + local domain index=1 + _nc_set trusted_domains localhost 0 + for domain in $(app_domains); do + _nc_set trusted_domains "${domain}" "${index}" + index=$((index + 1)) + done + _nc_set overwrite.cli.url "https://$(app_domain)" + _nc_set overwriteprotocol https + _nc_set maintenance_window_start 1 + if [[ $(_nc_occ config:app:get core backgroundjobs_mode 2>/dev/null) != cron ]]; then + log_command _nc_occ background:cron || die "nextcloud: occ cannot switch background jobs to cron." + files_note_change "nextcloud background jobs by cron" + fi + return +} + +mod_configure() { + app_db_secrets postgres "${NC_USER}" + app_secret nextcloud-admin-password "${NC_USER}" 32 admin-password + _nc_compose | app_compose + app_up + _nc_install + _nc_settings + app_site "$(opt_int port)" + app_nextcloud_office + app_timer nextcloud-cron 5min "Nextcloud background jobs" app php -f /var/www/html/cron.php + return +} + +# _nc_dav METHOD PATH [CURL OPTION...]: a WebDAV request as the administrator; prints the body and the status. +_nc_dav() { + local method=$1 path=$2 netrc page + shift 2 + netrc=$(app_netrc "$(opt admin_user)" nextcloud-admin-password) + page=$(app_get "remote.php/dav/files/$(opt admin_user)/${path}" --netrc-file "${netrc}" -X "${method}" "$@") || true + rm -f -- "${netrc}" + printf '%s' "${page}" + return +} + +mod_verify() { + local page status probe + app_verify_stack + page=$(app_wait status.php 200) || die "nextcloud: https://$(app_domain)/status.php does not answer." + [[ $(jq -r '.installed' <<<"${page%$'\n'*}" 2>/dev/null) == true ]] || die "nextcloud: Nextcloud is not installed." + [[ $(jq -r '.maintenance' <<<"${page%$'\n'*}" 2>/dev/null) == false ]] || die "nextcloud: Nextcloud is in maintenance mode." + page=$(app_wait login 200 60) || die "nextcloud: the login page does not answer." + grep -q 'name="password"\|id="password"\|login' <<<"${page}" || die "nextcloud: https://$(app_domain)/login shows no login." + # Used, not only running: the administrator stores a file over WebDAV, reads it back and deletes it. + probe="lss-verify-$(date +%s)" + printf '%s' "${probe}" >"${TMPDIR:-/tmp}/${probe}.txt" + status=$(app_status "$(_nc_dav PUT "${probe}.txt" --data-binary "@${TMPDIR:-/tmp}/${probe}.txt")") + rm -f -- "${TMPDIR:-/tmp}/${probe}.txt" + [[ ${status} == 201 || ${status} == 204 ]] || die "nextcloud: WebDAV upload as $(opt admin_user) answers ${status}." + page=$(_nc_dav GET "${probe}.txt") + [[ ${page%$'\n'*} == "${probe}" ]] || die "nextcloud: the uploaded file does not come back over WebDAV." + status=$(app_status "$(_nc_dav DELETE "${probe}.txt")") + [[ ${status} == 204 ]] || die "nextcloud: deleting the test file answers ${status}." + svc_is_enabled lss-nextcloud-cron.timer || die "nextcloud: the timer lss-nextcloud-cron for the background jobs is off." + timeout 600 systemctl start lss-nextcloud-cron.service || die "nextcloud: the background jobs fail (journalctl -u lss-nextcloud-cron)." + if state_installed collabora; then + log_command _nc_occ richdocuments:activate-config || die "nextcloud: Nextcloud Office cannot reach Collabora Online." + fi + return +} + +mod_remove() { + app_timer_remove nextcloud-cron + app_remove + return +} diff --git a/modules/apps/self-updater.sh b/modules/apps/self-updater.sh new file mode 100644 index 0000000..cc0d4a8 --- /dev/null +++ b/modules/apps/self-updater.sh @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="self-updater" +MOD_GROUP="apps" +MOD_DESCRIPTION="Self-Updater 4.2 (VectoDE): keeps a checkout of a Git repository on this server current, restarts its service, checks its health and rolls back on failure" +MOD_REQUIRES="system node" +MOD_MODES="native" +MOD_OPTIONS="repo||Repository to deploy (HTTPS URL, git@host:owner/repo.git or a local path); required +branch|main|Branch to follow (track branch) +track|branch|branch (deploy the newest commit of the branch) or tag (deploy the newest release tag matching v*) +path|/srv/app|Directory of the deployed checkout; created on the first run, owned by the user self-updater +service|none|What restarts after a deployment: none, or systemd:UNIT (a sudo rule allows exactly that restart) +health_url||URL that must answer 200 after a deployment, otherwise the previous commit comes back; empty: no check +interval|60|Seconds between two polls of the repository (at least 15) +port|8093|Port of the webhook receiver (/hooks/self-updater, signed with the secret self-updater-webhook-secret) and the status endpoint (/status) on 127.0.0.1 +token||Access token for a private HTTPS repository (GitHub: fine-grained, Contents read), stored in the lss secrets (self-updater-token)" +MOD_SECRETS="self-updater-webhook-secret self-updater-token" +MOD_PORTS="none (the status endpoint and the webhook receiver listen on 127.0.0.1)" + +# A new version is a new entry self-updater- in data/checksums.txt and a new SU_VERSION. +SU_VERSION=4.2.1 +SU_USER=self-updater +SU_HOME=/var/lib/self-updater +SU_CONFIG_DIR=/etc/self-updater +SU_CONFIG=${SU_CONFIG_DIR}/updater.config.json +SU_ENV=${SU_CONFIG_DIR}/.env +SU_UNIT=self-updater +SU_SUDOERS=/etc/sudoers.d/lss-self-updater +SU_STATUS_PATH=/status + +mod_check() { + local repo path service interval + repo=$(opt repo) + [[ -n ${repo} ]] || die "LSS_SELF_UPDATER_REPO must name the repository to deploy (https://github.com/owner/repo.git)." + [[ ${repo} =~ ^(https://[A-Za-z0-9.-]+(:[0-9]+)?/[A-Za-z0-9._/~-]+|git@[A-Za-z0-9.-]+:[A-Za-z0-9._/~-]+|/[A-Za-z0-9._/-]+)$ ]] || + die "LSS_SELF_UPDATER_REPO: '${repo}' is not an HTTPS URL, git@host:owner/repo.git or an absolute path." + [[ $(opt branch) =~ ^[A-Za-z0-9._/-]+$ ]] || die "LSS_SELF_UPDATER_BRANCH: '$(opt branch)' is not a branch name." + [[ $(opt track) =~ ^(branch|tag)$ ]] || die "LSS_SELF_UPDATER_TRACK must be branch or tag." + path=$(opt path) + [[ ${path} =~ ^/[A-Za-z0-9._/-]+$ && /${path}/ != */../* ]] || die "LSS_SELF_UPDATER_PATH must be an absolute path." + case ${path%/} in + / | /etc* | /usr* | /bin* | /sbin* | /lib* | /boot* | /proc* | /sys* | /dev* | /root* | /var/lib/lss* | /opt/lss*) + die "LSS_SELF_UPDATER_PATH must not be a system directory (${path})." + ;; + *) ;; + esac + service=$(opt service) + [[ ${service} == none || ${service} =~ ^systemd:[A-Za-z0-9@._-]+$ ]] || + die "LSS_SELF_UPDATER_SERVICE must be none or systemd:UNIT." + [[ -z $(opt health_url) || $(opt health_url) =~ ^https?://[A-Za-z0-9.:\[\]-]+(/[A-Za-z0-9._/?=\&%~-]*)?$ ]] || + die "LSS_SELF_UPDATER_HEALTH_URL: '$(opt health_url)' is not an http(s) URL." + interval=$(opt_int interval) + ((interval >= 15)) || die "LSS_SELF_UPDATER_INTERVAL must be at least 15 seconds." + [[ $(opt token) != *[[:space:]]* ]] || die "LSS_SELF_UPDATER_TOKEN must not contain spaces." + app_check_port port + fetch_entry "self-updater-${SU_VERSION}" >/dev/null || + die "self-updater: data/checksums.txt has no checksum for Self-Updater ${SU_VERSION}." + return +} + +# _su_installed: the version of the installed self-updater, or nothing. +_su_installed() { + local package + package="$(npm root --global 2>/dev/null)/@vectode/self-updater/package.json" + [[ -f ${package} ]] || return 0 + jq -r '.version // empty' "${package}" 2>/dev/null || true + return +} + +# The release archive from GitHub, checked against data/checksums.txt and installed with npm (no install scripts; the +# package has no dependencies). Node.js comes from the node module. +mod_install() { + local tmp + pkg_install git jq sudo + if [[ $(_su_installed) != "${SU_VERSION}" ]]; then + tmp=$(mktemp -d) + fetch_verified "self-updater-${SU_VERSION}" "${tmp}/self-updater.tgz" + log_command npm install --global --ignore-scripts --no-audit --no-fund "${tmp}/self-updater.tgz" || + die "self-updater: npm cannot install Self-Updater ${SU_VERSION}." + rm -rf -- "${tmp}" + files_note_change "installed Self-Updater ${SU_VERSION}" + fi + mon_user "${SU_USER}" "${SU_HOME}" + return +} + +# _su_token_flag: true when the repository needs the token. +_su_token_flag() { + if [[ -n $(opt token) ]]; then printf 'true'; else printf 'false'; fi + return +} + +# _su_config: updater.config.json. Secrets stay in .env (${SELF_UPDATER_WEBHOOK_SECRET}, ${GITHUB_TOKEN}). +_su_config() { + local service unit='' + service=$(opt service) + [[ ${service} == none ]] || unit=${service#systemd:} + jq -n --arg repo "$(opt repo)" --arg branch "$(opt branch)" --arg track "$(opt track)" --arg path "$(opt path)" \ + --arg unit "${unit}" --arg health "$(opt health_url)" --argjson interval "$(opt_int interval)" \ + --argjson port "$(opt_int port)" --arg status "${SU_STATUS_PATH}" --arg token "$(_su_token_flag)" ' + { + version: 3, + repo: ({url: $repo, branch: $branch} + (if $token == "true" then {token: "${GITHUB_TOKEN}"} else {} end)), + track: {mode: $track, tagPattern: "v*"}, + workspace: {path: $path}, + service: (if $unit == "" then {type: "none"} else {type: "systemd", name: $unit, sudo: true} end), + rollback: {enabled: true}, + schedule: {enabled: true, intervalSeconds: $interval, jitterSeconds: 10}, + webhook: {enabled: true, host: "127.0.0.1", port: $port, statusPath: $status, secret: "${SELF_UPDATER_WEBHOOK_SECRET}"}, + logging: {level: "info", format: "text"} + } + (if $health == "" then {} else {healthCheck: {url: $health}} end)' + return +} + +# _su_env: the environment file with the secrets the configuration refers to. +_su_env() { + local secret token + secret_load self-updater-webhook-secret secret 48 + printf 'SELF_UPDATER_WEBHOOK_SECRET=%s\n' "${secret}" + token=$(opt token) + if [[ -n ${token} ]]; then + log_mask "${token}" + if ! secret_exists self-updater-token || [[ $(secret_get self-updater-token) != "${token}" ]]; then + secret_set self-updater-token "${token}" + fi + printf 'GITHUB_TOKEN=%s\n' "${token}" + fi + return +} + +_su_unit() { + local path nnp=yes + path=$(opt path) + # sudo for the restart of a systemd unit needs new privileges. + [[ $(opt service) == none ]] || nnp=no + cat </dev/null || die "self-updater: visudo rejects ${SU_SUDOERS}." + fi + log_command runuser -u "${SU_USER}" -- env SELF_UPDATER_CONFIG="${SU_CONFIG}" HOME="${SU_HOME}" \ + self-updater validate --config "${SU_CONFIG}" --env-file "${SU_ENV}" || + die "self-updater: the configuration does not validate; see the log." + _su_unit | svc_unit "${SU_UNIT}.service" + local -a sandbox=("ReadWritePaths=${SU_CONFIG_DIR}" "ReadWritePaths=${path}" "ReadWritePaths=${SU_HOME}") + if [[ ${service} != none ]]; then + # sudo needs its setuid bit and to read /etc/sudoers.d. + sandbox+=(NoNewPrivileges=no RestrictSUIDSGID=no) + fi + svc_sandbox "${SU_UNIT}.service" "${sandbox[@]}" + svc_enable "${SU_UNIT}" + if [[ $(files_changes_of self-updater) != "${before}" ]]; then + svc_restart "${SU_UNIT}" || die "self-updater: ${SU_UNIT} does not start; see journalctl -u ${SU_UNIT}." + fi + return +} + +# _su_status: the JSON of the status endpoint, or nothing. +_su_status() { + curl -fs --max-time 10 "http://127.0.0.1:$(opt_int port)${SU_STATUS_PATH}" 2>/dev/null || true + return +} + +mod_verify() { + local status='' i commit + [[ $(_su_installed) == "${SU_VERSION}" ]] || die "self-updater: Self-Updater ${SU_VERSION} is not installed." + svc_is_active "${SU_UNIT}" || die "self-updater: ${SU_UNIT} is not running; see journalctl -u ${SU_UNIT}." + [[ $(stat -c '%a %U:%G' "${SU_ENV}") == "640 root:${SU_USER}" ]] || die "self-updater: ${SU_ENV} must be mode 640, root:${SU_USER}." + # The first deployment runs right after the start (--immediate): a clone of the repository into the workspace. + for ((i = 0; i < 100; i++)); do + status=$(_su_status) + commit=$(jq -r '.workspaces[0].deployed.commit // empty' <<<"${status}" 2>/dev/null) || commit='' + [[ -z ${commit} ]] || break + sleep 3 + done + [[ -n ${status} ]] || die "self-updater: the status endpoint http://127.0.0.1:$(opt_int port)${SU_STATUS_PATH} does not answer." + [[ -n ${commit} ]] || + die "self-updater: nothing is deployed from $(opt repo) yet (last attempt: $(jq -c '.workspaces[0].lastAttempt' <<<"${status}" 2>/dev/null))." + [[ $(runuser -u "${SU_USER}" -- git -C "$(opt path)" rev-parse HEAD 2>/dev/null) == "${commit}"* ]] || + die "self-updater: $(opt path) is not at the deployed commit ${commit}." + return +} + +mod_remove() { + svc_disable "${SU_UNIT}" + svc_dropin_remove "${SU_UNIT}.service" lss-sandbox + file_remove "/etc/systemd/system/${SU_UNIT}.service" + svc_daemon_reload + file_remove "${SU_SUDOERS}" + file_remove "${SU_CONFIG}" + file_remove "${SU_ENV}" + if [[ -n $(_su_installed) ]]; then + log_command npm uninstall --global @vectode/self-updater || die "self-updater: npm cannot remove Self-Updater." + files_note_change "removed Self-Updater" + fi + # The deployed checkout and the state of the updater stay unless --purge. + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + rm -rf -- "${SU_CONFIG_DIR}" "${SU_HOME}" + secret_remove self-updater-webhook-secret + secret_remove self-updater-token + files_note_change "deleted the state of Self-Updater" + return +} diff --git a/modules/apps/uptime-kuma.sh b/modules/apps/uptime-kuma.sh new file mode 100644 index 0000000..a3c33f0 --- /dev/null +++ b/modules/apps/uptime-kuma.sh @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="uptime-kuma" +MOD_GROUP="apps" +MOD_DESCRIPTION="Uptime Kuma 2 (status pages and uptime checks) as a rootless Docker Compose stack behind the web server, with SQLite and the first administrator already created" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Uptime Kuma site, separated by spaces; empty: uptime-kuma. +port|8084|Port of the Uptime Kuma container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the first administrator (the password is the secret uptime-kuma-admin-password) +memory|512m|Memory limit of the Uptime Kuma container" +MOD_SECRETS="uptime-kuma-admin-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Uptime Kuma 2.5.6, the slim rootless variant (runs as node, 1000:1000; no bundled Chromium or MariaDB), the +# multi-architecture index (amd64 and arm64), checked 2026-10-15. +KUMA_IMAGE=docker.io/louislam/uptime-kuma@sha256:6eb4e96a7cbdb8a5d6779aa26dce2cb97dcea65b486856b9b2280f73a1d0adf4 +KUMA_USER=1000:1000 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-zA-Z][a-zA-Z0-9._-]{1,31}$ ]] || die "LSS_UPTIME_KUMA_ADMIN_USER must be a login name." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_UPTIME_KUMA_MEMORY must be a size such as 512m or 1g." + return +} + +mod_install() { + app_compose_ready + return +} + +_kuma_compose() { + cat < { console.error(message); process.exit(1); }; +const call = (event, ...args) => new Promise((resolve) => socket.emit(event, ...args, resolve)); +setTimeout(() => fail("timeout"), 120000); +socket.on("connect_error", (error) => fail(`connect: ${error.message}`)); +socket.on("connect", async () => { + if (action === "setup") { + const needed = await call("needSetup"); + if (!needed) { console.log("exists"); process.exit(0); } + const result = await call("setup", username, password); + if (!result.ok) fail(`setup: ${result.msg}`); + console.log("created"); + process.exit(0); + } + const login = await call("login", { username, password, token: "" }); + if (!login.ok) fail(`login: ${login.msg}`); + const added = await call("add", { + type: "http", name: "lss verify", url: "http://127.0.0.1:3001/", method: "GET", interval: 20, + retryInterval: 20, resendInterval: 0, maxretries: 0, timeout: 16, maxredirects: 10, + accepted_statuscodes: ["200-399"], notificationIDList: {}, ignoreTls: false, upsideDown: false, + expiryNotification: false, kafkaProducerBrokers: [], kafkaProducerSaslOptions: { mechanism: "None" }, + conditions: [], active: true, + }); + if (!added.ok) fail(`add monitor: ${added.msg}`); + socket.on("heartbeat", async (beat) => { + if (beat.monitorID !== added.monitorID) return; + const removed = await call("deleteMonitor", added.monitorID); + if (!removed.ok) fail(`delete monitor: ${removed.msg}`); + if (beat.status !== 1) fail(`the monitor reports down: ${beat.msg}`); + console.log("up"); + process.exit(0); + }); +}); +EOF + return +} + +# _kuma ACTION: runs the client in the container. +_kuma() { + app_exec app node /opt/lss/lss-kuma.js "$1" + return +} + +mod_configure() { + local result + app_secret uptime-kuma-admin-password "${KUMA_USER}" 32 admin-password + _kuma_script | app_file config/lss-kuma.js "${KUMA_USER}" 0444 + _kuma_compose | app_compose + app_up + app_site "$(opt_int port)" + app_wait '' 302 >/dev/null || die "uptime-kuma: https://$(app_domain)/ does not answer." + result=$(_kuma setup 2>&1) || die "uptime-kuma: the administrator cannot be created: ${result}" + if [[ ${result} == *created* ]]; then + log_info "uptime-kuma: created the administrator $(opt admin_user); password in $(secret_path uptime-kuma-admin-password)" + files_note_change "created the Uptime Kuma administrator" + fi + return +} + +mod_verify() { + local page result + app_verify_stack + page=$(app_wait '' 302) || die "uptime-kuma: https://$(app_domain)/ does not answer." + page=$(app_wait dashboard 200 60) || die "uptime-kuma: the dashboard does not answer." + grep -qi 'uptime kuma' <<<"${page}" || die "uptime-kuma: https://$(app_domain)/dashboard is not Uptime Kuma." + # Used, not only running: the administrator logs in and a monitor checks Uptime Kuma itself. + result=$(_kuma check 2>&1) || die "uptime-kuma: the check as $(opt admin_user) failed: ${result}" + [[ ${result} == *up* ]] || die "uptime-kuma: the check as $(opt admin_user) failed: ${result}" + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/vaultwarden.sh b/modules/apps/vaultwarden.sh new file mode 100644 index 0000000..c632335 --- /dev/null +++ b/modules/apps/vaultwarden.sh @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="vaultwarden" +MOD_GROUP="apps" +MOD_DESCRIPTION="Vaultwarden 1.37 (password manager, compatible with the Bitwarden apps) as a Docker Compose stack behind the web server; no open registration, users are invited from the admin page" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Vaultwarden site, separated by spaces; empty: vaultwarden. +port|8085|Port of the Vaultwarden container on 127.0.0.1, which the web server forwards to +memory|512m|Memory limit of the Vaultwarden container" +MOD_SECRETS="vaultwarden-admin-token" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Vaultwarden 1.37.4, the multi-architecture index (amd64 and arm64), checked 2026-10-15. +VW_IMAGE=docker.io/vaultwarden/server@sha256:efb3cde962015fcc036b2ea625242248611943b27212ebf4392841fb30fad055 +# Not the root user of the image: an unprivileged user that owns only the data volume. +VW_USER=1000:1000 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_VAULTWARDEN_MEMORY must be a size such as 512m or 1g." + return +} + +mod_install() { + app_compose_ready + # argon2 hashes the admin token (EPEL on the RHEL family). + pkg_enable_epel + pkg_install argon2 + return +} + +_vw_compose() { + cat </dev/null || die "vaultwarden: https://$(app_domain)/alive does not answer." + log_info "vaultwarden: the admin page is https://$(app_domain)/admin; token in $(secret_path vaultwarden-admin-token)" + return +} + +# _vw_admin JAR PATH [CURL OPTION...]: a request to the admin page with the session in the cookie file JAR. +_vw_admin() { + local jar=$1 path=$2 + shift 2 + app_get "admin/${path}" -b "${jar}" -c "${jar}" "$@" + return +} + +mod_verify() { + local page jar email id status + app_verify_stack + page=$(app_wait alive 200) || die "vaultwarden: https://$(app_domain)/alive does not answer." + page=$(app_wait '' 200 60) || die "vaultwarden: the web vault does not answer." + grep -qi 'bitwarden\|vaultwarden' <<<"${page}" || die "vaultwarden: https://$(app_domain)/ is not the web vault." + # Used, not only running: log in to the admin page with the token, invite a user, find and delete it. + jar=$(mktemp) + email="lss-verify-$(date +%s)@example.com" + status=$(app_status "$(_vw_admin "${jar}" '' -X POST --data-urlencode "token@$(secret_path vaultwarden-admin-token)")") + if [[ ${status} != 200 && ${status} != 303 ]] || ! grep -q 'VW_ADMIN' "${jar}"; then + rm -f -- "${jar}" + die "vaultwarden: the admin token is not accepted (HTTP ${status})." + fi + page=$(_vw_admin "${jar}" invite -X POST -H 'Content-Type: application/json' -d "{\"email\":\"${email}\"}") + if [[ $(app_status "${page}") != 200 ]]; then + rm -f -- "${jar}" + die "vaultwarden: inviting a user answers $(app_status "${page}")." + fi + page=$(_vw_admin "${jar}" users -H 'Accept: application/json') + id=$(jq -r --arg email "${email}" '.[] | select(.email == $email) | .id' <<<"${page%$'\n'*}" 2>/dev/null) || id='' + if [[ -z ${id} ]]; then + rm -f -- "${jar}" + die "vaultwarden: the invited user is missing from the user list." + fi + status=$(app_status "$(_vw_admin "${jar}" "users/${id}/delete" -X POST)") + rm -f -- "${jar}" + [[ ${status} == 200 ]] || die "vaultwarden: deleting the invited user answers ${status}." + return +} + +mod_remove() { + app_remove + [[ ${LSS_PURGE:-0} != 1 ]] || state_set vaultwarden token_sum '' + return +} diff --git a/modules/apps/wordpress.sh b/modules/apps/wordpress.sh new file mode 100644 index 0000000..b39d003 --- /dev/null +++ b/modules/apps/wordpress.sh @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="wordpress" +MOD_GROUP="apps" +MOD_DESCRIPTION="WordPress 7.1 with MariaDB as a Docker Compose stack behind the web server, installed with a first administrator; the theme and plugin editor is off" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the WordPress site, separated by spaces; empty: wordpress. +port|8088|Port of the WordPress container on 127.0.0.1, which the web server forwards to +title|WordPress|Title of the site +admin_user|lssadmin|Name of the first administrator (the password is the secret wordpress-admin-password) +admin_email||E-mail address of the first administrator; empty: ADMIN_USER@ +memory|768m|Memory limit of the WordPress container" +MOD_SECRETS="wordpress-admin-password wordpress-db-password wordpress-db-root-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# WordPress 7.1.3 with PHP 8.4 (apache), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +WP_IMAGE=docker.io/library/wordpress@sha256:309307430abf80009378a63a79df73554a6449b700fac59a991b4959e6dda8ca +# www-data in the image; the container runs as this user from the start, so it needs no capabilities. +WP_USER=33:33 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{2,59}$ && $(opt admin_user) != admin ]] || + die "LSS_WORDPRESS_ADMIN_USER must be a lowercase login name other than admin (the first name attackers try)." + [[ -z $(opt admin_email) || $(opt admin_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_WORDPRESS_ADMIN_EMAIL must be an e-mail address." + [[ -n $(opt title) && $(opt title) != *[\'\"\\]* ]] || die "LSS_WORDPRESS_TITLE must not be empty or contain quotes or backslashes." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_WORDPRESS_MEMORY must be a size such as 768m or 1g." + return +} + +mod_install() { + app_compose_ready + return +} + +# _wp_compose: the stack. wp-config.php reads the database password from WORDPRESS_DB_PASSWORD_FILE on every request; +# the extra settings fix the address, force HTTPS for the login and turn off the file editor of the dashboard. +_wp_compose() { + local domain + domain=$(app_domain) + cat </dev/null || die "wordpress: https://$(app_domain)/ does not answer." + _wp_install + return +} + +mod_verify() { + local page jar status + app_verify_stack + page=$(app_wait wp-login.php 200) || die "wordpress: the login page does not answer." + grep -q 'name="pwd"' <<<"${page}" || die "wordpress: https://$(app_domain)/wp-login.php shows no login." + page=$(app_get wp-json/) || true + [[ $(app_status "${page}") == 200 && $(jq -r '.home' <<<"${page%$'\n'*}" 2>/dev/null) == "https://$(app_domain)" ]] || + die "wordpress: the REST API does not report https://$(app_domain) as the address of the site." + # Used, not only running: the administrator logs in and opens the dashboard. + jar=$(mktemp) + page=$(app_get wp-login.php -b "${jar}" -c "${jar}" -b 'wordpress_test_cookie=WP%20Cookie%20check' -X POST \ + --data-urlencode "log=$(opt admin_user)" --data-urlencode "pwd@$(secret_path wordpress-admin-password)" \ + -d testcookie=1 --data-urlencode "redirect_to=https://$(app_domain)/wp-admin/") || true + status=$(app_status "${page}") + if [[ ${status} != 302 ]]; then + rm -f -- "${jar}" + die "wordpress: $(opt admin_user) cannot log in (HTTP ${status})." + fi + page=$(app_get wp-admin/ -b "${jar}") || true + rm -f -- "${jar}" + if [[ $(app_status "${page}") != 200 ]] || ! grep -q 'id="wpadminbar"' <<<"${page}"; then + die "wordpress: the dashboard does not open after the login." + fi + return +} + +mod_remove() { + app_remove + return +} diff --git a/profiles/all.conf b/profiles/all.conf index 249fae7..467f481 100644 --- a/profiles/all.conf +++ b/profiles/all.conf @@ -6,5 +6,6 @@ # # For the nightly test and as a reference. ssh-2fa, clamav and usbguard stay opt-in as in minimal-secure. k3s # conflicts with docker (both bring their own containerd and claim the forwarding rules) and has its own profile, -# k8s-node; podman is the alternative to docker. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server mailserver webmail node-exporter prometheus alertmanager grafana loki alloy netdata smartd +# k8s-node; podman is the alternative to docker. The apps need about 4 GB of memory on top of the rest, and +# self-updater needs the repository it deploys (LSS_SELF_UPDATER_REPO). +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server mailserver webmail node-exporter prometheus alertmanager grafana loki alloy netdata smartd nextcloud collabora forgejo uptime-kuma vaultwarden keycloak matomo wordpress n8n minio self-updater diff --git a/profiles/devops.conf b/profiles/devops.conf index 5a53f83..7dd0c14 100644 --- a/profiles/devops.conf +++ b/profiles/devops.conf @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: MIT -# Profile: DevOps server: minimal-secure plus Docker and the monitoring stack (Prometheus, Alertmanager, Grafana, Loki) +# Profile: DevOps server: minimal-secure plus Docker, the Git forge Forgejo behind nginx and the monitoring stack (Prometheus, Alertmanager, Grafana, Loki) # # Every interface stays on 127.0.0.1 (SSH tunnel) until LSS__ACCESS=vpn with the wireguard module or -# LSS__WEB_ADMIN=true with a web server says otherwise. netdata and restic are opt-in. A Git forge (Forgejo) -# and CI runners join this profile with their modules in a later stage. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard docker node-exporter prometheus alertmanager loki alloy grafana smartd +# LSS__WEB_ADMIN=true with a web server says otherwise; Forgejo is the one site on port 443 +# (LSS_FORGEJO_DOMAINS, by default forgejo. in front of the server's name). netdata and restic are opt-in. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard docker nginx forgejo node-exporter prometheus alertmanager loki alloy grafana smartd diff --git a/profiles/office.conf b/profiles/office.conf index 007fa7e..1b90819 100644 --- a/profiles/office.conf +++ b/profiles/office.conf @@ -2,10 +2,12 @@ # # SPDX-License-Identifier: MIT -# Profile: Office server: minimal-secure plus a mail server with webmail behind nginx, certificates and backups +# Profile: Office server: minimal-secure plus mail with webmail, Nextcloud with Collabora Online, Vaultwarden, certificates and backups # # LSS_MAILSERVER_DOMAINS (and LSS_MAILSERVER_USERS) must be set; lss prints the DNS records to set and keeps them in # /etc/lss/mailserver-dns.txt. With LSS_TLS_DOMAINS naming the mail host name, LSS_MAILSERVER_CERTIFICATE=default -# gives Postfix and Dovecot the Let's Encrypt certificate. restic backs up the mailboxes, the DKIM keys and the -# webmail database daily. Nextcloud, Collabora and Vaultwarden join this profile with the application modules. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mailserver webmail restic +# gives Postfix and Dovecot the Let's Encrypt certificate. Nextcloud, Collabora Online and Vaultwarden run as Docker +# Compose stacks behind nginx, each on its own host name (LSS_NEXTCLOUD_DOMAINS, LSS_COLLABORA_DOMAINS, +# LSS_VAULTWARDEN_DOMAINS; by default nextcloud., collabora. and vaultwarden. in front of the server's name), which must +# point to the server. restic backs up the mailboxes, the DKIM keys and the webmail database daily. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mailserver webmail docker nextcloud collabora vaultwarden restic diff --git a/tests/integration/run.sh b/tests/integration/run.sh index 2c91e81..77f78ec 100755 --- a/tests/integration/run.sh +++ b/tests/integration/run.sh @@ -3,15 +3,17 @@ # # SPDX-License-Identifier: MIT -# Integration test: tests/integration/run.sh SYSTEM|host MODULE... +# Integration test: tests/integration/run.sh SYSTEM|host [--base MODULE,...] MODULE... # tests/integration/run.sh SYSTEM|host --profile NAME [LYNIS_MIN_INDEX] # # For every module: install (with its requirements), verify, install again (every module in the run must report # "unchanged"), and remove. A comma joins modules that are tested together (nginx,tls,vhost: installed at once, -# removed in reverse order). With --profile: install the whole profile, verify, install again, then run the Lynis -# audit and fail below LYNIS_MIN_INDEX. SYSTEM is a directory in tests/integration/images (a systemd container), -# vm-NAME for a KVM virtual machine from tests/integration/vm-images.conf, or "host" for the machine itself (modules -# that need a real kernel; used on throwaway CI virtual machines only). +# removed in reverse order). --base installs modules the others need first (docker and a web server for the apps), +# verifies them and removes them after the last one, so the figures of each module leave them out. With --profile: +# install the whole profile, verify, install again, then run the Lynis audit and fail below LYNIS_MIN_INDEX. SYSTEM +# is a directory in tests/integration/images (a systemd container), vm-NAME for a KVM virtual machine from +# tests/integration/vm-images.conf, or "host" for the machine itself (modules that need a real kernel; used on +# throwaway CI virtual machines only). set -Eeuo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." @@ -76,6 +78,10 @@ start_target() { fi in_target 'ln -sf /opt/lss-src/lss /usr/local/sbin/lss && mkdir -p /etc/lss' in_target "cat > /etc/lss/lss.conf" <"${settings}" + # Settings of one job only (one per line), such as the docker mode of a module that runs natively elsewhere. + if [[ -n ${LSS_TEST_SETTINGS:-} ]]; then + in_target "cat >> /etc/lss/lss.conf" <<<"${LSS_TEST_SETTINGS}" + fi # A setting that no module reads is a typo in the test settings. if in_target 'lss --no-color list 2>&1' | grep 'is not a setting'; then exit 1 @@ -248,6 +254,20 @@ if [[ " $* " == *[\ ,]tls[\ ,]* ]]; then group_end fi +base=() +if [[ $1 == --base ]]; then + IFS=',' read -ra base <<<"${2:?usage: run.sh SYSTEM|host --base MODULE,... MODULE...}" + shift 2 + group_start "${system}: base ${base[*]}" + before=$(resources_snapshot) + started=${SECONDS} + memory_watch_start + in_target "${LSS_RUN} --yes --no-color install ${base[*]}" + in_target "${LSS_RUN} --yes --no-color verify" + resources_report "${system} ${base[*]}" "${before}" "${started}" + group_end +fi + for item in "$@"; do IFS=',' read -ra set <<<"${item}" group_start "${system}: ${set[*]}" @@ -267,4 +287,12 @@ for item in "$@"; do fi group_end done -echo "${system}: ${*} passed" +if [[ ${#base[@]} -gt 0 && ${LSS_TEST_KEEP:-0} != 1 ]]; then + group_start "${system}: remove base ${base[*]}" + install_again "${base[@]}" + for ((i = ${#base[@]} - 1; i >= 0; i--)); do + in_target "${LSS_RUN} --yes --no-color remove ${base[i]}" + done + group_end +fi +echo "${system}: ${base[*]} ${*} passed" diff --git a/tests/integration/settings.conf b/tests/integration/settings.conf index 7f10778..4e68d71 100644 --- a/tests/integration/settings.conf +++ b/tests/integration/settings.conf @@ -53,3 +53,20 @@ LSS_MAILSERVER_ALIASES=info:alice,bob@example.org # so mail-relay names it lss-test.localdomain). LSS_GUARD_EMAIL=root@lss-test.localdomain LSS_GUARD_DAILY_SUMMARY=true + +# Apps: each on a host name of its own below example.com (certificates from Pebble where tls is installed; the tests +# reach the sites through --resolve, the containers through extra_hosts). Self-Updater deploys its own repository by +# release tag. +LSS_NEXTCLOUD_DOMAINS=cloud.example.com +LSS_COLLABORA_DOMAINS=office.example.com +LSS_FORGEJO_DOMAINS=git.example.com +LSS_UPTIME_KUMA_DOMAINS=status.example.com +LSS_VAULTWARDEN_DOMAINS=vault.example.com +LSS_KEYCLOAK_DOMAINS=sso.example.com +LSS_MATOMO_DOMAINS=stats.example.com +LSS_WORDPRESS_DOMAINS=blog.example.com +LSS_N8N_DOMAINS=n8n.example.com +LSS_MINIO_DOMAINS=s3.example.com +LSS_MINIO_CONSOLE_DOMAINS=s3-console.example.com +LSS_SELF_UPDATER_REPO=https://github.com/VectoDE/Self-Updater.git +LSS_SELF_UPDATER_TRACK=tag diff --git a/tests/integration/vm.sh b/tests/integration/vm.sh index 638a3c5..5fbb9cf 100644 --- a/tests/integration/vm.sh +++ b/tests/integration/vm.sh @@ -9,6 +9,11 @@ VM_DIR='' VM_PORT=${LSS_VM_SSH_PORT:-2222} +# Size of the VM: the app stacks and the profile all need more than the default (LSS_VM_MEMORY in MiB, LSS_VM_CPUS, +# LSS_VM_DISK as for qemu-img). +VM_MEMORY=${LSS_VM_MEMORY:-4096} +VM_CPUS=${LSS_VM_CPUS:-2} +VM_DISK=${LSS_VM_DISK:-20G} vm_ssh() { ssh -i "${VM_DIR}/key" -p "${VM_PORT}" -o BatchMode=yes -o StrictHostKeyChecking=no \ @@ -51,7 +56,7 @@ vm_start() { read -r image group <<<"$(vm_image "${name}")" [[ -n ${image} ]] || exit 1 VM_DIR=$(mktemp -d) - qemu-img create -q -f qcow2 -b "${image}" -F qcow2 "${VM_DIR}/disk.qcow2" 20G + qemu-img create -q -f qcow2 -b "${image}" -F qcow2 "${VM_DIR}/disk.qcow2" "${VM_DISK}" ssh-keygen -q -t ed25519 -N '' -C lss-vm-test -f "${VM_DIR}/key" cat >"${VM_DIR}/user-data" <"${VM_DIR}/meta-data" cloud-localds "${VM_DIR}/seed.iso" "${VM_DIR}/user-data" "${VM_DIR}/meta-data" - qemu-system-x86_64 -enable-kvm -cpu host -machine q35 -m 4096 -smp 2 \ + qemu-system-x86_64 -enable-kvm -cpu host -machine q35 -m "${VM_MEMORY}" -smp "${VM_CPUS}" \ -drive "file=${VM_DIR}/disk.qcow2,if=virtio" -drive "file=${VM_DIR}/seed.iso,if=virtio,format=raw" \ -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${VM_PORT}-:22" \ -device qemu-xhci -device usb-kbd \ diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats new file mode 100644 index 0000000..abf883e --- /dev/null +++ b/tests/unit/apps.bats @@ -0,0 +1,450 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lib/apps.sh: the Compose stacks of the app modules (files, secrets for a container user, start, site, checks, +# removal) against a stub docker. + +setup() { + load helpers + lss_test_setup + use_os debian-12 + shopt -s lastpipe + LSS_MODULE=demo + MOD_OPTIONS="domains||Host names +port|8081|Port on 127.0.0.1" + CTR_CONFIG_DIR=${BATS_TEST_TMPDIR}/containers + APP_DOCKER_CONFIG=${BATS_TEST_TMPDIR}/daemon.json + APP_SUBUID=${BATS_TEST_TMPDIR}/subuid + APP_SUBGID=${BATS_TEST_TMPDIR}/subgid + printf '{"icc": false}\n' >"${APP_DOCKER_CONFIG}" + ENGINE_LOG=${BATS_TEST_TMPDIR}/engine.log + : >"${ENGINE_LOG}" + WRITES=${BATS_TEST_TMPDIR}/writes + : >"${WRITES}" + # file_write as the test user; the owner it was asked for goes to $WRITES ("== MODE OWNER DEST"). + eval "_lss_file_write() $(declare -f file_write | tail -n +2)" + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + printf '== %s %s %s\n' "$1" "$2" "$3" >>"${WRITES}" + _lss_file_write "$1" "${OWNER}" "$3" + return + } +} + +# docker stub: records every call; "compose ... config --services" prints $SERVICES, "compose ... ps" prints +# $RUNNING, "compose ... config --format json" an empty stack, "compose version" and "info" succeed. +docker() { + printf 'docker %s\n' "$*" >>"${ENGINE_LOG}" + case $* in + *"config --services"*) [[ -z ${SERVICES:-} ]] || tr ' ' '\n' <<<"${SERVICES}" ;; + *"ps --status running --services"*) [[ -z ${RUNNING:-} ]] || tr ' ' '\n' <<<"${RUNNING}" ;; + *"config --format json"*) printf '{"services": {}}\n' ;; + *) ;; + esac + return "${DOCKER_EXIT:-0}" +} + +@test "the stack of a module has its project name and directory" { + [ "$(app_project)" = lss-demo ] + [ "$(app_dir)" = "${CTR_CONFIG_DIR}/lss-demo" ] + [ "$(app_compose_file)" = "${CTR_CONFIG_DIR}/lss-demo/compose.yaml" ] +} + +@test "host names and ports are checked" { + app_valid_domain cloud.example.com + run app_valid_domain localhost + [ "${status}" -eq 1 ] + run app_valid_domain Cloud.Example.com + [ "${status}" -eq 1 ] + config_set LSS_DEMO_DOMAINS 'a.example.com b.example.org' + app_check_domains + config_set LSS_DEMO_DOMAINS 'a.example.com bad_name' + run app_check_domains + [ "${status}" -eq 1 ] + [[ ${output} == *"LSS_DEMO_DOMAINS: 'bad_name' is not a host name like demo.example.com"* ]] + app_check_port port + config_set LSS_DEMO_PORT 80 + run app_check_port port + [[ ${output} == *"must be a port from 1024 to 65535"* ]] + config_set LSS_DEMO_PORT "${WEB_ADMIN_PORT}" + run app_check_port port + [[ ${output} == *"must differ from the admin port"* ]] +} + +@test "without host names the app is named after the module below the server's name" { + # shellcheck disable=SC2329 # called by app_domains + hostname() { + printf 'Server1.Example.com\n' + } + [ "$(app_domains)" = demo.server1.example.com ] + config_set LSS_DEMO_DOMAINS 'cloud.example.com www.example.com' + [ "$(app_domains)" = 'cloud.example.com www.example.com' ] + [ "$(app_domain)" = cloud.example.com ] +} + +@test "container users are mapped into Docker's user namespace" { + [ "$(app_host_owner 1000)" = 1000:1000 ] + [ "$(app_host_owner 33:44)" = 33:44 ] + printf '{"userns-remap": "default"}\n' >"${APP_DOCKER_CONFIG}" + printf 'admin:100000:65536\ndockremap:231072:65536\n' >"${APP_SUBUID}" + printf 'dockremap:231072:65536\n' >"${APP_SUBGID}" + [ "$(app_host_owner 0)" = 231072:231072 ] + [ "$(app_host_owner 70:71)" = 231142:231143 ] + printf '{"userns-remap": "remap:remap"}\n' >"${APP_DOCKER_CONFIG}" + run app_host_owner 0 + [ "${status}" -eq 1 ] + [[ ${output} == *"has no ID range for remap"* ]] + run app_host_owner www-data + [[ ${output} == *"'www-data' is not UID or UID:GID"* ]] +} + +@test "secrets are copied for the container user without a final newline" { + printf '{"userns-remap": "default"}\n' >"${APP_DOCKER_CONFIG}" + printf 'dockremap:100000:65536\n' | tee "${APP_SUBUID}" >"${APP_SUBGID}" + app_secret demo-db-password 999:70 40 + local copy=${CTR_CONFIG_DIR}/lss-demo/secrets/demo-db-password + grep -qxF "== 0400 100999:100070 ${copy}" "${WRITES}" + [ "$(stat -c '%a' "${copy}")" = 400 ] + cmp -s "${copy}" "${LSS_SECRETS_DIR}/demo-db-password" + [ "$(wc -c <"${copy}")" -eq 40 ] + [ "${APP_CHANGED}" = 1 ] + APP_CHANGED=0 + app_secret demo-db-password 999:70 40 + [ "${APP_CHANGED}" = 0 ] + app_secret demo-db-password 999:70 40 db-password.app + [ -f "${CTR_CONFIG_DIR}/lss-demo/secrets/db-password.app" ] + run app_file ../escape 0 0400 /dev/null || skip "jq is not installed" + LSS_MODULE=docker + state_set docker status installed + LSS_MODULE=demo + printf 'services: {}\n' | app_compose + [ "${APP_COMPOSE_CHANGED}" = 1 ] + grep -qxF "== 0600 root:root $(app_compose_file)" "${WRITES}" + SERVICES='app db' RUNNING='app db' + app_up + grep -q -- "up --detach --remove-orphans --quiet-pull$" "${ENGINE_LOG}" + [ "${APP_COMPOSE_CHANGED}" = 0 ] + : >"${ENGINE_LOG}" + app_up + run ! grep -q -- " up " "${ENGINE_LOG}" + RUNNING='db' + app_up + grep -q -- " up --detach" "${ENGINE_LOG}" + : >"${ENGINE_LOG}" + RUNNING='app db' + APP_CHANGED=1 + app_up + grep -q -- "up --detach --remove-orphans --quiet-pull --force-recreate$" "${ENGINE_LOG}" + [ "${APP_CHANGED}" = 0 ] + DOCKER_EXIT=1 run app_up + [[ ${output} == *"docker compose is missing"* ]] +} + +@test "app_running needs every service of the stack" { + SERVICES='' RUNNING='' + run app_running + [ "${status}" -eq 1 ] + SERVICES='db app' RUNNING='app db' + app_running + SERVICES='db app' RUNNING='app' + run app_running + [ "${status}" -eq 1 ] + app_exec app occ status + grep -qxF "docker compose --project-name lss-demo -f ${CTR_CONFIG_DIR}/lss-demo/compose.yaml exec -T app occ status" "${ENGINE_LOG}" +} + +@test "pages come from the web server of this server, with their status" { + config_set LSS_DEMO_DOMAINS cloud.example.com + # shellcheck disable=SC2329 # called by app_get + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf 'login\n%s' "${CURL_STATUS:-200}" + return 0 + } + local page + page=$(app_get /status.php) + [ "$(app_status "${page}")" = 200 ] + grep -qF -- "--resolve cloud.example.com:443:127.0.0.1 https://cloud.example.com/status.php" "${STUB_LOG}" + page=$(app_wait login 200 5) + [[ ${page} == *login* ]] + # shellcheck disable=SC2329 # called by app_wait + sleep() { + return 0 + } + CURL_STATUS=502 run app_wait login 200 1 + [ "${status}" -eq 1 ] + [[ ${output} == *"https://cloud.example.com/login answers 502 instead of 200"* ]] +} + +@test "the site proxies to 127.0.0.1 for every host name and gets a certificate with tls" { + config_set LSS_DEMO_DOMAINS 'cloud.example.com www.example.com' + # shellcheck disable=SC2329 # called by app_site + web_cert_placeholder() { + printf 'placeholder %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_site + web_site_write() { + printf 'site %s\n' "$*" >>"${STUB_LOG}" + LSS_FILE_CHANGED=1 + } + # shellcheck disable=SC2329 # called by app_site + web_reload() { + printf 'reload\n' >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_site + web_cert_issue() { + printf 'issue %s\n' "$*" >>"${STUB_LOG}" + } + app_site 8081 + grep -qxF 'placeholder demo' "${STUB_LOG}" + grep -qxF 'site demo proxy http://127.0.0.1:8081 cloud.example.com www.example.com' "${STUB_LOG}" + grep -qxF reload "${STUB_LOG}" + run ! grep -q '^issue' "${STUB_LOG}" + state_set tls status installed + app_site 8081 + grep -qxF 'issue demo cloud.example.com www.example.com' "${STUB_LOG}" +} + +@test "verify checks the files, the rules, the containers and the headers of the site" { + command -v jq >/dev/null || skip "jq is not installed" + # shellcheck disable=SC2329 # called by app_verify_stack + stat() { + printf '600 root\n' + } + run app_verify_stack + [[ ${output} == *"compose.yaml is missing"* ]] + printf 'services: {}\n' | app_compose + SERVICES='app' RUNNING='' + run app_verify_stack + [[ ${output} == *"a container of lss-demo is not running"* ]] + RUNNING='app' + # shellcheck disable=SC2329 # called by app_verify_stack + curl() { + printf 'HTTP/2 200\r\n%s\r\n\n200' "${HEADER:-}" + return 0 + } + run app_verify_stack + [[ ${output} == *"sends no Strict-Transport-Security header"* ]] + HEADER='strict-transport-security: max-age=63072000' + app_verify_stack +} + +@test "removal stops the stack and keeps the data unless --purge" { + MOD_SECRETS="demo-db-password" + app_secret demo-db-password 0 + printf 'services: {}\n' | app_compose + # shellcheck disable=SC2329 # called by app_remove + web_server() { + printf 'nginx' + } + # shellcheck disable=SC2329 # called by app_remove + web_site_remove() { + printf 'site-remove %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_remove + web_reload() { + return 0 + } + app_remove + grep -q -- "down --remove-orphans$" "${ENGINE_LOG}" + grep -qxF 'site-remove demo' "${STUB_LOG}" + [ ! -e "$(app_dir)" ] + secret_exists demo-db-password + printf 'services: {}\n' | app_compose + LSS_PURGE=1 app_remove + grep -q -- "down --remove-orphans --volumes$" "${ENGINE_LOG}" + run secret_exists demo-db-password + [ "${status}" -eq 1 ] +} + +@test "database services run unprivileged on the internal network, with their password from a file" { + run app_postgres cloud + [[ ${output} == *"image: ${APP_POSTGRES_IMAGE}"* ]] + [[ ${output} == *'user: "70:70"'* ]] + [[ ${output} == *"POSTGRES_USER: cloud"*"POSTGRES_PASSWORD_FILE: /run/secrets/db-password"* ]] + [[ ${output} == *"secrets/db-password.db:/run/secrets/db-password:ro,z"* ]] + [[ ${output} == *"networks: [backend]"*"pg_isready"* ]] + [[ ${output} == *"cap_drop: [ALL]"*"read_only: true"* ]] + run app_mariadb shop + [[ ${output} == *'user: "999:999"'* ]] + [[ ${output} == *"MARIADB_PASSWORD_FILE: /run/secrets/db-password"*"MARIADB_ROOT_PASSWORD_FILE: /run/secrets/db-root-password"* ]] + [[ ${output} == *"healthcheck.sh"* ]] + run app_networks + [[ ${output} == *"backend:"*"internal: true"* ]] + run app_init data /data 1000:1000 + [[ ${output} == *'entrypoint: ["chown", "1000:1000", "/data"]'* ]] + [[ ${output} == *"network_mode: none"*"cap_drop: [ALL]"*"cap_add: [CHOWN]"*"read_only: true"* ]] + [[ ${output} == *'restart: "no"'* ]] +} + +@test "database secrets go to the database container and the app, each for its own user" { + app_db_secrets postgres 33:33 + local dir=${CTR_CONFIG_DIR}/lss-demo/secrets + grep -qxF "== 0400 70:70 ${dir}/db-password.db" "${WRITES}" + grep -qxF "== 0400 33:33 ${dir}/db-password.app" "${WRITES}" + cmp -s "${dir}/db-password.db" "${LSS_SECRETS_DIR}/demo-db-password" + run ! grep -q db-root-password "${WRITES}" + app_db_secrets mariadb 1000 app-db + grep -qxF "== 0400 999:999 ${dir}/db-root-password.db" "${WRITES}" + grep -qxF "== 0400 1000:1000 ${dir}/app-db" "${WRITES}" + secret_exists demo-db-root-password +} + +@test "the one-shot init service does not count as a stopped service" { + SERVICES='app init' RUNNING='app' + app_running + SERVICES='app db init' RUNNING='db' + run app_running + [ "${status}" -eq 1 ] +} + +@test "logins for curl go through a private netrc file" { + config_set LSS_DEMO_DOMAINS cloud.example.com + secret_set demo-admin-password 'S3cret' + local file + file=$(app_netrc admin demo-admin-password) + [ "$(stat -c '%a' "${file}")" = 600 ] + [ "$(cat "${file}")" = 'machine cloud.example.com login admin password S3cret' ] + rm -f -- "${file}" +} + +@test "the host names of another app come from its settings or the server's name" { + # shellcheck disable=SC2329 # called by app_domains_of + hostname() { + printf 'srv.example.com\n' + } + [ "$(app_domains_of collabora)" = collabora.srv.example.com ] + config_set LSS_COLLABORA_DOMAINS 'office.example.com docs.example.com' + [ "$(app_domains_of collabora)" = 'office.example.com docs.example.com' ] +} + +@test "Nextcloud Office is linked once both apps are there, trusting the certificate of the web server" { + command -v jq >/dev/null || skip "jq is not installed" + config_set LSS_COLLABORA_DOMAINS office.example.com + LSS_MODULE=collabora + app_nextcloud_office + [ ! -s "${ENGINE_LOG}" ] + state_set nextcloud status installed + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + mkdir -p "${WEB_TLS_DIR}/default" + printf 'CERT\n' >"${WEB_TLS_DIR}/default/fullchain.pem" + app_nextcloud_office + local occ="docker compose --project-name lss-nextcloud -f ${CTR_CONFIG_DIR}/lss-nextcloud/compose.yaml exec -T app php occ" + grep -qxF "${occ} app:install richdocuments" "${ENGINE_LOG}" + grep -qxF "${occ} config:app:set richdocuments wopi_url --value=https://office.example.com" "${ENGINE_LOG}" + grep -qF "exec -T app sh -c cat >/tmp/lss-web.pem" "${ENGINE_LOG}" + grep -qxF "${occ} security:certificates:import /tmp/lss-web.pem" "${ENGINE_LOG}" + grep -qxF "${occ} richdocuments:activate-config" "${ENGINE_LOG}" + [ -n "$(state_get nextcloud trusted_cert)" ] + : >"${ENGINE_LOG}" + app_nextcloud_office + run ! grep -q 'certificates:import' "${ENGINE_LOG}" + # With the tls module, only a private CA needs trusting. + state_set tls status installed + [ -z "$(_app_nc_trust)" ] + printf 'CA\n' >"${BATS_TEST_TMPDIR}/ca.pem" + config_set LSS_TLS_CA_BUNDLE "${BATS_TEST_TMPDIR}/ca.pem" + [ "$(_app_nc_trust)" = "${BATS_TEST_TMPDIR}/ca.pem" ] + DOCKER_EXIT=1 run app_nextcloud_office + [ "${status}" -eq 1 ] + [[ ${output} == *"Nextcloud Office (richdocuments) cannot be installed"* ]] +} + +@test "timers run a command in a container of the stack and go away on removal" { + # shellcheck disable=SC2329 # called by app_timer + svc_unit() { + printf -- '--- %s\n' "$1" >>"${STUB_LOG}" + cat >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer + svc_enable() { + printf 'enable %s\n' "$1" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer_remove + svc_disable() { + printf 'disable %s\n' "$1" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer_remove + file_remove() { + printf 'remove %s\n' "$1" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer_remove + svc_daemon_reload() { + return 0 + } + app_timer demo-cron 5min "Demo jobs" app php -f cron.php + grep -qxF -- '--- lss-demo-cron.service' "${STUB_LOG}" + grep -qxF "ExecStart=docker compose --project-name lss-demo -f ${CTR_CONFIG_DIR}/lss-demo/compose.yaml exec -T app php -f cron.php" "${STUB_LOG}" + grep -qxF 'OnUnitActiveSec=5min' "${STUB_LOG}" + grep -qxF 'enable lss-demo-cron.timer' "${STUB_LOG}" + app_timer_remove demo-cron + grep -qxF 'disable lss-demo-cron.timer' "${STUB_LOG}" + grep -qxF 'remove /etc/systemd/system/lss-demo-cron.service' "${STUB_LOG}" +} + +@test "a second site of an app has its own name and host names, and goes away with its certificate" { + # shellcheck disable=SC2329 # called by app_site + web_cert_placeholder() { + return 0 + } + # shellcheck disable=SC2329 # called by app_site + web_site_write() { + printf 'site %s\n' "$*" >>"${STUB_LOG}" + LSS_FILE_CHANGED=0 + } + app_site 8092 demo-console console.example.com + grep -qxF 'site demo-console proxy http://127.0.0.1:8092 console.example.com' "${STUB_LOG}" + # shellcheck disable=SC2329 # called by app_site_remove + web_server() { + printf 'caddy' + } + # shellcheck disable=SC2329 # called by app_site_remove + web_site_remove() { + printf 'site-remove %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_site_remove + web_reload() { + return 0 + } + # shellcheck disable=SC2329 # called by app_site_remove + certbot() { + printf 'certbot %s\n' "$*" >>"${STUB_LOG}" + } + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + WEB_CERT_LIVE=${BATS_TEST_TMPDIR}/live + mkdir -p "${WEB_CERT_LIVE}/demo-console" + app_site_remove demo-console + grep -qxF 'site-remove demo-console' "${STUB_LOG}" + grep -qxF 'certbot delete --non-interactive --cert-name demo-console' "${STUB_LOG}" +} + +@test "pages of a second host name of the app" { + config_set LSS_DEMO_DOMAINS cloud.example.com + # shellcheck disable=SC2329 # called by app_get + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf 'ok\n200' + } + APP_GET_DOMAIN=console.example.com app_get login >/dev/null + grep -qF -- "--resolve console.example.com:443:127.0.0.1 https://console.example.com/login" "${STUB_LOG}" +} diff --git a/tests/unit/helpers.bash b/tests/unit/helpers.bash index 028bb71..0a067a1 100644 --- a/tests/unit/helpers.bash +++ b/tests/unit/helpers.bash @@ -22,7 +22,7 @@ lss_test_setup() { export OWNER : >"${STUB_LOG}" local library - for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring module ui; do + for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring apps module ui; do # shellcheck source=/dev/null source "${REPO_ROOT}/lib/${library}.sh" done From be8ecfe60c7b70e6c45b0889d7d64a088ceaf797 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 19:38:56 +0200 Subject: [PATCH 02/28] refactor(apps): publish apps through one web_proxy_route helper An optional edge proxy (Traefik, #27) may later own ports 80 and 443 and route to the apps itself. Every app now becomes reachable through web_proxy_route/web_proxy_route_remove in lib/web.sh, which today writes the reverse-proxy site of the web server, so that switch touches one place. Also fixes found while reviewing the apps against their sources: Uptime Kuma's deleteMonitor takes three arguments, Vaultwarden's user deletion needs a JSON content type, and Matomo waits for a static file instead of the tracker, which answers differently before the installation. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/apps.sh | 24 +++++------------------- lib/web.sh | 32 ++++++++++++++++++++++++++++++++ modules/apps/matomo.sh | 2 +- modules/apps/uptime-kuma.sh | 7 ++++--- modules/apps/vaultwarden.sh | 2 +- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/lib/apps.sh b/lib/apps.sh index a4a59b5..3cee1ec 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -335,19 +335,13 @@ app_timer_remove() { return } -# app_site PORT [NAME DOMAIN...]: the HTTPS site NAME (default: the module) for the host names DOMAIN (default: those of -# the app), proxied to 127.0.0.1:PORT, with a certificate from the tls module when it is installed. The web server -# reloads when the site changed. +# app_site PORT [NAME DOMAIN...]: the HTTPS route NAME (default: the module) for the host names DOMAIN (default: those +# of the app) to 127.0.0.1:PORT, through web_proxy_route (lib/web.sh). app_site() { local port=$1 name=${2:-${LSS_MODULE}} local -a domains=("${@:3}") [[ ${#domains[@]} -gt 0 ]] || read -ra domains <<<"$(app_domains)" - web_cert_placeholder "${name}" - web_site_write "${name}" proxy "http://127.0.0.1:${port}" "${domains[@]}" - [[ ${LSS_FILE_CHANGED} == 0 ]] || web_reload - if state_installed tls; then - web_cert_issue "${name}" "${domains[@]}" - fi + web_proxy_route "${name}" "http://127.0.0.1:${port}" "${domains[@]}" return } @@ -532,18 +526,10 @@ app_netrc() { return } -# app_site_remove NAME: removes the site NAME of app_site with its certificates. +# app_site_remove NAME: removes the route NAME of app_site. app_site_remove() { local name=$1 - if [[ -n $(web_server) ]]; then - web_site_remove "${name}" - web_reload - fi - file_remove "${WEB_TLS_DIR}/${name}" - if [[ -d ${WEB_CERT_LIVE}/${name} ]] && command -v certbot >/dev/null 2>&1; then - log_command certbot delete --non-interactive --cert-name "${name}" || - log_warn "${name}: certbot could not delete the certificate ${name}." - fi + web_proxy_route_remove "${name}" return } diff --git a/lib/web.sh b/lib/web.sh index 0a07392..e60338c 100644 --- a/lib/web.sh +++ b/lib/web.sh @@ -631,6 +631,38 @@ web_site_remove() { return } +# web_proxy_route NAME UPSTREAM DOMAIN...: publishes a local service (UPSTREAM, such as http://127.0.0.1:8081) as the +# HTTPS route NAME for the host names DOMAIN, with a certificate from the tls module when it is installed. The one +# place where an app becomes reachable from outside: today it writes a reverse-proxy site of the web server, and an +# edge proxy in front of the web server can take its routes over here. Reloads the web server when the site changed. +web_proxy_route() { + local name=$1 upstream=$2 + shift 2 + [[ $# -gt 0 ]] || die "web: the route ${name} has no host names." + web_cert_placeholder "${name}" + web_site_write "${name}" proxy "${upstream}" "$@" + [[ ${LSS_FILE_CHANGED} == 0 ]] || web_reload + if state_installed tls; then + web_cert_issue "${name}" "$@" + fi + return +} + +# web_proxy_route_remove NAME: removes the route of web_proxy_route with its certificates. +web_proxy_route_remove() { + local name=$1 + if [[ -n $(web_server) ]]; then + web_site_remove "${name}" + web_reload + fi + file_remove "${WEB_TLS_DIR}/${name}" + if [[ -d ${WEB_CERT_LIVE}/${name} ]] && command -v certbot >/dev/null 2>&1; then + log_command certbot delete --non-interactive --cert-name "${name}" || + log_warn "${name}: certbot could not delete the certificate ${name}." + fi + return +} + # web_admin_write NAME PATH php|proxy|strip TARGET: publishes an admin interface below /PATH/ on the admin server # (WEB_ADMIN_PORT). TARGET is a directory with PHP files or an upstream URL. proxy passes the whole path on (the # upstream serves below /PATH/), strip removes /PATH first (for upstreams that only serve their root, such as Netdata). diff --git a/modules/apps/matomo.sh b/modules/apps/matomo.sh index 97af363..c23ec82 100644 --- a/modules/apps/matomo.sh +++ b/modules/apps/matomo.sh @@ -180,7 +180,7 @@ mod_configure() { _mt_compose | app_compose app_up app_site "$(opt_int port)" - app_wait matomo.php 200 >/dev/null || die "matomo: https://$(app_domain)/matomo.php does not answer." + app_wait matomo.js 200 >/dev/null || die "matomo: https://$(app_domain)/matomo.js does not answer." _mt_install _mt_set General force_ssl 1 _mt_set General assume_secure_protocol 1 diff --git a/modules/apps/uptime-kuma.sh b/modules/apps/uptime-kuma.sh index a3c33f0..50de00b 100644 --- a/modules/apps/uptime-kuma.sh +++ b/modules/apps/uptime-kuma.sh @@ -93,12 +93,12 @@ socket.on("connect", async () => { retryInterval: 20, resendInterval: 0, maxretries: 0, timeout: 16, maxredirects: 10, accepted_statuscodes: ["200-399"], notificationIDList: {}, ignoreTls: false, upsideDown: false, expiryNotification: false, kafkaProducerBrokers: [], kafkaProducerSaslOptions: { mechanism: "None" }, - conditions: [], active: true, + conditions: [], rabbitmqNodes: [], active: true, }); if (!added.ok) fail(`add monitor: ${added.msg}`); socket.on("heartbeat", async (beat) => { if (beat.monitorID !== added.monitorID) return; - const removed = await call("deleteMonitor", added.monitorID); + const removed = await call("deleteMonitor", added.monitorID, false); if (!removed.ok) fail(`delete monitor: ${removed.msg}`); if (beat.status !== 1) fail(`the monitor reports down: ${beat.msg}`); console.log("up"); @@ -111,7 +111,8 @@ EOF # _kuma ACTION: runs the client in the container. _kuma() { - app_exec app node /opt/lss/lss-kuma.js "$1" + local action=$1 + app_exec app node /opt/lss/lss-kuma.js "${action}" return } diff --git a/modules/apps/vaultwarden.sh b/modules/apps/vaultwarden.sh index c632335..0887ab2 100644 --- a/modules/apps/vaultwarden.sh +++ b/modules/apps/vaultwarden.sh @@ -130,7 +130,7 @@ mod_verify() { rm -f -- "${jar}" die "vaultwarden: the invited user is missing from the user list." fi - status=$(app_status "$(_vw_admin "${jar}" "users/${id}/delete" -X POST)") + status=$(app_status "$(_vw_admin "${jar}" "users/${id}/delete" -X POST -H 'Content-Type: application/json')") rm -f -- "${jar}" [[ ${status} == 200 ]] || die "vaultwarden: deleting the invited user answers ${status}." return From 5aef35267d6aacb53017a4f3f57fda6de32128fc Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 20:16:35 +0200 Subject: [PATCH 03/28] fix(apps): read the user namespace setting of Docker without jq The coverage run uses a kcov image without jq, where the mapping of container users silently fell back to no remapping. The docker module writes "userns-remap" on a line of its own, so sed reads it everywhere. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/apps.sh | 4 +++- tests/unit/apps.bats | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/apps.sh b/lib/apps.sh index 3cee1ec..43681f8 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -92,7 +92,9 @@ app_compose_ready() { # when the daemon runs without user namespace remapping. _app_userns_base() { local file=$1 user base - user=$(jq -r '."userns-remap" // empty' "${APP_DOCKER_CONFIG:-/etc/docker/daemon.json}" 2>/dev/null) || user='' + # The daemon.json of the docker module has the setting on a line of its own; read without jq. + user=$(sed -n 's/.*"userns-remap"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "${APP_DOCKER_CONFIG:-/etc/docker/daemon.json}" 2>/dev/null | + head -n 1) || user='' if [[ -z ${user} ]]; then printf '0' return 0 diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats index abf883e..e798e0a 100644 --- a/tests/unit/apps.bats +++ b/tests/unit/apps.bats @@ -339,7 +339,6 @@ docker() { } @test "Nextcloud Office is linked once both apps are there, trusting the certificate of the web server" { - command -v jq >/dev/null || skip "jq is not installed" config_set LSS_COLLABORA_DOMAINS office.example.com LSS_MODULE=collabora app_nextcloud_office From c593afa0243ed52c487104e527868052e8badf5c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 21:28:50 +0200 Subject: [PATCH 04/28] fix(apps): Keycloak discovery behind Caddy, Collabora TLS check, Kuma start Three causes of the failing app jobs in CI: - Caddy answered 404 for every dot path except /.well-known/ at the root, so Keycloak's /realms/master/.well-known/openid-configuration never got through. nginx and Apache already let .well-known through at any depth; Caddy now does the same, and other dot files stay hidden. - Without the tls module, every route used a copy of the default certificate, which only names the host. Nextcloud checks the host name when it calls Collabora, so trusting that certificate could not work. A route now gets a self-signed placeholder for its own names (renewed when the names change or expiry is near; a certificate from the tls module is never replaced), and Nextcloud trusts the certificate of the collabora route. - Uptime Kuma registers its Socket.IO handlers only after answering the connection, so the first question could get lost. The script now asks again until it gets an answer. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/apps.sh | 6 ++--- lib/web.sh | 47 ++++++++++++++++++++++++++++++------- modules/apps/uptime-kuma.sh | 13 ++++++++-- modules/web/caddy.sh | 5 ++-- tests/unit/apps.bats | 6 ++--- tests/unit/web.bats | 31 ++++++++++++++++++++++++ 6 files changed, 89 insertions(+), 19 deletions(-) diff --git a/lib/apps.sh b/lib/apps.sh index 43681f8..763a928 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -245,15 +245,15 @@ _app_nc_occ() { } # _app_nc_trust: the certificate Nextcloud must trust to reach the Collabora site: the CA of a private ACME server -# (LSS_TLS_CA_BUNDLE) or, without the tls module, the self-signed default certificate of the web server; nothing for +# (LSS_TLS_CA_BUNDLE) or, without the tls module, the self-signed certificate of the collabora site; nothing for # Let's Encrypt. Prints the file, or nothing. _app_nc_trust() { local bundle if state_installed tls; then bundle=$(config_option tls ca_bundle '') [[ -z ${bundle} || ! -s ${bundle} ]] || printf '%s' "${bundle}" - elif [[ -s ${WEB_TLS_DIR}/default/fullchain.pem ]]; then - printf '%s/default/fullchain.pem' "${WEB_TLS_DIR}" + elif [[ -s ${WEB_TLS_DIR}/collabora/fullchain.pem ]]; then + printf '%s/collabora/fullchain.pem' "${WEB_TLS_DIR}" fi return } diff --git a/lib/web.sh b/lib/web.sh index e60338c..b1bd569 100644 --- a/lib/web.sh +++ b/lib/web.sh @@ -161,26 +161,55 @@ web_default_cert() { return 0 fi name=$(hostname -f 2>/dev/null || hostname) + _web_self_signed "${dir}" "${name}" + files_note_change "self-signed default certificate" + return +} + +# _web_self_signed DIR DOMAIN...: a self-signed certificate for the names DOMAIN (the first one is the subject) in +# DIR/fullchain.pem and DIR/privkey.pem, valid for 825 days; the key is readable by WEB_GROUP. +_web_self_signed() { + local dir=$1 tmp san domain + shift + san='' + for domain in "$@"; do + san+="${san:+,}DNS:${domain}" + done tmp=$(mktemp -d) - openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 825 -subj "/CN=${name}" \ - -addext "subjectAltName=DNS:${name}" -keyout "${tmp}/privkey.pem" -out "${tmp}/fullchain.pem" >/dev/null 2>&1 || - die "web: could not create the self-signed default certificate." - install -d -m 0755 "${WEB_TLS_DIR}" + if ! openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 825 -subj "/CN=$1" \ + -addext "subjectAltName=${san}" -keyout "${tmp}/privkey.pem" -out "${tmp}/fullchain.pem" >/dev/null 2>&1; then + rm -rf -- "${tmp}" + die "web: could not create a self-signed certificate for $1." + fi # Certificates are state, not configuration: the tls module's deploy hook replaces them, so they are not # recorded as managed files (lss verify would report every renewal as a change made outside lss). - install -d -m 0755 "${dir}" + install -d -m 0755 "${WEB_TLS_DIR}" "${dir}" install -m 0640 -o root -g "${WEB_GROUP}" "${tmp}/privkey.pem" "${dir}/privkey.pem" install -m 0644 -o root -g root "${tmp}/fullchain.pem" "${dir}/fullchain.pem" - files_note_change "self-signed default certificate" rm -rf -- "${tmp}" return } -# web_cert_placeholder NAME: until NAME has a certificate of its own, its directory holds a copy of the default -# certificate, so a site can point at WEB_TLS_DIR/NAME from the start and keeps that path after the real one arrives. +# web_cert_placeholder NAME [DOMAIN...]: until NAME has a certificate of its own, its directory holds a copy of the +# default certificate, so a site can point at WEB_TLS_DIR/NAME from the start and keeps that path after the real one +# arrives. With DOMAIN, the placeholder is a self-signed certificate for these names instead, so clients that trust +# it (a Nextcloud that calls Collabora through the web server) can check the host name; it follows the names and is +# renewed within 30 days of expiry, and a certificate of the tls module (lineage NAME) is never replaced. web_cert_placeholder() { local name=$1 dir + shift dir="${WEB_TLS_DIR}/${name}" + if [[ $# -gt 0 ]]; then + [[ ! -s ${WEB_CERT_LIVE}/${name}/fullchain.pem ]] || return 0 + if [[ -s ${dir}/fullchain.pem && -s ${dir}/privkey.pem ]] && + [[ $(web_cert_domains "${dir}/fullchain.pem") == "$(printf '%s\n' "$@" | sort -u)" ]] && + openssl x509 -checkend $((30 * 86400)) -noout -in "${dir}/fullchain.pem" >/dev/null 2>&1; then + return 0 + fi + _web_self_signed "${dir}" "$@" + files_note_change "self-signed certificate for ${name}" + return + fi [[ -s ${dir}/fullchain.pem && -s ${dir}/privkey.pem ]] && return 0 install -d -m 0755 "${dir}" install -m 0644 -o root -g root "${WEB_TLS_DIR}/default/fullchain.pem" "${dir}/fullchain.pem" @@ -639,7 +668,7 @@ web_proxy_route() { local name=$1 upstream=$2 shift 2 [[ $# -gt 0 ]] || die "web: the route ${name} has no host names." - web_cert_placeholder "${name}" + web_cert_placeholder "${name}" "$@" web_site_write "${name}" proxy "${upstream}" "$@" [[ ${LSS_FILE_CHANGED} == 0 ]] || web_reload if state_installed tls; then diff --git a/modules/apps/uptime-kuma.sh b/modules/apps/uptime-kuma.sh index 50de00b..fc70541 100644 --- a/modules/apps/uptime-kuma.sh +++ b/modules/apps/uptime-kuma.sh @@ -75,18 +75,27 @@ const password = fs.readFileSync("/run/secrets/admin-password", "utf8").trim(); const socket = io("http://127.0.0.1:3001", { transports: ["websocket"], reconnection: false }); const fail = (message) => { console.error(message); process.exit(1); }; const call = (event, ...args) => new Promise((resolve) => socket.emit(event, ...args, resolve)); +// The server registers its handlers only after it has answered the connection, so the first question is asked +// again until an answer comes. +const ask = async (event, ...args) => { + for (let attempt = 0; attempt < 20; attempt++) { + const answer = await Promise.race([call(event, ...args), new Promise((resolve) => setTimeout(resolve, 3000))]); + if (answer !== undefined) return answer; + } + fail(`no answer to ${event}`); +}; setTimeout(() => fail("timeout"), 120000); socket.on("connect_error", (error) => fail(`connect: ${error.message}`)); socket.on("connect", async () => { if (action === "setup") { - const needed = await call("needSetup"); + const needed = await ask("needSetup"); if (!needed) { console.log("exists"); process.exit(0); } const result = await call("setup", username, password); if (!result.ok) fail(`setup: ${result.msg}`); console.log("created"); process.exit(0); } - const login = await call("login", { username, password, token: "" }); + const login = await ask("login", { username, password, token: "" }); if (!login.ok) fail(`login: ${login.msg}`); const added = await call("add", { type: "http", name: "lss verify", url: "http://127.0.0.1:3001/", method: "GET", interval: 20, diff --git a/modules/web/caddy.sh b/modules/web/caddy.sh index 13825f2..73c2eee 100644 --- a/modules/web/caddy.sh +++ b/modules/web/caddy.sh @@ -78,11 +78,12 @@ $(_caddy_headers) } } -# Dot files and directories (.git, .env, .htpasswd) are never served; ACME challenges below .well-known are. +# Dot files and directories (.git, .env, .htpasswd) are never served; .well-known paths (ACME challenges, OpenID and +# similar discovery documents of proxied apps) are, as with nginx and Apache. (lss_hidden) { @lss_hidden { path */.* - not path /.well-known/* + not path */.well-known/* } respond @lss_hidden 404 } diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats index e798e0a..84da4a8 100644 --- a/tests/unit/apps.bats +++ b/tests/unit/apps.bats @@ -338,15 +338,15 @@ docker() { [ "$(app_domains_of collabora)" = 'office.example.com docs.example.com' ] } -@test "Nextcloud Office is linked once both apps are there, trusting the certificate of the web server" { +@test "Nextcloud Office is linked once both apps are there, trusting the certificate of the collabora site" { config_set LSS_COLLABORA_DOMAINS office.example.com LSS_MODULE=collabora app_nextcloud_office [ ! -s "${ENGINE_LOG}" ] state_set nextcloud status installed WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls - mkdir -p "${WEB_TLS_DIR}/default" - printf 'CERT\n' >"${WEB_TLS_DIR}/default/fullchain.pem" + mkdir -p "${WEB_TLS_DIR}/collabora" + printf 'CERT\n' >"${WEB_TLS_DIR}/collabora/fullchain.pem" app_nextcloud_office local occ="docker compose --project-name lss-nextcloud -f ${CTR_CONFIG_DIR}/lss-nextcloud/compose.yaml exec -T app php occ" grep -qxF "${occ} app:install richdocuments" "${ENGINE_LOG}" diff --git a/tests/unit/web.bats b/tests/unit/web.bats index 1957d10..473f326 100644 --- a/tests/unit/web.bats +++ b/tests/unit/web.bats @@ -241,6 +241,37 @@ EOF [ "$(files_changes_of core)" = 1 ] } +@test "a route's placeholder is self-signed for its names, follows them and leaves a tls certificate alone" { + as_test_user + fake_openssl + web_cert_placeholder office office.example.com docs.example.com + grep -q '^openssl req -x509 .*-subj /CN=office.example.com -addext subjectAltName=DNS:office.example.com,DNS:docs.example.com ' "${STUB_LOG}" + grep -q "^install -m 0640 -o root -g ${WEB_GROUP} .*/privkey.pem ${WEB_TLS_DIR}/office/privkey.pem$" "${STUB_LOG}" + [ -s "${WEB_TLS_DIR}/office/fullchain.pem" ] + [ "$(files_changes_of core)" = 1 ] + # The same names: kept. + : >"${STUB_LOG}" + export STUB_SAN='DNS:office.example.com, DNS:docs.example.com' + web_cert_placeholder office office.example.com docs.example.com + run ! grep -q '^openssl req' "${STUB_LOG}" + [ "$(files_changes_of core)" = 1 ] + # Other names: a new one. + web_cert_placeholder office office.example.com + grep -q '^openssl req .*subjectAltName=DNS:office.example.com ' "${STUB_LOG}" + [ "$(files_changes_of core)" = 2 ] + # A certificate of the tls module stays. + : >"${STUB_LOG}" + mkdir -p "${WEB_CERT_LIVE}/office" + printf 'issued\n' >"${WEB_CERT_LIVE}/office/fullchain.pem" + web_cert_placeholder office other.example.com + run ! grep -q '^openssl req' "${STUB_LOG}" + # openssl failing stops with an error. + export STUB_EXIT_openssl_req=1 + run web_cert_placeholder shop shop.example.com + [ "${status}" -eq 1 ] + [[ ${output} == *"could not create a self-signed certificate for shop.example.com"* ]] +} + @test "the default certificate is self-signed for the host name with a key for the web group" { as_test_user fake_openssl From 084ae8eeaa83ef454375d7a9f4a64305785dcc98 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 21:33:17 +0200 Subject: [PATCH 05/28] refactor(apps): SonarCloud findings in the installers and the certificate helper The Matomo and WordPress installers loaded the administrator password into a variable they never used. curl reads the password from the secret file, so the installers now only make sure the secret exists. The helper for self-signed certificates names its subject in a local variable instead of using $1 directly. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/web.sh | 6 +++--- modules/apps/matomo.sh | 4 ++-- modules/apps/wordpress.sh | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/web.sh b/lib/web.sh index b1bd569..c4086dd 100644 --- a/lib/web.sh +++ b/lib/web.sh @@ -169,17 +169,17 @@ web_default_cert() { # _web_self_signed DIR DOMAIN...: a self-signed certificate for the names DOMAIN (the first one is the subject) in # DIR/fullchain.pem and DIR/privkey.pem, valid for 825 days; the key is readable by WEB_GROUP. _web_self_signed() { - local dir=$1 tmp san domain + local dir=$1 subject=$2 tmp san domain shift san='' for domain in "$@"; do san+="${san:+,}DNS:${domain}" done tmp=$(mktemp -d) - if ! openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 825 -subj "/CN=$1" \ + if ! openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 825 -subj "/CN=${subject}" \ -addext "subjectAltName=${san}" -keyout "${tmp}/privkey.pem" -out "${tmp}/fullchain.pem" >/dev/null 2>&1; then rm -rf -- "${tmp}" - die "web: could not create a self-signed certificate for $1." + die "web: could not create a self-signed certificate for ${subject}." fi # Certificates are state, not configuration: the tls module's deploy hook replaces them, so they are not # recorded as managed files (lss verify would report every renewal as a change made outside lss). diff --git a/modules/apps/matomo.sh b/modules/apps/matomo.sh index c23ec82..d62b44e 100644 --- a/modules/apps/matomo.sh +++ b/modules/apps/matomo.sh @@ -117,14 +117,14 @@ _mt_post() { # curl as files, so they never show up in the process list; "**********" makes the installer use the database # password from MATOMO_DATABASE_PASSWORD_FILE. _mt_install() { - local jar email site domain password + local jar email site domain _mt_installed && return 0 domain=$(app_domain) email=$(opt admin_email) [[ -n ${email} ]] || email=$(opt admin_user)@${domain} site=$(opt site_url) [[ -n ${site} ]] || site=https://${domain} - secret_load matomo-admin-password password 32 + secret_get matomo-admin-password 32 >/dev/null || die "matomo: the password of the super user cannot be created." log_info "matomo: installing Matomo (super user $(opt admin_user), password in $(secret_path matomo-admin-password))" jar=$(mktemp) if ! { diff --git a/modules/apps/wordpress.sh b/modules/apps/wordpress.sh index b39d003..5957f15 100644 --- a/modules/apps/wordpress.sh +++ b/modules/apps/wordpress.sh @@ -94,11 +94,11 @@ _wp_installed() { # _wp_install: step 2 of the installer; the password goes to curl as a file. _wp_install() { - local email page password + local email page _wp_installed && return 0 email=$(opt admin_email) [[ -n ${email} ]] || email=$(opt admin_user)@$(app_domain) - secret_load wordpress-admin-password password 32 + secret_get wordpress-admin-password 32 >/dev/null || die "wordpress: the password of the administrator cannot be created." log_info "wordpress: installing WordPress (administrator $(opt admin_user), password in $(secret_path wordpress-admin-password))" page=$(app_get 'wp-admin/install.php?step=2' -X POST --data-urlencode "weblog_title=$(opt title)" \ --data-urlencode "user_name=$(opt admin_user)" --data-urlencode "admin_password@$(secret_path wordpress-admin-password)" \ From 616c6ea74fde8629bc09f17391c796275cffc42c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 21:41:01 +0200 Subject: [PATCH 06/28] test: expectations for route placeholders with host names web_proxy_route now passes the host names of a route to its placeholder certificate, and the self-signed helper names the subject in its error message. Two tests still expected the old call and the old message. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- tests/unit/apps.bats | 2 +- tests/unit/web.bats | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats index 84da4a8..547166d 100644 --- a/tests/unit/apps.bats +++ b/tests/unit/apps.bats @@ -216,7 +216,7 @@ docker() { printf 'issue %s\n' "$*" >>"${STUB_LOG}" } app_site 8081 - grep -qxF 'placeholder demo' "${STUB_LOG}" + grep -qxF 'placeholder demo cloud.example.com www.example.com' "${STUB_LOG}" grep -qxF 'site demo proxy http://127.0.0.1:8081 cloud.example.com www.example.com' "${STUB_LOG}" grep -qxF reload "${STUB_LOG}" run ! grep -q '^issue' "${STUB_LOG}" diff --git a/tests/unit/web.bats b/tests/unit/web.bats index 473f326..25b4704 100644 --- a/tests/unit/web.bats +++ b/tests/unit/web.bats @@ -313,7 +313,7 @@ EOF export STUB_EXIT_openssl_req=1 run web_default_cert [ "${status}" -eq 1 ] - [[ ${output} == *'web: could not create the self-signed default certificate.'* ]] + [[ ${output} == *'web: could not create a self-signed certificate for web.example.com.'* ]] } @test "the names of a certificate are read from its subjectAltName" { From 65b906b6024dd5e311af8e46b6de7c653563e41c Mon Sep 17 00:00:00 2001 From: VectoDE Date: Thu, 15 Oct 2026 23:33:17 +0200 Subject: [PATCH 07/28] feat(apps): back up app data, check Nextcloud's certificate in Collabora, cover arm64 The restic backup left the app stacks out: their data sits in Docker volumes, and a live copy of a database's files may not load again. Every app now registers what lss-app-backup saves of it before each restic run: - PostgreSQL and MariaDB are dumped inside their container; - the SQLite apps (Forgejo, Vaultwarden, Uptime Kuma) are copied while the app is paused for a moment; - file volumes and native data directories go to restic as they are. lss verify runs this step for each app, so a dump that breaks shows up before the night's backup. Collabora no longer turns off the certificate check without the tls module. It loads the host's CAs and the trust anchor of the Nextcloud route (the self-signed route certificate, or the private ACME CA) as its storage CA file, and verify asks Collabora's own WOPI access check whether it reaches Nextcloud with a valid certificate. When Nextcloud is installed after Collabora, it hands its certificate over and starts Collabora again. The arm64 runners have no KVM, so the VM jobs cannot cover the apps there. A new job runs the three app sets on the arm64 runner itself (Ubuntu 24.04), after removing the runner's own Docker. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- .github/workflows/integration.yml | 44 +++++++ README.md | 11 +- lib/apps.sh | 121 +++++++++++++++-- libexec/lss-app-backup | 212 ++++++++++++++++++++++++++++++ libexec/lss-backup | 17 ++- modules/apps/collabora.sh | 22 ++-- modules/apps/forgejo.sh | 2 + modules/apps/keycloak.sh | 2 + modules/apps/matomo.sh | 2 + modules/apps/minio.sh | 4 + modules/apps/n8n.sh | 2 + modules/apps/nextcloud.sh | 3 + modules/apps/self-updater.sh | 4 + modules/apps/uptime-kuma.sh | 2 + modules/apps/vaultwarden.sh | 2 + modules/apps/wordpress.sh | 2 + modules/backup/restic.sh | 6 +- tests/unit/apps.bats | 110 +++++++++++++++- tests/unit/backup.bats | 152 ++++++++++++++++++++- 19 files changed, 690 insertions(+), 30 deletions(-) create mode 100755 libexec/lss-app-backup diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 96c41da..d6e874f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -245,3 +245,47 @@ jobs: run: | read -ra modules <<<"$MODULES" tests/integration/run.sh "$SYSTEM" --base "$BASE" "${modules[@]}" + + # The app stacks on arm64. The arm64 runners have no KVM, so the stacks run on the runner itself (host), Ubuntu + # 24.04: the multi-architecture images, the database containers and the web servers on aarch64. The runner's own + # Docker goes first, so the docker module sets Docker up as on a fresh server. + apps-arm64: + name: ubuntu-24.04 arm64 / apps ${{ matrix.set }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + set: [office, dev, web] + include: + - set: office + base: docker,nginx + modules: nextcloud collabora,nextcloud vaultwarden + - set: dev + base: docker,apache + modules: forgejo uptime-kuma n8n + - set: web + base: docker,caddy,tls + modules: keycloak matomo wordpress minio + settings: LSS_MINIO_MODE=docker + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Remove the runner's Docker + run: | + sudo systemctl stop docker.socket docker.service containerd.service 2>/dev/null || true + mapfile -t packages < <(dpkg-query -W -f "\${Package}\n" 'moby-*' 'docker-*' containerd 'containerd.io' 2>/dev/null) + [ "${#packages[@]}" -eq 0 ] || sudo NEEDRESTART_SUSPEND=1 apt-get -qq purge -y "${packages[@]}" >/dev/null + sudo rm -rf /var/lib/docker /var/lib/containerd /etc/docker + - name: Install, verify, reinstall, remove + env: + BASE: ${{ matrix.base }} + MODULES: ${{ matrix.modules }} + LSS_TEST_SETTINGS: ${{ matrix.settings }} + run: | + read -ra modules <<<"$MODULES" + tests/integration/run.sh host --base "$BASE" "${modules[@]}" diff --git a/README.md b/README.md index e90eb04..70e8177 100644 --- a/README.md +++ b/README.md @@ -325,8 +325,15 @@ and writes a bucket and an object to MinIO. Nextcloud and Collabora Online link in either order. Data lives in Docker volumes named after the stack (`lss-nextcloud_html`, `lss-nextcloud_db`, ...). `lss remove` -keeps them; `--purge` deletes them with the secrets. Back them up with `docker run` or a restic path of your own; -the `restic` module does not include them yet. +keeps them; `--purge` deletes them with the secrets. Before every restic backup, `/usr/local/lib/lss/app-backup` saves +what each app registered in `/etc/lss/app-backup`. It dumps the PostgreSQL and MariaDB databases (Nextcloud, Keycloak, +n8n, Matomo, WordPress) and copies the SQLite apps (Forgejo, Vaultwarden, Uptime Kuma) while the app is paused for a +moment, both to `/var/backups/lss-apps`. Restic backs up that directory together with the file volumes, which stay where +they are (Nextcloud's files, MinIO's objects). `lss verify` runs this step for each app, so a broken dump shows up before +the night's backup. + +Nextcloud and Collabora Online call each other through the web server, and both check the other's certificate. Without +the `tls` module, every site gets a self-signed certificate for its own names, and each app trusts the other's site. `minio` is the MinIO server code, maintained as [PGSTY Silo](https://github.com/pgsty/silo) since MinIO archived its community edition in 2026. The S3 API answers on `LSS_MINIO_DOMAINS`, the web console on diff --git a/lib/apps.sh b/lib/apps.sh index 763a928..bc8b1e1 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -218,6 +218,9 @@ app_exec() { # Nextcloud Office: the collabora module and the nextcloud module link each other, whichever comes second. APP_NEXTCLOUD_PROJECT=lss-nextcloud +APP_COLLABORA_PROJECT=lss-collabora +# The certificates Collabora checks Nextcloud's against (mounted into the container; see app_collabora_trust). +APP_COLLABORA_TRUST=trust/ca.pem # app_domains_of MODULE: the host names of another app module (its option domains, or MODULE.). app_domains_of() { @@ -244,20 +247,53 @@ _app_nc_occ() { return } -# _app_nc_trust: the certificate Nextcloud must trust to reach the Collabora site: the CA of a private ACME server -# (LSS_TLS_CA_BUNDLE) or, without the tls module, the self-signed certificate of the collabora site; nothing for -# Let's Encrypt. Prints the file, or nothing. -_app_nc_trust() { - local bundle +# _app_route_trust ROUTE: the certificate an app must trust to reach the route ROUTE of this server through the web +# server: the CA of a private ACME server (LSS_TLS_CA_BUNDLE) or, without the tls module, the self-signed certificate +# of the route; nothing for Let's Encrypt. Prints the file, or nothing. +_app_route_trust() { + local route=$1 bundle if state_installed tls; then bundle=$(config_option tls ca_bundle '') [[ -z ${bundle} || ! -s ${bundle} ]] || printf '%s' "${bundle}" - elif [[ -s ${WEB_TLS_DIR}/collabora/fullchain.pem ]]; then - printf '%s/collabora/fullchain.pem' "${WEB_TLS_DIR}" + elif [[ -s ${WEB_TLS_DIR}/${route}/fullchain.pem ]]; then + printf '%s/%s/fullchain.pem' "${WEB_TLS_DIR}" "${route}" fi return } +# app_collabora_trust: the certificates Collabora checks the one of Nextcloud against when it fetches and saves +# documents (WOPI): the CAs of the host and the trust anchor of the Nextcloud route (_app_route_trust), in +# APP_COLLABORA_TRUST of the collabora stack. LSS_FILE_CHANGED tells whether it changed. +app_collabora_trust() { + local dir="${CTR_CONFIG_DIR}/${APP_COLLABORA_PROJECT}" bundle trust content='' + for bundle in /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt; do + if [[ -s ${bundle} ]]; then + content=$(<"${bundle}") + break + fi + done + trust=$(_app_route_trust nextcloud) + [[ -z ${trust} ]] || content+=${content:+$'\n'}$(<"${trust}") + # Collabora refuses an empty file; the default certificate of the web server always exists. + [[ -n ${content} ]] || content=$(<"${WEB_TLS_DIR}/default/fullchain.pem") + install -d -m 0700 "${dir}" "$(dirname "${dir}/${APP_COLLABORA_TRUST}")" + printf '%s\n' "${content}" | file_write 0444 root:root "${dir}/${APP_COLLABORA_TRUST}" + return +} + +# app_collabora_check: Collabora reaches Nextcloud through the web server and accepts its certificate (the WOPI +# access check of Collabora, the same connection it opens for every document). +app_collabora_check() { + local nextcloud collabora page + read -r nextcloud _ <<<"$(app_domains_of nextcloud)" + read -r collabora _ <<<"$(app_domains_of collabora)" + page=$(APP_GET_DOMAIN=${collabora} app_get hosting/wopiAccessCheck -X POST -H 'Content-Type: application/json' \ + --data "{\"callbackUrl\":\"https://${nextcloud}/status.php\"}") || true + [[ $(app_status "${page}") == 200 && ${page%$'\n'*} =~ \"status\":\ *\"Ok\" ]] || + die "${LSS_MODULE}: Collabora Online cannot reach Nextcloud at https://${nextcloud} with a valid certificate (${page%$'\n'*})." + return +} + # app_nextcloud_office: with both modules installed (or the other one being installed), Nextcloud Office # (richdocuments) points at the Collabora site and checks that it reaches it. app_nextcloud_office() { @@ -277,7 +313,7 @@ app_nextcloud_office() { die "${LSS_MODULE}: occ cannot set the address of Collabora Online." files_note_change "linked Nextcloud Office to ${url}" fi - trust=$(_app_nc_trust) + trust=$(_app_route_trust collabora) if [[ -n ${trust} ]]; then sum=$(sha256sum "${trust}" | cut -d ' ' -f 1) if [[ $(state_get nextcloud trusted_cert) != "${sum}" ]]; then @@ -291,6 +327,16 @@ app_nextcloud_office() { fi log_command _app_nc_occ richdocuments:activate-config || die "${LSS_MODULE}: Nextcloud Office cannot reach Collabora Online at ${url}; see the log." + # Collabora installed first knew no certificate of Nextcloud yet: it gets it now and starts again. + if [[ ${LSS_MODULE} == nextcloud ]]; then + app_collabora_trust + if [[ ${LSS_FILE_CHANGED} == 1 ]]; then + log_info "Starting ${APP_COLLABORA_PROJECT} again, so it trusts the certificate of Nextcloud" + ctr_compose_up "${APP_COLLABORA_PROJECT}" "${CTR_CONFIG_DIR}/${APP_COLLABORA_PROJECT}/compose.yaml" --force-recreate + APP_GET_DOMAIN=${first} app_wait '' 200 >/dev/null || die "${LSS_MODULE}: Collabora Online does not answer at ${url}." + fi + fi + app_collabora_check return } @@ -397,6 +443,64 @@ app_verify_stack() { return } +# The data of the apps for the restic backup: each app registers what lss-app-backup (libexec) saves of it before every +# backup, whether or not the restic module is installed yet. +APP_BACKUP_SCRIPT=/usr/local/lib/lss/app-backup +APP_BACKUP_REGISTRY=/etc/lss/app-backup +APP_BACKUP_DIR=/var/backups/lss-apps + +# app_backup KEY=VALUE...: registers the data of the app with lss-app-backup: DB=postgres|mariadb (the service "db", +# dumped) with DB_USER, COPY=VOLUMES (copied while the app is paused: SQLite), LIVE=VOLUMES (backed up in place), +# PATHS=DIRECTORIES (native installations); volumes and directories separated by spaces. +app_backup() { + local pair key + local -A values=([DB]=none [DB_USER]='' [COPY]='' [LIVE]='' [PATHS]='') + for pair in "$@"; do + key=${pair%%=*} + [[ ${pair} == *=* && ${key} =~ ^[A-Z_]+$ && -v values[${key}] && ${pair#*=} =~ ^[A-Za-z0-9_./\ -]*$ ]] || + die "${LSS_MODULE}: app_backup does not know '${pair}'." + values[${key}]=${pair#*=} + done + file_write 0755 root:root "${APP_BACKUP_SCRIPT}" <"${LSS_ROOT}/libexec/lss-app-backup" + install -d -m 0700 "${APP_BACKUP_REGISTRY}" + file_write 0600 root:root "${APP_BACKUP_REGISTRY}/${LSS_MODULE}.conf" </dev/null 2>&1 && docker info >/dev/null 2>&1; then local -a options=(--remove-orphans) [[ ${LSS_PURGE:-0} != 1 ]] || options+=(--volumes) diff --git a/libexec/lss-app-backup b/libexec/lss-app-backup new file mode 100755 index 0000000..13e4012 --- /dev/null +++ b/libexec/lss-app-backup @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lss app-backup: the data of the app stacks (Docker Compose, lib/apps.sh) in a form restic can back up consistently. +# Installed by the app modules as /usr/local/lib/lss/app-backup; lss-backup runs it before every backup, and lss +# verify of an app runs it for that app (app-backup NAME...). +# +# Every app registers itself in /etc/lss/app-backup/NAME.conf (KEY=value lines, parsed, never sourced): +# PROJECT the Compose project (lss-NAME) COMPOSE its compose.yaml +# DB postgres, mariadb or none: the service "db" is dumped (pg_dumpall, mariadb-dump of every database) +# DB_USER the database user of the PostgreSQL service (its superuser in the official image) +# COPY volumes copied while the service "app" is paused for a moment (SQLite databases next to their files) +# LIVE volumes backed up where they are (files only; the database of the stack is in the dump) +# PATHS host directories of the app (native installations) +# The dumps and copies go to /var/backups/lss-apps/NAME (root only), replaced on every run; restic keeps the history. +# The paths restic backs up (that directory, the LIVE volumes, PATHS) are listed in /var/lib/lss/app-backup/paths, +# and the result of each app in /var/lib/lss/app-backup/status. +set -Eeuo pipefail +shopt -s inherit_errexit + +REGISTRY=${LSS_APP_BACKUP_REGISTRY:-/etc/lss/app-backup} +OUT_ROOT=${LSS_APP_BACKUP_DIR:-/var/backups/lss-apps} +STATE=${LSS_APP_BACKUP_STATE:-/var/lib/lss/app-backup} +DOCKER=${LSS_DOCKER:-docker} + +say() { + printf 'lss-app-backup: %s\n' "$*" + return +} + +# setting FILE KEY [DEFAULT]: the value of KEY in the registration FILE. +setting() { + local file=$1 key=$2 default=${3-} line + while IFS= read -r line || [[ -n ${line} ]]; do + if [[ ${line} == "${key}="* ]]; then + printf '%s' "${line#*=}" + return 0 + fi + done <"${file}" + printf '%s' "${default}" + return +} + +# compose FILE ARGS...: docker compose for the stack of the registration FILE. +compose() { + local file=$1 + shift + "${DOCKER}" compose --project-name "$(setting "${file}" PROJECT)" -f "$(setting "${file}" COMPOSE)" "$@" + return +} + +# mountpoint FILE VOLUME: the directory of the named volume VOLUME of the stack on the host. +mountpoint() { + local file=$1 volume=$2 path + path=$("${DOCKER}" volume inspect --format '{{.Mountpoint}}' "$(setting "${file}" PROJECT)_${volume}") || return 1 + [[ ${path} == /* && -d ${path} ]] || return 1 + printf '%s' "${path}" + return +} + +# dump_db FILE OUT: the database of the stack as SQL. The MariaDB root password goes to the dump tool through its +# environment inside the container, never on a command line. +dump_db() { + local file=$1 out=$2 + case $(setting "${file}" DB none) in + postgres) + compose "${file}" exec -T db pg_dumpall --username="$(setting "${file}" DB_USER)" --clean --if-exists >"${out}" + ;; + mariadb) + # shellcheck disable=SC2016 # expanded inside the container + compose "${file}" exec -T db sh -c 'MYSQL_PWD=$(cat /run/secrets/db-root-password) exec mariadb-dump --user=root \ + --all-databases --single-transaction --routines --events --triggers --hex-blob' >"${out}" + ;; + *) return 1 ;; + esac + return +} + +# copy_volumes FILE DIR: the COPY volumes into DIR/VOLUME, with the stack paused so that files and SQLite databases +# match; the stack runs on even when a copy fails. +copy_volumes() { + local file=$1 dir=$2 volume source attempt status=0 + local -a volumes=() + read -ra volumes <<<"$(setting "${file}" COPY)" + ((${#volumes[@]} > 0)) || return 0 + # Only the app pauses; its database (if any) is dumped, and the one-shot "init" service has exited. + compose "${file}" pause app >/dev/null 2>&1 || return 1 + for volume in "${volumes[@]}"; do + if source=$(mountpoint "${file}" "${volume}"); then + rm -rf -- "${dir}/${volume}.part" + if cp -a -- "${source}" "${dir}/${volume}.part"; then + rm -rf -- "${dir:?}/${volume}" + mv -f -- "${dir}/${volume}.part" "${dir}/${volume}" + else + rm -rf -- "${dir}/${volume}.part" + status=1 + fi + else + status=1 + fi + done + for attempt in 1 2 3; do + compose "${file}" unpause app >/dev/null 2>&1 && break + ((attempt < 3)) || { + say "the app of $(setting "${file}" PROJECT) stays paused; run docker compose unpause app." >&2 + status=1 + } + sleep 2 + done + return "${status}" +} + +# backup_app FILE: the dump, the copies and the paths of one app; prints the paths for restic. +backup_app() { + local file=$1 name dir out volume path failed=0 + local -a volumes=() paths=() + name=$(basename "${file}" .conf) + dir=${OUT_ROOT}/${name} + mkdir -p "${dir}" + chmod 0700 "${dir}" + if [[ $(setting "${file}" DB none) != none ]]; then + out=${dir}/db.sql + # The dump goes to a temporary file first, so a failed run never replaces the last good dump. + if dump_db "${file}" "${out}.part" && [[ -s ${out}.part ]]; then + mv -f -- "${out}.part" "${out}" + else + rm -f -- "${out}.part" + say "the database dump of ${name} failed; ${out} keeps the previous one." >&2 + failed=1 + fi + fi + if ! copy_volumes "${file}" "${dir}"; then + say "the copy of the data of ${name} failed; ${dir} keeps the previous one." >&2 + failed=1 + fi + read -ra volumes <<<"$(setting "${file}" LIVE)" + for volume in "${volumes[@]}"; do + if path=$(mountpoint "${file}" "${volume}"); then + printf '%s\n' "${path}" + else + say "the volume ${volume} of ${name} is missing." >&2 + failed=1 + fi + done + read -ra paths <<<"$(setting "${file}" PATHS)" + for path in "${paths[@]}"; do + [[ ! -e ${path} ]] || printf '%s\n' "${path}" + done + return "${failed}" +} + +# run_backups [NAME...]: every registered app (or the ones named); fails when one of them failed. +run_backups() { + local file name failed=0 count=0 + local -a files=() + if (($# > 0)); then + for name in "$@"; do + [[ ${name} =~ ^[a-z0-9][a-z0-9-]*$ && -f ${REGISTRY}/${name}.conf ]] || { + say "${name} is not registered in ${REGISTRY}." >&2 + return 1 + } + files+=("${REGISTRY}/${name}.conf") + done + else + for file in "${REGISTRY}"/*.conf; do + [[ -f ${file} ]] && files+=("${file}") + done + fi + mkdir -p "${OUT_ROOT}" "${STATE}" + chmod 0700 "${OUT_ROOT}" "${STATE}" + : >"${STATE}/paths.tmp" + : >"${STATE}/status.tmp" + for file in "${files[@]}"; do + name=$(basename "${file}" .conf) + count=$((count + 1)) + if backup_app "${file}" >>"${STATE}/paths.tmp"; then + printf '%s ok %s\n' "${name}" "$(date +%s)" >>"${STATE}/status.tmp" + say "saved the data of ${name}." + else + printf '%s failed %s\n' "${name}" "$(date +%s)" >>"${STATE}/status.tmp" + failed=1 + fi + done + # A run for some apps only (lss verify) leaves the list of the full run alone. + if (($# == 0)); then + if ((count > 0)); then + printf '%s\n' "${OUT_ROOT}" >>"${STATE}/paths.tmp" + fi + mv -f -- "${STATE}/paths.tmp" "${STATE}/paths" + mv -f -- "${STATE}/status.tmp" "${STATE}/status" + ((count > 0)) || say "0 apps are registered here; nothing to save." + else + rm -f -- "${STATE}/paths.tmp" "${STATE}/status.tmp" + fi + return "${failed}" +} + +main() { + [[ ${EUID} -eq 0 ]] || { + say "must run as root." >&2 + exit 1 + } + umask 077 + run_backups "$@" +} + +# The unit tests source this file without running it. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + main "$@" +fi diff --git a/libexec/lss-backup b/libexec/lss-backup index 42aa9db..f4403b9 100755 --- a/libexec/lss-backup +++ b/libexec/lss-backup @@ -9,7 +9,8 @@ # of the restic process alone, never through its command line; the database dumps and every other child never see # them. # -# backup run database dumps (db-dumps module), backup, forget and prune by the retention, check +# backup run database dumps (db-dumps module) and app data (app modules), backup, forget and prune by +# the retention, check # backup init creates the repository unless it exists # backup restore-test restores the canary file of the latest backup and compares it, then reads a part of the data # backup probe a small backup of the canary file and its restore test (lss verify, before the first run) @@ -26,6 +27,8 @@ STATE=${LSS_BACKUP_STATE:-/var/lib/lss/restic} TEXTFILE_DIR=${LSS_TEXTFILE_DIR:-/var/lib/prometheus/node-exporter} DB_DUMPS=${LSS_DB_DUMPS:-/usr/local/lib/lss/db-dumps} DB_DUMPS_CONF=${LSS_DB_DUMPS_CONF:-/etc/lss/db-dumps.conf} +APP_BACKUP=${LSS_APP_BACKUP:-/usr/local/lib/lss/app-backup} +APP_BACKUP_PATHS=${LSS_APP_BACKUP_STATE:-/var/lib/lss/app-backup}/paths CACHE=${LSS_BACKUP_CACHE:-/var/cache/restic} RESTIC=${LSS_RESTIC:-restic} CANARY_DIR=${STATE}/canary @@ -169,11 +172,15 @@ metrics() { return } -# backup_paths: the configured paths that exist (restic counts a missing one as an error). +# backup_paths: the configured paths and those of the app stacks (lss-app-backup) that exist (restic counts a missing +# one as an error). backup_paths() { local path local -a paths=() read -ra paths <<<"$(setting BACKUP_PATHS /etc)" + if [[ -f ${APP_BACKUP_PATHS} ]]; then + mapfile -t -O "${#paths[@]}" paths <"${APP_BACKUP_PATHS}" + fi for path in "${paths[@]}"; do if [[ -e ${path} ]]; then printf '%s\n' "${path}" @@ -216,6 +223,12 @@ cmd_run() { say "the database dumps failed; the backup goes on with the files." } fi + if [[ -x ${APP_BACKUP} ]]; then + "${APP_BACKUP}" || { + status=1 + say "saving the data of the apps failed; the backup goes on with the rest." + } + fi prepare cmd_init mapfile -t paths < <(backup_paths) diff --git a/modules/apps/collabora.sh b/modules/apps/collabora.sh index 2304bb9..204c5db 100644 --- a/modules/apps/collabora.sh +++ b/modules/apps/collabora.sh @@ -43,26 +43,17 @@ _cool_wopi_hosts() { return } -# _cool_verify_tls: Collabora checks the certificate of Nextcloud only when it comes from a public CA (the tls module -# without a private ACME server); the image has no way to add another CA, and the connection stays on this server. -_cool_verify_tls() { - if state_installed tls && [[ -z $(config_option tls ca_bundle '') ]]; then - printf 'true' - else - printf 'false' - fi - return -} - # _cool_compose: the stack. Collabora runs without capabilities and without new privileges, so it cannot build its # chroot jails; COOL_FORCE_LANDLOCK makes it isolate the document processes with Landlock instead. The root file # system stays writable because the jails are created below /opt/cool. The admin console is off: it would need a -# password in the environment. +# password in the environment. Collabora always checks the certificate of Nextcloud (storage.ssl.as_scheme: an https +# WOPI address means a checked TLS connection), against the CAs of the image and app_collabora_trust. _cool_compose() { local domain params domain=$(app_domain) params="--o:ssl.enable=false --o:ssl.termination=true --o:admin_console.enable=false" - params+=" --o:ssl.ssl_verification=$(_cool_verify_tls) --o:logging.level=warning" + params+=" --o:ssl.ssl_verification=true --o:storage.ssl.as_scheme=true" + params+=" --o:storage.ssl.ca_file_path=/etc/coolwsd/lss-ca.pem --o:logging.level=warning" cat </dev/null || die "forgejo: https://$(app_domain)/api/healthz does not answer." @@ -143,6 +144,7 @@ mod_verify() { local page repo user user=$(opt admin_user) app_verify_stack + app_verify_backup page=$(app_wait api/healthz 200) || die "forgejo: https://$(app_domain)/api/healthz does not answer." [[ $(jq -r '.status' <<<"${page%$'\n'*}" 2>/dev/null) =~ ^(pass|warn)$ ]] || die "forgejo: the health check fails." page=$(app_wait user/login 200 60) || die "forgejo: the login page does not answer." diff --git a/modules/apps/keycloak.sh b/modules/apps/keycloak.sh index a30af69..a26fcdd 100644 --- a/modules/apps/keycloak.sh +++ b/modules/apps/keycloak.sh @@ -90,6 +90,7 @@ mod_configure() { app_secret keycloak-db-password "${APP_POSTGRES_USER}" 32 db-password.db _kc_config _kc_compose | app_compose + app_backup DB=postgres DB_USER=keycloak app_up app_site "$(opt_int port)" app_wait realms/master/.well-known/openid-configuration 200 >/dev/null || @@ -111,6 +112,7 @@ _kc_token() { mod_verify() { local page token headers user id status app_verify_stack + app_verify_backup page=$(app_wait realms/master/.well-known/openid-configuration 200) || die "keycloak: the master realm does not answer." [[ $(jq -r '.issuer' <<<"${page%$'\n'*}" 2>/dev/null) == "https://$(app_domain)/realms/master" ]] || die "keycloak: the issuer is not https://$(app_domain)/realms/master (KC_HOSTNAME or the proxy headers are wrong)." diff --git a/modules/apps/matomo.sh b/modules/apps/matomo.sh index d62b44e..bd5e223 100644 --- a/modules/apps/matomo.sh +++ b/modules/apps/matomo.sh @@ -178,6 +178,7 @@ _mt_token() { mod_configure() { app_db_secrets mariadb "${MT_USER}" _mt_compose | app_compose + app_backup DB=mariadb LIVE=html app_up app_site "$(opt_int port)" app_wait matomo.js 200 >/dev/null || die "matomo: https://$(app_domain)/matomo.js does not answer." @@ -202,6 +203,7 @@ _mt_api() { mod_verify() { local page action app_verify_stack + app_verify_backup page=$(app_wait 'index.php?module=Login' 200) || die "matomo: the login page does not answer." grep -q 'form_password\|name="form_password"' <<<"${page}" || die "matomo: https://$(app_domain)/index.php?module=Login shows no login." # Used, not only running: a page view goes in through the tracker and comes out of the reporting API. diff --git a/modules/apps/minio.sh b/modules/apps/minio.sh index c4368f2..e6f77c4 100644 --- a/modules/apps/minio.sh +++ b/modules/apps/minio.sh @@ -173,9 +173,11 @@ mod_configure() { printf '%s' "$(opt root_user)" | app_file config/root-user "${SILO_USER}" 0444 app_secret minio-root-password "${SILO_USER}" 40 root-password _minio_compose | app_compose + app_backup LIVE=data app_up else _minio_native + app_backup "PATHS=${SILO_DATA}" fi app_site "$(opt_int api_port)" app_site "$(opt_int console_port)" minio-console "${console[@]}" @@ -205,6 +207,7 @@ _minio_s3() { mod_verify() { local page bucket probe user status user=$(opt root_user) + app_verify_backup if _minio_docker; then app_verify_stack else @@ -244,6 +247,7 @@ mod_remove() { return 0 fi app_site_remove minio + app_backup_remove svc_disable silo.service svc_dropin_remove silo.service lss-sandbox file_remove "${SILO_ETC}" diff --git a/modules/apps/n8n.sh b/modules/apps/n8n.sh index 8affceb..2fe648d 100644 --- a/modules/apps/n8n.sh +++ b/modules/apps/n8n.sh @@ -138,6 +138,7 @@ mod_configure() { app_db_secrets postgres "${N8N_USER}" app_secret n8n-encryption-key "${N8N_USER}" 48 encryption-key _n8n_compose | app_compose + app_backup DB=postgres DB_USER=n8n LIVE=data app_up app_site "$(opt_int port)" app_wait healthz/readiness 200 >/dev/null || die "n8n: https://$(app_domain)/healthz/readiness does not answer." @@ -156,6 +157,7 @@ _n8n_rest() { mod_verify() { local page jar body id name app_verify_stack + app_verify_backup page=$(app_wait healthz/readiness 200) || die "n8n: https://$(app_domain)/healthz/readiness does not answer." page=$(app_wait signin 200 60) || die "n8n: the sign-in page does not answer." grep -qi 'n8n' <<<"${page}" || die "n8n: https://$(app_domain)/signin is not n8n." diff --git a/modules/apps/nextcloud.sh b/modules/apps/nextcloud.sh index 63f7c23..f8fb0ae 100644 --- a/modules/apps/nextcloud.sh +++ b/modules/apps/nextcloud.sh @@ -154,6 +154,7 @@ mod_configure() { app_db_secrets postgres "${NC_USER}" app_secret nextcloud-admin-password "${NC_USER}" 32 admin-password _nc_compose | app_compose + app_backup DB=postgres DB_USER=nextcloud LIVE=html app_up _nc_install _nc_settings @@ -177,6 +178,7 @@ _nc_dav() { mod_verify() { local page status probe app_verify_stack + app_verify_backup page=$(app_wait status.php 200) || die "nextcloud: https://$(app_domain)/status.php does not answer." [[ $(jq -r '.installed' <<<"${page%$'\n'*}" 2>/dev/null) == true ]] || die "nextcloud: Nextcloud is not installed." [[ $(jq -r '.maintenance' <<<"${page%$'\n'*}" 2>/dev/null) == false ]] || die "nextcloud: Nextcloud is in maintenance mode." @@ -196,6 +198,7 @@ mod_verify() { timeout 600 systemctl start lss-nextcloud-cron.service || die "nextcloud: the background jobs fail (journalctl -u lss-nextcloud-cron)." if state_installed collabora; then log_command _nc_occ richdocuments:activate-config || die "nextcloud: Nextcloud Office cannot reach Collabora Online." + app_collabora_check fi return } diff --git a/modules/apps/self-updater.sh b/modules/apps/self-updater.sh index cc0d4a8..d55fad5 100644 --- a/modules/apps/self-updater.sh +++ b/modules/apps/self-updater.sh @@ -194,6 +194,8 @@ mod_configure() { sandbox+=(NoNewPrivileges=no RestrictSUIDSGID=no) fi svc_sandbox "${SU_UNIT}.service" "${sandbox[@]}" + # The backup takes the deployed checkout and the state of the updater (its home) along. + app_backup "PATHS=${path} ${SU_HOME}" svc_enable "${SU_UNIT}" if [[ $(files_changes_of self-updater) != "${before}" ]]; then svc_restart "${SU_UNIT}" || die "self-updater: ${SU_UNIT} does not start; see journalctl -u ${SU_UNIT}." @@ -211,6 +213,7 @@ mod_verify() { local status='' i commit [[ $(_su_installed) == "${SU_VERSION}" ]] || die "self-updater: Self-Updater ${SU_VERSION} is not installed." svc_is_active "${SU_UNIT}" || die "self-updater: ${SU_UNIT} is not running; see journalctl -u ${SU_UNIT}." + app_verify_backup [[ $(stat -c '%a %U:%G' "${SU_ENV}") == "640 root:${SU_USER}" ]] || die "self-updater: ${SU_ENV} must be mode 640, root:${SU_USER}." # The first deployment runs right after the start (--immediate): a clone of the repository into the workspace. for ((i = 0; i < 100; i++)); do @@ -235,6 +238,7 @@ mod_remove() { file_remove "${SU_SUDOERS}" file_remove "${SU_CONFIG}" file_remove "${SU_ENV}" + app_backup_remove if [[ -n $(_su_installed) ]]; then log_command npm uninstall --global @vectode/self-updater || die "self-updater: npm cannot remove Self-Updater." files_note_change "removed Self-Updater" diff --git a/modules/apps/uptime-kuma.sh b/modules/apps/uptime-kuma.sh index fc70541..a156dfa 100644 --- a/modules/apps/uptime-kuma.sh +++ b/modules/apps/uptime-kuma.sh @@ -130,6 +130,7 @@ mod_configure() { app_secret uptime-kuma-admin-password "${KUMA_USER}" 32 admin-password _kuma_script | app_file config/lss-kuma.js "${KUMA_USER}" 0444 _kuma_compose | app_compose + app_backup COPY=data app_up app_site "$(opt_int port)" app_wait '' 302 >/dev/null || die "uptime-kuma: https://$(app_domain)/ does not answer." @@ -144,6 +145,7 @@ mod_configure() { mod_verify() { local page result app_verify_stack + app_verify_backup page=$(app_wait '' 302) || die "uptime-kuma: https://$(app_domain)/ does not answer." page=$(app_wait dashboard 200 60) || die "uptime-kuma: the dashboard does not answer." grep -qi 'uptime kuma' <<<"${page}" || die "uptime-kuma: https://$(app_domain)/dashboard is not Uptime Kuma." diff --git a/modules/apps/vaultwarden.sh b/modules/apps/vaultwarden.sh index 0887ab2..455adc5 100644 --- a/modules/apps/vaultwarden.sh +++ b/modules/apps/vaultwarden.sh @@ -90,6 +90,7 @@ _vw_admin_token() { mod_configure() { _vw_admin_token _vw_compose | app_compose + app_backup COPY=data app_up app_site "$(opt_int port)" app_wait alive 200 >/dev/null || die "vaultwarden: https://$(app_domain)/alive does not answer." @@ -108,6 +109,7 @@ _vw_admin() { mod_verify() { local page jar email id status app_verify_stack + app_verify_backup page=$(app_wait alive 200) || die "vaultwarden: https://$(app_domain)/alive does not answer." page=$(app_wait '' 200 60) || die "vaultwarden: the web vault does not answer." grep -qi 'bitwarden\|vaultwarden' <<<"${page}" || die "vaultwarden: https://$(app_domain)/ is not the web vault." diff --git a/modules/apps/wordpress.sh b/modules/apps/wordpress.sh index 5957f15..33cc4b9 100644 --- a/modules/apps/wordpress.sh +++ b/modules/apps/wordpress.sh @@ -113,6 +113,7 @@ _wp_install() { mod_configure() { app_db_secrets mariadb "${WP_USER}" _wp_compose | app_compose + app_backup DB=mariadb LIVE=html app_up app_site "$(opt_int port)" app_wait wp-includes/images/w-logo-blue.png 200 >/dev/null || die "wordpress: https://$(app_domain)/ does not answer." @@ -123,6 +124,7 @@ mod_configure() { mod_verify() { local page jar status app_verify_stack + app_verify_backup page=$(app_wait wp-login.php 200) || die "wordpress: the login page does not answer." grep -q 'name="pwd"' <<<"${page}" || die "wordpress: https://$(app_domain)/wp-login.php shows no login." page=$(app_get wp-json/) || true diff --git a/modules/backup/restic.sh b/modules/backup/restic.sh index 185dff5..869511c 100644 --- a/modules/backup/restic.sh +++ b/modules/backup/restic.sh @@ -121,12 +121,14 @@ mod_configure() { fi file_write 0755 root:root "${BACKUP_SCRIPT}" <"${LSS_ROOT}/libexec/lss-backup" _restic_conf | file_write 0600 root:root "${BACKUP_CONF}" - # Reads everything, writes only its state, its cache, the dumps, the metrics and a local repository. + # Reads everything, writes only its state, its cache, the dumps (of the databases and of the app stacks, which it + # reaches through the Docker socket), the metrics and a local repository. svc_unit "${BACKUP_UNIT}.service" < <( printf '# Managed by lss (module restic).\n[Unit]\nDescription=lss backup with restic\n' printf 'After=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\n' printf 'ExecStart=%s run\nNice=10\nIOSchedulingClass=idle\n' "${BACKUP_SCRIPT}" - svc_scanner_sandbox strict "${BACKUP_STATE}" "${BACKUP_CACHE}" /var/backups/lss-db "${MON_TEXTFILE_DIR}" \ + svc_scanner_sandbox strict "${BACKUP_STATE}" "${BACKUP_CACHE}" /var/backups/lss-db /var/backups/lss-apps \ + /var/lib/lss/app-backup "${MON_TEXTFILE_DIR}" \ ${writable:+"${writable}"} ) svc_unit "${BACKUP_UNIT}.timer" <"${APP_DOCKER_CONFIG}" ENGINE_LOG=${BATS_TEST_TMPDIR}/engine.log : >"${ENGINE_LOG}" + APP_BACKUP_SCRIPT=${BATS_TEST_TMPDIR}/app-backup + APP_BACKUP_REGISTRY=${BATS_TEST_TMPDIR}/app-backup.d + APP_BACKUP_DIR=${BATS_TEST_TMPDIR}/lss-apps WRITES=${BATS_TEST_TMPDIR}/writes : >"${WRITES}" # file_write as the test user; the owner it was asked for goes to $WRITES ("== MODE OWNER DEST"). @@ -340,6 +343,11 @@ docker() { @test "Nextcloud Office is linked once both apps are there, trusting the certificate of the collabora site" { config_set LSS_COLLABORA_DOMAINS office.example.com + # shellcheck disable=SC2329 # called by app_collabora_check + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf '{"status": "Ok"}\n200' + } LSS_MODULE=collabora app_nextcloud_office [ ! -s "${ENGINE_LOG}" ] @@ -354,16 +362,17 @@ docker() { grep -qF "exec -T app sh -c cat >/tmp/lss-web.pem" "${ENGINE_LOG}" grep -qxF "${occ} security:certificates:import /tmp/lss-web.pem" "${ENGINE_LOG}" grep -qxF "${occ} richdocuments:activate-config" "${ENGINE_LOG}" + grep -q '^curl .*--resolve office.example.com:443:127.0.0.1 -X POST .*https://office.example.com/hosting/wopiAccessCheck$' "${STUB_LOG}" [ -n "$(state_get nextcloud trusted_cert)" ] : >"${ENGINE_LOG}" app_nextcloud_office run ! grep -q 'certificates:import' "${ENGINE_LOG}" # With the tls module, only a private CA needs trusting. state_set tls status installed - [ -z "$(_app_nc_trust)" ] + [ -z "$(_app_route_trust collabora)" ] printf 'CA\n' >"${BATS_TEST_TMPDIR}/ca.pem" config_set LSS_TLS_CA_BUNDLE "${BATS_TEST_TMPDIR}/ca.pem" - [ "$(_app_nc_trust)" = "${BATS_TEST_TMPDIR}/ca.pem" ] + [ "$(_app_route_trust collabora)" = "${BATS_TEST_TMPDIR}/ca.pem" ] DOCKER_EXIT=1 run app_nextcloud_office [ "${status}" -eq 1 ] [[ ${output} == *"Nextcloud Office (richdocuments) cannot be installed"* ]] @@ -447,3 +456,100 @@ docker() { APP_GET_DOMAIN=console.example.com app_get login >/dev/null grep -qF -- "--resolve console.example.com:443:127.0.0.1 https://console.example.com/login" "${STUB_LOG}" } + +@test "Collabora trusts the host CAs and the Nextcloud route, and starts again when Nextcloud comes second" { + config_set LSS_NEXTCLOUD_DOMAINS cloud.example.com + config_set LSS_COLLABORA_DOMAINS office.example.com + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + mkdir -p "${WEB_TLS_DIR}/nextcloud" "${WEB_TLS_DIR}/default" + printf 'DEFAULT\n' >"${WEB_TLS_DIR}/default/fullchain.pem" + local trust="${CTR_CONFIG_DIR}/lss-collabora/trust/ca.pem" + # Collabora alone: the CAs of the host (or the default certificate on a host without a bundle). + app_collabora_trust + [ -s "${trust}" ] + grep -qxF "== 0444 root:root ${trust}" "${WRITES}" + run ! grep -qxF NEXTCLOUD "${trust}" + # With the Nextcloud route: its self-signed certificate too. + printf 'NEXTCLOUD\n' >"${WEB_TLS_DIR}/nextcloud/fullchain.pem" + app_collabora_trust + [ "${LSS_FILE_CHANGED}" = 1 ] + grep -qxF NEXTCLOUD "${trust}" + app_collabora_trust + [ "${LSS_FILE_CHANGED}" = 0 ] + # Nextcloud second: Collabora gets the new certificate and starts again before the access check. + # shellcheck disable=SC2329 # called by app_nextcloud_office + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf '{"status": "Ok"}\n200' + } + command -v jq >/dev/null || skip "jq is not installed" + state_set docker status installed + state_set collabora status installed + printf 'services: {}\n' >"${CTR_CONFIG_DIR}/lss-collabora/compose.yaml" + printf 'NEXTCLOUD 2\n' >"${WEB_TLS_DIR}/nextcloud/fullchain.pem" + LSS_MODULE=nextcloud + app_nextcloud_office + grep -q "compose --project-name lss-collabora .* up .*--force-recreate" "${ENGINE_LOG}" + grep -q '^curl .* https://office.example.com/hosting/wopiAccessCheck$' "${STUB_LOG}" + grep -qxF 'NEXTCLOUD 2' "${trust}" +} + +@test "the WOPI access check of Collabora must reach Nextcloud with a valid certificate" { + config_set LSS_NEXTCLOUD_DOMAINS cloud.example.com + config_set LSS_COLLABORA_DOMAINS office.example.com + # shellcheck disable=SC2329 # called by app_collabora_check + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf '%s\n%s' "${ANSWER}" "${CODE:-200}" + } + ANSWER='{"status": "Ok"}' app_collabora_check + grep -qF -- '--data {"callbackUrl":"https://cloud.example.com/status.php"}' "${STUB_LOG}" + ANSWER='{"status": "SelfSignedCertificate"}' run app_collabora_check + [ "${status}" -eq 1 ] + [[ ${output} == *'cannot reach Nextcloud at https://cloud.example.com with a valid certificate ({"status": "SelfSignedCertificate"}'* ]] + ANSWER='{"status": "Ok"}' CODE=502 run app_collabora_check + [ "${status}" -eq 1 ] +} + +@test "an app registers its data for the backup, which verify saves and removal forgets" { + app_backup DB=postgres DB_USER=demo LIVE=html 'COPY=data cache' + local conf="${APP_BACKUP_REGISTRY}/demo.conf" + grep -qxF "== 0755 root:root ${APP_BACKUP_SCRIPT}" "${WRITES}" + cmp "${REPO_ROOT}/libexec/lss-app-backup" "${APP_BACKUP_SCRIPT}" + grep -qxF "== 0600 root:root ${conf}" "${WRITES}" + grep -qxF PROJECT=lss-demo "${conf}" + grep -qxF "COMPOSE=${CTR_CONFIG_DIR}/lss-demo/compose.yaml" "${conf}" + grep -qxF DB=postgres "${conf}" + grep -qxF DB_USER=demo "${conf}" + grep -qxF 'COPY=data cache' "${conf}" + grep -qxF LIVE=html "${conf}" + grep -qxF PATHS= "${conf}" + run app_backup DATABASE=postgres + [[ ${output} == *"app_backup does not know 'DATABASE=postgres'"* ]] + run app_backup 'PATHS=/srv;rm' + [ "${status}" -eq 1 ] + # verify runs the script for this app; a database app needs a dump. + printf '#!/bin/sh\necho "save $*" >>"%s"\n' "${STUB_LOG}" >"${APP_BACKUP_SCRIPT}" + chmod +x "${APP_BACKUP_SCRIPT}" + run app_verify_backup + [ "${status}" -eq 1 ] + [[ ${output} == *"the database dump for the backup is empty"* ]] + grep -qxF 'save demo' "${STUB_LOG}" + mkdir -p "${APP_BACKUP_DIR}/demo" + printf 'SQL\n' >"${APP_BACKUP_DIR}/demo/db.sql" + app_verify_backup + printf '#!/bin/sh\nexit 1\n' >"${APP_BACKUP_SCRIPT}" + run app_verify_backup + [[ ${output} == *"saving the data of the app for the backup fails"* ]] + # Removal keeps the saved data unless --purge; the script goes with the last app. + app_backup_remove + [ ! -e "${conf}" ] + [ ! -e "${APP_BACKUP_REGISTRY}" ] + [ ! -e "${APP_BACKUP_SCRIPT}" ] + [ -s "${APP_BACKUP_DIR}/demo/db.sql" ] + run app_verify_backup + [[ ${output} == *"the backup would leave the app out"* ]] + app_backup COPY=data + LSS_PURGE=1 app_backup_remove + [ ! -e "${APP_BACKUP_DIR}/demo" ] +} diff --git a/tests/unit/backup.bats b/tests/unit/backup.bats index 472da90..ef3ee51 100644 --- a/tests/unit/backup.bats +++ b/tests/unit/backup.bats @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -# The backup scripts (libexec/lss-backup, libexec/lss-db-dumps) against a restic stub, and the checks of the restic +# The backup scripts (libexec/lss-backup, libexec/lss-db-dumps, libexec/lss-app-backup) against a restic stub, and the checks of the restic # module. setup() { @@ -19,6 +19,10 @@ setup() { export LSS_DB_DUMPS_CONF=${BATS_TEST_TMPDIR}/db-dumps.conf export LSS_DB_DUMPS_STATE=${BATS_TEST_TMPDIR}/db-dumps-state export LSS_RESTIC=restic_stub + export LSS_APP_BACKUP=${BATS_TEST_TMPDIR}/app-backup + export LSS_APP_BACKUP_STATE=${BATS_TEST_TMPDIR}/app-backup-state + export LSS_APP_BACKUP_REGISTRY=${BATS_TEST_TMPDIR}/app-backup.d + export LSS_APP_BACKUP_DIR=${BATS_TEST_TMPDIR}/lss-apps mkdir -p "${LSS_TEXTFILE_DIR}" "${LSS_SECRETS_DIR}" printf 'BACKUP_REPOSITORY=/srv/repo\nBACKUP_PATHS=/etc /does-not-exist\nBACKUP_EXCLUDE=*.tmp\n' >"${LSS_BACKUP_CONF}" printf 'password' >"${LSS_SECRETS_DIR}/restic-password" @@ -334,3 +338,149 @@ load_dumps() { [ "${status}" -eq 1 ] [ ! -e "${BATS_TEST_TMPDIR}/w.rdb" ] } + +@test "a run saves the data of the apps first and backs up the paths they list" { + load_backup + mkdir -p "${LSS_APP_BACKUP_STATE}" "${BATS_TEST_TMPDIR}/volume" + printf '#!/bin/sh\necho apps >>"%s"\n' "${STUB_LOG}" >"${LSS_APP_BACKUP}" + chmod +x "${LSS_APP_BACKUP}" + printf '%s\n' "${BATS_TEST_TMPDIR}/volume" /does-not-exist-either >"${LSS_APP_BACKUP_STATE}/paths" + cmd_run + grep -qxF apps "${STUB_LOG}" + grep -qxF "restic backup --tag lss --exclude-caches --exclude *.tmp /etc ${BATS_TEST_TMPDIR}/volume ${LSS_BACKUP_STATE}/canary" "${STUB_LOG}" + printf '#!/bin/sh\nexit 1\n' >"${LSS_APP_BACKUP}" + run cmd_run + [ "${status}" -eq 1 ] + [[ ${output} == *"saving the data of the apps failed; the backup goes on with the rest."* ]] +} + +load_apps() { + # shellcheck source=/dev/null + source "${REPO_ROOT}/libexec/lss-app-backup" + DOCKER=docker_stub + mkdir -p "${REGISTRY}" "${BATS_TEST_TMPDIR}/volumes/lss-demo_data" "${BATS_TEST_TMPDIR}/volumes/lss-demo_html" + printf 'app.db\n' >"${BATS_TEST_TMPDIR}/volumes/lss-demo_data/app.db" + return +} + +# docker stub for lss-app-backup: records the call; volume inspect prints the directory below volumes/ (missing +# ones fail), exec prints $DUMP (fails with $DUMP_EXIT), pause and unpause fail with $PAUSE_EXIT. +docker_stub() { + printf 'docker %s\n' "$*" >>"${STUB_LOG}" + case $* in + 'volume inspect'*) + [[ -d ${BATS_TEST_TMPDIR}/volumes/${*: -1} ]] || return 1 + printf '%s\n' "${BATS_TEST_TMPDIR}/volumes/${*: -1}" + ;; + *' exec '*) + printf '%s' "${DUMP:-}" + return "${DUMP_EXIT:-0}" + ;; + *' pause '* | *' unpause '*) return "${PAUSE_EXIT:-0}" ;; + *) ;; + esac + return 0 +} + +register() { + printf 'PROJECT=lss-%s\nCOMPOSE=/etc/lss/containers/lss-%s/compose.yaml\n%s\n' "$1" "$1" "$2" >"${REGISTRY}/$1.conf" + return +} + +@test "app data: database dumps, copies of paused apps and the paths restic takes as they are" { + load_apps + register demo $'DB=postgres\nDB_USER=demo\nCOPY=data\nLIVE=html\nPATHS=/etc /does-not-exist' + DUMP='SQL' run_backups + local compose='docker compose --project-name lss-demo -f /etc/lss/containers/lss-demo/compose.yaml' + grep -qxF "${compose} exec -T db pg_dumpall --username=demo --clean --if-exists" "${STUB_LOG}" + grep -qxF "${compose} pause app" "${STUB_LOG}" + grep -qxF "${compose} unpause app" "${STUB_LOG}" + [ "$(cat "${OUT_ROOT}/demo/db.sql")" = SQL ] + [ "$(cat "${OUT_ROOT}/demo/data/app.db")" = app.db ] + [ "$(stat -c '%a' "${OUT_ROOT}/demo")" = 700 ] + [ "$(cat "${STATE}/paths")" = "$(printf '%s\n' "${BATS_TEST_TMPDIR}/volumes/lss-demo_html" /etc "${OUT_ROOT}")" ] + grep -q '^demo ok [0-9]*$' "${STATE}/status" + # MariaDB: mariadb-dump as root with the password from the secret file, inside the container. + register shop $'DB=mariadb' + DUMP='SQL' run_backups shop + grep -qF 'compose --project-name lss-shop -f /etc/lss/containers/lss-shop/compose.yaml exec -T db sh -c MYSQL_PWD=$(cat /run/secrets/db-root-password) exec mariadb-dump --user=root' "${STUB_LOG}" + [ -s "${OUT_ROOT}/shop/db.sql" ] +} + +@test "a failed dump or copy keeps the previous one, unpauses the app and fails the run" { + load_apps + register demo $'DB=postgres\nDB_USER=demo\nCOPY=data' + mkdir -p "${OUT_ROOT}/demo/data" + printf 'OLD\n' >"${OUT_ROOT}/demo/db.sql" + printf 'OLD\n' >"${OUT_ROOT}/demo/data/app.db" + # shellcheck disable=SC2329 # called by the functions under test + cp() { + return 1 + } + DUMP='PART' DUMP_EXIT=1 run run_backups + [ "${status}" -eq 1 ] + [[ ${output} == *"the database dump of demo failed; ${OUT_ROOT}/demo/db.sql keeps the previous one."* ]] + [[ ${output} == *"the copy of the data of demo failed; ${OUT_ROOT}/demo keeps the previous one."* ]] + [ "$(cat "${OUT_ROOT}/demo/db.sql")" = OLD ] + [ "$(cat "${OUT_ROOT}/demo/data/app.db")" = OLD ] + [ ! -e "${OUT_ROOT}/demo/db.sql.part" ] + grep -q 'unpause app' "${STUB_LOG}" + grep -q '^demo failed ' "${STATE}/status" +} + +@test "an app that cannot be paused or unpaused, and a missing volume, fail the run" { + load_apps + # shellcheck disable=SC2329 # called by the functions under test + sleep() { + : + } + register demo $'COPY=data' + PAUSE_EXIT=1 run run_backups demo + [ "${status}" -eq 1 ] + [[ ${output} == *"the copy of the data of demo failed"* ]] + run ! grep -q 'unpause' "${STUB_LOG}" + register demo $'COPY=data\nLIVE=gone' + # Pausing works, unpausing never does. + # shellcheck disable=SC2329 # called by the functions under test + compose() { + [[ $2 != unpause ]] + } + run run_backups demo + [[ ${output} == *"the app of lss-demo stays paused; run docker compose unpause app."* ]] + [[ ${output} == *"the volume gone of demo is missing."* ]] + register demo $'COPY=gone' + run run_backups demo + [ "${status}" -eq 1 ] +} + +@test "a run for named apps leaves the full list alone; unknown names and an empty registry are reported" { + load_apps + register demo $'LIVE=html' + run_backups + local before + before=$(cat "${STATE}/paths") + register other $'LIVE=data' + mkdir -p "${BATS_TEST_TMPDIR}/volumes/lss-other_data" + run_backups other + [ "$(cat "${STATE}/paths")" = "${before}" ] + run run_backups '../etc' + [ "${status}" -eq 1 ] + [[ ${output} == *"../etc is not registered"* ]] + rm -f -- "${REGISTRY}"/*.conf + run run_backups + [ "${status}" -eq 0 ] + [[ ${output} == *"0 apps are registered here; nothing to save."* ]] + [ ! -s "${STATE}/paths" ] + register bad $'DB=sqlite' + run run_backups bad + [ "${status}" -eq 1 ] +} + +@test "the app backup script needs root" { + if [[ ${EUID} -eq 0 ]]; then + skip "runs as root" + fi + run bash "${REPO_ROOT}/libexec/lss-app-backup" + [ "${status}" -eq 1 ] + [[ ${output} == *"must run as root."* ]] +} From 0e3f18258e780cf792d0772f0afa0c578f44b243 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 01:54:03 +0200 Subject: [PATCH 08/28] fix(apps): Matomo idempotency, shared SELinux label for init volumes, S3 body hash Three failures from the CI run of 65b906b: - Matomo: the console writes a setting as a quoted string ("0"), so the check for the plain value never matched, and every install set enable_browser_archiving_triggering again. The check now accepts both forms. - Vaultwarden on Rocky 9 and AlmaLinux 10: the init service could not chown the data volume. Each container of a stack gets its own SELinux MCS categories, and the volume carried the app's. init and the app now mount the shared volume with ":z" (shared label); this also affects MinIO in Docker mode. - MinIO (native, Debian 12): storing an object answered 403. Older curl sends no x-amz-content-sha256 with --aws-sigv4, and the server then checks against the hash of an empty body. verify sends the hash of the body explicitly and puts the S3 error code in the message. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/apps.sh | 5 +++-- modules/apps/matomo.sh | 3 ++- modules/apps/minio.sh | 16 +++++++++++++--- modules/apps/vaultwarden.sh | 2 +- tests/unit/apps.bats | 2 ++ 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/apps.sh b/lib/apps.sh index bc8b1e1..d2cbdb7 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -588,7 +588,8 @@ app_db_secrets() { # app_init VOLUME PATH UID:GID: the service "init", which gives the mount point PATH of the named volume VOLUME (not # what is inside) to the container user UID:GID before the app starts; for images whose data directory belongs to root # while the app runs as another user. It runs as root with only CAP_CHOWN, without a network, and exits. The app waits -# for it with "depends_on: init: condition: service_completed_successfully". +# for it with "depends_on: init: condition: service_completed_successfully". Both mount the volume with ":z" (the shared +# SELinux label): each container of a stack gets its own MCS categories, which would lock the other one out. app_init() { local volume=$1 path=$2 ids=$3 cat </dev/null; then + # Matomo's ini writer puts strings in quotes (the console sets "0"), the installer writes plain numbers. + if ! app_exec app grep -Eqx "${key} = \"?${value}\"?" /var/www/html/config/config.ini.php 2>/dev/null; then log_command _mt_console config:set --section="${section}" --key="${key}" --value="${value}" || die "matomo: the console cannot set ${section}.${key}." files_note_change "matomo setting ${section}.${key}" diff --git a/modules/apps/minio.sh b/modules/apps/minio.sh index e6f77c4..e3ebeee 100644 --- a/modules/apps/minio.sh +++ b/modules/apps/minio.sh @@ -122,7 +122,7 @@ services: - 127.0.0.1:$(opt_int api_port):9000 - 127.0.0.1:$(opt_int console_port):9001 volumes: - - data:/data + - data:/data:z $(app_mount config/root-user /run/secrets/access_key) $(app_mount secrets/root-password /run/secrets/secret_key) tmpfs: @@ -204,6 +204,12 @@ _minio_s3() { return } +# _minio_error PAGE: the S3 error code in the XML answer PAGE (such as SignatureDoesNotMatch). +_minio_error() { + sed -n 's:.*\([A-Za-z]*\).*:\1:p' <<<"$1" | head -n 1 + return +} + mod_verify() { local page bucket probe user status user=$(opt root_user) @@ -228,9 +234,13 @@ mod_verify() { printf '%s' "${bucket}" >"${probe}" status=$(app_status "$(_minio_s3 "${user}" minio-root-password PUT "${bucket}")") [[ ${status} == 200 ]] || die "minio: creating the bucket ${bucket} answers ${status}." - status=$(app_status "$(_minio_s3 "${user}" minio-root-password PUT "${bucket}/probe.txt" --data-binary "@${probe}")") + # The hash of the body goes along explicitly: curl before 8.x leaves x-amz-content-sha256 out, and the server then + # checks the signature against the hash of an empty body. + page=$(_minio_s3 "${user}" minio-root-password PUT "${bucket}/probe.txt" --data-binary "@${probe}" \ + -H "x-amz-content-sha256: $(sha256sum "${probe}" | cut -d ' ' -f 1)") rm -f -- "${probe}" - [[ ${status} == 200 ]] || die "minio: storing an object answers ${status}." + status=$(app_status "${page}") + [[ ${status} == 200 ]] || die "minio: storing an object answers ${status} ($(_minio_error "${page}"))." page=$(_minio_s3 "${user}" minio-root-password GET "${bucket}/probe.txt") [[ ${page%$'\n'*} == "${bucket}" ]] || die "minio: the stored object does not come back." status=$(app_status "$(_minio_s3 "${user}" minio-root-password DELETE "${bucket}/probe.txt")") diff --git a/modules/apps/vaultwarden.sh b/modules/apps/vaultwarden.sh index 455adc5..b0bdc5f 100644 --- a/modules/apps/vaultwarden.sh +++ b/modules/apps/vaultwarden.sh @@ -57,7 +57,7 @@ services: ports: - 127.0.0.1:$(opt_int port):8080 volumes: - - data:/data + - data:/data:z $(app_mount secrets/admin-token /run/secrets/admin-token) tmpfs: - /tmp:mode=1777,size=64m diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats index 9e111dc..d8f8bc7 100644 --- a/tests/unit/apps.bats +++ b/tests/unit/apps.bats @@ -296,6 +296,8 @@ docker() { [[ ${output} == *"backend:"*"internal: true"* ]] run app_init data /data 1000:1000 [[ ${output} == *'entrypoint: ["chown", "1000:1000", "/data"]'* ]] + # Shared SELinux label: the app mounts the volume with another MCS category than init. + [[ ${output} == *"- data:/data:z"* ]] [[ ${output} == *"network_mode: none"*"cap_drop: [ALL]"*"cap_add: [CHOWN]"*"read_only: true"* ]] [[ ${output} == *'restart: "no"'* ]] } From 71373741796451ea269d964003d7cdfbd1fbd9c6 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 02:41:28 +0200 Subject: [PATCH 09/28] fix(container): retry pulling the images of a Compose project Docker Hub answered one pull of the arm64 apps job with HTTP 500, and the n8n installation was rolled back over that single answer. ctr_compose_up now pulls the missing images first, three attempts 15 s apart, and only then starts the stack, so a passing registry hiccup no longer undoes an installation while a registry that stays down still fails loudly. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/container.sh | 16 ++++++++++++++-- tests/unit/apps.bats | 1 + tests/unit/container.bats | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/lib/container.sh b/lib/container.sh index 61f3741..2554266 100644 --- a/lib/container.sh +++ b/lib/container.sh @@ -451,14 +451,26 @@ ctr_compose_check() { return 1 } +# Attempts and the pause in seconds between them for pulling the images of a Compose project. Registries answer +# 500 or 429 now and then; one such answer should not undo an installation. +CTR_PULL_TRIES=3 +CTR_PULL_DELAY=15 + # ctr_compose_up PROJECT FILE [OPTION...]: starts a Compose project after ctr_compose_check passed (Docker only). The -# OPTIONs go to "docker compose up" (--force-recreate after a mounted file changed). +# images that are missing are pulled first, with retries. The OPTIONs go to "docker compose up" (--force-recreate +# after a mounted file changed). ctr_compose_up() { - local project=$1 file=$2 + local project=$1 file=$2 try=1 shift 2 [[ $(ctr_engine) == docker ]] || die "container: Compose projects need the docker module." [[ ${project} =~ ^[a-z0-9][a-z0-9_-]*$ ]] || die "container: '${project}' is not a valid Compose project name." ctr_compose_check "${file}" "${project}" || die "container: ${file} breaks the container rules; see above." + until log_command docker compose --project-name "${project}" -f "${file}" pull --quiet --policy missing; do + ((try < CTR_PULL_TRIES)) || die "container: the images of ${project} cannot be pulled; see the log." + log_warn "container: pulling the images of ${project} failed (attempt ${try} of ${CTR_PULL_TRIES}); retrying in ${CTR_PULL_DELAY} s." + sleep "${CTR_PULL_DELAY}" + try=$((try + 1)) + done log_command docker compose --project-name "${project}" -f "${file}" up --detach --remove-orphans --quiet-pull "$@" || die "container: docker compose cannot start ${project}; see the log." return diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats index d8f8bc7..c917cad 100644 --- a/tests/unit/apps.bats +++ b/tests/unit/apps.bats @@ -15,6 +15,7 @@ setup() { MOD_OPTIONS="domains||Host names port|8081|Port on 127.0.0.1" CTR_CONFIG_DIR=${BATS_TEST_TMPDIR}/containers + CTR_PULL_DELAY=0 APP_DOCKER_CONFIG=${BATS_TEST_TMPDIR}/daemon.json APP_SUBUID=${BATS_TEST_TMPDIR}/subuid APP_SUBGID=${BATS_TEST_TMPDIR}/subgid diff --git a/tests/unit/container.bats b/tests/unit/container.bats index 0394e32..de67287 100644 --- a/tests/unit/container.bats +++ b/tests/unit/container.bats @@ -395,12 +395,50 @@ record_writes() { run ctr_compose_up 'Bad Name' "${file}" [ "${status}" -eq 1 ] ctr_compose_up shop "${file}" + grep -qxF "docker compose --project-name shop -f ${file} pull --quiet --policy missing" "${BATS_TEST_TMPDIR}/engine.log" grep -qxF "docker compose --project-name shop -f ${file} up --detach --remove-orphans --quiet-pull" "${BATS_TEST_TMPDIR}/engine.log" LSS_MODULE=podman run ctr_compose_up shop "${file}" [[ ${output} == *"need the docker module"* ]] } +@test "Compose projects retry a failed image pull and give up after the last attempt" { + command -v jq >/dev/null || skip "jq is not installed" + local file=${BATS_TEST_TMPDIR}/compose.yaml + : >"${file}" + CTR_PULL_DELAY=0 + printf '1 +' >"${BATS_TEST_TMPDIR}/fails" + # shellcheck disable=SC2329 # called by ctr_compose_up + docker() { + local left + printf 'docker %s +' "$*" >>"${BATS_TEST_TMPDIR}/engine.log" + if [[ $* == *config* ]]; then + printf '{"services": {}} +' + elif [[ $* == *" pull "* ]]; then + left=$(cat "${BATS_TEST_TMPDIR}/fails") + ((left > 0)) || return 0 + printf '%s +' $((left - 1)) >"${BATS_TEST_TMPDIR}/fails" + return 1 + fi + return 0 + } + ctr_compose_up shop "${file}" + [ "$(grep -c ' pull ' "${BATS_TEST_TMPDIR}/engine.log")" -eq 2 ] + grep -q ' up --detach' "${BATS_TEST_TMPDIR}/engine.log" + : >"${BATS_TEST_TMPDIR}/engine.log" + printf '9 +' >"${BATS_TEST_TMPDIR}/fails" + run ctr_compose_up shop "${file}" + [ "${status}" -eq 1 ] + [[ ${output} == *"images of shop cannot be pulled"* ]] + [ "$(grep -c ' pull ' "${BATS_TEST_TMPDIR}/engine.log")" -eq "${CTR_PULL_TRIES}" ] + run ! grep -q ' up --detach' "${BATS_TEST_TMPDIR}/engine.log" +} + @test "Trivy is kept when it is current and removed with the last engine" { CTR_TRIVY=${BATS_TEST_TMPDIR}/trivy CTR_TRIVY_CACHE=${BATS_TEST_TMPDIR}/trivy-cache From 2395b2d1e806f17f316653d024b9f4db1cfa43d2 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 02:43:16 +0200 Subject: [PATCH 10/28] style(minio): name the parameter of _minio_error SonarCloud asks for positional parameters to be bound to a named local (shelldre:S7679); the other helpers of the module already do so. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/apps/minio.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/apps/minio.sh b/modules/apps/minio.sh index e3ebeee..fdeedfd 100644 --- a/modules/apps/minio.sh +++ b/modules/apps/minio.sh @@ -206,7 +206,8 @@ _minio_s3() { # _minio_error PAGE: the S3 error code in the XML answer PAGE (such as SignatureDoesNotMatch). _minio_error() { - sed -n 's:.*\([A-Za-z]*\).*:\1:p' <<<"$1" | head -n 1 + local page=$1 + sed -n 's:.*\([A-Za-z]*\).*:\1:p' <<<"${page}" | head -n 1 return } From 2bda346838e5f7abd6596aedb25dec7ced89e6f0 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 06:07:53 +0200 Subject: [PATCH 11/28] fix(wordpress): ask the REST API through rest_route and log in without the test cookie A fresh site has plain permalinks, so /wp-json/ never reaches the REST API and the check failed on every system. index.php?rest_route=/ works with every permalink setting. The login no longer posts testcookie=1: curl cannot send the jar and a fixed cookie together on every version, and without the field WordPress skips that browser check. The start check waits for a stylesheet instead of a PNG, which bash read with a null-byte warning. The REST error now names the status and address. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/apps/wordpress.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/apps/wordpress.sh b/modules/apps/wordpress.sh index 33cc4b9..1471f72 100644 --- a/modules/apps/wordpress.sh +++ b/modules/apps/wordpress.sh @@ -116,25 +116,28 @@ mod_configure() { app_backup DB=mariadb LIVE=html app_up app_site "$(opt_int port)" - app_wait wp-includes/images/w-logo-blue.png 200 >/dev/null || die "wordpress: https://$(app_domain)/ does not answer." + app_wait wp-includes/css/buttons.min.css 200 >/dev/null || die "wordpress: https://$(app_domain)/ does not answer." _wp_install return } mod_verify() { - local page jar status + local page jar status home app_verify_stack app_verify_backup page=$(app_wait wp-login.php 200) || die "wordpress: the login page does not answer." grep -q 'name="pwd"' <<<"${page}" || die "wordpress: https://$(app_domain)/wp-login.php shows no login." - page=$(app_get wp-json/) || true - [[ $(app_status "${page}") == 200 && $(jq -r '.home' <<<"${page%$'\n'*}" 2>/dev/null) == "https://$(app_domain)" ]] || - die "wordpress: the REST API does not report https://$(app_domain) as the address of the site." - # Used, not only running: the administrator logs in and opens the dashboard. + # rest_route works with plain permalinks too; /wp-json/ needs pretty permalinks, which a fresh site lacks. + page=$(app_get 'index.php?rest_route=/') || true + home=$(jq -r '.home // empty' <<<"${page%$'\n'*}" 2>/dev/null) || home='' + [[ $(app_status "${page}") == 200 && ${home} == "https://$(app_domain)" ]] || + die "wordpress: the REST API answers $(app_status "${page}") with the address '${home}' instead of https://$(app_domain)." + # Used, not only running: the administrator logs in and opens the dashboard. Without testcookie WordPress skips its + # check for a browser that blocks cookies; the session cookies land in the jar. jar=$(mktemp) - page=$(app_get wp-login.php -b "${jar}" -c "${jar}" -b 'wordpress_test_cookie=WP%20Cookie%20check' -X POST \ + page=$(app_get wp-login.php -b "${jar}" -c "${jar}" -X POST \ --data-urlencode "log=$(opt admin_user)" --data-urlencode "pwd@$(secret_path wordpress-admin-password)" \ - -d testcookie=1 --data-urlencode "redirect_to=https://$(app_domain)/wp-admin/") || true + --data-urlencode "redirect_to=https://$(app_domain)/wp-admin/") || true status=$(app_status "${page}") if [[ ${status} != 302 ]]; then rm -f -- "${jar}" From f5aea00acf0dfbf96e8f5b1d8354a1efa0c45132 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 06:07:55 +0200 Subject: [PATCH 12/28] test(integration): trust the root Pebble issues from Pebble serves its API with the minica certificate but signs issued certificates with a root it creates at start. Nextcloud and Collabora check each other's certificate against LSS_TLS_CA_BUNDLE, so with only minica in it the office profile failed with "unable to get local issuer certificate". The bundle now holds minica and every issuing root, as a step-ca root would cover both in a real private ACME setup. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- tests/integration/pebble.sh | 16 ++++++++++++++++ tests/integration/settings.conf | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/integration/pebble.sh b/tests/integration/pebble.sh index 53beb6f..198cbba 100644 --- a/tests/integration/pebble.sh +++ b/tests/integration/pebble.sh @@ -16,6 +16,8 @@ c87fb918d9bac8db11aa54493b5f0cf6e0725a3fe73f617d1c64737e1ab4caf9 test/certs/loca 0977979255b0e0721c17335c056b627f66a43cf56e5a52f3cda81fb111ecf00e test/certs/localhost/key.pem" # Where the target keeps Pebble; LSS_TLS_CA_BUNDLE in settings.conf points at the CA certificate here. PEBBLE_DIR=/etc/lss-pebble +# The CA bundle LSS_TLS_CA_BUNDLE points at: the minica certificate of the API and the roots Pebble issues from. +PEBBLE_BUNDLE=${PEBBLE_DIR}/ca-bundle.pem pebble_start() { local arch sum work file @@ -61,5 +63,19 @@ EOF --setenv=PEBBLE_WFE_NONCEREJECT=0 ${PEBBLE_DIR}/pebble -config ${PEBBLE_DIR}/config.json for i in \$(seq 30); do curl -fsS --cacert ${PEBBLE_DIR}/pebble.minica.pem -o /dev/null https://localhost:14000/dir && exit 0; sleep 1; done journalctl -u lss-pebble --no-pager; exit 1" + pebble_bundle + return +} + +# pebble_bundle: PEBBLE_BUNDLE, the CA bundle of settings.conf. Pebble serves its API with the minica certificate but +# signs the certificates it issues with a root it creates at every start; apps that check each other's certificates +# (Nextcloud and Collabora) need that root, as they would need the root of a step-ca. Roots of earlier starts stay in +# the bundle, so certificates issued before a restart keep their anchor. +pebble_bundle() { + in_target "set -e + [ -s ${PEBBLE_BUNDLE} ] || cp ${PEBBLE_DIR}/pebble.minica.pem ${PEBBLE_BUNDLE} + curl -fsS --retry 3 --cacert ${PEBBLE_DIR}/pebble.minica.pem -o ${PEBBLE_DIR}/root.pem https://localhost:15000/roots/0 + grep -q 'BEGIN CERTIFICATE' ${PEBBLE_DIR}/root.pem + grep -qxF \"\$(sed -n 2p ${PEBBLE_DIR}/root.pem)\" ${PEBBLE_BUNDLE} || { echo; cat ${PEBBLE_DIR}/root.pem; } >>${PEBBLE_BUNDLE}" return } diff --git a/tests/integration/settings.conf b/tests/integration/settings.conf index 4e68d71..65aec54 100644 --- a/tests/integration/settings.conf +++ b/tests/integration/settings.conf @@ -12,7 +12,7 @@ LSS_SYSTEM_UPGRADE=false LSS_AIDE_EXCLUDE=/opt /usr/local /usr/share/dotnet /usr/share/swift /usr/share/miniconda /usr/lib/jvm /usr/lib/google-cloud-sdk /home /snap /var/lib/snapd # Web group: certificates from Pebble inside the target (tests/integration/pebble.sh), which accepts any host name. LSS_TLS_SERVER=https://localhost:14000/dir -LSS_TLS_CA_BUNDLE=/etc/lss-pebble/pebble.minica.pem +LSS_TLS_CA_BUNDLE=/etc/lss-pebble/ca-bundle.pem LSS_TLS_DOMAINS=lss-test.example.com LSS_VHOST_SITES=demo:static:demo.example.com,www.demo.example.com app:proxy:app.example.com:http://127.0.0.1:9 LSS_HAPROXY_SERVICES=echo:9190/local:127.0.0.1:9,127.0.0.1:10 From 4a7b5247d31af597aeb25355902253485a214db6 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 06:07:58 +0200 Subject: [PATCH 13/28] fix(mail): let the Postfix master and Dovecot start their workers under SELinux On AlmaLinux 10 with SELinux enforcing the profile office hung in mailserver: the sandbox sets NoNewPrivileges, and SELinux then denies postfix_master_t the change into postfix_qmgr_t, postfix_pickup_t and postfix_smtpd_t (avc: nnp_transition), so no daemon ever started. Dovecot changes into dovecot_auth_t the same way. The mail tests run in containers without SELinux, so this never showed there. svc_spawner_directives turns NoNewPrivileges off for such masters under SELinux only, as svc_scanner_sandbox already does for the scanners: the masters run as root and drop their rights themselves, and the policy confines every worker. The rest of the sandbox stays. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/mail.sh | 7 +++++-- lib/service.sh | 12 ++++++++++++ modules/mail/mailserver.sh | 6 ++++-- tests/unit/mail.bats | 5 +++++ tests/unit/system.bats | 11 +++++++++++ 5 files changed, 37 insertions(+), 4 deletions(-) diff --git a/lib/mail.sh b/lib/mail.sh index 5b84cec..172e682 100644 --- a/lib/mail.sh +++ b/lib/mail.sh @@ -116,10 +116,13 @@ EOF } # mail_postfix_sandbox [DIRECTIVE...]: the systemd sandbox of the Postfix master and its daemons. They write the queue, -# their state and the local mailboxes only. Postfix needs /dev/log of the host for syslog, so /dev stays. +# their state and the local mailboxes only. Postfix needs /dev/log of the host for syslog, so /dev stays; the master +# starts its daemons in their own SELinux domains (svc_spawner_directives). mail_postfix_sandbox() { + local -a spawner + mapfile -t spawner < <(svc_spawner_directives) svc_sandbox "$(mail_postfix_template)" PrivateDevices=no ReadWritePaths=/var/spool/postfix ReadWritePaths=/var/lib/postfix \ - "$@" + "${spawner[@]}" "$@" return } diff --git a/lib/service.sh b/lib/service.sh index 2302e93..a8732ac 100644 --- a/lib/service.sh +++ b/lib/service.sh @@ -144,6 +144,18 @@ _svc_selinux_nnp() { return } +# svc_spawner_directives: sandbox directives (one per line) for a master process that starts its workers in SELinux +# domains of their own (the Postfix master: postfix_qmgr_t, postfix_smtpd_t; Dovecot: dovecot_auth_t). NoNewPrivileges +# forbids those changes (avc: nnp_transition) and the workers never start, so under SELinux it is off: the master runs +# as root and drops its rights itself, and the policy confines every worker. Without SELinux it prints nothing. +svc_spawner_directives() { + if _svc_selinux_on; then + printf '%s +' NoNewPrivileges=no + fi + return 0 +} + # svc_scanner_sandbox full|strict [PATH...]: [Service] lines that confine lss's own scanner units. They read the whole # system but write only to the PATHs (with strict) or below /var (with full). The real /tmp, /proc/kallsyms, the # kernel modules and the kernel log stay visible: a private /tmp would hide it from ClamAV and rkhunter, and rkhunter, diff --git a/modules/mail/mailserver.sh b/modules/mail/mailserver.sh index 51e8bc7..b7274ab 100644 --- a/modules/mail/mailserver.sh +++ b/modules/mail/mailserver.sh @@ -367,7 +367,7 @@ EOF } _ms_dovecot() { - local cert=$1 before config users output + local cert=$1 before config users output spawner before=$(files_changes_of mailserver) users=$(_ms_users_file) file_write 0640 root:dovecot "${MS_USERS}" <<<"${users}" @@ -380,8 +380,10 @@ _ms_dovecot() { # Dovecot puts its sockets for Postfix there; Postfix 3.10 creates the directory only on its first start, which # comes later, and the sandbox needs it to exist. Postfix's own owner and mode. [[ -d /var/spool/postfix/private ]] || install -d -m 0700 -o postfix -g root /var/spool/postfix/private + # Dovecot starts its login, auth and delivery processes in their own SELinux domains (svc_spawner_directives). + mapfile -t spawner < <(svc_spawner_directives) svc_sandbox dovecot.service "ReadWritePaths=${MS_VMAIL}" ReadWritePaths=-/var/lib/dovecot \ - ReadWritePaths=/var/spool/postfix/private + ReadWritePaths=/var/spool/postfix/private "${spawner[@]}" svc_enable dovecot if [[ $(files_changes_of mailserver) != "${before}" ]]; then svc_restart dovecot || die "mailserver: Dovecot does not start with the new configuration." diff --git a/tests/unit/mail.bats b/tests/unit/mail.bats index 22cc077..7211265 100644 --- a/tests/unit/mail.bats +++ b/tests/unit/mail.bats @@ -70,6 +70,11 @@ load_module() { run mail_postfix_sandbox ReadWritePaths=/var/mail [ "${lines[0]}" = 'postfix@.service' ] [[ ${output} == *$'PrivateDevices=no\nReadWritePaths=/var/spool/postfix\nReadWritePaths=/var/lib/postfix\nReadWritePaths=/var/mail'* ]] + run ! grep -q NoNewPrivileges <<<"${output}" + # Under SELinux the master must start qmgr, pickup and smtpd in their own domains. + export STUB_EXIT_selinuxenabled=0 + run mail_postfix_sandbox ReadWritePaths=/var/mail + [[ ${output} == *$'ReadWritePaths=/var/lib/postfix\nNoNewPrivileges=no\nReadWritePaths=/var/mail'* ]] } @test "Postfix starts once, restarts only after a change and stops the run on a configuration it rejects" { diff --git a/tests/unit/system.bats b/tests/unit/system.bats index 30baf05..ef9c6ff 100644 --- a/tests/unit/system.bats +++ b/tests/unit/system.bats @@ -656,6 +656,17 @@ EOF run ! grep -qxF 'NoNewPrivileges=yes' <<<"${output}" } +@test "svc_spawner_directives lets a master start its workers in their SELinux domains" { + export STUB_EXIT_selinuxenabled=1 + run svc_spawner_directives + [ "${status}" -eq 0 ] + [ -z "${output}" ] + export STUB_EXIT_selinuxenabled=0 + run svc_spawner_directives + [ "${status}" -eq 0 ] + [ "${output}" = NoNewPrivileges=no ] +} + @test "svc_sandbox writes the baseline and the service's own lines, without new privileges outside SELinux" { # shellcheck disable=SC2329 # called by the function under test svc_dropin() { From 903a30788c58f6fee85dae83a5d259316796356a Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 08:28:39 +0200 Subject: [PATCH 14/28] fix(collabora): say who refused the PDF conversion The conversion check of the profile all failed with a bare HTTP 403, which Collabora, nginx and ModSecurity can all answer. The error now carries the start of the answer and, with the modsecurity module, the rule IDs and messages of the latest audit entries (app_waf_hits), so the next failure points at its cause instead of needing another CI round. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/apps.sh | 13 +++++++++++++ modules/apps/collabora.sh | 7 +++++-- tests/unit/apps.bats | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/apps.sh b/lib/apps.sh index d2cbdb7..3077bd7 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -403,6 +403,19 @@ app_get() { return } +# app_waf_hits: the rule IDs and messages of the latest ModSecurity audit entries (modsecurity module), so a check that +# the web application firewall turned away says which rules fired; prints nothing without ModSecurity. +app_waf_hits() { + local server log + state_installed modsecurity || return 0 + server=$(web_server) || return 0 + log=$(web_log_dir "${server}")/modsecurity/audit.log + [[ -r ${log} ]] || return 0 + tail -n 300 "${log}" | grep -oE '\[id "[0-9]+"\]|\[msg "[^"]{1,120}"\]|"ruleId":"[0-9]+"|"message":"[^"]{1,120}"' | + tail -n 12 | paste -sd ' ' - + return 0 +} + # app_status PAGE: the HTTP status at the end of the output of app_get. app_status() { local page=$1 diff --git a/modules/apps/collabora.sh b/modules/apps/collabora.sh index 204c5db..02cc5aa 100644 --- a/modules/apps/collabora.sh +++ b/modules/apps/collabora.sh @@ -104,7 +104,7 @@ mod_configure() { } mod_verify() { - local page probe pdf status + local page probe pdf status answer waf app_verify_stack page=$(app_wait '' 200) || die "collabora: https://$(app_domain)/ does not answer." [[ ${page%$'\n'*} == OK ]] || die "collabora: https://$(app_domain)/ answers '${page%$'\n'*}' instead of OK." @@ -116,8 +116,11 @@ mod_verify() { printf 'lss verify %s\n' "$(date +%s)" >"${probe}" status=$(app_status "$(app_get cool/convert-to/pdf -F "data=@${probe};filename=lss-verify.txt" -o "${pdf}" --max-time 120)") if [[ ${status} != 200 || $(head -c 4 "${pdf}") != %PDF ]]; then + # The answer tells Collabora's own refusal from one of the web server or its firewall. + answer=$(head -c 300 "${pdf}" | tr -c '[:print:]' ' ' | sed 's/<[^>]*>/ /g; s/ */ /g') rm -f -- "${probe}" "${pdf}" - die "collabora: converting a document to PDF failed (HTTP ${status})." + waf=$(app_waf_hits) + die "collabora: converting a document to PDF failed (HTTP ${status}: ${answer:-empty answer})${waf:+; ModSecurity: ${waf}}." fi rm -f -- "${probe}" "${pdf}" if state_installed nextcloud; then diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats index c917cad..c218787 100644 --- a/tests/unit/apps.bats +++ b/tests/unit/apps.bats @@ -556,3 +556,19 @@ docker() { LSS_PURGE=1 app_backup_remove [ ! -e "${APP_BACKUP_DIR}/demo" ] } + +@test "app_waf_hits names the ModSecurity rules of the latest audit entries" { + run app_waf_hits + [ "${status}" -eq 0 ] + [ -z "${output}" ] + state_set nginx status installed + state_set modsecurity status installed + # shellcheck disable=SC2329 # called by app_waf_hits + web_log_dir() { printf '%s' "${BATS_TEST_TMPDIR}/log"; } + mkdir -p "${BATS_TEST_TMPDIR}/log/modsecurity" + printf '%s\n' '---H---' 'ModSecurity: Warning. [file "x.conf"] [line "1"] [id "932180"] [rev ""] [msg "Restricted File Upload Attempt"]' \ + 'ModSecurity: Access denied with code 403 (phase 2). [id "949110"] [msg "Inbound Anomaly Score Exceeded (Total Score: 5)"]' \ + >"${BATS_TEST_TMPDIR}/log/modsecurity/audit.log" + run app_waf_hits + [ "${output}" = '[id "932180"] [msg "Restricted File Upload Attempt"] [id "949110"] [msg "Inbound Anomaly Score Exceeded (Total Score: 5)"]' ] +} From a53ec0b20bb15018ef3e4ddac9106c2df1293da7 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 08:40:29 +0200 Subject: [PATCH 15/28] fix(apps): write timer and service units with the real program path The second install of the office profile on AlmaLinux 10 rewrote lss-nextcloud-cron.service: command -v docker answered /bin/docker in one run and /usr/bin/docker in the next, because /bin links to /usr/bin and the PATH differed. The units now take the resolved path, as aide, rkhunter and clamav already do, so a second run reports unchanged. The Self-Updater unit resolves its directory the same way. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/apps.sh | 2 ++ modules/apps/self-updater.sh | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/apps.sh b/lib/apps.sh index 3077bd7..121042b 100644 --- a/lib/apps.sh +++ b/lib/apps.sh @@ -345,7 +345,9 @@ app_nextcloud_office() { app_timer() { local name=$1 interval=$2 description=$3 service=$4 docker shift 4 + # The real path: with /bin a link to /usr/bin, command -v answers either, depending on PATH, and the unit would change. docker=$(command -v docker) || die "${LSS_MODULE}: docker is missing." + [[ ${docker} != /* ]] || docker=$(readlink -f "${docker}") svc_unit "lss-${name}.service" < Date: Fri, 16 Oct 2026 09:45:07 +0200 Subject: [PATCH 16/28] fix(modsecurity): stop ModSecurity 3 from refusing every multipart upload The profile all failed the Collabora PDF conversion with 403; the new diagnostics named rule 200004, "possible unmatched boundary". ModSecurity 3 sets MULTIPART_UNMATCHED_BOUNDARY to 2 for every well-formed body that ends with its final boundary and to 1 only for a suspicious one, and its own modsecurity.conf-recommended therefore tests "@eq 1". lss used the "!@eq 0" of ModSecurity 2, so nginx turned away every multipart upload: file uploads, forms with files and the conversion API alike. nginx now gets "@eq 1"; Apache runs ModSecurity 2, which only ever sets 1, and keeps the strict test. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/web/modsecurity.sh | 14 ++++++++++++-- tests/unit/web.bats | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/web/modsecurity.sh b/modules/web/modsecurity.sh index dc69d2e..c6ac387 100644 --- a/modules/web/modsecurity.sh +++ b/modules/web/modsecurity.sh @@ -479,7 +479,7 @@ mod_install() { # _modsecurity_engine_conf SERVER: the engine settings after modsecurity.conf-recommended of ModSecurity. _modsecurity_engine_conf() { - local server=$1 bytes nofiles json=0 + local server=$1 bytes nofiles json=0 unmatched bytes=$(_modsecurity_bytes "$(_modsecurity_max_body)") # Bodies without uploads (forms, JSON) up to 1 MiB: the recommended 128 KiB turns away ordinary JSON documents. nofiles=$((bytes < 1048576 ? bytes : 1048576)) @@ -507,7 +507,17 @@ EOF cat <<'EOF' SecRule REQBODY_ERROR "!@eq 0" "id:'200002',phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2" SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:'200003',phase:2,t:none,log,deny,status:400,msg:'Multipart request body failed strict validation: PE %{REQBODY_PROCESSOR_ERROR}, BQ %{MULTIPART_BOUNDARY_QUOTED}, BW %{MULTIPART_BOUNDARY_WHITESPACE}, DB %{MULTIPART_DATA_BEFORE}, DA %{MULTIPART_DATA_AFTER}, HF %{MULTIPART_HEADER_FOLDING}, LF %{MULTIPART_LF_LINE}, SM %{MULTIPART_MISSING_SEMICOLON}, IQ %{MULTIPART_INVALID_QUOTING}, IP %{MULTIPART_INVALID_PART}, IH %{MULTIPART_INVALID_HEADER_FOLDING}, FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'" -SecRule MULTIPART_UNMATCHED_BOUNDARY "!@eq 0" "id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'" +EOF + # ModSecurity 3 (nginx) sets MULTIPART_UNMATCHED_BOUNDARY to 2 for every well-formed body that ends with its final + # boundary and to 1 only for a possible unmatched one; "!@eq 0" of ModSecurity 2 would turn away every upload, so + # libmodsecurity's own recommendation tests for 1. + if [[ ${server} == nginx ]]; then + unmatched='@eq 1' + else + unmatched='!@eq 0' + fi + printf '%s\n' "SecRule MULTIPART_UNMATCHED_BOUNDARY \"${unmatched}\" \"id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'\"" + cat <<'EOF' SecPcreMatchLimit 1000 SecPcreMatchLimitRecursion 1000 SecRule TX:/^MSC_/ "!@streq 0" "id:'200005',phase:2,t:none,log,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'" diff --git a/tests/unit/web.bats b/tests/unit/web.bats index 031023d..10ec8b1 100644 --- a/tests/unit/web.bats +++ b/tests/unit/web.bats @@ -1068,3 +1068,19 @@ EOF [ "${status}" -eq 1 ] [[ ${output} == *"nginx-dev 1.28.3-2ubuntu1.11 has no nginx sources in ${MODSEC_NGINX_DEV_SRC}."* ]] } + +@test "ModSecurity 3 lets well-formed multipart bodies through, ModSecurity 2 keeps the strict test" { + use_os debian-13 + # shellcheck source=modules/web/modsecurity.sh + source "${REPO_ROOT}/modules/web/modsecurity.sh" + LSS_MODULE=modsecurity + # shellcheck disable=SC2329 # called by the function under test + _modsecurity_json() { return 1; } + run _modsecurity_engine_conf nginx + [ "${status}" -eq 0 ] + grep -qF "SecRule MULTIPART_UNMATCHED_BOUNDARY \"@eq 1\" \"id:'200004'," <<<"${output}" + grep -qxF 'SecPcreMatchLimit 1000' <<<"${output}" + run _modsecurity_engine_conf apache + [ "${status}" -eq 0 ] + grep -qF "SecRule MULTIPART_UNMATCHED_BOUNDARY \"!@eq 0\" \"id:'200004'," <<<"${output}" +} From 0a702a1206c363a3336116f8ebaaaa2d7b09cde0 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 09:45:10 +0200 Subject: [PATCH 17/28] docs(readme): resource figures of the apps and the office and devops profiles The figures come from the RESOURCES lines of the green apps, profile and container jobs of runs 38840181620, 38849074335 and 38863689930: the largest value across the seven systems (Rocky 9 outliers aside, as for the other tables), rounded up, with the dnf peak shown apart where the RHEL family needs clearly more. The note says where the apps are measured and what "while running" means for a stack. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- README.md | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0f10f1a..d8c400b 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,19 @@ every module on each tested system. | Profile `minimal-secure` | 1 GB; 2 GB on the RHEL family | 4 GB | 3 to 5 min plus AIDE | | `minimal-secure` with `crowdsec` | 1.5 GB; 2 GB on the RHEL family | 4.5 GB | about 30 s more | | `minimal-secure` with `clamav` | 2.5 GB | 4.5 GB | about 1 min more | -| Profile `office` | not measured yet | not measured yet | not measured yet | -| Profile `devops` | not measured yet | not measured yet | not measured yet | +| Profile `office` | 1.5 GB; 2 GB on the RHEL family | 10 GB | 9 to 16 min | +| Profile `devops` | 1 GB; 1.5 GB on the RHEL family | 7 GB | 6 to 11 min | One CPU core is enough; more cores mainly shorten the first AIDE database build. The disk figure includes the swap file (`LSS_SWAP_SIZE`, 1 GB in the measurement), the journal (up to `LSS_LOGGING_MAX_USE`, 1 GB) and the audit logs (up to 10 × 32 MiB). The RHEL family needs more memory because dnf loads its repository metadata during every install, up to 0.7 GB. +The `office` and `devops` profiles are measured in the CI VMs of Debian 13 and AlmaLinux 10. Their disk figure includes +the container images of the apps. After the install, `office` keeps about 0.9 GiB resident (Nextcloud, Collabora +Online, Vaultwarden, the mail server and the web server) and `devops` about 0.8 GiB; plan more for real use, since +Collabora needs memory for every open document. + The first AIDE run reads every file under the directories it watches, so its time depends on the server: 12 to 90 seconds on a fresh system, 16 minutes on a CI machine full of preinstalled toolchains. It runs with the lowest CPU and I/O priority. @@ -70,7 +75,7 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `lynis` | 25 s | 80 MiB | 5 MiB | weekly audit | | `usbguard` | 10 s | 90 MiB | 10 MiB | usbguard-daemon | | `guard` | 5 s | 90 MiB | – | daily check | -| `docker` | not measured yet | not measured yet | not measured yet | dockerd and containerd | +| `docker` | 90 s with the web server | 660 MiB with the web server | 940 MiB with the web server | dockerd and containerd, about 260 MiB with the web server | | `podman` | not measured yet | not measured yet | not measured yet | – (no daemon) | | `portainer` | not measured yet | not measured yet | not measured yet | Portainer container, up to 512 MiB | | `k3s` | not measured yet | not measured yet | not measured yet | k3s server, about 1 GiB with Traefik | @@ -90,20 +95,24 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `mail-relay` | 35 s | 250 MiB | 180 MiB (dnf), else 135 MiB | Postfix (master, queue manager, pickup) | | `mailserver` | 90 s | 800 MiB (dnf), else 225 MiB | 235 MiB (dnf), else 120 MiB | Postfix, Dovecot and Rspamd, up to 180 MiB | | `webmail` | 135 s with nginx, php and mailserver | 390 MiB together | 265 MiB together | Roundcube in PHP-FPM; with nginx and the mail server up to 290 MiB | -| `nextcloud` | not measured yet | not measured yet | not measured yet | Nextcloud (Apache) and PostgreSQL, up to 1.5 GiB | -| `collabora` | not measured yet | not measured yet | not measured yet | Collabora Online, up to 2 GiB | -| `forgejo` | not measured yet | not measured yet | not measured yet | Forgejo, up to 1 GiB | -| `uptime-kuma` | not measured yet | not measured yet | not measured yet | Uptime Kuma, up to 512 MiB | -| `vaultwarden` | not measured yet | not measured yet | not measured yet | Vaultwarden, up to 512 MiB | -| `keycloak` | not measured yet | not measured yet | not measured yet | Keycloak and PostgreSQL, up to 2 GiB | -| `matomo` | not measured yet | not measured yet | not measured yet | Matomo (Apache) and MariaDB, up to 1.5 GiB | -| `wordpress` | not measured yet | not measured yet | not measured yet | WordPress (Apache) and MariaDB, up to 1.3 GiB | -| `n8n` | not measured yet | not measured yet | not measured yet | n8n and PostgreSQL, up to 1.5 GiB | -| `minio` | not measured yet | not measured yet | not measured yet | Silo, up to 1 GiB in a container | -| `self-updater` | not measured yet | not measured yet | not measured yet | Self-Updater, about 60 MiB | +| `nextcloud` | 295 s | 810 MiB (dnf), else 595 MiB | 2.8 GiB | Nextcloud (Apache) and PostgreSQL, about 420 MiB (limit 1.5 GiB) | +| `collabora` | 85 s | 830 MiB | 1.4 GiB | Collabora Online, about 640 MiB (limit 2 GiB) | +| `forgejo` | 35 s | 300 MiB | 195 MiB | Forgejo, about 300 MiB (limit 1 GiB) | +| `uptime-kuma` | 60 s | 285 MiB | 665 MiB | Uptime Kuma, about 170 MiB (limit 512 MiB) | +| `vaultwarden` | 35 s | 730 MiB (dnf), else 255 MiB | 380 MiB | Vaultwarden, about 125 MiB (limit 512 MiB) | +| `keycloak` | 85 s | 960 MiB | 1.0 GiB | Keycloak and PostgreSQL, about 900 MiB (limit 2 GiB) | +| `matomo` | 75 s | 450 MiB | 1.3 GiB | Matomo (Apache) and MariaDB, about 280 MiB (limit 1.5 GiB) | +| `wordpress` | 45 s | 360 MiB | 580 MiB | WordPress (Apache) and MariaDB, about 260 MiB (limit 1.3 GiB) | +| `n8n` | 200 s | 1.2 GiB (dnf), else 935 MiB | 1.8 GiB | n8n and PostgreSQL, about 680 MiB (limit 1.5 GiB) | +| `minio` | 30 s | 235 MiB | 160 MiB | Silo, about 210 MiB (limit 1 GiB in a container) | +| `self-updater` | 65 s with node | 475 MiB with node | 440 MiB with node | Self-Updater (Node.js) | Modules the tests install together are measured together. The Rocky 9 VM in CI is slow throughout (eight minutes even for `wireguard`); those outliers are left out, like one Rocky 9 container run of `mail-relay` that took six minutes. +The apps run as Docker stacks in the x86_64 VMs of all seven systems and on an Ubuntu 24.04 arm64 runner; their disk +figure includes the container images. `collabora` is measured with `nextcloud` already installed. `minio` in native +mode and `self-updater` also run in the containers of all seven systems on both architectures. For an app, "while +running" is what its stack holds after `lss verify`, and the limit is the memory limit lss sets for its containers. Web, runtime and database modules, measured the same way. Modules that only work together are tested together; their row is the cost of the whole set, including the web server or PHP it needs. From 6f4e5bd16551fe933d13664f32c91904083d0c47 Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 10:34:48 +0200 Subject: [PATCH 18/28] fix(matomo): name the installer step that gets no answer In the profile all, behind nginx with ModSecurity, the Matomo installer stopped after two seconds with "see above" and nothing above: a GET step had failed inside curl, whose error -s hides. Each step now logs its curl exit status and error text, the failure adds the latest ModSecurity rule hits, and a step may take five minutes, since creating the tables on a server with sixty modules can outlast the 30 s of app_get. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- modules/apps/matomo.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/apps/matomo.sh b/modules/apps/matomo.sh index cfeb136..e722d68 100644 --- a/modules/apps/matomo.sh +++ b/modules/apps/matomo.sh @@ -91,12 +91,14 @@ _mt_installed() { return } -# _mt_step JAR STEP [CURL OPTION...]: one step of the web installer; prints the body and the status. +# _mt_step JAR STEP [CURL OPTION...]: one step of the web installer; prints the body and the status. Creating the +# tables takes long on a busy server, so a step may take up to five minutes (the later --max-time wins in curl). _mt_step() { - local jar=$1 step=$2 + local jar=$1 step=$2 code=0 shift 2 - app_get "index.php?module=Installation&action=${step}" -b "${jar}" -c "${jar}" "$@" - return + app_get "index.php?module=Installation&action=${step}" -b "${jar}" -c "${jar}" --max-time 300 --show-error "$@" || code=$? + ((code == 0)) || log_error "matomo: the installer step ${step:-welcome} gets no answer (curl exit status ${code})." + return "${code}" } # _mt_post JAR STEP FIELD...: posts the form of STEP; fails unless Matomo moves on (redirect) or shows a page without a @@ -117,7 +119,7 @@ _mt_post() { # curl as files, so they never show up in the process list; "**********" makes the installer use the database # password from MATOMO_DATABASE_PASSWORD_FILE. _mt_install() { - local jar email site domain + local jar email site domain waf _mt_installed && return 0 domain=$(app_domain) email=$(opt admin_email) @@ -142,7 +144,8 @@ _mt_install() { _mt_post "${jar}" finished -d anonymise_ip=1 }; then rm -f -- "${jar}" - die "matomo: the installation failed; see above." + waf=$(app_waf_hits) + die "matomo: the installation failed; see above${waf:+ (ModSecurity: ${waf})}." fi rm -f -- "${jar}" _mt_installed || die "matomo: Matomo is not installed after the web installer." From c747211e15e183bac16a758472a5d0320dec82fa Mon Sep 17 00:00:00 2001 From: VectoDE Date: Fri, 16 Oct 2026 11:03:25 +0200 Subject: [PATCH 19/28] fix(firewall): run ufw again when it loses the iptables race with k3s The containers jobs with firewall and k3s fail now and then on the second install with "ERROR: initcaps" (debian-13 and ubuntu-24.04, runs 38826334577 and 38874654698): before each change ufw tests iptables with a chain of its own, and k3s rewriting its chains at the same moment makes that test fail ("CHAIN_DEL failed: No such file or directory"). A ufw function now repeats such a command up to three times, two seconds apart; every other failure and all output pass through unchanged. The function hides the binary from command -v, so the checks for an installed ufw use type -P. Refs #8 Co-Authored-By: Claude Opus 5.5 Signed-off-by: VectoDE --- lib/firewall.sh | 29 +++++++++++++++++++++++++-- modules/security/firewall.sh | 2 +- tests/unit/system.bats | 38 +++++++++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/lib/firewall.sh b/lib/firewall.sh index 37d9cd0..e15584a 100644 --- a/lib/firewall.sh +++ b/lib/firewall.sh @@ -20,6 +20,31 @@ fw_backend() { return } +# Attempts and the pause in seconds for a ufw command that loses a race in iptables. +FW_UFW_TRIES=3 +FW_UFW_DELAY=2 + +# ufw ARG...: ufw, again after a short pause when it fails with "initcaps". Before every change ufw tests the +# capabilities of iptables with a chain of its own; when k3s (or Docker) rewrites its chains at the same moment, that +# test fails ("CHAIN_DEL failed") although nothing is wrong. Other failures and the output pass through unchanged. +# The function hides the command from command -v; "type -P ufw" tells whether ufw is installed. +ufw() { + local try=1 code err + err=$(mktemp) + while :; do + code=0 + command ufw "$@" 2>"${err}" || code=$? + if ((code == 0 || try >= FW_UFW_TRIES)) || ! grep -q initcaps "${err}"; then + break + fi + try=$((try + 1)) + sleep "${FW_UFW_DELAY}" + done + cat "${err}" >&2 + rm -f -- "${err}" + return "${code}" +} + fw_vpn_network() { config_get LSS_WIREGUARD_NETWORK 10.66.0.0/24 return @@ -42,7 +67,7 @@ fw_vpn_address() { fw_is_enabled() { if [[ $(fw_backend) == ufw ]]; then - command -v ufw >/dev/null 2>&1 && ufw status 2>/dev/null | grep -q '^Status: active' + type -P ufw >/dev/null && ufw status 2>/dev/null | grep -q '^Status: active' else svc_is_active firewalld fi @@ -120,7 +145,7 @@ _fw_apply() { # The backend tool is installed (the firewall module installs it). fw_available() { if [[ $(fw_backend) == ufw ]]; then - command -v ufw >/dev/null 2>&1 + type -P ufw >/dev/null else command -v firewall-offline-cmd >/dev/null 2>&1 fi diff --git a/modules/security/firewall.sh b/modules/security/firewall.sh index ecf29ce..5ed4553 100644 --- a/modules/security/firewall.sh +++ b/modules/security/firewall.sh @@ -147,7 +147,7 @@ mod_verify() { mod_remove() { fw_rollback_cancel if [[ $(fw_backend) == ufw ]]; then - if command -v ufw >/dev/null 2>&1; then + if type -P ufw >/dev/null; then ufw --force disable >/dev/null fi svc_disable ufw diff --git a/tests/unit/system.bats b/tests/unit/system.bats index ef9c6ff..08d463c 100644 --- a/tests/unit/system.bats +++ b/tests/unit/system.bats @@ -274,7 +274,7 @@ time.sleep(30)' >"${port_file}" 2>/dev/null 3>&- & @test "without a firewall tool rules are only recorded and applied later" { use_os debian-12 PATH=$(printf '%s' "${PATH}" | tr ':' '\n' | grep -v fixtures/bin | paste -sd:) - command -v ufw >/dev/null && skip "ufw is installed on this machine" + type -P ufw >/dev/null && skip "ufw is installed on this machine" LSS_MODULE=web fw_allow 443/tcp [ "$(cat "${LSS_STATE_DIR}/firewall/web")" = 443/tcp ] @@ -804,3 +804,39 @@ EOF grep -qxF 'Suites: trixie/' "${BATS_TEST_TMPDIR}/writes" run ! grep -q '^Components:' "${BATS_TEST_TMPDIR}/writes" } + +@test "ufw runs again when it loses the iptables race (initcaps) and passes other failures through" { + local bin=${BATS_TEST_TMPDIR}/racebin + mkdir -p "${bin}" + FW_UFW_DELAY=0 + printf '1\n' >"${BATS_TEST_TMPDIR}/fails" + cat >"${bin}/ufw" <