diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1d65e36..82378da 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] - group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network, mail] + group: [base, security, scanners, web, edge, runtime, database, database-alt, container, backup, backup-alt, monitoring, network, mail] arch: [x64, arm64] include: - group: base @@ -43,6 +43,15 @@ jobs: # Each web server alone, then with certificates (Pebble) and sites; then HAProxy. - group: web modules: nginx,tls,vhost apache,tls,vhost caddy,tls,vhost haproxy nginx,modsecurity apache,modsecurity + # Traefik in front: alone with its own ACME client (Pebble), then before each web server in backend mode with + # sites, an admin tool, fail2ban or CrowdSec, and every WAF mode. LSS_KEY=VALUE applies to its set only. + - group: edge + modules: >- + traefik,LSS_TRAEFIK_ACME=true + nginx,tls,vhost,php,adminer,fail2ban,traefik,LSS_TRAEFIK_ADMIN_HOST=admin.example.com + apache,tls,vhost,modsecurity,traefik,LSS_TRAEFIK_WAF=both + traefik,caddy,php,adminer,crowdsec,LSS_TRAEFIK_WAF=off + apache,modsecurity,traefik,LSS_TRAEFIK_WAF=modsecurity - group: runtime modules: php,composer nginx,php node python ruby java go dotnet # Conflicting engines (mariadb/mysql, redis/valkey) run in separate jobs: a removal keeps their data. @@ -173,7 +182,7 @@ jobs: if [[ $RUN == kernel ]]; then tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard wireguard elif [[ $RUN == containers ]]; then - tests/integration/run.sh "$SYSTEM" firewall,docker,portainer firewall,k3s + tests/integration/run.sh "$SYSTEM" firewall,docker,portainer firewall,docker,traefik,LSS_TRAEFIK_DOCKER=true firewall,k3s else case $SYSTEM:$RUN in vm-debian-*:"profile minimal-secure") minimum=82 ;; diff --git a/CHANGELOG.md b/CHANGELOG.md index e5aa3d0..1cade12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,13 @@ now or later. (NodeSource), `python` (venv, pip, pipx), `ruby` (Bundler), `java` (OpenJDK, Temurin where missing), `go` (official archive) and `dotnet` (.NET SDK, telemetry off), all from repositories with pinned keys or checksum-verified downloads. +- `traefik`: Traefik as an edge proxy on ports 80, 443 and 443/udp (HTTP/3) with security headers, HSTS, a JSON + access log, fail2ban or CrowdSec, and the OWASP Coraza WAF (or ModSecurity behind it). nginx, Apache or Caddy move + to 127.0.0.1 behind it and keep their sites, PHP and admin tools. Applications get routes through + `web_proxy_route`, which falls back to a reverse-proxy site without Traefik; the Docker provider is opt-in and goes + through a read-only socket proxy. On amd64 Coraza waits for a Traefik release with wazero 1.9 (wazero 1.8.0 crashes + Traefik there); until then lss installs `modsecurity` for nginx or Apache and routes the applications through the + web server, so every request still passes a WAF. - `modsecurity`: the ModSecurity web application firewall with the OWASP Core Rule Set for nginx and Apache on every supported system, with a module built from verified sources where the distribution has none. - Database modules: `mariadb`, `mysql`, `postgresql`, `redis`, `valkey` and `mongodb`, on localhost with hardened diff --git a/README.md b/README.md index 9c3ca81..ea6bd03 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `lynis` | 25 s | 80 MiB | 5 MiB | weekly audit | | `usbguard` | 10 s | 90 MiB | 10 MiB | usbguard-daemon | | `guard` | 5 s | 90 MiB | – | daily check | +| `traefik` | 35 s | 300 MiB; 1.8 GiB with Coraza (arm64) | 330 MiB | Traefik, 100 MiB; 1.75 GiB with Coraza | | `docker` | not measured yet | not measured yet | not measured yet | dockerd and containerd | | `podman` | not measured yet | not measured yet | not measured yet | – (no daemon) | | `portainer` | not measured yet | not measured yet | not measured yet | Portainer container, up to 512 MiB | @@ -198,7 +199,8 @@ change, run `lss configure MODULE`. `usbguard`, `guard`. **Web:** `nginx`, `apache` or `caddy`, `tls` (certificates from Let's Encrypt), `vhost` (sites), `haproxy`, -`modsecurity` (web application firewall with the OWASP Core Rule Set for nginx or Apache). +`modsecurity` (web application firewall with the OWASP Core Rule Set for nginx or Apache), `traefik` (edge proxy in +front of the web server and the applications). See [Traefik](#traefik). **Runtimes:** `php` (PHP-FPM with a hardened pool for the web server), `composer`, `node` (Node.js 22, 24 or 26 from NodeSource), `python` (venv, pip, pipx), `ruby` (Bundler), `java` (OpenJDK 21, Temurin where the distribution has @@ -232,8 +234,9 @@ Dovecot and Rspamd for your domains), `webmail` (Roundcube on the web server). S ### Web sites -One web server owns ports 80 and 443. It answers only for the sites you set up; requests for any other host name get -no answer, and port 80 serves nothing but ACME challenges and redirects to HTTPS. A site is one entry in +One web server owns ports 80 and 443, unless `traefik` sits in front of it (see [Traefik](#traefik)). It answers +only for the sites you set up; requests for any other host name get no answer, and port 80 serves nothing but ACME +challenges and redirects to HTTPS. A site is one entry in `LSS_VHOST_SITES`: a name, `static`, `php` or `proxy`, its host names and, if needed, a document root or backend URL. With the `tls` module each site gets its own certificate from Let's Encrypt; until it arrives, and without `tls`, the site uses a self-signed certificate. The names must point to the server and port 80 must be reachable from the @@ -244,6 +247,47 @@ Admin interfaces such as phpMyAdmin listen on `127.0.0.1:8090`. Reach them throu `LSS__ADMIN_PUBLIC=true` to publish port 8090 with TLS and the password from `lss secrets show web-admin-password`. +### Traefik + +`traefik` puts Traefik in front of everything that speaks HTTP. It takes ports 80/tcp, 443/tcp and 443/udp (HTTP/3), +redirects port 80 to HTTPS, adds the same security headers and HSTS as the web servers, writes a JSON access log +and filters every request through the OWASP Coraza WAF with the Core Rule Set (on arm64; on amd64 see below). The +web server stays: nginx, Apache or Caddy moves to `127.0.0.1:8008` and `127.0.0.1:8443` and keeps serving the sites +of `vhost`, PHP and the admin tools. It takes the client address from Traefik's `X-Forwarded-For`, so its logs, fail2ban and ModSecurity still see +the client. Removing `traefik` gives ports 80 and 443 back to the web server. + +On amd64 Coraza stays switched off for now: every current Traefik release ships wazero 1.8.0, which crashes Traefik at +random on amd64 with Go 1.24 or later (fixed in wazero 1.9.0). arm64 is unaffected. With `LSS_TRAEFIK_WAF=coraza` +(the default) or `both`, lss puts ModSecurity in Coraza's place on amd64: + +- with nginx or Apache, lss installs the `modsecurity` module itself (once; removing `traefik` removes it again, unless + you installed it yourself) and sends the routes of applications through the web server too, so every request passes + the WAF: Traefik, then nginx or Apache with ModSecurity, then the application on 127.0.0.1. Run `lss configure traefik` + after installing nginx or Apache later, so it adds ModSecurity then. +- Caddy has no ModSecurity connector. Requests reach Caddy and the applications unfiltered, and lss says so on every + run; put nginx or Apache in Caddy's place for a WAF. +- Containers of the Docker provider are routed to directly, without a WAF in front of them. + +Once a Traefik release carries wazero 1.9, lss switches back to Coraza in Traefik and direct routes on amd64. +`LSS_TRAEFIK_WAF=modsecurity`, `both` on arm64 and `off` keep their meaning. + +Applications get a host name of their own through a route: `LSS_TRAEFIK_ROUTES=app:app.example.com:127.0.0.1:3000`, +or from a module with `web_proxy_route NAME DOMAIN 127.0.0.1:PORT`. Without Traefik the same route becomes a +reverse-proxy site of the web server. Certificates come from the `tls` module, or from Traefik itself with +`LSS_TRAEFIK_ACME=true` (TLS-ALPN challenge on port 443). + +| Setting | Default | Meaning | +| ---------------------------- | -------- | --------------------------------------------------------------------------------------- | +| `LSS_TRAEFIK_WAF` | `coraza` | `coraza` in Traefik (arm64), `modsecurity` in the web server behind it, `both`, or `off` | +| `LSS_TRAEFIK_PARANOIA` | `1` | Paranoia level of the Core Rule Set in Coraza (1 to 4) | +| `LSS_TRAEFIK_ADMIN_HOST` | empty | Publishes the admin port (phpMyAdmin, Adminer, pgAdmin) under this name with a password | +| `LSS_TRAEFIK_DOCKER` | `false` | Routes containers labelled `traefik.enable=true`, through a read-only socket proxy | +| `LSS_TRAEFIK_DASHBOARD_PORT` | `8099` | Dashboard and read-only API on 127.0.0.1 only (`ssh -L 8099:127.0.0.1:8099 server`) | + +The Docker provider never sees the engine's socket. A small proxy container +([wollomatic/socket-proxy](https://github.com/wollomatic/socket-proxy), pinned by digest) holds it, runs without a +network and without capabilities, and answers only the calls Traefik needs to list and inspect containers. + ### Containers `docker` installs Docker CE from Docker's repository (Debian 12 and 13, Ubuntu 22.04, 24.04 and 26.04, the RHEL 9 and diff --git a/data/checksums.txt b/data/checksums.txt index 5c5f5be..5cdf159 100644 --- a/data/checksums.txt +++ b/data/checksums.txt @@ -106,3 +106,11 @@ alertmanager-0.34.1 x86_64 265b9d1e55ef0d5306a436018af6d2b686c2ce051f03d968f7464 alertmanager-0.34.1 aarch64 d98d6cbaf52151c7e76e24355fec88b11cebcb9875d4cdd8b76ddce7a7e5535c https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-arm64.tar.gz node_exporter-1.12.1 x86_64 b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz node_exporter-1.12.1 aarch64 ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-arm64.tar.gz + +# Traefik 3.7.15: SHA-256 from traefik_v3.7.15_checksums.txt of the release, which matches the digests GitHub +# publishes for the assets (gh api repos/traefik/traefik/releases/tags/v3.7.15, checked 2026-10-15). +traefik-3.7.15 x86_64 5b9f51199fcb7a3b5f7ae0f9dae8dcf01efadafd92c5fca96ea2822eca81f8e6 https://github.com/traefik/traefik/releases/download/v3.7.15/traefik_v3.7.15_linux_amd64.tar.gz +traefik-3.7.15 aarch64 1866d8e831b95d14b599bb59eb20d2033b979c0cb1e347b48a0866a9b810454e https://github.com/traefik/traefik/releases/download/v3.7.15/traefik_v3.7.15_linux_arm64.tar.gz +# Coraza http-wasm plugin for Traefik 0.3.0 (Coraza with the OWASP Core Rule Set 4.0.0 compiled in). The release of +# 2024 has no published digest; the SHA-256 is that of the asset downloaded on 2026-10-15, which pins it from then on. +coraza-http-wasm-0.3.0 any b96680b46e61287faae243e360db5de34e4f315e2a8ee105fedddc176e79b1e7 https://github.com/jcchavezs/coraza-http-wasm-traefik/releases/download/v0.3.0/coraza-http-wasm-v0.3.0.zip diff --git a/docs/hardening.md b/docs/hardening.md index e7c61e2..e6be9bb 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -39,6 +39,7 @@ Docker, databases and its own sudo rules) 78. CI checks that it stays at 82 on D | `phpmyadmin` | Checksum-verified release without the setup wizard; configuration readable by root and PHP-FPM only, with a generated cookie secret; only the local database server, no root and no empty-password logins; no version check and no error reports; only on the admin port | (operations) | | `adminer` | Checksum-verified release; a login plugin allows only the configured local servers and drivers; only on the admin port | (operations) | | `pgadmin` | Pinned, checksum-verified wheels in an isolated environment; own system user; systemd sandbox without capabilities; bound to 127.0.0.1; no upgrade check, no psql tool, no browsing of binary paths; secure cookies when the admin port is public | (operations) | +| `traefik` | Release binary pinned and checked against `data/checksums.txt` (amd64 and arm64); own system user, systemd sandbox with `CAP_NET_BIND_SERVICE` as the only capability; ports 80, 443 and 443/udp with a redirect to HTTPS, TLS 1.2 and 1.3 with the cipher suites of the web servers, the same security headers and HSTS, no `Server` header; the OWASP Coraza WAF with the Core Rule Set from a pinned, checksum-verified local plugin (never the plugin catalog); dashboard and API on 127.0.0.1 only; JSON access log read by fail2ban (failed logins, WAF blocks) or CrowdSec; the web server behind it on 127.0.0.1 only; the Docker provider (opt-in) through a read-only socket proxy, never the engine socket; configuration readable by root and Traefik only, no secrets in it (the admin password only as a bcrypt hash in a file of its own) | OWASP CRS, Mozilla TLS guidelines, OWASP Secure Headers, CIS Docker 5 (socket) | | `modsecurity` | Requests filtered by ModSecurity with the OWASP Core Rule Set (anomaly scoring, paranoia level 1 by default) and blocked with 403; matches go to an audit log without request bodies | OWASP CRS | | `mariadb` | Bound to 127.0.0.1, `local-infile=0`, root only through the Unix socket, no anonymous accounts and no test database, one least-privilege user per application, systemd sandbox | CIS MariaDB | | `mysql` | Bound to 127.0.0.1 (the X protocol too), `local-infile=0`, root through `auth_socket` (RHEL: a generated password in `/root/.my.cnf` 0600); root never exists without a password; systemd sandbox | CIS MySQL | @@ -76,6 +77,9 @@ These settings differ from a strict reading of the benchmarks. Each one has a se | Docker user namespace remapping | on | Images that expect to own host paths as root need it off. | `LSS_DOCKER_USERNS_REMAP=false` | | Docker and k3s together | not possible | Each brings its own containerd and claims the forwarding rules; `profiles/all.conf` has Docker, `profiles/k8s-node.conf` has k3s. | – | | Kubernetes API | 127.0.0.1 | kubectl through an SSH tunnel; a cluster with more nodes or remote tooling needs `vpn` or `public`. | `LSS_K3S_API_ACCESS` | +| Certificate check from Traefik to the web server | off, on 127.0.0.1 only | Traefik reaches the web server on `127.0.0.1:8443`, which presents the site's certificate, not one for 127.0.0.1. The connection never leaves the loopback interface. | – | +| Apache SNI policy behind Traefik | `SSLVHostSNIPolicy authonly` | Traefik reaches Apache on 127.0.0.1 without SNI, so the TLS handshake runs in the default site and the Host header picks the site. Apache 2.4.65 and later refuse that switch with 421 when the TLS settings of the two sites differ. `authonly` still refuses it when client authentication differs, and lss sets up none. It applies only in backend mode. | – | +| Coraza plugin | default on arm64; ModSecurity in its place on amd64 | The WebAssembly build of Coraza is young (its authors call it early stage), so `lss verify` sends an SQL injection and a path traversal through it and expects 403 for both. On amd64 Traefik 3.7.15 crashes with it at random: its wazero 1.8.0 does not save a register that `runtime.memmove` of Go 1.24 and later overwrites (fixed in wazero 1.9.0, wazero/wazero#2378). There lss installs `modsecurity` for nginx or Apache, routes the applications through the web server and verifies a 403 for every vhost site and every route through Traefik. Caddy has no ModSecurity connector, so behind Caddy and for Docker-provider containers nothing filters on amd64; lss warns on every run. | `LSS_TRAEFIK_WAF=modsecurity` | | Traefik in k3s | on | The bundled ingress controller serves ports 80 and 443 for the cluster; without it you bring your own. | `LSS_K3S_TRAEFIK=false` | ## What lss does not do diff --git a/lib/container.sh b/lib/container.sh index b6cc622..68ac1e2 100644 --- a/lib/container.sh +++ b/lib/container.sh @@ -16,8 +16,9 @@ # - the socket of a container engine is never mounted, except into the management tool (CTR_SOCKET_CONTAINERS) # Containers that may mount the socket of the engine. Whoever controls that socket controls the host; Portainer is a -# management tool and needs it, nothing else does. -CTR_SOCKET_CONTAINERS="lss-portainer" +# management tool and needs it, and the socket proxy of the traefik module passes only read-only container calls +# on to Traefik (its Docker provider). Nothing else gets it. +CTR_SOCKET_CONTAINERS="lss-portainer lss-traefik-docker" # Capabilities no container of lss gets, whatever a module asks for: each one hands over the kernel or other processes. CTR_DENIED_CAPS="SYS_ADMIN SYS_MODULE SYS_RAWIO SYS_PTRACE SYS_BOOT SYS_TIME DAC_READ_SEARCH MAC_ADMIN MAC_OVERRIDE BPF PERFMON NET_ADMIN" CTR_DEFAULT_MEMORY=512m diff --git a/lib/step.sh b/lib/step.sh index 757da7a..51efc5b 100755 --- a/lib/step.sh +++ b/lib/step.sh @@ -15,7 +15,7 @@ set -Eeuo pipefail # change notes it sets reach the caller. shopt -s inherit_errexit lastpipe -for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring module; do +for library in log os config files template secrets state pkg repo fetch service firewall container web traefik mail monitoring module; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/lib/traefik.sh b/lib/traefik.sh new file mode 100644 index 0000000..b1019b4 --- /dev/null +++ b/lib/traefik.sh @@ -0,0 +1,353 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Traefik as the edge proxy (module traefik). Traefik owns ports 80, 443 and 443/udp; the web server, when there is +# one, runs behind it in backend mode (lib/web.sh, web_backend) and Traefik sends every host name without a route of +# its own there. Routes of applications come from web_proxy_route (lib/web.sh). +# +# /etc/traefik/traefik.yml static configuration (module traefik) +# /etc/traefik/dynamic/lss.yml middlewares (headers, WAF, redirect) and the probe route (module traefik) +# /etc/traefik/dynamic/tls.yml certificates of WEB_TLS_DIR and the TLS settings (derived, see below) +# /etc/traefik/dynamic/backend.yml routes to the web server in backend mode (derived) +# /etc/traefik/dynamic/route-NAME.yml application routes of web_proxy_route (derived) +# +# The derived files follow the state of other modules (certificates, the web server, recorded routes) and are written +# by whichever module changes that state, so they carry no checksum of one module: lss verify would report every new +# site as a change made outside lss. Traefik watches the directory and loads a change without a restart. + +# shellcheck disable=SC2034 # read by the traefik module +TRAEFIK_DIR=/etc/traefik +TRAEFIK_DYNAMIC=/etc/traefik/dynamic +TRAEFIK_USER=lss-traefik +# shellcheck disable=SC2034 # read by the traefik module +TRAEFIK_UNIT=lss-traefik +TRAEFIK_LOG_DIR=/var/log/traefik +TRAEFIK_FAIL2BAN_JAIL=/etc/fail2ban/jail.d/lss-traefik.local +TRAEFIK_FAIL2BAN_FILTER_AUTH=/etc/fail2ban/filter.d/lss-traefik-auth.conf +TRAEFIK_FAIL2BAN_FILTER_WAF=/etc/fail2ban/filter.d/lss-traefik-waf.conf +TRAEFIK_CROWDSEC_ACQUIS=/etc/crowdsec/acquis.d/lss-traefik.yaml +# TLS 1.2 suites of WEB_TLS_CIPHERS in Go's names; TLS 1.3 has no settings in Go. +TRAEFIK_CIPHERS="TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" + +# traefik_present: Traefik is installed or being installed (a module's state is recorded only after its install). +traefik_present() { + [[ ${LSS_MODULE:-} == traefik ]] || state_installed traefik + return +} + +# traefik_active: routes go to Traefik. False while the traefik module hands everything back to the web server +# (LSS_WEB_BACKEND=off during its removal). +traefik_active() { + [[ ${LSS_WEB_BACKEND:-} != off ]] || return 1 + traefik_present + return +} + +# Coraza runs in Traefik's WebAssembly runtime (wazero). Every Traefik release up to 3.7.15 and 3.6.25 ships wazero +# 1.8.0, whose amd64 compiler calls runtime.memmove without saving the DX register that Go 1.24 and later clobber +# there (fixed in wazero 1.9.0, wazero/wazero#2378); Traefik then dies at random with "unsafe.Slice: len out of +# range" or "split stack overflow" below memmove. On amd64 Coraza waits for a Traefik release with wazero 1.9 or +# later, and ModSecurity in the web server takes its place (traefik_waf_backend); arm64 is unaffected. Set to true +# together with the Traefik version that carries the fix: Coraza and direct routes then return on amd64. +TRAEFIK_CORAZA_AMD64=false + +# traefik_coraza_runs: Traefik can load the Coraza plugin on this architecture. +traefik_coraza_runs() { + case ${OS_ARCH:-$(uname -m)} in + aarch64 | arm64) return 0 ;; + *) ;; + esac + [[ ${TRAEFIK_CORAZA_AMD64} == true ]] + return +} + +# traefik_coraza_wanted: LSS_TRAEFIK_WAF asks for Coraza (coraza, the default, or both). +traefik_coraza_wanted() { + case $(config_option traefik waf coraza) in + coraza | both) return 0 ;; + *) return 1 ;; + esac +} + +# traefik_waf_fallback: Coraza is asked for and cannot run here. +traefik_waf_fallback() { + traefik_coraza_wanted && ! traefik_coraza_runs + return +} + +# traefik_waf_backend: the routes of applications pass through the web server, whose ModSecurity stands in for Coraza +# (Traefik -> nginx or Apache with ModSecurity -> application). Caddy has no ModSecurity, so routes stay direct there. +# TRAEFIK_BACKEND_GONE is set while the web server is being removed. +traefik_waf_backend() { + [[ -z ${TRAEFIK_BACKEND_GONE:-} ]] || return 1 + if ! traefik_active || ! traefik_waf_fallback; then + return 1 + fi + case $(web_server) in + nginx | apache) return 0 ;; + *) return 1 ;; + esac +} + +# _traefik_ready: Traefik's dynamic directory exists, so derived files can be written. +_traefik_ready() { + traefik_active && [[ -d ${TRAEFIK_DYNAMIC} ]] + return +} + +# _traefik_write DEST: a derived file of the dynamic configuration from stdin (mode 0640, group TRAEFIK_USER), +# replaced only when it differs. Sets LSS_FILE_CHANGED. +_traefik_write() { + local dest=$1 tmp + LSS_FILE_CHANGED=0 + tmp=$(mktemp "${dest}.lss.XXXXXX") + cat >"${tmp}" + if [[ -f ${dest} ]] && cmp -s "${tmp}" "${dest}" && + [[ $(stat -c '%a %U:%G' "${dest}") == "640 root:${TRAEFIK_USER}" ]]; then + rm -f -- "${tmp}" + return 0 + fi + chmod 0640 "${tmp}" + chown "root:${TRAEFIK_USER}" "${tmp}" + mv -f -- "${tmp}" "${dest}" + if command -v restorecon >/dev/null 2>&1; then + restorecon -F "${dest}" 2>/dev/null || true + fi + # shellcheck disable=SC2034 # read by callers + LSS_FILE_CHANGED=1 + files_note_change "wrote ${dest}" + return +} + +# _traefik_drop FILE: removes a derived file. +_traefik_drop() { + local file=$1 + [[ -e ${file} ]] || return 0 + rm -f -- "${file}" + files_note_change "removed ${file}" + return +} + +# traefik_tls_conf: the content of tls.yml: every certificate below WEB_TLS_DIR (Traefik picks one by the SNI name), the +# default certificate for every other name, and the TLS settings of the web servers (TLS 1.2 and 1.3, the cipher suites +# of WEB_TLS_CIPHERS). +traefik_tls_conf() { + local dir cipher + printf '# Managed by lss (Traefik). The certificates of %s; written again when one is added or removed.\ntls:\n' \ + "${WEB_TLS_DIR}" + printf ' certificates:\n' + for dir in "${WEB_TLS_DIR}"/*/; do + dir=${dir%/} + [[ -s ${dir}/fullchain.pem && -s ${dir}/privkey.pem ]] || continue + printf ' - certFile: %s/fullchain.pem\n keyFile: %s/privkey.pem\n' "${dir}" "${dir}" + done + printf ' stores:\n default:\n defaultCertificate:\n' + printf ' certFile: %s/default/fullchain.pem\n keyFile: %s/default/privkey.pem\n' "${WEB_TLS_DIR}" \ + "${WEB_TLS_DIR}" + printf ' options:\n default:\n minVersion: VersionTLS12\n cipherSuites:\n' + for cipher in ${TRAEFIK_CIPHERS}; do + printf ' - %s\n' "${cipher}" + done + return +} + +# traefik_tls_sync: writes tls.yml when Traefik is in front. +traefik_tls_sync() { + _traefik_ready || return 0 + traefik_tls_conf | _traefik_write "${TRAEFIK_DYNAMIC}/tls.yml" + return +} + +# traefik_backend_conf: the content of backend.yml: every host name without a route of its own goes to the web +# server's HTTPS port, ACME challenges on port 80 to its HTTP port (the tls module's webroot challenges). Both stay on +# the loopback interface; the backend presents the certificate of the site, which is not issued for 127.0.0.1, so +# Traefik does not check it on this hop (insecureSkipVerify concerns 127.0.0.1 only). +traefik_backend_conf() { + cat <".*"DownstreamStatus":401[,}] +ignoreregex = +EOF + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + file_write 0644 root:root "${TRAEFIK_FAIL2BAN_FILTER_WAF}" <<'EOF' +# Managed by lss (Traefik). A 403 answer is a request the WAF (or the application) refused. +[Definition] +failregex = ^\{.*"ClientHost":"".*"DownstreamStatus":403[,}] +ignoreregex = +EOF + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + file_write 0644 root:root "${TRAEFIK_FAIL2BAN_JAIL}" </dev/null 2>&1 || die "traefik: the fail2ban jails for Traefik do not load." + svc_restart fail2ban + fi + return +} + +# _traefik_crowdsec LOG: the CrowdSec collection for Traefik and its access log. +_traefik_crowdsec() { + local log=$1 changed=0 + if ! cscli collections list -o raw 2>/dev/null | cut -d ',' -f 1 | grep -qxF crowdsecurity/traefik; then + cscli collections install crowdsecurity/traefik >/dev/null 2>&1 || + die "traefik: CrowdSec could not install crowdsecurity/traefik." + files_note_change "crowdsec collection crowdsecurity/traefik" + changed=1 + fi + printf '# Managed by lss (Traefik).\nfilenames:\n - %s\nlabels:\n type: traefik\n' "${log}" | + file_write 0644 root:root "${TRAEFIK_CROWDSEC_ACQUIS}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + if [[ ${changed} == 1 ]] && svc_is_active crowdsec; then + svc_restart crowdsec + fi + return +} + +# traefik_ids_remove: drops Traefik's jails and log acquisition. +traefik_ids_remove() { + if [[ -e ${TRAEFIK_FAIL2BAN_JAIL} ]]; then + file_remove "${TRAEFIK_FAIL2BAN_JAIL}" + file_remove "${TRAEFIK_FAIL2BAN_FILTER_AUTH}" + file_remove "${TRAEFIK_FAIL2BAN_FILTER_WAF}" + if svc_is_active fail2ban; then + svc_restart fail2ban + fi + fi + if [[ -e ${TRAEFIK_CROWDSEC_ACQUIS} ]]; then + file_remove "${TRAEFIK_CROWDSEC_ACQUIS}" + if svc_is_active crowdsec; then + svc_restart crowdsec + fi + fi + return +} diff --git a/lib/web.sh b/lib/web.sh index 50c6476..fa0c0e2 100644 --- a/lib/web.sh +++ b/lib/web.sh @@ -4,7 +4,7 @@ # Web layer shared by the web server modules (nginx, apache, caddy) and everything that publishes something through # them (tls, vhost, php, phpmyadmin, adminer, pgadmin, apps). One web server owns ports 80 and 443; the others -# conflict with it. +# conflict with it. With Traefik in front (lib/traefik.sh) it moves to 127.0.0.1 (web_backend) and Traefik owns them. # # Conventions: # WEB_GROUP group of every web server user; PHP-FPM's socket and the TLS keys are readable by it @@ -27,6 +27,10 @@ WEB_ACME_ROOT=/var/www/lss-acme WEB_DEFAULT_ROOT=/var/www/lss-default # shellcheck disable=SC2034 # read by the web server and admin interface modules WEB_ADMIN_PORT=8090 +# Behind Traefik (backend mode) the web server listens on these ports of 127.0.0.1 only, as Traefik owns 80 and 443. +# Both carry the SELinux type http_port_t, so every web server may bind them without a port label of their own. +WEB_BACKEND_HTTP_PORT=8008 +WEB_BACKEND_HTTPS_PORT=8443 WEB_ADMIN_HTPASSWD=/etc/lss/web/admin.htpasswd WEB_ADMIN_CADDY_AUTH=/etc/lss/web/admin-auth.caddy # Mozilla "intermediate" (TLS 1.2 and 1.3) without the DHE suites, which would need a DH parameter file. @@ -54,6 +58,30 @@ web_server() { return 0 } +# web_backend: the web server runs behind Traefik (backend mode): on 127.0.0.1 only, on WEB_BACKEND_HTTP_PORT and +# WEB_BACKEND_HTTPS_PORT. The traefik module sets LSS_WEB_BACKEND=on or off while it switches the web server; at any +# other time an installed Traefik decides. +web_backend() { + case ${LSS_WEB_BACKEND:-} in + on) return 0 ;; + off) return 1 ;; + *) ;; + esac + state_installed traefik + return +} + +# web_http_port / web_https_port: where the web server serves HTTP and HTTPS in its current mode. +web_http_port() { + if web_backend; then printf '%s' "${WEB_BACKEND_HTTP_PORT}"; else printf '80'; fi + return +} + +web_https_port() { + if web_backend; then printf '%s' "${WEB_BACKEND_HTTPS_PORT}"; else printf '443'; fi + return +} + # web_unit SERVER: systemd unit of a web server. web_unit() { local server=$1 @@ -261,10 +289,13 @@ web_reload() { if svc_is_active "$(web_unit "${server}")"; then svc_reload "$(web_unit "${server}")" fi + # A new site brings a certificate directory, which Traefik has to load as well. + traefik_tls_sync return } -# web_headers SERVER: the security headers every HTTPS site sends. +# web_headers SERVER: the security headers every HTTPS site sends (SERVER traefik: the customResponseHeaders entries +# of Traefik's headers middleware). web_headers() { local server=$1 header local -a headers=( @@ -279,6 +310,7 @@ web_headers() { case ${server} in nginx) printf '\tadd_header %s always;\n' "${header}" ;; apache) printf '\tHeader always set %s\n' "${header}" ;; + traefik) printf ' %s: %s\n' "${header%% *}" "${header#* }" ;; *) printf '\t\t%s\n' "${header}" ;; esac done @@ -310,6 +342,25 @@ _web_listen_nginx() { return } +# web_nginx_listen http|https [FLAGS]: the listen lines of the HTTP or HTTPS port in the current mode: every address +# on 80 and 443, or only 127.0.0.1 on the backend ports behind Traefik. The nginx module writes them to +# listen-http.conf and listen-https.conf, which every site includes, so a switch of the mode leaves the sites alone. +web_nginx_listen() { + local kind=$1 flags=${2:-} port + if [[ ${kind} == https ]]; then + flags="${flags:+${flags} }$(web_nginx_ssl_flags)" + port=$(web_https_port) + else + port=$(web_http_port) + fi + if web_backend; then + printf '\tlisten 127.0.0.1:%s%s;\n' "${port}" "${flags:+ ${flags}}" + else + _web_listen_nginx "${port}" "${flags}" + fi + return +} + # _web_tls_nginx CERTDIR: certificate lines of an nginx server block. _web_tls_nginx() { local dir=$1 @@ -373,18 +424,15 @@ EOF _web_site_nginx() { local name=$1 kind=$2 target=$3 shift 3 - local domains="$*" cert + local domains="$*" cert dir cert=$(web_cert_dir "${name}") - printf '# Managed by lss (site %s).\nserver {\n' "${name}" - _web_listen_nginx 80 + dir=$(web_conf_dir nginx) + printf '# Managed by lss (site %s).\nserver {\n\tinclude %s/listen-http.conf;\n' "${name}" "${dir}" printf '\tserver_name %s;\n' "${domains}" printf '\tlocation /.well-known/acme-challenge/ {\n\t\troot %s;\n\t}\n' "${WEB_ACME_ROOT}" # shellcheck disable=SC2016 # $host and $request_uri are nginx variables printf '\tlocation / {\n\t\treturn 301 https://$host$request_uri;\n\t}\n}\n\nserver {\n' - _web_listen_nginx 443 "$(web_nginx_ssl_flags)" - if web_nginx_http2_directive; then - printf '\thttp2 on;\n' - fi + printf '\tinclude %s/listen-https.conf;\n' "${dir}" printf '\tserver_name %s;\n' "${domains}" _web_tls_nginx "${cert}" web_headers nginx @@ -480,7 +528,7 @@ _web_site_apache() { cert=$(web_cert_dir "${name}") cat < + ServerName ${first} ${aliases:+ ServerAlias ${aliases} } Alias /.well-known/acme-challenge/ ${WEB_ACME_ROOT}/.well-known/acme-challenge/ @@ -489,7 +537,7 @@ ${aliases:+ ServerAlias ${aliases} RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [R=301,L] - + ServerName ${first} ${aliases:+ ServerAlias ${aliases} } Protocols h2 http/1.1 @@ -501,7 +549,7 @@ EOF printf '\tSSLUseStapling on\n' fi web_headers apache - printf '\tErrorLog %s\n\tCustomLog %s combined\n' "$(web_log_file apache error)" "$(web_log_file apache access)" + printf '\tErrorLog %s\n\tCustomLog %s lss_combined\n' "$(web_log_file apache error)" "$(web_log_file apache access)" _web_body_apache "${kind}" "${target}" printf '\n' return @@ -554,9 +602,11 @@ _web_site_caddy() { http_names+="http://${domain} " https_names+="https://${domain} " done - printf '# Managed by lss (site %s).\n%s{\n' "${name}" "${http_names}" + # lss_bind (Caddyfile of the caddy module) keeps the listeners on 127.0.0.1 behind Traefik; http_port and + # https_port there move them to the backend ports. + printf '# Managed by lss (site %s).\n%s{\n\timport lss_bind\n' "${name}" "${http_names}" printf '\thandle /.well-known/acme-challenge/* {\n\t\troot * %s\n\t\tfile_server\n\t}\n' "${WEB_ACME_ROOT}" - printf '\thandle {\n\t\tredir https://{host}{uri} permanent\n\t}\n}\n\n%s{\n' "${https_names}" + printf '\thandle {\n\t\tredir https://{host}{uri} permanent\n\t}\n}\n\n%s{\n\timport lss_bind\n' "${https_names}" printf '\ttls %s/fullchain.pem %s/privkey.pem\n\timport lss_tls_headers\n\timport lss_hidden\n\tlog\n' \ "${cert}" "${cert}" case ${kind} in @@ -648,6 +698,137 @@ web_site_remove() { return } +# --- Routes of applications ------------------------------------------------------------------------------------- +# +# web_proxy_route NAME DOMAIN UPSTREAM is how a module publishes an application under a host name. DOMAIN is one host +# name or several separated by commas; UPSTREAM is HOST:PORT (normally 127.0.0.1:PORT) or an http(s):// URL. The route +# is recorded below LSS_STATE_DIR/web-routes and applied for whatever is in front: +# Traefik installed a file-provider route (/etc/traefik/dynamic/route-NAME.yml) with the security headers and +# the WAF of the HTTPS entry point +# Traefik, Coraza out a Traefik route to the web server and its reverse-proxy site, so the web server's ModSecurity +# filters the route where Coraza cannot run (traefik_waf_backend in lib/traefik.sh) +# otherwise a reverse-proxy site of the installed web server (web_site_write NAME proxy) +# Either way the route gets a certificate: the placeholder at first, then one from the tls module when it is +# installed (or Traefik's own ACME client, LSS_TRAEFIK_ACME). When Traefik is installed or removed later, it applies +# every recorded route again in the new place. web_proxy_route_remove NAME takes a route away. + +_web_routes_dir() { + printf '%s/web-routes' "${LSS_STATE_DIR:-/var/lib/lss}" + return +} + +# _web_route_parse NAME DOMAIN UPSTREAM: checks the arguments and sets WEB_ROUTE_DOMAINS (array) and WEB_ROUTE_URL. +_web_route_parse() { + local name=$1 domains=$2 given=$3 upstream=$3 domain + _web_name_check "${name}" + IFS=', ' read -ra WEB_ROUTE_DOMAINS <<<"${domains}" + ((${#WEB_ROUTE_DOMAINS[@]} > 0)) || die "web: the route ${name} has no host name." + for domain in "${WEB_ROUTE_DOMAINS[@]}"; do + [[ ${domain} =~ ^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z][a-z0-9-]*[a-z0-9]$ ]] || + die "web: '${domain}' of the route ${name} is not a host name like app.example.com." + done + [[ ${upstream} == http://* || ${upstream} == https://* ]] || upstream="http://${upstream}" + [[ ${upstream} =~ ^https?://([A-Za-z0-9.-]+|\[[0-9A-Fa-f:]+\]):[0-9]{1,5}(/[A-Za-z0-9._~/-]*)?$ ]] || + die "web: the upstream '${given}' of the route ${name} is not HOST:PORT or an http(s):// URL with a port." + WEB_ROUTE_URL=${upstream} + return +} + +# web_proxy_route NAME DOMAIN UPSTREAM: publishes UPSTREAM under DOMAIN over HTTPS (see above). Idempotent. +web_proxy_route() { + local name=$1 file + _web_route_parse "$@" + file="$(_web_routes_dir)/${name}" + mkdir -p "$(_web_routes_dir)" + if [[ $(cat "${file}" 2>/dev/null) != "${LSS_MODULE:-lss} ${WEB_ROUTE_DOMAINS[*]} ${WEB_ROUTE_URL}" ]]; then + printf '%s %s %s\n' "${LSS_MODULE:-lss}" "${WEB_ROUTE_DOMAINS[*]}" "${WEB_ROUTE_URL}" >"${file}" + files_note_change "route ${name}" + fi + _web_route_apply "${name}" + return +} + +# _web_route_apply NAME: writes the recorded route NAME for Traefik or the web server, in the name of the module that +# recorded it (its managed files and change notes). +_web_route_apply() { + local name=$1 owner url + local -a domains=() + read -r owner url < <(awk '{ print $1, $NF }' "$(_web_routes_dir)/${name}") + read -ra domains < <(awk '{ $1 = ""; $NF = ""; print }' "$(_web_routes_dir)/${name}") + LSS_MODULE=${owner} _web_route_write "${name}" "${owner}" "${url}" "${domains[@]}" + return +} + +# _web_route_write NAME OWNER URL DOMAIN...: the work of _web_route_apply, run with LSS_MODULE set to OWNER. +_web_route_write() { + local name=$1 owner=$2 url=$3 before + shift 3 + before=$(files_changes_of "${owner}") + [[ ! -s ${WEB_TLS_DIR}/default/fullchain.pem ]] || web_cert_placeholder "${name}" + if traefik_waf_backend; then + web_site_write "${name}" proxy "${url}" "$@" + traefik_route_write "${name}" "https://127.0.0.1:${WEB_BACKEND_HTTPS_PORT}" "$@" + elif traefik_active; then + web_site_remove "${name}" + traefik_route_write "${name}" "${url}" "$@" + else + [[ -n $(web_server) ]] || die "web: the route ${name} needs Traefik or a web server (nginx, apache or caddy)." + traefik_route_remove "${name}" + web_site_write "${name}" proxy "${url}" "$@" + fi + if [[ $(files_changes_of "${owner}") != "${before}" ]]; then + web_reload + traefik_tls_sync + fi + if state_installed tls; then + web_cert_issue "${name}" "$@" + fi + return +} + +# web_routes_apply: applies every recorded route again (after Traefik was installed or removed). +web_routes_apply() { + local file + for file in "$(_web_routes_dir)"/*; do + [[ -f ${file} ]] || continue + _web_route_apply "${file##*/}" + done + return +} + +# web_route_names [OWNER]: the recorded routes, of one module or of all. +web_route_names() { + local owner=${1:-} file + for file in "$(_web_routes_dir)"/*; do + [[ -f ${file} ]] || continue + if [[ -z ${owner} || $(cut -d ' ' -f 1 "${file}") == "${owner}" ]]; then + printf '%s\n' "${file##*/}" + fi + done + return +} + +# web_proxy_route_remove NAME: takes the route away from Traefik and the web server, with its certificate. +web_proxy_route_remove() { + local name=$1 + _web_name_check "${name}" + traefik_route_remove "${name}" + web_site_remove "${name}" + file_remove "${WEB_TLS_DIR}/${name}" + if [[ -d ${WEB_CERT_LIVE}/${name} ]] && command -v certbot >/dev/null 2>&1; then + log_command certbot delete --non-interactive --cert-name "${name}" || + die "web: certbot could not delete the certificate ${name}." + files_note_change "certificate ${name} deleted" + fi + if [[ -f $(_web_routes_dir)/${name} ]]; then + rm -f -- "$(_web_routes_dir)/${name}" + files_note_change "removed route ${name}" + fi + web_reload + traefik_tls_sync + return +} + # web_admin_write NAME PATH php|proxy|strip TARGET: publishes an admin interface below /PATH/ on the admin server # (WEB_ADMIN_PORT). TARGET is a directory with PHP files or an upstream URL. proxy passes the whole path on (the # upstream serves below /PATH/), strip removes /PATH first (for upstreams that only serve their root, such as Netdata). @@ -735,6 +916,8 @@ web_admin_htpasswd() { # installed, because its state is recorded only after it. web_ids_configure() { local server logs + # Traefik's access log as well, when it is in front. + traefik_ids_configure server=$(web_server) case ${LSS_MODULE:-} in nginx | apache | caddy) server=${LSS_MODULE} ;; @@ -814,9 +997,10 @@ EOF ;; *) file_write 0644 root:root "${WEB_FAIL2BAN_FILTER}" <<'EOF' -# Managed by lss (web server caddy). Caddy logs JSON; a 401 answer is a failed basic-auth login. +# Managed by lss (web server caddy). Caddy logs JSON; a 401 answer is a failed basic-auth login. client_ip is the +# client behind Traefik as well (trusted_proxies), remote_ip would be Traefik itself. [Definition] -failregex = ^.*"remote_ip":"".*"status":401\b +failregex = ^.*"client_ip":"".*"status":401\b ignoreregex = datepattern = "ts":{EPOCH} EOF diff --git a/lss b/lss index 532763c..9641b0d 100755 --- a/lss +++ b/lss @@ -14,7 +14,7 @@ LSS_ROOT=$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd) LSS_VERSION=$(<"${LSS_ROOT}/VERSION") export LSS_ROOT LSS_VERSION -for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring module ui; do +for library in log os config files template secrets state pkg repo fetch service firewall container web traefik monitoring module ui; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/modules/web/apache.sh b/modules/web/apache.sh index 2140dc3..6b11e28 100644 --- a/modules/web/apache.sh +++ b/modules/web/apache.sh @@ -4,14 +4,14 @@ MOD_NAME="apache" MOD_GROUP="web" -MOD_DESCRIPTION="Apache httpd web server: HTTPS only (TLS 1.2 and 1.3), HTTP/2, security headers, admin port on localhost" +MOD_DESCRIPTION="Apache httpd web server: HTTPS only (TLS 1.2 and 1.3), HTTP/2, security headers, admin port on localhost; behind Traefik on 127.0.0.1 only" MOD_REQUIRES="system" MOD_CONFLICTS="nginx caddy" MOD_OPTIONS="max_body|64m|Largest request body (uploads), for example 64m or 1g admin_public|false|Serve the admin interfaces (phpMyAdmin, Adminer, pgAdmin) on port 8090 to everyone, with TLS and a password (user admin, lss secrets show web-admin-password); otherwise only on 127.0.0.1 (use an SSH tunnel)" MOD_SECRETS="web-admin-password" -APACHE_MODULES_DEB="ssl headers rewrite proxy proxy_http proxy_fcgi proxy_wstunnel setenvif http2 socache_shmcb auth_basic authn_file" +APACHE_MODULES_DEB="ssl headers rewrite proxy proxy_http proxy_fcgi proxy_wstunnel setenvif http2 socache_shmcb auth_basic authn_file remoteip" _apache_root() { if os_is_debian; then printf '/etc/apache2'; else printf '/etc/httpd'; fi @@ -60,9 +60,32 @@ _apache_stapling_cache() { return } +# The listeners: ports 80 and 443 on every address, or the backend ports on 127.0.0.1 behind Traefik. The sites name +# their port through ${LSS_HTTP_PORT} and ${LSS_HTTPS_PORT}, so a switch of the mode leaves them alone. The stock +# Listen lines (ports.conf, httpd.conf, ssl.conf) are switched off. +_apache_listen_conf() { + local address='' + web_backend && address=127.0.0.1: + printf 'Define LSS_HTTP_PORT %s\nDefine LSS_HTTPS_PORT %s\n' "$(web_http_port)" "$(web_https_port)" + printf 'Listen %s%s\nListen %s%s https\n' "${address}" "$(web_http_port)" "${address}" "$(web_https_port)" + # %a is the client: behind Traefik mod_remoteip takes it from X-Forwarded-For, which only Traefik on the loopback + # interface may set; logs, fail2ban and ModSecurity then see the client instead of Traefik. + # shellcheck disable=SC2016 # Apache format strings + printf 'LogFormat "%%a %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-Agent}i\\"" lss_combined\n' + if web_backend; then + printf 'RemoteIPHeader X-Forwarded-For\nRemoteIPInternalProxy 127.0.0.1 ::1\n' + # Traefik connects to 127.0.0.1 without SNI, so the handshake runs in the default site and Host picks the site + # afterwards. Apache 2.4.65 and later refuse that switch (421) when the TLS settings of the two sites differ in + # anything (certificate, stapling); "authonly" compares only client authentication, which lss never uses. + printf '\n\tSSLVHostSNIPolicy authonly\n\n' + fi + return +} + _apache_global_conf() { cat < + ServerName lss-default.invalid DocumentRoot ${WEB_DEFAULT_ROOT} Alias /.well-known/acme-challenge/ ${WEB_ACME_ROOT}/.well-known/acme-challenge/ @@ -112,7 +135,7 @@ _apache_default_conf() { - + ServerName lss-default.invalid SSLEngine on SSLCertificateFile ${WEB_TLS_DIR}/default/fullchain.pem @@ -150,7 +173,7 @@ _apache_admin_conf() { EOF fi web_headers apache | grep -v Strict-Transport-Security - printf '\tErrorLog %s\n\tCustomLog %s combined\n' "$(web_log_file apache error)" "$(web_log_file apache access)" + printf '\tErrorLog %s\n\tCustomLog %s lss_combined\n' "$(web_log_file apache error)" "$(web_log_file apache access)" printf '\tIncludeOptional lss/admin/*.conf\n\n' return } @@ -178,19 +201,30 @@ _apache_debian_enable() { # RHEL family: mod_ssl's ssl.conf carries a test virtual host and welcome.conf the test page; lss replaces both. _apache_rhel_files() { file_write 0644 root:root /etc/httpd/conf.d/ssl.conf <<'EOF' -# Managed by lss (module apache). The TLS settings and virtual hosts are in conf.d/lss.conf and lss/. -Listen 443 https +# Managed by lss (module apache). The TLS settings, the Listen lines and the virtual hosts are in conf.d/lss.conf +# and lss/. SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog SSLSessionCache shmcb:/run/httpd/sslcache(512000) SSLSessionCacheTimeout 300 SSLCryptoDevice builtin EOF file_write 0644 root:root /etc/httpd/conf.d/welcome.conf <<<'# Managed by lss (module apache). No test page.' + # httpd.conf listens on port 80 itself; conf.d/lss.conf has the Listen lines of both modes instead. + if grep -qx 'Listen 80' /etc/httpd/conf/httpd.conf; then + sed 's|^Listen 80$|# Listen 80: lss sets the Listen lines in conf.d/lss.conf.|' /etc/httpd/conf/httpd.conf | + file_write 0644 root:root /etc/httpd/conf/httpd.conf + fi + return +} + +# Debian: ports.conf carries the stock Listen lines; conf-available/zz-lss.conf has those of both modes instead. +_apache_debian_ports() { + file_write 0644 root:root /etc/apache2/ports.conf <<<'# Managed by lss (module apache). The Listen lines are in conf-available/zz-lss.conf.' return } mod_configure() { - local dir unit changed=0 + local dir unit before changed=0 dir=$(web_conf_dir apache) unit=$(web_unit apache) web_prepare apache @@ -201,15 +235,14 @@ mod_configure() { _apache_admin_conf | file_write 0644 root:root "${dir}/admin.conf" [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 file_write 0644 root:root "${WEB_ACME_ROOT}/.well-known/acme-challenge/lss-probe" <<<"lss" + before=$(files_changes_of apache) if os_is_debian; then - local before - before=$(files_changes_of apache) + _apache_debian_ports _apache_debian_enable - [[ $(files_changes_of apache) == "${before}" ]] || changed=1 else _apache_rhel_files - [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 fi + [[ $(files_changes_of apache) == "${before}" ]] || changed=1 if opt_bool admin_public; then web_admin_htpasswd apache fw_allow "${WEB_ADMIN_PORT}/tcp" @@ -228,16 +261,20 @@ mod_configure() { svc_restart "${unit}" fi web_ids_configure + traefik_backend_sync apache return } mod_verify() { + local http https + http=$(web_http_port) + https=$(web_https_port) svc_is_active "$(web_unit apache)" || die "apache: the service is not running." web_test apache || die "apache: the configuration does not load." - [[ $(curl -fsS --max-time 10 http://127.0.0.1/.well-known/acme-challenge/lss-probe 2>/dev/null) == lss ]] || - die "apache: port 80 does not serve the ACME challenge path." - openssl s_client -connect 127.0.0.1:443 -servername lss-probe /dev/null | grep -q 'BEGIN CERTIFICATE' || - die "apache: port 443 does not answer with TLS." + [[ $(curl -fsS --max-time 10 "http://127.0.0.1:${http}/.well-known/acme-challenge/lss-probe" 2>/dev/null) == lss ]] || + die "apache: port ${http} does not serve the ACME challenge path." + openssl s_client -connect "127.0.0.1:${https}" -servername lss-probe /dev/null | grep -q 'BEGIN CERTIFICATE' || + die "apache: port ${https} does not answer with TLS." curl -s --max-time 10 -o /dev/null "http://127.0.0.1:${WEB_ADMIN_PORT}/" || opt_bool admin_public || die "apache: the admin port ${WEB_ADMIN_PORT} does not answer on 127.0.0.1." return @@ -246,6 +283,7 @@ mod_verify() { mod_remove() { local unit unit=$(web_unit apache) + traefik_backend_sync '' svc_disable "${unit}" svc_dropin_remove "${unit}.service" lss-sandbox web_ids_remove diff --git a/modules/web/caddy.sh b/modules/web/caddy.sh index 14626fc..c0f21b3 100644 --- a/modules/web/caddy.sh +++ b/modules/web/caddy.sh @@ -4,7 +4,7 @@ MOD_NAME="caddy" MOD_GROUP="web" -MOD_DESCRIPTION="Caddy web server from its official repository: HTTPS only, security headers, JSON access log, admin port on localhost" +MOD_DESCRIPTION="Caddy web server from its official repository: HTTPS only, security headers, JSON access log, admin port on localhost; behind Traefik on 127.0.0.1 only" MOD_REQUIRES="system" MOD_CONFLICTS="nginx apache" MOD_OPTIONS="max_body|64m|Largest request body (uploads), for example 64m or 1g @@ -47,14 +47,41 @@ _caddy_headers() { return } +# Behind Traefik: the backend ports for every http:// and https:// address, the listeners on 127.0.0.1 (lss_bind, +# which every site imports) and the client address from Traefik's X-Forwarded-For (client_ip in the access log). +_caddy_backend_options() { + if web_backend; then + printf '\thttp_port %s\n\thttps_port %s\n' "${WEB_BACKEND_HTTP_PORT}" "${WEB_BACKEND_HTTPS_PORT}" + fi + return +} + +_caddy_backend_proxies() { + if web_backend; then + printf '\t\ttrusted_proxies static 127.0.0.1/32 ::1/128\n' + fi + return +} + +_caddy_bind_snippet() { + printf '(lss_bind) {\n' + if web_backend; then + printf '\tbind 127.0.0.1\n' + fi + printf '}\n' + return +} + _caddy_file() { cat </dev/null) == lss ]] && break + [[ $(curl -fsS --max-time 5 "http://127.0.0.1:${http}/.well-known/acme-challenge/lss-probe" 2>/dev/null) == lss ]] && break sleep 1 done - [[ $(curl -fsS --max-time 10 http://127.0.0.1/.well-known/acme-challenge/lss-probe 2>/dev/null) == lss ]] || - die "caddy: port 80 does not serve the ACME challenge path." - openssl s_client -connect 127.0.0.1:443 -servername lss-probe /dev/null | grep -q 'BEGIN CERTIFICATE' || - die "caddy: port 443 does not answer with TLS." + [[ $(curl -fsS --max-time 10 "http://127.0.0.1:${http}/.well-known/acme-challenge/lss-probe" 2>/dev/null) == lss ]] || + die "caddy: port ${http} does not serve the ACME challenge path." + openssl s_client -connect "127.0.0.1:${https}" -servername lss-probe /dev/null | grep -q 'BEGIN CERTIFICATE' || + die "caddy: port ${https} does not answer with TLS." curl -s --max-time 10 -o /dev/null "http://127.0.0.1:${WEB_ADMIN_PORT}/" || opt_bool admin_public || die "caddy: the admin port ${WEB_ADMIN_PORT} does not answer on 127.0.0.1." return } mod_remove() { + traefik_backend_sync '' svc_disable caddy svc_dropin_remove caddy.service lss-sandbox web_ids_remove diff --git a/modules/web/nginx.sh b/modules/web/nginx.sh index d0c7d66..696526a 100644 --- a/modules/web/nginx.sh +++ b/modules/web/nginx.sh @@ -4,7 +4,7 @@ MOD_NAME="nginx" MOD_GROUP="web" -MOD_DESCRIPTION="nginx web server: HTTPS only (TLS 1.2 and 1.3), security headers, closed default server, admin port on localhost" +MOD_DESCRIPTION="nginx web server: HTTPS only (TLS 1.2 and 1.3), security headers, closed default server, admin port on localhost; behind Traefik on 127.0.0.1 only" MOD_REQUIRES="system" MOD_CONFLICTS="apache caddy" MOD_OPTIONS="max_body|64m|Largest request body (uploads), for example 64m or 1g @@ -86,10 +86,10 @@ EOF # Unknown host names get no answer (444) on both ports; port 80 still serves ACME challenges for every name. _nginx_default_conf() { printf '# Managed by lss (module nginx). Requests for host names without a site.\nserver {\n' - _web_listen_nginx 80 default_server + web_nginx_listen http default_server printf '\tserver_name _;\n\tlocation /.well-known/acme-challenge/ {\n\t\troot %s;\n\t}\n' "${WEB_ACME_ROOT}" printf '\tlocation / {\n\t\treturn 444;\n\t}\n}\n\nserver {\n' - _web_listen_nginx 443 "default_server $(web_nginx_ssl_flags)" + web_nginx_listen https default_server printf '\tserver_name _;\n' _web_tls_nginx "${WEB_TLS_DIR}/default" printf '\treturn 444;\n}\n' @@ -111,6 +111,36 @@ _nginx_admin_conf() { return } +# The listeners every site includes: ports 80 and 443 on every address, or the backend ports on 127.0.0.1 behind +# Traefik. +_nginx_listen_http() { + printf '# Managed by lss (module nginx). The HTTP listener of every site.\n' + web_nginx_listen http + return +} + +_nginx_listen_https() { + printf '# Managed by lss (module nginx). The HTTPS listener of every site.\n' + web_nginx_listen https + if web_nginx_http2_directive; then + printf '\thttp2 on;\n' + fi + return +} + +# Behind Traefik the client address comes from X-Forwarded-For, which Traefik sets; only requests from the loopback +# interface may carry it. Logs, fail2ban and ModSecurity then see the client instead of Traefik. +_nginx_realip_conf() { + cat <<'EOF' +# Managed by lss (module nginx). The web server runs behind Traefik. +set_real_ip_from 127.0.0.1; +set_real_ip_from ::1; +real_ip_header X-Forwarded-For; +real_ip_recursive on; +EOF + return +} + _nginx_proxy_conf() { cat <<'EOF' # Managed by lss (module nginx). Included by every proxied location. @@ -137,6 +167,17 @@ mod_configure() { [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 _nginx_proxy_conf | file_write 0644 root:root "${NGINX_DIR}/proxy.conf" [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + _nginx_listen_http | file_write 0644 root:root "${NGINX_DIR}/listen-http.conf" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + _nginx_listen_https | file_write 0644 root:root "${NGINX_DIR}/listen-https.conf" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + if web_backend; then + _nginx_realip_conf | file_write 0644 root:root "${NGINX_DIR}/http/realip.conf" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + elif [[ -e ${NGINX_DIR}/http/realip.conf ]]; then + file_remove "${NGINX_DIR}/http/realip.conf" + changed=1 + fi file_write 0644 root:root "${WEB_ACME_ROOT}/.well-known/acme-challenge/lss-probe" <<<"lss" if opt_bool admin_public; then web_admin_htpasswd nginx @@ -152,22 +193,27 @@ mod_configure() { svc_restart nginx fi web_ids_configure + traefik_backend_sync nginx return } mod_verify() { + local http https + http=$(web_http_port) + https=$(web_https_port) svc_is_active nginx || die "nginx: the service is not running." web_test nginx || die "nginx: the configuration does not load." - [[ $(curl -fsS --max-time 10 http://127.0.0.1/.well-known/acme-challenge/lss-probe 2>/dev/null) == lss ]] || - die "nginx: port 80 does not serve the ACME challenge path." - openssl s_client -connect 127.0.0.1:443 -servername lss-probe /dev/null | grep -q 'BEGIN CERTIFICATE' || - die "nginx: port 443 does not answer with TLS." + [[ $(curl -fsS --max-time 10 "http://127.0.0.1:${http}/.well-known/acme-challenge/lss-probe" 2>/dev/null) == lss ]] || + die "nginx: port ${http} does not serve the ACME challenge path." + openssl s_client -connect "127.0.0.1:${https}" -servername lss-probe /dev/null | grep -q 'BEGIN CERTIFICATE' || + die "nginx: port ${https} does not answer with TLS." curl -s --max-time 10 -o /dev/null "http://127.0.0.1:${WEB_ADMIN_PORT}/" || opt_bool admin_public || die "nginx: the admin port ${WEB_ADMIN_PORT} does not answer on 127.0.0.1." return } mod_remove() { + traefik_backend_sync '' svc_disable nginx svc_dropin_remove nginx.service lss-sandbox file_remove "${NGINX_DIR}" diff --git a/modules/web/tls.sh b/modules/web/tls.sh index 7ec0fa4..bdf8abd 100644 --- a/modules/web/tls.sh +++ b/modules/web/tls.sh @@ -88,10 +88,12 @@ for unit in nginx apache2 httpd haproxy postfix postfix@- dovecot; do systemctl reload "\${unit}" fi done -# Caddy keeps certificates it loaded from files until it restarts. -if systemctl is-active --quiet caddy; then - systemctl restart caddy -fi +# Caddy and Traefik keep certificates they loaded from files until they restart. +for unit in caddy lss-traefik; do + if systemctl is-active --quiet "\${unit}"; then + systemctl restart "\${unit}" + fi +done EOF return } diff --git a/modules/web/traefik.sh b/modules/web/traefik.sh new file mode 100644 index 0000000..ad0b60c --- /dev/null +++ b/modules/web/traefik.sh @@ -0,0 +1,887 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="traefik" +MOD_GROUP="web" +MOD_DESCRIPTION="Traefik 3.7 edge proxy in front of the web server and the applications: HTTPS with HTTP/3, security headers, OWASP Coraza WAF, JSON access log, dashboard on localhost" +MOD_REQUIRES="system" +MOD_OPTIONS="waf|coraza|Web application firewall: coraza (OWASP Coraza with the Core Rule Set in Traefik, for every route; on amd64 held back until Traefik ships wazero 1.9: lss then installs modsecurity for nginx or Apache and routes the applications through it), modsecurity (the modsecurity module in the backend web server, for what goes through it), both, or off +paranoia|1|Paranoia level of the Core Rule Set in Coraza, 1 to 4: higher levels catch more attacks and also block more legitimate requests +routes||Routes Traefik serves, separated by spaces: NAME:DOMAIN[,DOMAIN...]:HOST:PORT, for example app:app.example.com:127.0.0.1:3000 (Traefik forwards to http://HOST:PORT) +admin_host||Host name under which Traefik publishes the admin interfaces of the web server (phpMyAdmin, Adminer, pgAdmin) over HTTPS with a password (user admin, lss secrets show web-admin-password); empty keeps them on the admin port only +acme|false|Traefik requests certificates for its routes itself (TLS-ALPN challenge on port 443; the names must point to this server); otherwise the tls module or the self-signed default certificate +acme_email||Address for Traefik's ACME account (optional) +acme_server|https://acme-v02.api.letsencrypt.org/directory|ACME directory URL of Traefik's ACME client +acme_ca_bundle||CA certificate to trust for a private ACME server (Pebble, step-ca); empty for Let's Encrypt +docker|false|Docker provider: containers with the label traefik.enable=true are routed. Traefik reads the container engine through a socket proxy that allows only listing and inspecting containers, never the socket itself +dashboard_port|8099|Port of the dashboard and the read-only API on 127.0.0.1 (never public)" +MOD_SECRETS="web-admin-password" +MOD_PORTS="80/tcp 443/tcp 443/udp" + +# Traefik from its GitHub release (data/checksums.txt, amd64 and arm64), the Coraza plugin from its GitHub release: +# a WebAssembly build of Coraza with the OWASP Core Rule Set 4.0 inside, loaded as a local plugin (Traefik never +# downloads it from the plugin catalog). A new version is a new entry in data/checksums.txt and a new version here. +TRAEFIK_VERSION=3.7.15 +TRAEFIK_CORAZA_VERSION=0.3.0 +TRAEFIK_CORAZA_MODULE=github.com/jcchavezs/coraza-http-wasm-traefik +TRAEFIK_BIN=/usr/local/bin/traefik +TRAEFIK_CACHE=/var/cache/lss-traefik +# Traefik looks for local plugins below plugins-local/src of its working directory, which stays read-only for it. +TRAEFIK_HOME=/usr/local/lib/lss-traefik +TRAEFIK_PLUGIN_DIR=${TRAEFIK_HOME}/plugins-local/src/${TRAEFIK_CORAZA_MODULE} +TRAEFIK_STATE=/var/lib/lss-traefik +# shellcheck disable=SC2153 # TRAEFIK_DIR is set in lib/traefik.sh +TRAEFIK_CONF=${TRAEFIK_DIR}/traefik.yml +TRAEFIK_HTPASSWD=${TRAEFIK_DIR}/admin.htpasswd +TRAEFIK_ACME_CA=${TRAEFIK_DIR}/acme-ca.pem +TRAEFIK_LOGROTATE=/etc/logrotate.d/lss-traefik +# The probe route: Traefik's own ping behind the HTTPS entry point with its middlewares, for verify. +TRAEFIK_PROBE_HOST=lss-probe.invalid +# The SQL injection verify sends through the WAF (1' OR '1'='1, URL-encoded); the WAF must answer 403. +TRAEFIK_SQLI='1%27%20OR%20%271%27%3D%271' +# Socket proxy for the Docker provider (opt-in): Traefik gets a socket that answers only a few read-only calls. +TRAEFIK_PROXY_NAME=lss-traefik-docker +# wollomatic/socket-proxy 1.13.1, the multi-architecture index (amd64 and arm64) on GHCR, checked 2026-10-15. +TRAEFIK_PROXY_IMAGE=ghcr.io/wollomatic/socket-proxy@sha256:e78e68ba3ea4a6bf8ee47865ac1d1f98d525c3660df948554e766283c40d3f4b +TRAEFIK_PROXY_DIR=/run/lss-traefik-docker +TRAEFIK_PROXY_SOCKET=${TRAEFIK_PROXY_DIR}/docker.sock +TRAEFIK_PROXY_TMPFILES=/etc/tmpfiles.d/lss-traefik-docker.conf +# Listing, inspecting and the event stream of containers, the version, the engine information and the ping (what +# Traefik's Docker provider calls); nothing that changes anything. +TRAEFIK_PROXY_ALLOW='^/(v[0-9.]+/)?(_ping|version|info|events|containers/json|containers/[A-Za-z0-9_.-]+/json)$' + +mod_check() { + local port waf paranoia route bundle + waf=$(opt waf) + [[ ${waf} =~ ^(coraza|modsecurity|both|off)$ ]] || die "LSS_TRAEFIK_WAF must be coraza, modsecurity, both or off." + paranoia=$(opt_int paranoia) + ((paranoia >= 1 && paranoia <= 4)) || die "LSS_TRAEFIK_PARANOIA must be 1, 2, 3 or 4." + port=$(opt_int dashboard_port) + ((port >= 1024 && port <= 65535)) || die "LSS_TRAEFIK_DASHBOARD_PORT must be a port number from 1024 to 65535." + case ${port} in + "${WEB_ADMIN_PORT}" | "${WEB_BACKEND_HTTP_PORT}" | "${WEB_BACKEND_HTTPS_PORT}") + die "LSS_TRAEFIK_DASHBOARD_PORT must differ from ${WEB_ADMIN_PORT}, ${WEB_BACKEND_HTTP_PORT} and ${WEB_BACKEND_HTTPS_PORT}." + ;; + *) ;; + esac + for route in $(opt routes); do + _traefik_route_parse "${route}" + done + if [[ -n $(opt admin_host) ]]; then + [[ $(opt admin_host) =~ ^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z][a-z0-9-]*[a-z0-9]$ ]] || + die "LSS_TRAEFIK_ADMIN_HOST '$(opt admin_host)' is not a host name like admin.example.com." + fi + opt_bool acme || true + opt_bool docker || true + [[ -z $(opt acme_email) || $(opt acme_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[^@[:space:]]+$ ]] || + die "LSS_TRAEFIK_ACME_EMAIL '$(opt acme_email)' is not an e-mail address." + [[ $(opt acme_server) =~ ^https://[^[:space:]]+$ ]] || die "LSS_TRAEFIK_ACME_SERVER must be an https:// URL." + bundle=$(opt acme_ca_bundle) + [[ -z ${bundle} || -s ${bundle} ]] || die "LSS_TRAEFIK_ACME_CA_BUNDLE: ${bundle} does not exist." + fetch_entry "traefik-${TRAEFIK_VERSION}" >/dev/null || + die "traefik: data/checksums.txt has no checksum for Traefik ${TRAEFIK_VERSION} on ${OS_ARCH:-$(uname -m)}." + fetch_entry "coraza-http-wasm-${TRAEFIK_CORAZA_VERSION}" >/dev/null || + die "traefik: data/checksums.txt has no checksum for the Coraza plugin ${TRAEFIK_CORAZA_VERSION}." + return +} + +# _traefik_route_parse SPEC: NAME:DOMAIN[,DOMAIN...]:HOST:PORT; sets TRAEFIK_ROUTE_NAME, TRAEFIK_ROUTE_DOMAINS and +# TRAEFIK_ROUTE_UPSTREAM. +_traefik_route_parse() { + local spec=$1 + IFS=':' read -r TRAEFIK_ROUTE_NAME TRAEFIK_ROUTE_DOMAINS TRAEFIK_ROUTE_UPSTREAM <<<"${spec}" + [[ ${TRAEFIK_ROUTE_NAME} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + die "LSS_TRAEFIK_ROUTES: '${spec}' does not start with a route name (a-z, 0-9, -)." + [[ ${TRAEFIK_ROUTE_UPSTREAM} =~ ^[A-Za-z0-9.-]+:[0-9]{1,5}$ ]] || + die "LSS_TRAEFIK_ROUTES: the route ${TRAEFIK_ROUTE_NAME} needs HOST:PORT at the end, like 127.0.0.1:3000." + # The same checks as for every application route. + _web_route_parse "${TRAEFIK_ROUTE_NAME}" "${TRAEFIK_ROUTE_DOMAINS}" "${TRAEFIK_ROUTE_UPSTREAM}" + return +} + +# Coraza in Traefik where it runs (traefik_coraza_runs in lib/traefik.sh: arm64, amd64 once Traefik ships wazero 1.9). +_traefik_coraza() { + traefik_coraza_wanted && traefik_coraza_runs + return +} + +# ModSecurity in the backend web server when asked for, and in place of Coraza where Coraza cannot run. +_traefik_modsecurity() { + if [[ $(opt waf) == modsecurity || $(opt waf) == both ]]; then + return 0 + fi + traefik_waf_fallback && state_installed modsecurity + return +} + +# _traefik_wait: waits until Traefik answers its ping (after a start or a restart). +_traefik_wait() { + local port + port=$(opt_int dashboard_port) + for _ in {1..30}; do + curl -fsS --max-time 2 -o /dev/null "http://127.0.0.1:${port}/ping" 2>/dev/null && return 0 + sleep 1 + done + _traefik_die "the ping on 127.0.0.1:${port} does not answer." + return 1 +} + +# _traefik_waf_fallback: where Coraza is asked for and cannot run (amd64, see lib/traefik.sh), ModSecurity in the web +# server takes its place: lss installs the modsecurity module (once, and removes it again with Traefik) and +# traefik_waf_backend sends the routes of the applications through the web server, so every request passes a WAF. +# Caddy has no ModSecurity connector; without a web server there is nothing to filter in. +_traefik_waf_fallback() { + local server reason='traefik: Coraza waits for a Traefik release with wazero 1.9 on amd64' + traefik_waf_fallback || return 0 + server=$(web_server) + case ${server} in + nginx | apache) + if state_installed modsecurity; then + log_warn "${reason}; ModSecurity in ${server} filters every request, and the routes of applications pass through ${server} for it." + else + log_warn "${reason}; lss installs modsecurity so that ${server} filters every request, the routes of applications included." + _traefik_wait + module_run modsecurity check || die "traefik: the settings of modsecurity do not pass its check." + LSS_WEB_BACKEND=on module_apply modsecurity + state_set traefik waf_dependency modsecurity + fi + ;; + caddy) + log_warn "${reason}, and Caddy has no ModSecurity connector; requests reach Caddy and the applications unfiltered (with nginx or Apache in place of Caddy, lss adds ModSecurity)." + ;; + *) + log_warn "${reason}; requests reach the applications unfiltered until nginx or Apache is installed (then lss configure traefik adds ModSecurity)." + ;; + esac + if opt_bool docker; then + log_warn "traefik: containers of the Docker provider are routed to directly, without ModSecurity in front of them." + fi + return +} + +# _traefik_waf_fallback_remove: the modsecurity module goes when lss installed it in place of Coraza. +_traefik_waf_fallback_remove() { + [[ $(state_get traefik waf_dependency) == modsecurity ]] || return 0 + state_installed modsecurity || return 0 + log_info "traefik: removing modsecurity, which lss installed in place of Coraza." + module_remove modsecurity + return +} + +_traefik_arch() { + case ${OS_ARCH:-$(uname -m)} in + aarch64 | arm64) printf 'arm64' ;; + *) printf 'amd64' ;; + esac + return +} + +# The release archive, checked against data/checksums.txt; the binary is replaced when its version differs. +_traefik_binary() { + local archive tmp current='' file + archive="${TRAEFIK_CACHE}/traefik-${TRAEFIK_VERSION}-$(_traefik_arch).tar.gz" + fetch_verified "traefik-${TRAEFIK_VERSION}" "${archive}" + for file in "${TRAEFIK_CACHE}"/traefik-*; do + [[ ! -f ${file} || ${file} == "${archive}" ]] || rm -f -- "${file}" + done + if [[ -x ${TRAEFIK_BIN} ]]; then + current=$("${TRAEFIK_BIN}" version 2>/dev/null | awk '$1 == "Version:" { print $2 }') || current='' + fi + [[ ${current} != "${TRAEFIK_VERSION}" ]] || return 0 + tmp=$(mktemp -d) + tar -xzf "${archive}" -C "${tmp}" traefik || die "traefik: the release archive has no traefik binary." + install -m 0755 -o root -g root "${tmp}/traefik" "${TRAEFIK_BIN}" + rm -rf -- "${tmp}" + if command -v restorecon >/dev/null 2>&1; then + restorecon -F "${TRAEFIK_BIN}" 2>/dev/null || true + fi + files_note_change "installed Traefik ${TRAEFIK_VERSION}" + return +} + +# The Coraza plugin: its manifest and the WebAssembly module from the checked release archive. +_traefik_plugin() { + local archive file + archive="${TRAEFIK_CACHE}/coraza-http-wasm-${TRAEFIK_CORAZA_VERSION}.zip" + fetch_verified "coraza-http-wasm-${TRAEFIK_CORAZA_VERSION}" "${archive}" + for file in .traefik.yml coraza-http-wasm.wasm LICENSE; do + unzip -p "${archive}" "${file}" | file_write 0644 root:root "${TRAEFIK_PLUGIN_DIR}/${file}" + done + return +} + +_traefik_plugin_remove() { + local file + for file in .traefik.yml coraza-http-wasm.wasm LICENSE; do + file_remove "${TRAEFIK_PLUGIN_DIR}/${file}" + done + file_remove "${TRAEFIK_HOME}/plugins-local" + return +} + +mod_install() { + pkg_install curl tar unzip openssl + if [[ -n $(opt admin_host) ]]; then + # htpasswd hashes the admin password with bcrypt, which Traefik's basic auth reads. + pkg_install_family apache2-utils httpd-tools + fi + _traefik_binary + if _traefik_coraza; then + _traefik_plugin + fi + return +} + +_traefik_user() { + if ! getent passwd "${TRAEFIK_USER}" >/dev/null; then + useradd --system --user-group --home-dir "${TRAEFIK_STATE}" --no-create-home --shell /usr/sbin/nologin \ + --comment "Traefik (lss)" "${TRAEFIK_USER}" + files_note_change "created user ${TRAEFIK_USER}" + fi + # The TLS keys of WEB_TLS_DIR are readable by the web group. + web_group_ensure "${TRAEFIK_USER}" + return +} + +# Request checks of every entry point: headers whose names alias another one (X_Forwarded_For for X-Forwarded-For, +# which PHP and CGI read as the same variable) are dropped, so a client cannot pass on a header Traefik manages; +# encoded backslashes and null characters in the path are refused (400). +_traefik_entry_http() { + cat <<'EOF' + http: + aliasHeadersStrategy: delete + encodedCharacters: + allowEncodedBackslash: false + allowEncodedNullCharacter: false +EOF + return +} + +# The static configuration: entry points, providers, logs, the ACME client and the local Coraza plugin. +_traefik_static() { + local email + cat </dev/null 2>&1; then + return 0 + fi + htpasswd -niB -C 12 admin <<<"${password}" | grep -v '^$' | file_write 0640 "root:${TRAEFIK_USER}" "${TRAEFIK_HTPASSWD}" + return +} + +_traefik_admin() { + local host + host=$(opt admin_host) + if [[ -n ${host} && -z $(web_server) ]]; then + log_warn "traefik: LSS_TRAEFIK_ADMIN_HOST waits for a web server with the admin port (nginx, apache or caddy); run lss configure traefik after its install." + host='' + fi + if [[ -z ${host} ]]; then + file_remove "${TRAEFIK_DYNAMIC}/admin.yml" + file_remove "${TRAEFIK_HTPASSWD}" + _traefik_admin_cert_remove + return 0 + fi + _traefik_htpasswd + web_cert_placeholder lss-admin + _traefik_admin_conf | file_write 0640 "root:${TRAEFIK_USER}" "${TRAEFIK_DYNAMIC}/admin.yml" + if state_installed tls; then + web_cert_issue lss-admin "${host}" + fi + return +} + +# _traefik_admin_cert_remove: the certificate of the admin host goes with it, also certbot's lineage, which the tls +# module would otherwise renew and miss in the web server's directory. +_traefik_admin_cert_remove() { + file_remove "${WEB_TLS_DIR}/lss-admin" + if [[ -d ${WEB_CERT_LIVE}/lss-admin ]] && command -v certbot >/dev/null 2>&1; then + log_command certbot delete --non-interactive --cert-name lss-admin || + die "traefik: certbot could not delete the certificate lss-admin." + files_note_change "certificate lss-admin deleted" + fi + return +} + +# The socket proxy for the Docker provider: a container that holds the engine's socket and offers Traefik a socket of +# its own with TRAEFIK_PROXY_ALLOW only. It runs as root in the group of Traefik (the socket it creates belongs to +# that group), without capabilities and without a network. +_traefik_docker() { + local engine gid + engine=$(ctr_engine) || die "traefik: LSS_TRAEFIK_DOCKER=true needs a container engine (lss install docker or podman)." + if [[ ${engine} == podman ]]; then + svc_enable podman.socket + fi + gid=$(getent group "${TRAEFIK_USER}" | cut -d ':' -f 3) + printf '# Managed by lss (module traefik).\nd %s 0750 root %s -\n' "${TRAEFIK_PROXY_DIR}" "${TRAEFIK_USER}" | + file_write 0644 root:root "${TRAEFIK_PROXY_TMPFILES}" + systemd-tmpfiles --create "${TRAEFIK_PROXY_TMPFILES}" >/dev/null 2>&1 || install -d -m 0750 -o root -g "${TRAEFIK_USER}" "${TRAEFIK_PROXY_DIR}" + ctr_run "${TRAEFIK_PROXY_NAME}" "${TRAEFIK_PROXY_IMAGE}" --socket --user "0:${gid}" --network none \ + --volume "${TRAEFIK_PROXY_DIR}:${TRAEFIK_PROXY_DIR}" --memory 64m --pids 64 -- \ + "-proxysocketendpoint=${TRAEFIK_PROXY_SOCKET}" -proxysocketendpointfilemode=0660 \ + "-allowGET=${TRAEFIK_PROXY_ALLOW}" '-allowHEAD=^/(v[0-9.]+/)?_ping$' -watchdoginterval=60 -stoponwatchdog \ + -shutdowngracetime=5 + return +} + +_traefik_docker_remove() { + if [[ -f /etc/systemd/system/lss-ctr-${TRAEFIK_PROXY_NAME}.service ]] || ctr_engine >/dev/null 2>&1; then + ctr_remove "${TRAEFIK_PROXY_NAME}" + fi + file_remove "${TRAEFIK_PROXY_TMPFILES}" + if [[ -d ${TRAEFIK_PROXY_DIR} ]]; then + rm -rf -- "${TRAEFIK_PROXY_DIR}" + files_note_change "removed ${TRAEFIK_PROXY_DIR}" + fi + return +} + +_traefik_unit() { + cat </dev/null 2>&1 || true + endscript +} +EOF + return +} + +# The routes of LSS_TRAEFIK_ROUTES through web_proxy_route; routes that left the setting since the last run go. +_traefik_routes() { + local spec name + local -A wanted=() + for spec in $(opt routes); do + _traefik_route_parse "${spec}" + web_proxy_route "${TRAEFIK_ROUTE_NAME}" "${TRAEFIK_ROUTE_DOMAINS}" "${TRAEFIK_ROUTE_UPSTREAM}" + wanted[${TRAEFIK_ROUTE_NAME}]=1 + done + for name in $(web_route_names traefik); do + [[ -n ${wanted[${name}]+x} ]] || web_proxy_route_remove "${name}" + done + return +} + +mod_configure() { + local server changed=0 bundle + server=$(web_server) + _traefik_user + web_default_cert + install -d -m 0750 -o root -g "${TRAEFIK_USER}" "${TRAEFIK_DIR}" "${TRAEFIK_DYNAMIC}" + install -d -m 0700 -o "${TRAEFIK_USER}" -g "${TRAEFIK_USER}" "${TRAEFIK_STATE}" + install -d -m 0750 -o "${TRAEFIK_USER}" -g "${TRAEFIK_USER}" "${TRAEFIK_LOG_DIR}" + install -d -m 0755 "${TRAEFIK_HOME}" + if _traefik_coraza; then + _traefik_plugin + else + _traefik_plugin_remove + fi + [[ $(files_changes_of traefik) == 0 ]] || changed=1 + bundle=$(opt acme_ca_bundle) + if [[ -n ${bundle} ]]; then + file_write 0644 root:root "${TRAEFIK_ACME_CA}" <"${bundle}" + else + file_remove "${TRAEFIK_ACME_CA}" + fi + _traefik_static | file_write 0640 "root:${TRAEFIK_USER}" "${TRAEFIK_CONF}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + _traefik_dynamic | file_write 0640 "root:${TRAEFIK_USER}" "${TRAEFIK_DYNAMIC}/lss.yml" + _traefik_admin + # Traefik reads the password file only when it loads the route. + [[ $(files_changes_of traefik) == 0 ]] || changed=1 + if opt_bool docker; then + _traefik_docker + else + _traefik_docker_remove + fi + traefik_tls_sync + _traefik_logrotate | file_write 0644 root:root "${TRAEFIK_LOGROTATE}" + fw_allow 80/tcp 443/tcp 443/udp + _traefik_unit | svc_unit "${TRAEFIK_UNIT}.service" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + svc_sandbox "${TRAEFIK_UNIT}.service" "ReadWritePaths=${TRAEFIK_STATE}" "ReadWritePaths=${TRAEFIK_LOG_DIR}" \ + ProtectProc=invisible ProcSubset=pid SystemCallFilter=@system-service SystemCallErrorNumber=EPERM + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + # The web server leaves ports 80 and 443 to Traefik first and listens on 127.0.0.1 behind it. + if [[ -n ${server} ]]; then + LSS_WEB_BACKEND=on module_run "${server}" configure || + die "traefik: ${server} could not switch to the backend ports ${WEB_BACKEND_HTTP_PORT} and ${WEB_BACKEND_HTTPS_PORT}." + fi + traefik_backend_sync "${server}" + if [[ ${changed} == 1 ]] && svc_is_active "${TRAEFIK_UNIT}"; then + svc_restart "${TRAEFIK_UNIT}" || die "traefik: ${TRAEFIK_UNIT} does not start; see journalctl -u ${TRAEFIK_UNIT}." + fi + svc_enable "${TRAEFIK_UNIT}" + _traefik_waf_fallback + # Routes of the applications move from the web server to Traefik; then the routes of this module. + web_routes_apply + _traefik_routes + traefik_ids_configure + return +} + +# _traefik_https HOST PATH [CURL OPTION...]: a request to https://HOST/PATH on this server's port 443; prints the +# response headers, the body and a last line with the status. +_traefik_https() { + local host=$1 path=$2 + shift 2 + curl -sk --max-time 15 -D - --resolve "${host}:443:127.0.0.1" -w '\n%{http_code}' "$@" "https://${host}/${path}" || true + return +} + +_traefik_status() { + local answer=$1 + printf '%s' "${answer##*$'\n'}" + return +} + +# _traefik_listen_local PORT: everything that listens on TCP PORT listens on the loopback interface only. +_traefik_listen_local() { + local port=$1 addresses + addresses=$(ss -Hltn "sport = :${port}" | awk '{ print $4 }') + [[ -n ${addresses} ]] || return 1 + ! grep -qvE '^(127\.0\.0\.1|\[::1\]):' <<<"${addresses}" + return +} + +_traefik_verify_waf() { + local answer + if _traefik_coraza; then + answer=$(_traefik_https "${TRAEFIK_PROBE_HOST}" "?id=${TRAEFIK_SQLI}") + [[ $(_traefik_status "${answer}") == 403 ]] || + die "traefik: Coraza let an SQL injection through (status $(_traefik_status "${answer}"), not 403)." + answer=$(_traefik_https "${TRAEFIK_PROBE_HOST}" "?file=../../../../etc/passwd") + [[ $(_traefik_status "${answer}") == 403 ]] || + die "traefik: Coraza let a path traversal to /etc/passwd through (status $(_traefik_status "${answer}"), not 403)." + else + answer=$(_traefik_https "${TRAEFIK_PROBE_HOST}" "?id=${TRAEFIK_SQLI}") + [[ $(_traefik_status "${answer}") == 200 ]] || + die "traefik: without Coraza the probe route answers $(_traefik_status "${answer}"), not 200." + fi + if _traefik_modsecurity; then + state_installed modsecurity || + die "traefik: LSS_TRAEFIK_WAF=$(opt waf) needs the modsecurity module in the web server (lss install modsecurity), or choose coraza." + # Through Traefik to the backend web server, where ModSecurity decides. + answer=$(curl -s --max-time 15 -o /dev/null -w '%{http_code}' -H 'Host: localhost' \ + "http://127.0.0.1/.well-known/acme-challenge/lss-probe?id=${TRAEFIK_SQLI}" || true) + [[ ${answer} == 403 ]] || die "traefik: ModSecurity behind Traefik let an SQL injection through (status ${answer}, not 403)." + fi + return +} + +# Every site of the vhost module answers through Traefik with the security headers. +_traefik_verify_sites() { + local spec domains domain answer + state_installed vhost || return 0 + for spec in $(config_get LSS_VHOST_SITES); do + IFS=':' read -r _ _ domains _ <<<"${spec}" + domain=${domains%%,*} + answer=$(_traefik_https "${domain}" '') + [[ $(_traefik_status "${answer}") =~ ^[1-5][0-9][0-9]$ ]] || + die "traefik: the site https://${domain}/ does not answer through Traefik." + grep -qi '^strict-transport-security: max-age=63072000' <<<"${answer}" || + die "traefik: https://${domain}/ has no Strict-Transport-Security header." + if _traefik_coraza || _traefik_modsecurity; then + answer=$(_traefik_https "${domain}" "?id=${TRAEFIK_SQLI}") + [[ $(_traefik_status "${answer}") == 403 ]] || + die "traefik: the WAF let an SQL injection to https://${domain}/ through (status $(_traefik_status "${answer}"), not 403)." + fi + done + return +} + +# Every recorded route answers (502 counts as an answer: the application may be down); with Traefik's ACME client +# its certificate comes from the ACME server. +_traefik_verify_routes() { + local name domain answer default i served + default=$(openssl x509 -noout -fingerprint -sha256 -in "${WEB_TLS_DIR}/default/fullchain.pem" 2>/dev/null || true) + for name in $(web_route_names); do + domain=$(awk '{ print $2 }' "$(_web_routes_dir)/${name}") + [[ -f ${TRAEFIK_DYNAMIC}/route-${name}.yml ]] || die "traefik: the route ${name} has no Traefik configuration." + answer='' + for ((i = 0; i < 30; i++)); do + answer=$(_traefik_https "${domain}" '') + [[ $(_traefik_status "${answer}") == 404 ]] || break + sleep 1 + done + [[ $(_traefik_status "${answer}") =~ ^[1-5][0-9][0-9]$ && $(_traefik_status "${answer}") != 404 ]] || + die "traefik: the route ${name} (https://${domain}/) does not answer through Traefik." + # Coraza at the edge, or ModSecurity in the web server the route passes through (traefik_waf_backend). + if _traefik_coraza || traefik_waf_backend; then + answer=$(_traefik_https "${domain}" "?id=${TRAEFIK_SQLI}") + [[ $(_traefik_status "${answer}") == 403 ]] || + die "traefik: the WAF let an SQL injection to the route ${name} through (status $(_traefik_status "${answer}"), not 403)." + fi + opt_bool acme || continue + for ((i = 0; i < 90; i++)); do + served=$(openssl s_client -connect 127.0.0.1:443 -servername "${domain}" /dev/null | + openssl x509 -noout -fingerprint -sha256 2>/dev/null || true) + [[ -z ${served} || ${served} == "${default}" ]] || break + sleep 2 + done + [[ -n ${served} && ${served} != "${default}" ]] || + die "traefik: the ACME server issued no certificate for ${domain}; see journalctl -u ${TRAEFIK_UNIT}." + done + return +} + +# The admin interfaces under LSS_TRAEFIK_ADMIN_HOST: refused without the password, served with it. +_traefik_verify_admin() { + local host password answer tool path + host=$(opt admin_host) + [[ -n ${host} && -n $(web_server) ]] || return 0 + answer=$(_traefik_https "${host}" '') + [[ $(_traefik_status "${answer}") == 401 ]] || + die "traefik: https://${host}/ answers $(_traefik_status "${answer}") without a password, not 401." + secret_load web-admin-password password 32 + for tool in adminer:adminer/ phpmyadmin:phpmyadmin/ pgadmin:pgadmin4/login; do + state_installed "${tool%%:*}" || continue + path=${tool#*:} + answer=$(_traefik_https "${host}" "${path}" -K - <<<"user = \"admin:${password}\"") + [[ $(_traefik_status "${answer}") == 200 ]] || + die "traefik: https://${host}/${path} answers $(_traefik_status "${answer}") with the password, not 200." + done + return +} + +# _traefik_proxy_status CURL-ARGS...: the HTTP status the socket proxy gives Traefik's user for a request. +_traefik_proxy_status() { + runuser -u "${TRAEFIK_USER}" -- curl -s --max-time 10 -o /dev/null -w '%{http_code}' \ + --unix-socket "${TRAEFIK_PROXY_SOCKET}" "$@" || true + return +} + +# The socket proxy answers Traefik's read-only calls and refuses the rest. +_traefik_verify_docker() { + local status + opt_bool docker || return 0 + ctr_running "${TRAEFIK_PROXY_NAME}" || die "traefik: the socket proxy ${TRAEFIK_PROXY_NAME} is not running." + [[ -S ${TRAEFIK_PROXY_SOCKET} ]] || die "traefik: the socket proxy has no socket at ${TRAEFIK_PROXY_SOCKET}." + status=$(_traefik_proxy_status http://localhost/containers/json) + [[ ${status} == 200 ]] || die "traefik: the socket proxy does not list the containers for Traefik (status ${status})." + status=$(_traefik_proxy_status -X POST http://localhost/containers/create) + [[ ${status} =~ ^40[0-9]$ ]] || die "traefik: the socket proxy lets a container be created (status ${status})." + status=$(_traefik_proxy_status http://localhost/images/json) + [[ ${status} =~ ^40[0-9]$ ]] || die "traefik: the socket proxy answers more than container calls (status ${status})." + return +} + +# _traefik_die MESSAGE: stops verify with MESSAGE and the last lines Traefik wrote to the journal, which hold the reason +# (a configuration it refuses, a port it cannot bind). +_traefik_die() { + local message=$1 line + while IFS= read -r line; do + log_warn "traefik: journal: ${line}" + done < <(journalctl -u "${TRAEFIK_UNIT}" -n 80 --no-pager -o cat 2>/dev/null || true) + die "traefik: ${message}" + return 1 +} + +mod_verify() { + local port i answer='' server + port=$(opt_int dashboard_port) + svc_is_active "${TRAEFIK_UNIT}" || _traefik_die "${TRAEFIK_UNIT} is not running." + for ((i = 0; i < 30; i++)); do + answer=$(curl -s --max-time 5 "http://127.0.0.1:${port}/ping" 2>/dev/null || true) + [[ ${answer} != OK ]] || break + sleep 1 + done + [[ ${answer} == OK ]] || _traefik_die "the ping on 127.0.0.1:${port} does not answer." + _traefik_listen_local "${port}" || die "traefik: the dashboard port ${port} listens beyond 127.0.0.1." + curl -fsS --max-time 10 -o /dev/null "http://127.0.0.1:${port}/api/version" || + _traefik_die "the API on 127.0.0.1:${port} does not answer." + # The probe route goes through the middlewares of the HTTPS entry point; a new file takes a moment to load. + for ((i = 0; i < 30; i++)); do + answer=$(_traefik_https "${TRAEFIK_PROBE_HOST}" '') + [[ $(_traefik_status "${answer}") != 200 ]] || break + sleep 1 + done + [[ $(_traefik_status "${answer}") == 200 ]] || _traefik_die "port 443 does not answer for the probe route." + grep -qi '^strict-transport-security: max-age=63072000' <<<"${answer}" || + die "traefik: HTTPS answers have no Strict-Transport-Security header." + grep -qi '^x-content-type-options: nosniff' <<<"${answer}" || die "traefik: HTTPS answers have no X-Content-Type-Options header." + grep -qi '^alt-svc: .*h3=' <<<"${answer}" || die "traefik: HTTPS answers do not advertise HTTP/3 (Alt-Svc)." + [[ -n $(ss -Hlun 'sport = :443') ]] || die "traefik: nothing listens on 443/udp for HTTP/3." + answer=$(curl -s --max-time 10 -o /dev/null -w '%{http_code} %{redirect_url}' -H "Host: ${TRAEFIK_PROBE_HOST}" \ + http://127.0.0.1/lss-check || true) + [[ ${answer} == "301 https://${TRAEFIK_PROBE_HOST}/lss-check" ]] || + die "traefik: port 80 does not redirect to HTTPS (${answer})." + _traefik_verify_waf + server=$(web_server) + if [[ -n ${server} ]]; then + _traefik_listen_local "${WEB_BACKEND_HTTPS_PORT}" || + die "traefik: ${server} does not listen on 127.0.0.1:${WEB_BACKEND_HTTPS_PORT} only (backend mode)." + [[ $(curl -fsS --max-time 10 http://127.0.0.1/.well-known/acme-challenge/lss-probe 2>/dev/null) == lss ]] || + die "traefik: ACME challenges on port 80 do not reach ${server}." + fi + # HTTPS to the backend: the sites of the vhost module (a host name without a site gets no answer from the web + # server, so it shows nothing). + _traefik_verify_sites + _traefik_verify_routes + _traefik_verify_admin + _traefik_verify_docker + return +} + +mod_remove() { + local server name + # Everything goes back to the web server: routes, ports 80 and 443. + _traefik_waf_fallback_remove + export LSS_WEB_BACKEND=off + for name in $(web_route_names traefik); do + web_proxy_route_remove "${name}" + done + svc_disable "${TRAEFIK_UNIT}" + server=$(web_server) + if [[ -n ${server} ]]; then + module_run "${server}" configure || die "traefik: ${server} could not take ports 80 and 443 back." + web_routes_apply + elif [[ -n $(web_route_names) ]]; then + log_warn "traefik: the routes $(web_route_names | tr '\n' ' ')have no web server now; install nginx, apache or caddy to serve them." + fi + _traefik_docker_remove + traefik_ids_remove + svc_dropin_remove "${TRAEFIK_UNIT}.service" lss-sandbox + file_remove "/etc/systemd/system/${TRAEFIK_UNIT}.service" + svc_daemon_reload + file_remove "${TRAEFIK_LOGROTATE}" + file_remove "${TRAEFIK_DIR}" + file_remove "${TRAEFIK_HOME}" + file_remove "${TRAEFIK_BIN}" + _traefik_admin_cert_remove + if [[ -d ${TRAEFIK_CACHE} ]]; then + rm -rf -- "${TRAEFIK_CACHE}" + files_note_change "removed ${TRAEFIK_CACHE}" + fi + # Traefik's ACME account and certificates and the access logs stay unless the removal purges. + if [[ ${LSS_PURGE:-0} == 1 ]]; then + rm -rf -- "${TRAEFIK_STATE}" "${TRAEFIK_LOG_DIR}" + if getent passwd "${TRAEFIK_USER}" >/dev/null; then + userdel "${TRAEFIK_USER}" >/dev/null 2>&1 || true + fi + if getent group "${TRAEFIK_USER}" >/dev/null; then + groupdel "${TRAEFIK_USER}" >/dev/null 2>&1 || true + fi + files_note_change "removed Traefik's data" + fi + return +} diff --git a/profiles/all.conf b/profiles/all.conf index 249fae7..5bdd3ae 100644 --- a/profiles/all.conf +++ b/profiles/all.conf @@ -7,4 +7,4 @@ # For the nightly test and as a reference. ssh-2fa, clamav and usbguard stay opt-in as in minimal-secure. k3s # conflicts with docker (both bring their own containerd and claim the forwarding rules) and has its own profile, # k8s-node; podman is the alternative to docker. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server mailserver webmail node-exporter prometheus alertmanager grafana loki alloy netdata smartd +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity traefik php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server mailserver webmail node-exporter prometheus alertmanager grafana loki alloy netdata smartd diff --git a/tests/integration/run.sh b/tests/integration/run.sh index 737568c..72a1051 100755 --- a/tests/integration/run.sh +++ b/tests/integration/run.sh @@ -243,22 +243,36 @@ if [[ $1 == --profile ]]; then exit 0 fi -# The tls module gets its certificates from Pebble. -if [[ " $* " == *[\ ,]tls[\ ,]* ]]; then +# The tls module and Traefik's own ACME client get their certificates from Pebble. +if [[ " $* " == *[\ ,]tls[\ ,]* || " $* " == *[\ ,]traefik[\ ,]* ]]; then group_start "${system}: Pebble (ACME test server)" pebble_start group_end fi for item in "$@"; do - IFS=',' read -ra set <<<"${item}" - group_start "${system}: ${set[*]}" + # LSS_KEY=VALUE in a set is a setting for this set only (nginx,traefik,LSS_TRAEFIK_WAF=off), added to the test + # settings for its install, verify and removal. + IFS=',' read -ra parts <<<"${item}" + set=() + extra=() + for part in "${parts[@]}"; do + if [[ ${part} =~ ^LSS_[A-Z0-9_]+=[A-Za-z0-9_.:/@-]*$ ]]; then + extra+=("${part}") + else + set+=("${part}") + fi + done + group_start "${system}: ${set[*]}${extra[*]:+ (${extra[*]})}" + if ((${#extra[@]} > 0)); then + printf '%s\n' "${extra[@]}" | in_target "cat >> /etc/lss/lss.conf" + fi before=$(resources_snapshot) started=${SECONDS} memory_watch_start in_target "${LSS_RUN} --yes --no-color install ${set[*]}" in_target "${LSS_RUN} --yes --no-color verify" - resources_report "${system} ${set[*]}" "${before}" "${started}" + resources_report "${system} ${set[*]}${extra[*]:+ (${extra[*]})}" "${before}" "${started}" mail_flow "${set[@]}" install_again "${set[@]}" if [[ ${LSS_TEST_KEEP:-0} != 1 ]]; then @@ -266,6 +280,7 @@ for item in "$@"; do in_target "${LSS_RUN} --yes --no-color remove ${set[i]}" in_target "! lss status | grep -q '^${set[i]} '" done + in_target "cat > /etc/lss/lss.conf" <"${settings}" fi group_end done diff --git a/tests/integration/settings.conf b/tests/integration/settings.conf index 7f10778..00da851 100644 --- a/tests/integration/settings.conf +++ b/tests/integration/settings.conf @@ -16,6 +16,11 @@ LSS_TLS_CA_BUNDLE=/etc/lss-pebble/pebble.minica.pem LSS_TLS_DOMAINS=lss-test.example.com LSS_VHOST_SITES=demo:static:demo.example.com,www.demo.example.com app:proxy:app.example.com:http://127.0.0.1:9 LSS_HAPROXY_SERVICES=echo:9190/local:127.0.0.1:9,127.0.0.1:10 +# Traefik: a route to a port without a service (it answers 502, which still shows the route works) and Pebble for +# its own ACME client (the job that sets LSS_TRAEFIK_ACME=true). +LSS_TRAEFIK_ROUTES=echo:echo.example.com:127.0.0.1:9 +LSS_TRAEFIK_ACME_SERVER=https://localhost:14000/dir +LSS_TRAEFIK_ACME_CA_BUNDLE=/etc/lss-pebble/pebble.minica.pem # Database modules: one application user each, so verify also covers creating users, databases and their grants. # A small memory limit keeps redis and valkey from growing on the shared runner. diff --git a/tests/unit/container.bats b/tests/unit/container.bats index 0394e32..809e409 100644 --- a/tests/unit/container.bats +++ b/tests/unit/container.bats @@ -108,10 +108,10 @@ setup() { ctr_args docker web "${IMAGE}" --volume "${dir}/fine:/data" } -@test "only Portainer gets the socket, in the host user namespace, without SELinux separation on RHEL" { +@test "only Portainer and the socket proxy of Traefik get the socket, in the host user namespace, without SELinux separation on RHEL" { run ctr_args docker web "${IMAGE}" --socket [ "${status}" -eq 1 ] - [[ ${output} == *"only lss-portainer may have the socket"* ]] + [[ ${output} == *"only lss-portainer lss-traefik-docker may have the socket"* ]] ctr_args docker lss-portainer "${IMAGE}" --socket [[ ${CTR_ARGS[*]} == *"--volume /var/run/docker.sock:/var/run/docker.sock --userns host"* ]] [[ ${CTR_ARGS[*]} != *label=disable* ]] diff --git a/tests/unit/helpers.bash b/tests/unit/helpers.bash index 028bb71..243a4c9 100644 --- a/tests/unit/helpers.bash +++ b/tests/unit/helpers.bash @@ -22,7 +22,7 @@ lss_test_setup() { export OWNER : >"${STUB_LOG}" local library - for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring module ui; do + for library in log os config files template secrets state pkg repo fetch service firewall container web traefik mail monitoring module ui; do # shellcheck source=/dev/null source "${REPO_ROOT}/lib/${library}.sh" done diff --git a/tests/unit/traefik.bats b/tests/unit/traefik.bats new file mode 100644 index 0000000..aa6e051 --- /dev/null +++ b/tests/unit/traefik.bats @@ -0,0 +1,503 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Traefik as the edge proxy: the derived files of lib/traefik.sh (certificates, backend, routes, fail2ban and CrowdSec) +# and the configuration the traefik module writes (static configuration, middlewares, WAF, unit, routes). + +setup() { + load helpers + lss_test_setup + use_os debian-13 + shopt -s lastpipe + export TMPDIR=${BATS_TEST_TMPDIR} + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + TRAEFIK_DYNAMIC=${BATS_TEST_TMPDIR}/traefik/dynamic + TRAEFIK_LOG_DIR=${BATS_TEST_TMPDIR}/log/traefik + TRAEFIK_FAIL2BAN_JAIL=${BATS_TEST_TMPDIR}/fail2ban/jail.d/lss-traefik.local + TRAEFIK_FAIL2BAN_FILTER_AUTH=${BATS_TEST_TMPDIR}/fail2ban/filter.d/lss-traefik-auth.conf + TRAEFIK_FAIL2BAN_FILTER_WAF=${BATS_TEST_TMPDIR}/fail2ban/filter.d/lss-traefik-waf.conf + TRAEFIK_CROWDSEC_ACQUIS=${BATS_TEST_TMPDIR}/crowdsec/acquis.d/lss-traefik.yaml + mkdir -p "${WEB_TLS_DIR}" "${TRAEFIK_DYNAMIC}" "$(dirname "${TRAEFIK_FAIL2BAN_JAIL}")" \ + "$(dirname "${TRAEFIK_FAIL2BAN_FILTER_AUTH}")" "$(dirname "${TRAEFIK_CROWDSEC_ACQUIS}")" +} + +# The module as lss loads it for a step of its own. +load_module() { + # shellcheck source=modules/web/traefik.sh + source "${REPO_ROOT}/modules/web/traefik.sh" + LSS_MODULE=traefik + TRAEFIK_DYNAMIC=${BATS_TEST_TMPDIR}/traefik/dynamic + return +} + +# Lets the writers run without root: ownership and the SELinux label are left out, the mode stays; the derived files +# of Traefik look as if they belonged to root and Traefik's group. +as_test_user() { + eval "_lss_file_write() $(declare -f file_write | tail -n +2)" + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + _lss_file_write "$1" "${OWNER}" "$3" + return + } + # shellcheck disable=SC2329 # called by the functions under test + chown() { + printf 'chown %s\n' "$*" >>"${STUB_LOG}" + return + } + # shellcheck disable=SC2329 # called by the functions under test + stat() { + if [[ ${*: -1} == "${TRAEFIK_DYNAMIC}"/* ]]; then + printf '%s %s:%s' "$(command stat -c '%a' "${@: -1}")" root "${TRAEFIK_USER}" + else + command stat "$@" + fi + return + } + # shellcheck disable=SC2329 # called by the functions under test + install() { + local -a args=() + while [[ $# -gt 0 ]]; do + case $1 in + -o | -g) shift ;; + *) args+=("$1") ;; + esac + shift + done + command install "${args[@]}" + return + } + return +} + +use_module() { + state_set "$1" status installed + return +} + +# --- when Traefik is in front ------------------------------------------------------------------------------------- + +@test "Traefik is in front once installed or while it installs, and not while it hands the ports back" { + run ! traefik_active + LSS_MODULE=traefik + traefik_active + unset LSS_MODULE + use_module traefik + traefik_active + LSS_WEB_BACKEND=off + run ! traefik_active + traefik_present +} + +@test "without Traefik the derived files are left alone" { + traefik_tls_sync + traefik_backend_sync nginx + [ -z "$(ls -A "${TRAEFIK_DYNAMIC}")" ] + run traefik_route_write app http://127.0.0.1:3000 app.example.com + [ "${status}" -eq 1 ] + [[ ${output} == *"traefik: the route app needs Traefik's configuration in ${TRAEFIK_DYNAMIC}."* ]] +} + +# --- derived files ------------------------------------------------------------------------------------------------ + +@test "a derived file is written once per change, readable by Traefik's group only" { + as_test_user + use_module traefik + LSS_MODULE=core + traefik_backend_sync nginx + [ "${LSS_FILE_CHANGED}" = 1 ] + [ "$(command stat -c '%a' "${TRAEFIK_DYNAMIC}/backend.yml")" = 640 ] + grep -q '^chown root:lss-traefik .*/backend.yml.lss.' "${STUB_LOG}" + traefik_backend_sync nginx + [ "${LSS_FILE_CHANGED}" = 0 ] + [ "$(files_changes_of core)" = 1 ] + traefik_backend_sync '' + [ ! -e "${TRAEFIK_DYNAMIC}/backend.yml" ] + [ "$(files_changes_of core)" = 2 ] +} + +@test "the backend gets every other host name on HTTPS and the ACME challenges on port 80, on 127.0.0.1 only" { + run traefik_backend_conf + [[ ${output} == *$' lss-backend:\n entryPoints: [websecure]\n rule: PathPrefix(`/`)\n priority: 1'* ]] + [[ ${output} == *'rule: PathPrefix(`/.well-known/acme-challenge/`)'* ]] + [[ ${output} == *'- url: https://127.0.0.1:8443'* ]] + [[ ${output} == *'- url: http://127.0.0.1:8008'* ]] + [[ ${output} == *$' serversTransports:\n lss-backend:\n insecureSkipVerify: true'* ]] +} + +@test "every complete certificate directory is offered by SNI, the default certificate for every other name" { + mkdir -p "${WEB_TLS_DIR}/default" "${WEB_TLS_DIR}/shop" "${WEB_TLS_DIR}/half" + touch "${WEB_TLS_DIR}/half/fullchain.pem" + for dir in default shop; do + echo pem >"${WEB_TLS_DIR}/${dir}/fullchain.pem" + echo pem >"${WEB_TLS_DIR}/${dir}/privkey.pem" + done + run traefik_tls_conf + [[ ${output} == *" - certFile: ${WEB_TLS_DIR}/shop/fullchain.pem"$'\n'" keyFile: ${WEB_TLS_DIR}/shop/privkey.pem"* ]] + [[ ${output} == *"- certFile: ${WEB_TLS_DIR}/default/fullchain.pem"* ]] + [[ ${output} != *"${WEB_TLS_DIR}/half"* ]] + [[ ${output} == *$' defaultCertificate:\n certFile: '"${WEB_TLS_DIR}/default/fullchain.pem"* ]] + [[ ${output} == *'minVersion: VersionTLS12'* ]] + [ "$(grep -c '^ - TLS_ECDHE_' <<<"${output}")" = 6 ] +} + +@test "a route matches each of its host names and gets Traefik's ACME resolver only when it is switched on" { + run traefik_route_conf app http://127.0.0.1:3000 app.example.com www.app.example.com + [[ ${output} == *'rule: "Host(`app.example.com`) || Host(`www.app.example.com`)"'* ]] + [[ ${output} == *$' route-app:\n entryPoints: [websecure]'* ]] + [[ ${output} == *'- url: http://127.0.0.1:3000'* ]] + [[ ${output} != *certResolver* ]] + export LSS_TRAEFIK_ACME=true + run traefik_route_conf app http://127.0.0.1:3000 app.example.com + [[ ${output} == *$' tls:\n certResolver: lss'* ]] +} + +@test "a route file is written for Traefik and removed again" { + as_test_user + use_module traefik + LSS_MODULE=gitea + traefik_route_write gitea http://127.0.0.1:3000 git.example.com + grep -qF 'Host(`git.example.com`)' "${TRAEFIK_DYNAMIC}/route-gitea.yml" + traefik_route_remove gitea + [ ! -e "${TRAEFIK_DYNAMIC}/route-gitea.yml" ] + [ "$(files_changes_of gitea)" = 2 ] + traefik_route_remove gitea + [ "$(files_changes_of gitea)" = 2 ] +} + +# --- fail2ban and CrowdSec ------------------------------------------------------------------------------------------ + +@test "fail2ban bans failed logins and WAF blocks from Traefik's JSON access log" { + as_test_user + use_module traefik + use_module fail2ban + export STUB_ACTIVE_UNITS=fail2ban + traefik_ids_configure + grep -qxF '[lss-traefik-auth]' "${TRAEFIK_FAIL2BAN_JAIL}" + grep -qxF '[lss-traefik-waf]' "${TRAEFIK_FAIL2BAN_JAIL}" + grep -qxF "logpath = ${TRAEFIK_LOG_DIR}/access.log" "${TRAEFIK_FAIL2BAN_JAIL}" + [ -e "${TRAEFIK_LOG_DIR}/access.log" ] + grep -qxF 'fail2ban-client --test' "${STUB_LOG}" + grep -qxF 'systemctl restart fail2ban' "${STUB_LOG}" + run ! grep -q cscli "${STUB_LOG}" + # A line as Traefik writes it (keys sorted): the client is ClientHost, not the backend. + local line regex + line='{"ClientAddr":"203.0.113.7:51234","ClientHost":"203.0.113.7","DownstreamStatus":403,"RequestHost":"app.example.com"}' + regex=$(sed -n 's/^failregex = //p' "${TRAEFIK_FAIL2BAN_FILTER_WAF}") + regex=${regex///([0-9.]+)} + [[ ${line} =~ ${regex} ]] + [ "${BASH_REMATCH[1]}" = 203.0.113.7 ] + regex=$(sed -n 's/^failregex = //p' "${TRAEFIK_FAIL2BAN_FILTER_AUTH}") + regex=${regex///([0-9.]+)} + [[ ! ${line} =~ ${regex} ]] + : >"${STUB_LOG}" + traefik_ids_configure + run ! grep -qE 'fail2ban-client|restart' "${STUB_LOG}" +} + +@test "CrowdSec gets the Traefik collection and the access log, and both go when Traefik goes" { + as_test_user + use_module traefik + use_module crowdsec + export STUB_ACTIVE_UNITS=crowdsec + traefik_ids_configure + grep -qxF 'cscli collections install crowdsecurity/traefik' "${STUB_LOG}" + grep -qxF ' type: traefik' "${TRAEFIK_CROWDSEC_ACQUIS}" + grep -qxF " - ${TRAEFIK_LOG_DIR}/access.log" "${TRAEFIK_CROWDSEC_ACQUIS}" + grep -qxF 'systemctl restart crowdsec' "${STUB_LOG}" + : >"${STUB_LOG}" + LSS_WEB_BACKEND=off traefik_ids_configure + [ ! -e "${TRAEFIK_CROWDSEC_ACQUIS}" ] + grep -qxF 'systemctl restart crowdsec' "${STUB_LOG}" +} + +# --- the module ----------------------------------------------------------------------------------------------------- + +@test "the settings are checked before anything is installed" { + load_module + mod_check + export LSS_TRAEFIK_WAF=modsec + run mod_check + [[ ${output} == *'LSS_TRAEFIK_WAF must be coraza, modsecurity, both or off.'* ]] + export LSS_TRAEFIK_WAF=both LSS_TRAEFIK_PARANOIA=5 + run mod_check + [[ ${output} == *'LSS_TRAEFIK_PARANOIA must be 1, 2, 3 or 4.'* ]] + export LSS_TRAEFIK_PARANOIA=2 LSS_TRAEFIK_DASHBOARD_PORT=8443 + run mod_check + [[ ${output} == *'LSS_TRAEFIK_DASHBOARD_PORT must differ from 8090, 8008 and 8443.'* ]] + export LSS_TRAEFIK_DASHBOARD_PORT=8099 LSS_TRAEFIK_ROUTES='app:app.example.com:3000' + run mod_check + [[ ${output} == *'the route app needs HOST:PORT at the end, like 127.0.0.1:3000.'* ]] + export LSS_TRAEFIK_ROUTES='app:app.example.com,www.app.example.com:127.0.0.1:3000 api:api.example.com:10.0.0.5:8080' + mod_check + export LSS_TRAEFIK_ADMIN_HOST=admin + run mod_check + [[ ${output} == *"LSS_TRAEFIK_ADMIN_HOST 'admin' is not a host name like admin.example.com."* ]] + export LSS_TRAEFIK_ADMIN_HOST=admin.example.com LSS_TRAEFIK_ACME_SERVER=http://acme.example.com/dir + run mod_check + [[ ${output} == *'LSS_TRAEFIK_ACME_SERVER must be an https:// URL.'* ]] +} + +@test "Traefik and the Coraza plugin have checksums for both architectures" { + load_module + use_os debian-13 x86_64 + [[ $(fetch_entry "traefik-${TRAEFIK_VERSION}") == *_linux_amd64.tar.gz ]] + use_os debian-13 aarch64 + [[ $(fetch_entry "traefik-${TRAEFIK_VERSION}") == *_linux_arm64.tar.gz ]] + [ "$(_traefik_arch)" = arm64 ] + [[ $(fetch_entry "coraza-http-wasm-${TRAEFIK_CORAZA_VERSION}") == *"/v${TRAEFIK_CORAZA_VERSION}/"*.zip ]] +} + +@test "a route of the setting is split into its name, host names and upstream" { + load_module + _traefik_route_parse 'app:app.example.com,www.app.example.com:127.0.0.1:3000' + [ "${TRAEFIK_ROUTE_NAME}" = app ] + [ "${TRAEFIK_ROUTE_DOMAINS}" = app.example.com,www.app.example.com ] + [ "${TRAEFIK_ROUTE_UPSTREAM}" = 127.0.0.1:3000 ] + run _traefik_route_parse 'App:app.example.com:127.0.0.1:3000' + [ "${status}" -eq 1 ] + run _traefik_route_parse 'app:localhost:127.0.0.1:3000' + [ "${status}" -eq 1 ] +} + +@test "the static configuration: HTTP/3 on 443, the middlewares, the dashboard on 127.0.0.1 and a JSON access log" { + load_module + OS_ARCH=aarch64 + run _traefik_static + [[ ${output} == *$' web:\n address: ":80"'* ]] + [[ ${output} == *$' http3:\n advertisedPort: 443'* ]] + [[ ${output} == *$' - lss-headers@file\n - lss-waf@file\n'* ]] + [[ ${output} == *'address: "127.0.0.1:8099"'* ]] + [[ ${output} == *$'ping:\n entryPoint: traefik'* ]] + [[ ${output} == *"directory: ${TRAEFIK_DYNAMIC}"* ]] + [[ ${output} == *$' format: json\n fields:\n headers:\n defaultMode: drop'* ]] + [[ ${output} == *'moduleName: github.com/jcchavezs/coraza-http-wasm-traefik'* ]] + [[ ${output} == *'checkNewVersion: false'* ]] + [[ ${output} != *docker* ]] + [[ ${output} != *certificatesResolvers* ]] +} + +@test "without Coraza there is no plugin; Docker and ACME appear only when switched on" { + load_module + export LSS_TRAEFIK_WAF=modsecurity LSS_TRAEFIK_DOCKER=true LSS_TRAEFIK_ACME=true LSS_TRAEFIK_ACME_EMAIL=ops@example.com + run _traefik_static + [[ ${output} != *lss-waf* ]] + [[ ${output} != *localPlugins* ]] + # The socket proxy, never the engine's own socket. + [[ ${output} == *"endpoint: unix:///run/lss-traefik-docker/docker.sock"* ]] + [[ ${output} == *'exposedByDefault: false'* ]] + [[ ${output} != *docker.sock\"* ]] + [[ ${output} != */var/run/docker.sock* ]] + [[ ${output} == *$' tlsChallenge: {}\n email: ops@example.com'* ]] + [[ ${output} == *'caServer: https://acme-v02.api.letsencrypt.org/directory'* ]] +} + +@test "the middlewares: the security headers, the redirect to HTTPS and Coraza with the Core Rule Set" { + load_module + OS_ARCH=aarch64 + export LSS_TRAEFIK_PARANOIA=2 + run _traefik_dynamic + [[ ${output} == *' Strict-Transport-Security: "max-age=63072000"'* ]] + [[ ${output} == *$' Server: ""\n X-Powered-By: ""'* ]] + [[ ${output} == *$' redirectScheme:\n scheme: https\n permanent: true'* ]] + [[ ${output} == *'- Include @owasp_crs/*.conf'* ]] + [[ ${output} == *'setvar:tx.blocking_paranoia_level=2"'* ]] + [[ ${output} == *'- SecRuleEngine On'* ]] + [[ ${output} == *'rule: Host(`lss-probe.invalid`)'* ]] + export LSS_TRAEFIK_WAF=off + run _traefik_dynamic + [[ ${output} != *coraza* ]] +} + +@test "Coraza stays out on amd64 (wazero 1.8.0 in Traefik); ModSecurity takes its place" { + load_module + OS_ARCH=x86_64 + run _traefik_static + [[ ${output} != *lss-waf* ]] + [[ ${output} != *localPlugins* ]] + run _traefik_dynamic + [[ ${output} != *coraza* ]] + traefik_waf_fallback + run ! _traefik_modsecurity + state_set modsecurity status installed + _traefik_modsecurity + export LSS_TRAEFIK_WAF=both + traefik_waf_fallback + export LSS_TRAEFIK_WAF=modsecurity + run ! traefik_waf_fallback + export LSS_TRAEFIK_WAF=off + run ! traefik_waf_fallback + run ! _traefik_modsecurity + export LSS_TRAEFIK_WAF=coraza + TRAEFIK_CORAZA_AMD64=true + run ! traefik_waf_fallback + _traefik_coraza + TRAEFIK_CORAZA_AMD64=false OS_ARCH=aarch64 + _traefik_coraza + run ! traefik_waf_fallback +} + +@test "routes pass through the web server for ModSecurity only where Coraza cannot run and nginx or Apache serves" { + load_module + OS_ARCH=x86_64 + run ! traefik_waf_backend + state_set nginx status installed + traefik_waf_backend + TRAEFIK_BACKEND_GONE=1 + run ! traefik_waf_backend + unset TRAEFIK_BACKEND_GONE + LSS_WEB_BACKEND=off + run ! traefik_waf_backend + unset LSS_WEB_BACKEND + state_forget nginx + state_set caddy status installed + run ! traefik_waf_backend + state_forget caddy + state_set apache status installed + traefik_waf_backend + OS_ARCH=aarch64 + run ! traefik_waf_backend + OS_ARCH=x86_64 + export LSS_TRAEFIK_WAF=modsecurity + run ! traefik_waf_backend +} + +@test "a route through the web server points Traefik at the backend with its transport" { + load_module + run traefik_route_conf app "https://127.0.0.1:${WEB_BACKEND_HTTPS_PORT}" app.example.com + [[ ${output} == *$' loadBalancer:\n serversTransport: lss-backend\n servers:\n - url: https://127.0.0.1:8443'* ]] + run traefik_route_conf app http://127.0.0.1:3000 app.example.com + [[ ${output} != *serversTransport* ]] + [[ ${output} == *'- url: http://127.0.0.1:3000'* ]] +} + +@test "in place of Coraza lss installs modsecurity once, says why, and removes it with Traefik" { + load_module + OS_ARCH=x86_64 + # shellcheck disable=SC2329 # called by the functions under test + _traefik_wait() { return; } + # shellcheck disable=SC2329 # called by the functions under test + module_run() { + printf 'module_run %s\n' "$*" >>"${STUB_LOG}" + return + } + # shellcheck disable=SC2329 # called by the functions under test + module_apply() { + printf 'module_apply %s LSS_WEB_BACKEND=%s\n' "$*" "${LSS_WEB_BACKEND:-}" >>"${STUB_LOG}" + state_set "$1" status installed + return + } + # shellcheck disable=SC2329 # called by the functions under test + module_remove() { + printf 'module_remove %s\n' "$*" >>"${STUB_LOG}" + state_forget "$1" + return + } + run _traefik_waf_fallback + [[ ${output} == *'traefik: Coraza waits for a Traefik release with wazero 1.9 on amd64; requests reach the applications unfiltered until nginx or Apache is installed'* ]] + state_set caddy status installed + run _traefik_waf_fallback + [[ ${output} == *'Caddy has no ModSecurity connector'* ]] + state_forget caddy + state_set nginx status installed + _traefik_waf_fallback + grep -qxF 'module_run modsecurity check' "${STUB_LOG}" + grep -qxF 'module_apply modsecurity LSS_WEB_BACKEND=on' "${STUB_LOG}" + [ "$(state_get traefik waf_dependency)" = modsecurity ] + : >"${STUB_LOG}" + run _traefik_waf_fallback + [[ ${output} == *'ModSecurity in nginx filters every request, and the routes of applications pass through nginx for it.'* ]] + [[ ! -s ${STUB_LOG} ]] + _traefik_waf_fallback_remove + grep -qxF 'module_remove modsecurity' "${STUB_LOG}" + # A modsecurity of its own, installed by the user, stays. + state_set modsecurity status installed + state_set traefik waf_dependency '' + : >"${STUB_LOG}" + _traefik_waf_fallback_remove + [[ ! -s ${STUB_LOG} ]] + # With Coraza at work nothing happens. + OS_ARCH=aarch64 + run _traefik_waf_fallback + [ -z "${output}" ] +} + +@test "the unit runs as its own user with CAP_NET_BIND_SERVICE as its only capability" { + load_module + run _traefik_unit + [[ ${output} == *$'User=lss-traefik\nGroup=lss-traefik'* ]] + [[ ${output} == *$'AmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE'* ]] + [[ ${output} == *'ExecStart=/usr/local/bin/traefik --configFile=/etc/traefik/traefik.yml'* ]] + [[ ${output} != *LEGO_CA_CERTIFICATES* ]] + export LSS_TRAEFIK_ACME_CA_BUNDLE=/etc/lss-pebble/ca.pem + run _traefik_unit + [[ ${output} == *'Environment=LEGO_CA_CERTIFICATES=/etc/traefik/acme-ca.pem'* ]] +} + +@test "the admin host asks for the password and reaches a public admin port over HTTPS" { + load_module + use_module nginx + export LSS_TRAEFIK_ADMIN_HOST=admin.example.com + run _traefik_admin_conf + [[ ${output} == *'usersFile: /etc/traefik/admin.htpasswd'* ]] + [[ ${output} == *'rule: Host(`admin.example.com`)'* ]] + [[ ${output} == *'- url: http://127.0.0.1:8090'* ]] + [[ ${output} != *serversTransport* ]] + export LSS_NGINX_ADMIN_PUBLIC=true + run _traefik_admin_conf + [[ ${output} == *$' serversTransport: lss-backend\n servers:\n - url: https://127.0.0.1:8090'* ]] +} + +@test "the certificate of the admin host leaves with it, certbot's lineage included" { + load_module + WEB_CERT_LIVE=${BATS_TEST_TMPDIR}/letsencrypt/live + mkdir -p "${WEB_TLS_DIR}/lss-admin" "${WEB_CERT_LIVE}/lss-admin" + _traefik_admin_cert_remove + [[ ! -e ${WEB_TLS_DIR}/lss-admin ]] + grep -qxF 'certbot delete --non-interactive --cert-name lss-admin' "${STUB_LOG}" + : >"${STUB_LOG}" + rmdir "${WEB_CERT_LIVE}/lss-admin" + _traefik_admin_cert_remove + [[ ! -s ${STUB_LOG} ]] +} + +@test "the routes of the setting are recorded, and a route that left the setting is removed" { + load_module + # shellcheck disable=SC2329 # called by the functions under test + web_proxy_route() { + printf 'route %s\n' "$*" >>"${STUB_LOG}" + return + } + # shellcheck disable=SC2329 # called by the functions under test + web_route_names() { + printf 'app\nold\n' + return + } + # shellcheck disable=SC2329 # called by the functions under test + web_proxy_route_remove() { + printf 'remove %s\n' "$*" >>"${STUB_LOG}" + return + } + export LSS_TRAEFIK_ROUTES='app:app.example.com:127.0.0.1:3000' + _traefik_routes + grep -qxF 'route app app.example.com 127.0.0.1:3000' "${STUB_LOG}" + grep -qxF 'remove old' "${STUB_LOG}" + run ! grep -qxF 'remove app' "${STUB_LOG}" +} + +@test "the dashboard check accepts the loopback interface only" { + load_module + # shellcheck disable=SC2329 # called by the functions under test + ss() { + printf '%s\n' "${STUB_SS[@]}" + return + } + STUB_SS=('LISTEN 0 4096 127.0.0.1:8099 0.0.0.0:*') + _traefik_listen_local 8099 + STUB_SS=('LISTEN 0 4096 127.0.0.1:8099 0.0.0.0:*' 'LISTEN 0 4096 [::]:8099 [::]:*') + run ! _traefik_listen_local 8099 + STUB_SS=() + run ! _traefik_listen_local 8099 + [ "$(_traefik_status $'HTTP/2 403\r\n\r\nbody\n403')" = 403 ] +} diff --git a/tests/unit/web.bats b/tests/unit/web.bats index 2fe1caf..1764c88 100644 --- a/tests/unit/web.bats +++ b/tests/unit/web.bats @@ -441,7 +441,9 @@ EOF [[ ${output} == *'server_name shop.example.com www.shop.example.com;'* ]] [[ ${output} == *'return 301 https://$host$request_uri;'* ]] [[ ${output} == *"root ${WEB_ACME_ROOT};"* ]] - [[ ${output} == *'http2 on;'* ]] + [[ ${output} == *"include /etc/nginx/lss/listen-http.conf;"* ]] + [[ ${output} == *"include /etc/nginx/lss/listen-https.conf;"* ]] + [[ ${output} != *listen\ * ]] [[ ${output} == *"fastcgi_pass unix:${WEB_PHP_SOCKET};"* ]] # PATH_INFO after the script name (/static.php/skins/...), as Roundcube and Nextcloud use it. [[ ${output} == *'location ~ [^/]\.php(/|$) {'* ]] @@ -473,29 +475,69 @@ EOF [[ ${output} == *$'\tSSLUseStapling on'* ]] } -@test "nginx sites are written once per change for static files and proxies, on IPv4 and IPv6" { - as_test_user - use_server nginx +@test "nginx listens on every address on 80 and 443, or on 127.0.0.1 on the backend ports behind Traefik" { # shellcheck disable=SC2329 # called by the functions under test web_ipv6() { return 0; } + run web_nginx_listen http + [ "${output}" = $'\tlisten 80;\n\tlisten [::]:80;' ] + run web_nginx_listen https default_server + [ "${output}" = $'\tlisten 443 default_server ssl;\n\tlisten [::]:443 default_server ssl;' ] + NGINX_VERSION=1.22.1 + run web_nginx_listen https + [ "${output}" = $'\tlisten 443 ssl http2;\n\tlisten [::]:443 ssl http2;' ] + use_server traefik + run web_nginx_listen http default_server + [ "${output}" = $'\tlisten 127.0.0.1:8008 default_server;' ] + run web_nginx_listen https + [ "${output}" = $'\tlisten 127.0.0.1:8443 ssl http2;' ] + # While the traefik module hands the ports back, LSS_WEB_BACKEND wins over its state. + LSS_WEB_BACKEND=off + run web_nginx_listen http + [ "${output}" = $'\tlisten 80;\n\tlisten [::]:80;' ] +} + +@test "the backend mode follows Traefik's state unless LSS_WEB_BACKEND says otherwise" { + run ! web_backend + [ "$(web_http_port)" = 80 ] + [ "$(web_https_port)" = 443 ] + LSS_WEB_BACKEND=on + web_backend + [ "$(web_http_port)" = 8008 ] + [ "$(web_https_port)" = 8443 ] + unset LSS_WEB_BACKEND + use_server traefik + web_backend + LSS_WEB_BACKEND=off + run ! web_backend + [ "$(web_https_port)" = 443 ] +} + +@test "Traefik gets the same security headers as entries of its headers middleware" { + run web_headers traefik + [ "${#lines[@]}" -eq 6 ] + [ "${lines[0]}" = ' Strict-Transport-Security: "max-age=63072000"' ] + [ "${lines[1]}" = ' X-Content-Type-Options: "nosniff"' ] +} + +@test "nginx sites are written once per change for static files and proxies, with the shared listeners" { + as_test_user + use_server nginx local conf=${BATS_TEST_TMPDIR}/conf/nginx/sites web_site_write blog static /var/www/blog blog.example.com [ "${LSS_FILE_CHANGED}" = 1 ] grep -qxF "== 0644 root:root ${conf}/blog.conf" "${WRITES}" grep -qxF '# Managed by lss (site blog).' "${conf}/blog.conf" - grep -qxF $'\tlisten [::]:80;' "${conf}/blog.conf" - grep -qxF $'\tlisten [::]:443 ssl;' "${conf}/blog.conf" + grep -qxF $'\tinclude '"${BATS_TEST_TMPDIR}"'/conf/nginx/listen-http.conf;' "${conf}/blog.conf" + grep -qxF $'\tinclude '"${BATS_TEST_TMPDIR}"'/conf/nginx/listen-https.conf;' "${conf}/blog.conf" grep -qxF $'\troot /var/www/blog;' "${conf}/blog.conf" grep -qxF $'\tindex index.html;' "${conf}/blog.conf" grep -qxF $'\t\ttry_files $uri $uri/ =404;' "${conf}/blog.conf" grep -qxF $'\taccess_log /var/log/nginx/access.log;' "${conf}/blog.conf" web_site_write blog static /var/www/blog blog.example.com [ "${LSS_FILE_CHANGED}" = 0 ] - NGINX_VERSION=1.22.1 web_site_write app proxy http://127.0.0.1:3000 app.example.com [ "${LSS_FILE_CHANGED}" = 1 ] - grep -qxF $'\tlisten 443 ssl http2;' "${conf}/app.conf" - run ! grep -q 'http2 on;' "${conf}/app.conf" + run ! grep -qE '^[[:space:]]+listen |http2' "${conf}/app.conf" grep -qxF $'\t\tproxy_pass http://127.0.0.1:3000;' "${conf}/app.conf" grep -qxF $'\t\tinclude /etc/nginx/lss/proxy.conf;' "${conf}/app.conf" [ "$(files_changes_of core)" = 2 ] @@ -513,7 +555,11 @@ EOF grep -qxF $'\tDocumentRoot /var/www/blog' "${conf}/blog.conf" grep -qxF $'\t\tOptions -Indexes +FollowSymLinks' "${conf}/blog.conf" grep -qxF $'\tErrorLog /var/log/httpd/error_log' "${conf}/blog.conf" - grep -qxF $'\tCustomLog /var/log/httpd/access_log combined' "${conf}/blog.conf" + # lss_combined logs the client address that mod_remoteip takes from Traefik's X-Forwarded-For. + grep -qxF $'\tCustomLog /var/log/httpd/access_log lss_combined' "${conf}/blog.conf" + # The ports come from the Define lines of the global configuration, so the backend mode leaves the sites alone. + grep -qxF '' "${conf}/blog.conf" + grep -qxF '' "${conf}/blog.conf" grep -qxF $'\tSSLCertificateFile '"${WEB_TLS_DIR}/default/fullchain.pem" "${conf}/blog.conf" run ! grep -q SetHandler "${conf}/blog.conf" web_site_write shop php /var/www/shop shop.example.com www.shop.example.com @@ -546,6 +592,8 @@ EOF [[ ${output} == *"tls ${WEB_TLS_DIR}/default/fullchain.pem ${WEB_TLS_DIR}/default/privkey.pem"* ]] [[ ${output} == *'import lss_hidden'* ]] [[ ${output} == *'redir https://{host}{uri} permanent'* ]] + # lss_bind keeps both listeners on 127.0.0.1 behind Traefik. + [ "$(grep -c $'^\timport lss_bind$' <<<"${output}")" = 2 ] } @test "Caddy sites serve static files, PHP and proxies for every name" { @@ -819,6 +867,110 @@ EOF [[ ${output} == *'web: caddy could not hash the admin password.'* ]] } +# --- routes of applications ------------------------------------------------------------------------------------- + +# Records what the Traefik side of a route would do; lib/traefik.sh has its own tests. +stub_traefik_routes() { + # shellcheck disable=SC2329 # called by the functions under test + traefik_route_write() { + printf 'traefik_route_write %s\n' "$*" >>"${STUB_LOG}" + files_note_change "route-$1.yml" + return + } + # shellcheck disable=SC2329 # called by the functions under test + traefik_route_remove() { + printf 'traefik_route_remove %s\n' "$*" >>"${STUB_LOG}" + return + } + return +} + +@test "without Traefik a route becomes a proxy site of the web server, written and reloaded once per change" { + as_test_user + stub_traefik_routes + use_server nginx + LSS_MODULE=gitea + local site=${BATS_TEST_TMPDIR}/conf/nginx/sites/gitea.conf before + web_proxy_route gitea 'git.example.com, www.git.example.com' 127.0.0.1:3000 + [ "$(<"${LSS_STATE_DIR}/web-routes/gitea")" = 'gitea git.example.com www.git.example.com http://127.0.0.1:3000' ] + grep -qxF $'\tserver_name git.example.com www.git.example.com;' "${site}" + grep -qxF $'\t\tproxy_pass http://127.0.0.1:3000;' "${site}" + grep -qxF 'traefik_route_remove gitea' "${STUB_LOG}" + grep -q '^nginx -t' "${STUB_LOG}" + run ! grep -q traefik_route_write "${STUB_LOG}" + before=$(files_changes_of gitea) + : >"${STUB_LOG}" + web_proxy_route gitea 'git.example.com, www.git.example.com' 127.0.0.1:3000 + [ "$(files_changes_of gitea)" = "${before}" ] + run ! grep -q '^nginx -t' "${STUB_LOG}" + [ "$(web_route_names gitea)" = gitea ] + [ "$(web_route_names)" = gitea ] + [ -z "$(web_route_names other)" ] +} + +@test "with Traefik a route goes to its file provider and comes back to the web server when Traefik leaves" { + as_test_user + stub_traefik_routes + use_server nginx + use_server traefik + # Coraza at work (arm64): the route goes straight to the application. + OS_ARCH=aarch64 + LSS_MODULE=app + local site=${BATS_TEST_TMPDIR}/conf/nginx/sites/app.conf + web_proxy_route app app.example.com http://127.0.0.1:8080/ + grep -qxF 'traefik_route_write app http://127.0.0.1:8080/ app.example.com' "${STUB_LOG}" + [ ! -e "${site}" ] + # The traefik module re-applies every route in the name of its owner, with the ports handed back. + LSS_MODULE=traefik LSS_WEB_BACKEND=off web_routes_apply + grep -qxF 'traefik_route_remove app' "${STUB_LOG}" + grep -qxF $'\t\tproxy_pass http://127.0.0.1:8080/;' "${site}" + [ "$(files_changes_of traefik)" = 0 ] + web_proxy_route_remove app + [ ! -e "${site}" ] + [ ! -e "${LSS_STATE_DIR}/web-routes/app" ] + [ -z "$(web_route_names)" ] +} + +@test "where Coraza cannot run a route passes through the web server's ModSecurity, and goes direct when it leaves" { + as_test_user + stub_traefik_routes + use_server nginx + use_server traefik + OS_ARCH=x86_64 + LSS_MODULE=app + local site=${BATS_TEST_TMPDIR}/conf/nginx/sites/app.conf + web_proxy_route app app.example.com 127.0.0.1:3000 + grep -qxF "traefik_route_write app https://127.0.0.1:${WEB_BACKEND_HTTPS_PORT} app.example.com" "${STUB_LOG}" + grep -qxF $'\t\tproxy_pass http://127.0.0.1:3000;' "${site}" + # The web server goes away: Traefik routes to the application itself. + : >"${STUB_LOG}" + TRAEFIK_BACKEND_GONE=1 LSS_MODULE=nginx web_routes_apply + grep -qxF 'traefik_route_write app http://127.0.0.1:3000 app.example.com' "${STUB_LOG}" + [ ! -e "${site}" ] + # ModSecurity asked for explicitly keeps its meaning: what goes through the web server, routes direct. + : >"${STUB_LOG}" + export LSS_TRAEFIK_WAF=modsecurity + LSS_MODULE=nginx web_routes_apply + grep -qxF 'traefik_route_write app http://127.0.0.1:3000 app.example.com' "${STUB_LOG}" +} + +@test "a route needs host names, an upstream with a port and something to serve it" { + as_test_user + run web_proxy_route app 'app.example.com localhost' 127.0.0.1:3000 + [ "${status}" -eq 1 ] + [[ ${output} == *"web: 'localhost' of the route app is not a host name like app.example.com."* ]] + run web_proxy_route app app.example.com 3000 + [ "${status}" -eq 1 ] + [[ ${output} == *"web: the upstream '3000' of the route app is not HOST:PORT or an http(s):// URL with a port."* ]] + run web_proxy_route app app.example.com 'http://127.0.0.1:3000/;return' + [ "${status}" -eq 1 ] + run web_proxy_route ../app app.example.com 127.0.0.1:3000 + [ "${status}" -eq 1 ] + run web_proxy_route app app.example.com 127.0.0.1:3000 + [ "${status}" -eq 1 ] + [[ ${output} == *'web: the route app needs Traefik or a web server (nginx, apache or caddy).'* ]] +} + # --- fail2ban and CrowdSec -------------------------------------------------------------------------------------- @test "without a web server the jails and the log acquisition are removed" { @@ -888,6 +1040,8 @@ EOF web_ids_configure grep -qxF "== 0644 root:root ${WEB_FAIL2BAN_FILTER}" "${WRITES}" grep -qF '"status":401' "${WEB_FAIL2BAN_FILTER}" + # client_ip is the client behind Traefik as well (remote_ip would be Traefik itself). + grep -qF '"client_ip":""' "${WEB_FAIL2BAN_FILTER}" grep -qxF '[lss-caddy-auth]' "${WEB_FAIL2BAN_JAIL}" grep -qxF 'filter = lss-caddy-auth' "${WEB_FAIL2BAN_JAIL}" grep -qxF "logpath = ${BATS_TEST_TMPDIR}/log/caddy/access.log" "${WEB_FAIL2BAN_JAIL}"