From c4e1cdad36de8612f113f01509ec5706f5c09843 Mon Sep 17 00:00:00 2001 From: Gerrrt Date: Sun, 15 Nov 2026 23:21:07 +0000 Subject: [PATCH 1/4] docs(lab): correct the README's rule counts and client list, and check them The audit of alexander on 2026-11-13 found stacks/lab/README.md saying 9 and 12 rules against files holding 19 and 13, and naming three ingest clients and six tokens when there are six clients and seven. Correct both, and add a fourteenth check_docs assertion that reads each stack README's per-file rule count against its rule file, since the global totals passed throughout. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +++-- docs/changelog.md | 7 +++++++ scripts/check_docs.py | 36 +++++++++++++++++++++++++++++++++++- stacks/lab/README.md | 23 ++++++++++++++--------- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2189fa62..061ab603 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ documents for different readers. `promtool`, `amtool`, `alloy fmt`, a real Loki boot to parse the LogQL rules, dashboard-JSON and datasource checks, every dashboard's PromQL parsed, plus `gitleaks` over the full history. -- **CI that validates the documentation too.** Thirteen assertions cross-check +- **CI that validates the documentation too.** Fourteen assertions cross-check this prose against the configs it describes — counted claims (rules, dashboards, panels, Alloy agents, VLANs, ADRs, runbooks), the SNMP inventory and the compute table against `docs/network.md`, the host/stack @@ -120,7 +120,8 @@ documents for different readers. file's outstanding-purchase count against the roadmap's buy table, every critical alert's `runbook_url` against the runbook and heading it names, every relative Markdown link against the file and heading it points at, the - network diagram's PNG against its SVG, and + network diagram's PNG against its SVG, each stack README's per-file rule + counts against the rule files, and a ban on image versions in prose — Dependabot edits only `compose.yaml`, so a version written anywhere else is stale from the next bump. A document that disagrees with the repository fails the build. That opening count is diff --git a/docs/changelog.md b/docs/changelog.md index 0e6ec4ad..3d40a51f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -282,6 +282,13 @@ docstring gives: it is a record, not a claim about now. `fenrir`'s uses. It is not applied yet: the change on `alexander` is the operator's to make, by §2, and §7's `sshd -T` is the check. This follows the same weakness closed on `golem` by the audit of 2026-11-13. +- **`stacks/lab/README.md`'s rule counts are checked now.** The audit of + `alexander` on 2026-11-13 found its layout block saying "9 rules" and + "12 rules" against files holding 19 and 13, and its ingest section naming + three clients and six tokens when there are six clients and seven tokens. + Every global count in `check_docs.py` passed throughout, because those are + totals across all rule files. A fourteenth assertion now reads each stack + README's per-file count against its rule file. - **`garuda` is patched by hand, monthly, because nothing else patches it.** After the audit of 2026-11-13 closed, a re-check of the box found a gap that audit missed. `unattended-upgrades` runs daily on `garuda` and applies diff --git a/scripts/check_docs.py b/scripts/check_docs.py index e1f5aafe..a474f452 100755 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -17,7 +17,7 @@ The device list must live in exactly one place. It is currently spread across five ... and --check asserts the other copies still agree. -Thirteen assertions, each comparing prose against something machine-readable: +Fourteen assertions, each comparing prose against something machine-readable: 1. Counted claims rules, unit-test coverage, dashboards, panels, Alloy agents, ADRs, runbooks, and the size of this @@ -53,6 +53,11 @@ 13. Diagram render docs/diagrams/current/network.png changes whenever network.svg does, in the working tree and in the last commit that touched either. + 14. Stack rule files a stack README's "rules/.rules.yaml N rules" + against the rules in that file. The totals above + are global; this is the per-file count that stood + at 9 and 12 in stacks/lab/README.md while the files + held 19 and 13. Only present-tense documents are checked for their claims — the relative-link assertion (12) is the exception, and scans every tracked file including the @@ -1559,6 +1564,34 @@ def anchors_of(path: pathlib.Path) -> set[str]: return problems +# A stack README's layout block names each rule file with its count beside it: +# "rules/lab.rules.yaml 19 rules". check_counts cannot see these, because +# they are per-file and its facts are totals across every file. Found by the +# audit of alexander on 2026-11-13, which read "9 rules" and "12 rules" against +# files holding 19 and 13 while every global count passed. +STACK_RULE_FILE_CLAIM = re.compile(r"rules/([\w.-]+\.rules\.yaml)[ \t]+" + COUNT + WS + r"rules\b") + + +def check_stack_rule_files() -> list[str]: + problems = [] + for readme in sorted(REPO.glob("stacks/*/README.md")): + text = readme.read_text(encoding="utf-8") + rel = readme.relative_to(REPO) + for match in STACK_RULE_FILE_CLAIM.finditer(text): + name, claimed = match.group(1), number(match.group(2)) + n = text.count("\n", 0, match.start()) + 1 + files = sorted(readme.parent.glob(f"**/rules/{name}")) + if len(files) != 1: + problems.append( + f"{rel}:{n} counts rules/{name}, which names {len(files)} files under {readme.parent.relative_to(REPO)}/; want exactly one" + ) + continue + have = count_alerts(files) + count_recording_rules(files) + if claimed != have: + problems.append(f"{rel}:{n} claims {claimed} rules in {name}; the file has {have}") + return problems + + def check_diagram_render() -> list[str]: """The PNG is re-rendered whenever the SVG changes. @@ -1641,6 +1674,7 @@ def main() -> int: ("critical alerts' runbook_url against docs/runbooks/", check_runbook_urls), ("relative Markdown links resolve to a file and an anchor", check_relative_links), ("network.png rendered with network.svg", check_diagram_render), + ("stack READMEs' per-file rule counts against the rule files", check_stack_rule_files), ) # The registry is the source for README's "N assertions", so adding a diff --git a/stacks/lab/README.md b/stacks/lab/README.md index c915717e..872643be 100644 --- a/stacks/lab/README.md +++ b/stacks/lab/README.md @@ -59,16 +59,19 @@ prometheus/ prometheus.yaml six scrape jobs — this stack's four, the domain's windows_exporter and odin's Velociraptor; no alerting block, no file_sd - rules/lab.rules.yaml 9 rules — four for this stack watching itself, + rules/lab.rules.yaml 19 rules — four for this stack watching itself, two for the guests' disks, three for the domain - ADR-0029 sized - rules/soc.rules.yaml 12 rules — the SOC's indexer on odin, whose health is + ADR-0029 sized, seven for golem's backups + (ADR-0053) and three for fenrir's Zeek archive + prune (#850) + rules/soc.rules.yaml 13 rules — the SOC's indexer on odin, whose health is pushed here by stacks/soc's Alloy (ADR-0030); the manager's agents, queues and drops, from odin's wazuh-manager-state collector; and Zeek's conn stream, from fenrir's Alloy (#1038) - tests/lab.test.yaml promtool unit tests; all nineteen rules, firing + quiet - tests/soc.test.yaml the same for the twelve + tests/lab.test.yaml promtool unit tests; every rule in lab.rules.yaml, + firing + quiet + tests/soc.test.yaml the same for soc.rules.yaml loki/loki-config.yaml single-binary, filesystem, 15-day retention, no ruler grafana/ provisioning/ two datasources + dashboard provider @@ -100,9 +103,9 @@ delete API, and any query without the reader token. token early loses nothing. A proxy that goes up before its clients refuses their pushes until each one catches up. -1. **On `alexander`**, generate six tokens with `openssl rand -hex 32` and add +1. **On `alexander`**, generate seven tokens with `openssl rand -hex 32` and add them with `make secrets-edit STACK=lab`. The keys are `INGEST_TOKEN_ODIN`, - `_PHOENIX`, `_FENRIR`, `_GOLEM`, `_EDEN` and `_READER`, and + `_PHOENIX`, `_FENRIR`, `_GOLEM`, `_EDEN`, `_GARUDA` and `_READER`, and [`secrets/lab.example.yaml`](../../secrets/lab.example.yaml) says where each one goes. Do not `make up` yet. 2. **On `odin`**, set `INGEST_TOKEN` to `INGEST_TOKEN_ODIN` with `make @@ -133,8 +136,10 @@ curl -s -o /dev/null -w '%{http_code}\n' --cacert stacks/observability/alloy/ing curl -s -o /dev/null -w '%{http_code}\n' --cacert stacks/observability/alloy/ingest-ca.pem -X POST https://10.0.30.40:9443/api/v1/write ``` -The three clients must still deliver. In the lab's Grafana, `up` and the -newest log line for `odin`, `phoenix` and `fenrir` should be minutes old. +The six clients must still deliver. In the lab's Grafana, `up` and the +newest log line for `odin`, `phoenix`, `fenrir`, `golem`, `eden` and `garuda` +should be minutes old. `eden` is off between sessions, so its newest data is +from its last session. `docker logs lab-ingest-proxy` lists any refused request, with the client its token mapped to. A client still on its old config shows there as `-`. From d7c18239a58a0f0e79de5b5314ef4ddaff9cb22d Mon Sep 17 00:00:00 2001 From: Gerrrt Date: Mon, 16 Nov 2026 10:00:19 +0000 Subject: [PATCH 2/4] test(check_docs): fixtures for the per-file rule-count check check_stack_rule_files now takes the tree it reads, and --self-test runs it against a temp stack: a matching count, one in words, a stale count, a missing rule file and an ambiguous one. self-tests.sh already discovers check_docs.py's suite, so CI runs these. Co-Authored-By: Claude Opus 5.5 --- scripts/check_docs.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/check_docs.py b/scripts/check_docs.py index a474f452..81cb163b 100755 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -1572,18 +1572,18 @@ def anchors_of(path: pathlib.Path) -> set[str]: STACK_RULE_FILE_CLAIM = re.compile(r"rules/([\w.-]+\.rules\.yaml)[ \t]+" + COUNT + WS + r"rules\b") -def check_stack_rule_files() -> list[str]: +def check_stack_rule_files(repo: pathlib.Path = REPO) -> list[str]: problems = [] - for readme in sorted(REPO.glob("stacks/*/README.md")): + for readme in sorted(repo.glob("stacks/*/README.md")): text = readme.read_text(encoding="utf-8") - rel = readme.relative_to(REPO) + rel = readme.relative_to(repo) for match in STACK_RULE_FILE_CLAIM.finditer(text): name, claimed = match.group(1), number(match.group(2)) n = text.count("\n", 0, match.start()) + 1 files = sorted(readme.parent.glob(f"**/rules/{name}")) if len(files) != 1: problems.append( - f"{rel}:{n} counts rules/{name}, which names {len(files)} files under {readme.parent.relative_to(REPO)}/; want exactly one" + f"{rel}:{n} counts rules/{name}, which names {len(files)} files under {readme.parent.relative_to(repo)}/; want exactly one" ) continue have = count_alerts(files) + count_recording_rules(files) @@ -1793,6 +1793,33 @@ def adrs(status: str, first_mentions: bool) -> list[str]: ), ] + # Stack READMEs' per-file rule counts. Built as a temp tree so the glob and + # the count are exercised, not only the regex. + with tempfile.TemporaryDirectory() as d: + repo = pathlib.Path(d) + stack = repo / "stacks" / "demo" + (stack / "prometheus" / "rules").mkdir(parents=True) + (stack / "prometheus" / "rules" / "a.rules.yaml").write_text( + "groups:\n - rules:\n - alert: One\n - alert: Two\n - record: three\n" + ) + + def stack_rules(claim: str) -> str: + (stack / "README.md").write_text(claim + "\n") + return " | ".join(check_stack_rule_files(repo)) + + cases += [ + ("a matching per-file rule count passes", not stack_rules("rules/a.rules.yaml 3 rules")), + ("a count in words passes too", not stack_rules("rules/a.rules.yaml three rules")), + ( + "a stale count is caught", + "claims 2 rules in a.rules.yaml; the file has 3" in stack_rules("rules/a.rules.yaml 2 rules"), + ), + ("a missing rule file is caught", "names 0 files" in stack_rules("rules/gone.rules.yaml 1 rules")), + ] + (stack / "other" / "rules").mkdir(parents=True) + (stack / "other" / "rules" / "a.rules.yaml").write_text("groups: []\n") + cases.append(("an ambiguous rule file is caught", "names 2 files" in stack_rules("rules/a.rules.yaml 3 rules"))) + failed = 0 for name, ok in cases: failed += not ok From f7276e345a64d3a5229534fcd4d13eaa4f780fde Mon Sep 17 00:00:00 2001 From: Gerrrt Date: Sun, 15 Nov 2026 23:30:53 +0000 Subject: [PATCH 3/4] feat(lab): read the guests' patch and reboot state, and correct the Loki comment homelab-patch-state writes reboot and apt state on alexander, odin, garuda and eden, and no lab rule read it: the audit of alexander on 2026-11-13 found libc6 waiting on a reboot nothing could see. Add the estate's SecurityUpdatesPending, RebootRequired and PatchStateStopped to lab.rules.yaml with firing and quiet tests. PatchStateStopped also requires node metrics still arriving, so eden being off between sessions stays quiet. The loki service's comment said the scrape job observes readiness; it reads /metrics, which answers whatever /ready says. Say what is watched. Co-Authored-By: Claude Opus 5.5 --- docs/changelog.md | 10 ++ stacks/lab/README.md | 5 +- stacks/lab/compose.yaml | 7 +- stacks/lab/prometheus/rules/lab.rules.yaml | 76 +++++++++++++- stacks/lab/prometheus/tests/lab.test.yaml | 114 ++++++++++++++++++++- 5 files changed, 205 insertions(+), 7 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 3d40a51f..17640d78 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -282,6 +282,16 @@ docstring gives: it is a record, not a claim about now. `fenrir`'s uses. It is not applied yet: the change on `alexander` is the operator's to make, by §2, and §7's `sshd -T` is the check. This follows the same weakness closed on `golem` by the audit of 2026-11-13. +- **The lab's Prometheus now reads its guests' patch and reboot state.** + `homelab-patch-state` has written it on `alexander`, `odin`, `garuda` and + `eden` all along, and no lab rule read it. The audit of `alexander` on + 2026-11-13 found libc6 waiting for a reboot that nothing could see, and the + garuda entry below found the same gap. `lab.rules.yaml` gains the estate's + `SecurityUpdatesPending`, `RebootRequired` and `PatchStateStopped`, with + tests. `PatchStateStopped` also requires the host to still be pushing node + metrics, so `eden` being off between sessions stays quiet. None of them + pages (ADR-0020). The same audit found `compose.yaml` saying the `loki` + scrape job observes readiness. It does not, and the comment now says so. - **`stacks/lab/README.md`'s rule counts are checked now.** The audit of `alexander` on 2026-11-13 found its layout block saying "9 rules" and "12 rules" against files holding 19 and 13, and its ingest section naming diff --git a/stacks/lab/README.md b/stacks/lab/README.md index 872643be..5b8267f5 100644 --- a/stacks/lab/README.md +++ b/stacks/lab/README.md @@ -59,8 +59,9 @@ prometheus/ prometheus.yaml six scrape jobs — this stack's four, the domain's windows_exporter and odin's Velociraptor; no alerting block, no file_sd - rules/lab.rules.yaml 19 rules — four for this stack watching itself, - two for the guests' disks, three for the domain + rules/lab.rules.yaml 22 rules — four for this stack watching itself, + two for the guests' disks, three for their patch + and reboot state, three for the domain ADR-0029 sized, seven for golem's backups (ADR-0053) and three for fenrir's Zeek archive prune (#850) diff --git a/stacks/lab/compose.yaml b/stacks/lab/compose.yaml index 470d99e8..8d25933a 100644 --- a/stacks/lab/compose.yaml +++ b/stacks/lab/compose.yaml @@ -183,8 +183,11 @@ services: # --------------------------------------------------------------------------- # Log store. No healthcheck, for the reason written out against the `loki` # service in stacks/observability/compose.yaml: the image is distroless and - # holds no shell, wget or curl to probe with. Readiness is observed by the - # `loki` scrape job in prometheus.yaml instead. + # holds no shell, wget or curl to probe with. Nothing here observes /ready: + # the `loki` scrape job reads /metrics, which answers whatever /ready says, + # and the audit of alexander on 2026-11-13 found /ready returning 503 for + # four days while ingest ran normally. What is watched is the outcome — + # InstanceDown on the scrape, and LokiIngestionStalled on lines arriving. # --------------------------------------------------------------------------- loki: <<: *service-defaults diff --git a/stacks/lab/prometheus/rules/lab.rules.yaml b/stacks/lab/prometheus/rules/lab.rules.yaml index 8604251a..2037091c 100644 --- a/stacks/lab/prometheus/rules/lab.rules.yaml +++ b/stacks/lab/prometheus/rules/lab.rules.yaml @@ -1,13 +1,14 @@ --- # The lab stack watching itself. # -# Nineteen rules, and the number is still the point. The estate has 48; ADR-0007 +# Twenty-two rules, and the number is still the point. The estate has 48; ADR-0007 # names giving up "Loki, the dashboards and 40 alert rules" as the price of # keeping lab telemetry out of the estate, and this file does not pretend to # replace them. Four cover this stack's own working — the things that would make # every other signal here silently wrong. Two cover the disks of the guests that # push here, in the detail only their own node_exporter has; the estate pages on -# the same disks, read another way (ADR-0070). Three cover the domain ADR-0029 sized, +# the same disks, read another way (ADR-0070). Three cover the same guests' +# patch state, which only reaches this Prometheus. Three cover the domain ADR-0029 sized, # and they are here rather than in a stack of their own because this Prometheus # is the only thing on the segment that scrapes it. Seven cover golem's backups # (ADR-0053), because a verify nobody watches is a green check with nothing @@ -171,6 +172,77 @@ groups: annotations: summary: "{{ $labels.instance }}:{{ $labels.mountpoint }} below 10% free" + # The guests' patch state. homelab-patch-state writes it on every apt + # guest that pushes here (scripts/install-agent-collectors.sh), and until + # 2026-11-15 nothing in this file read it: the audit of alexander on + # 2026-11-13 found libc6 asking for a reboot that no rule could see, and + # the 2026-11-15 changelog found the same on garuda. The estate's three + # rules in host.rules.yaml cannot see these guests either, because none + # of them pushes to VLAN 99 (ADR-0007). + # + # Same names, expressions and windows as the estate's three, per the + # header; host.rules.yaml carries the reasoning for each window. One + # difference, in PatchStateStopped, is stated against it. + - alert: SecurityUpdatesPending + expr: homelab_apt_security_upgrades_pending > 0 + for: 7d + labels: + component: host + severity: warning + category: security + annotations: + summary: "{{ $labels.host }} has had {{ $value }} security update(s) pending for a week" + description: >- + Unattended-upgrades should have applied these. A week of them sitting + there means they are held, phased, or waiting on a reboot. Check with + `apt list --upgradable` and + `systemctl status unattended-upgrades.service`. On garuda a 0 is not + good news: Kali has no security pocket (schedule-maintenance.md). + + - alert: RebootRequired + expr: homelab_reboot_required == 1 + for: 3d + labels: + component: host + severity: warning + category: security + annotations: + summary: "{{ $labels.host }} has needed a reboot for three days" + description: >- + /var/run/reboot-required.pkgs names which packages asked, and + homelab_reboot_required_packages counts them. Pick a window outside + an exercise: rebooting alexander blinds the lab for its duration, + and rebooting odin stops the SOC. + + # The estate's expression, plus one condition: the host must still be + # pushing node metrics. eden is off between sessions (ADR-0029's per- + # session shape), and when it is off its patch state stops with + # everything else. Without the third line that is a permanently + # recurring alert for a correct state, which ADR-0017 refuses. A guest + # that is off entirely is the estate's business (ADR-0028); this asks + # only about a guest that is up and has stopped collecting. + - alert: PatchStateStopped + expr: >- + count by (host) (present_over_time(homelab_apt_upgrades_pending[1d] offset 6h)) + unless + count by (host) (present_over_time(homelab_apt_upgrades_pending[6h])) + and on (host) + count by (host) (present_over_time(node_boot_time_seconds[6h])) + for: 30m + labels: + component: host + severity: warning + category: security + annotations: + summary: "{{ $labels.host }} has stopped reporting patch state" + description: >- + It reported within the last day and has not in six hours, while its + node metrics still arrive, so SecurityUpdatesPending and + RebootRequired are both blind to this host until it comes back. + Check `systemctl status homelab-patch-state.timer` on the host and + that /var/lib/node_exporter/textfile_collector/apt-patch-state.prom + is being written and is world-readable. + # ----------------------------------------------------------------------- # The domain (ADR-0029). Scraped, never pushed — publishing a remote-write # receiver on the segment that exists to hold attackers would hand it a diff --git a/stacks/lab/prometheus/tests/lab.test.yaml b/stacks/lab/prometheus/tests/lab.test.yaml index 34bfe752..99f5dafd 100644 --- a/stacks/lab/prometheus/tests/lab.test.yaml +++ b/stacks/lab/prometheus/tests/lab.test.yaml @@ -11,7 +11,7 @@ # input, the second proves it does not produce one for the input production # actually supplies. # -# All nineteen rules in lab.rules.yaml are covered here. That is a deliberate +# All twenty-two rules in lab.rules.yaml are covered here. That is a deliberate # difference from the estate's tests, where four of the nine stack rules are # syntax-checked only: this file is small enough that "test the ones that were # hard" has no advantage over testing all of them, and a new stack has no @@ -738,3 +738,115 @@ tests: `make install-agent-collectors AGENT=atreus@10.0.30.90 ARGS='--only zeek-archive-prune'` (docs/runbooks/build-the-sensor-guest.md §4), and check that /var/lib/node_exporter/textfile_collector exists on fenrir. + + # --- SecurityUpdatesPending ---------------------------------------------- + # 2m, not the collector's daily cadence: the textfile is re-served on every + # scrape, and a fixture an hour apart goes stale between samples and resets + # `for: 7d` (host.test.yaml has the longer account of that first draft). + - interval: 2m + input_series: + - series: 'homelab_apt_security_upgrades_pending{instance="odin",job="odin-metrics",host="odin"}' + values: "3x5300" + alert_rule_test: + - eval_time: 175h + alertname: SecurityUpdatesPending + exp_alerts: + - exp_labels: + instance: odin + job: odin-metrics + host: odin + component: host + severity: warning + category: security + exp_annotations: + summary: "odin has had 3 security update(s) pending for a week" + description: >- + Unattended-upgrades should have applied these. A week of them + sitting there means they are held, phased, or waiting on a + reboot. Check with `apt list --upgradable` and `systemctl status + unattended-upgrades.service`. On garuda a 0 is not good news: + Kali has no security pocket (schedule-maintenance.md). + # Applied inside the week, the normal case. + - interval: 2m + input_series: + - series: 'homelab_apt_security_upgrades_pending{instance="odin",job="odin-metrics",host="odin"}' + values: "2x1500 0x3800" + alert_rule_test: + - eval_time: 175h + alertname: SecurityUpdatesPending + exp_alerts: [] + + # --- RebootRequired -------------------------------------------------------- + # alexander's own state on 2026-11-13: libc6 asked, and nothing rebooted. + - interval: 2m + input_series: + - series: 'homelab_reboot_required{instance="alexander",job="alexander-metrics",host="alexander"}' + values: "1x2400" + alert_rule_test: + - eval_time: 75h + alertname: RebootRequired + exp_alerts: + - exp_labels: + instance: alexander + job: alexander-metrics + host: alexander + component: host + severity: warning + category: security + exp_annotations: + summary: "alexander has needed a reboot for three days" + description: >- + /var/run/reboot-required.pkgs names which packages asked, and + homelab_reboot_required_packages counts them. Pick a window + outside an exercise: rebooting alexander blinds the lab for its + duration, and rebooting odin stops the SOC. + # Rebooted inside the three days. + - interval: 2m + input_series: + - series: 'homelab_reboot_required{instance="alexander",job="alexander-metrics",host="alexander"}' + values: "1x1000 0x3000" + alert_rule_test: + - eval_time: 130h + alertname: RebootRequired + exp_alerts: [] + + # --- PatchStateStopped ----------------------------------------------------- + # The collector stops after 20 hours while the host keeps pushing node + # metrics: a timer that died, on a guest that is up. + - interval: 5m + input_series: + - series: 'homelab_apt_upgrades_pending{instance="odin",job="odin-metrics",host="odin"}' + values: "3x240 _x120" + - series: 'node_boot_time_seconds{instance="odin",job="odin-metrics",host="odin"}' + values: "1000x360" + alert_rule_test: + - eval_time: 30h + alertname: PatchStateStopped + exp_alerts: + - exp_labels: + host: odin + component: host + severity: warning + category: security + exp_annotations: + summary: "odin has stopped reporting patch state" + description: >- + It reported within the last day and has not in six hours, while + its node metrics still arrive, so SecurityUpdatesPending and + RebootRequired are both blind to this host until it comes back. + Check `systemctl status homelab-patch-state.timer` on the host + and that + /var/lib/node_exporter/textfile_collector/apt-patch-state.prom + is being written and is world-readable. + # eden switched off after a session: patch state and node metrics stop + # together. That is the correct state for a per-session guest, so quiet. + - interval: 5m + input_series: + - series: 'homelab_apt_upgrades_pending{instance="eden",job="eden-metrics",host="eden"}' + values: "3x240 _x120" + - series: 'node_boot_time_seconds{instance="eden",job="eden-metrics",host="eden"}' + values: "1000x240 _x120" + alert_rule_test: + - eval_time: 30h + alertname: PatchStateStopped + exp_alerts: [] From 3747cef2d58d057ebc209b9279784dfc17c767b2 Mon Sep 17 00:00:00 2001 From: Gerrrt Date: Mon, 16 Nov 2026 10:08:31 +0000 Subject: [PATCH 4/4] fix(lab): PatchStateStopped also fires on a file that stopped being rewritten When homelab-patch-state.timer stops, apt-patch-state.prom stays on disk and its gauges keep being scraped, so the presence check never fired. The rule now also fires when node_textfile_mtime_seconds for that file is more than 26 hours old (daily 08:00 timer, 15 min jitter), still behind the node-metrics guard so eden being off stays quiet. Tests cover a stale file whose gauges keep arriving and a file rewritten on schedule. Co-Authored-By: Claude Opus 5.5 --- docs/changelog.md | 4 +- stacks/lab/prometheus/rules/lab.rules.yaml | 51 ++++++++++++++----- stacks/lab/prometheus/tests/lab.test.yaml | 59 +++++++++++++++++++--- 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 17640d78..c319dc3a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -289,7 +289,9 @@ docstring gives: it is a record, not a claim about now. garuda entry below found the same gap. `lab.rules.yaml` gains the estate's `SecurityUpdatesPending`, `RebootRequired` and `PatchStateStopped`, with tests. `PatchStateStopped` also requires the host to still be pushing node - metrics, so `eden` being off between sessions stays quiet. None of them + metrics, so `eden` being off between sessions stays quiet, and fires when + `apt-patch-state.prom` has not been rewritten in 26 hours, because a + stopped timer leaves the file's last gauges being served as current. None of them pages (ADR-0020). The same audit found `compose.yaml` saying the `loki` scrape job observes readiness. It does not, and the comment now says so. - **`stacks/lab/README.md`'s rule counts are checked now.** The audit of diff --git a/stacks/lab/prometheus/rules/lab.rules.yaml b/stacks/lab/prometheus/rules/lab.rules.yaml index 2037091c..c8033a58 100644 --- a/stacks/lab/prometheus/rules/lab.rules.yaml +++ b/stacks/lab/prometheus/rules/lab.rules.yaml @@ -214,18 +214,40 @@ groups: an exercise: rebooting alexander blinds the lab for its duration, and rebooting odin stops the SOC. - # The estate's expression, plus one condition: the host must still be - # pushing node metrics. eden is off between sessions (ADR-0029's per- - # session shape), and when it is off its patch state stops with - # everything else. Without the third line that is a permanently - # recurring alert for a correct state, which ADR-0017 refuses. A guest - # that is off entirely is the estate's business (ADR-0028); this asks - # only about a guest that is up and has stopped collecting. + # The estate's expression, plus two conditions. + # + # The host must still be pushing node metrics. eden is off between + # sessions (ADR-0029's per-session shape), and when it is off its patch + # state stops with everything else. Without the last line that is a + # permanently recurring alert for a correct state, which ADR-0017 + # refuses. A guest that is off entirely is the estate's business + # (ADR-0028); this asks only about a guest that is up and has stopped + # collecting. + # + # And the file must still be rewritten. Presence alone misses the + # commonest way this breaks: when homelab-patch-state.timer stops, + # apt-patch-state.prom stays on disk and the textfile collector keeps + # re-serving its last gauges, so the series never goes absent. The + # wiki drift check in the estate's host.rules.yaml says the same of its + # own file. Twenty-six hours against a daily 08:00 timer with fifteen + # minutes of jitter allows one late run and not a missed one. eden + # coming back after days off has a stale file until Persistent=true + # runs the missed timer at boot, inside the fifteen-minute jitter, and + # `for: 30m` waits that out. - alert: PatchStateStopped expr: >- - count by (host) (present_over_time(homelab_apt_upgrades_pending[1d] offset 6h)) - unless - count by (host) (present_over_time(homelab_apt_upgrades_pending[6h])) + ( + ( + count by (host) (present_over_time(homelab_apt_upgrades_pending[1d] offset 6h)) + unless + count by (host) (present_over_time(homelab_apt_upgrades_pending[6h])) + ) + or + ( + time() - max by (host) (node_textfile_mtime_seconds{file=~".*/apt-patch-state[.]prom"}) + > 26 * 3600 + ) + ) and on (host) count by (host) (present_over_time(node_boot_time_seconds[6h])) for: 30m @@ -236,12 +258,13 @@ groups: annotations: summary: "{{ $labels.host }} has stopped reporting patch state" description: >- - It reported within the last day and has not in six hours, while its - node metrics still arrive, so SecurityUpdatesPending and - RebootRequired are both blind to this host until it comes back. + Its patch state has either stopped arriving or is being re-served + from an apt-patch-state.prom not rewritten in over 26 hours, while + its node metrics still arrive. SecurityUpdatesPending and + RebootRequired are both blind to this host until it is fixed. Check `systemctl status homelab-patch-state.timer` on the host and that /var/lib/node_exporter/textfile_collector/apt-patch-state.prom - is being written and is world-readable. + is being rewritten daily and is world-readable. # ----------------------------------------------------------------------- # The domain (ADR-0029). Scraped, never pushed — publishing a remote-write diff --git a/stacks/lab/prometheus/tests/lab.test.yaml b/stacks/lab/prometheus/tests/lab.test.yaml index 99f5dafd..71c98147 100644 --- a/stacks/lab/prometheus/tests/lab.test.yaml +++ b/stacks/lab/prometheus/tests/lab.test.yaml @@ -831,13 +831,58 @@ tests: exp_annotations: summary: "odin has stopped reporting patch state" description: >- - It reported within the last day and has not in six hours, while - its node metrics still arrive, so SecurityUpdatesPending and - RebootRequired are both blind to this host until it comes back. - Check `systemctl status homelab-patch-state.timer` on the host - and that - /var/lib/node_exporter/textfile_collector/apt-patch-state.prom - is being written and is world-readable. + Its patch state has either stopped arriving or is being + re-served from an apt-patch-state.prom not rewritten in over 26 + hours, while its node metrics still arrive. + SecurityUpdatesPending and RebootRequired are both blind to + this host until it is fixed. Check + `systemctl status homelab-patch-state.timer` on the host and + that /var/lib/node_exporter/textfile_collector/apt-patch-state.prom + is being rewritten daily and is world-readable. + # The timer stops but the file stays: the gauges keep arriving, re-served + # from a file last written at t=0. Twenty-six hours on, it fires. + - interval: 5m + input_series: + - series: 'homelab_apt_upgrades_pending{instance="garuda",job="garuda-metrics",host="garuda"}' + values: "3x400" + - series: 'node_textfile_mtime_seconds{instance="garuda",job="garuda-metrics",host="garuda",file="/rootfs/var/lib/node_exporter/textfile_collector/apt-patch-state.prom"}' + values: "0x400" + - series: 'node_boot_time_seconds{instance="garuda",job="garuda-metrics",host="garuda"}' + values: "1000x400" + alert_rule_test: + - eval_time: 30h + alertname: PatchStateStopped + exp_alerts: + - exp_labels: + host: garuda + component: host + severity: warning + category: security + exp_annotations: + summary: "garuda has stopped reporting patch state" + description: >- + Its patch state has either stopped arriving or is being + re-served from an apt-patch-state.prom not rewritten in over 26 + hours, while its node metrics still arrive. + SecurityUpdatesPending and RebootRequired are both blind to + this host until it is fixed. Check + `systemctl status homelab-patch-state.timer` on the host and + that /var/lib/node_exporter/textfile_collector/apt-patch-state.prom + is being rewritten daily and is world-readable. + # The same host with the timer working: the file is rewritten each day at + # 24h, so at 30h it is six hours old. Quiet. + - interval: 5m + input_series: + - series: 'homelab_apt_upgrades_pending{instance="garuda",job="garuda-metrics",host="garuda"}' + values: "3x400" + - series: 'node_textfile_mtime_seconds{instance="garuda",job="garuda-metrics",host="garuda",file="/rootfs/var/lib/node_exporter/textfile_collector/apt-patch-state.prom"}' + values: "0x287 86400x113" + - series: 'node_boot_time_seconds{instance="garuda",job="garuda-metrics",host="garuda"}' + values: "1000x400" + alert_rule_test: + - eval_time: 30h + alertname: PatchStateStopped + exp_alerts: [] # eden switched off after a session: patch state and node metrics stop # together. That is the correct state for a per-session guest, so quiet. - interval: 5m