simonvanlierde · Oct 31, 2026 · Oct 31, 2026 · Oct 31, 2026 · Oct 31, 2026 · Oct 31, 2026
diff --git a/.env.example b/.env.example
@@ -22,12 +22,18 @@ GRAFANA_COOKIE_SECURE=false
 # openssl rand -hex 32
 OTLP_AUTH_TOKEN=local-dev-token
 
-# Where Grafana delivers alert notifications (any webhook: ntfy, Slack, …).
+# Where Grafana delivers alert notifications (self-hosted ntfy, hosted ntfy, Slack, …).
+# For self-hosted ntfy with basic auth publisher:
+#   ALERT_WEBHOOK_URL=http://grafana:password@ntfy:80/alerts?tpl=1&title={{.CommonAnnotations.summary}}&message={{.CommonAnnotations.description}}
 # Not optional: an empty value drops every alert silently while the heartbeat
 # below keeps reporting healthy. With the tunnel overlay active, `just up`
 # refuses to start without it.
 ALERT_WEBHOOK_URL=
 
+# Public URL Cloudflare Tunnel exposes ntfy on, e.g. https://ntfy.example.com
+# Defaults to http://localhost:80
+NTFY_BASE_URL=
+
 # Dead man's switch ping target (e.g. https://hc-ping.com/<uuid>). The Watchdog
 # alert posts here every 5m; alert externally when pings stop.
 HEARTBEAT_URL=

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,16 @@ Notable changes to this stack. Format follows
 
 ### Added
 
+- Prometheus Blackbox Exporter service for outside-in uptime probing of HTTP/S
+  endpoints (`prom/blackbox-exporter:v0.26.0`), reporting `probe_success` and latency
+  telemetry for research applications.
+- Multi-window SLO burn rate alerting (`ErrorBudgetBurnFast` and `ErrorBudgetBurnSlow`)
+  in `config/grafana/alerting/rules.yaml`, tracking 99.5% availability error budgets
+  across dual short and long evaluation windows.
+- `BlackboxProbeFailed` critical alert firing when an outside-in HTTP probe fails.
+- Self-hosted notification channel via `ntfy` (`binwiederhier/ntfy:v2.11.0`) with
+  zero-trust ACLs (`deny-all`), message caching, Prometheus metrics, and Cloudflare
+  Tunnel ingress to replace public push topics.
 - `CITATION.cff`, so GitHub offers a citation and a Zenodo archive of a release
   has the metadata it needs.
 - `SECURITY.md` says how to report a vulnerability and what is in scope;

diff --git a/compose.tunnel.yml b/compose.tunnel.yml
@@ -8,7 +8,7 @@ services:
   cloudflared:
     image: cloudflare/cloudflared:2026.10.3@sha256:faa8e0a1475f137c910be6025f43c8e4092fdd35461b31e19fd1390657a1ff39
     restart: unless-stopped
-    depends_on: [grafana, otel-collector]
+    depends_on: [grafana, otel-collector, ntfy]
     # Token via env, not --token: argv is readable from the host and from any
     # container with pid:host (node-exporter has it).
     command: tunnel --no-autoupdate run

diff --git a/compose.yml b/compose.yml
@@ -210,6 +210,38 @@ services:
     !!merge <<: *hardened
     networks: [default, backend]
 
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.26.0@sha256:92e05d5fe0df01d3980518dc42f07b778a8997048b57392ebc7f7391ebd7bb06
+    restart: unless-stopped
+    volumes:
+      - ./config/blackbox.yaml:/etc/blackbox_exporter/config.yml:ro
+    command: ["--config.file=/etc/blackbox_exporter/config.yml"]
+    healthcheck:
+      !!merge <<: *healthcheck
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:9115/-/healthy"]
+    mem_limit: 64m
+    pids_limit: 64
+    read_only: true
+    !!merge <<: *hardened
+    networks: [default, backend]
+
+  ntfy:
+    image: binwiederhier/ntfy:v2.11.0@sha256:4a7d0f0adc6d5d9fc36e64ab55ef676e76e124a2bdd50ce115b6d9c1c7430294
+    restart: unless-stopped
+    volumes:
+      - ./config/ntfy/server.yml:/etc/ntfy/server.yml:ro
+      - ntfy_data:/var/lib/ntfy
+    command: ["serve", "--config=/etc/ntfy/server.yml"]
+    healthcheck:
+      !!merge <<: *healthcheck
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:80/v1/health"]
+    environment:
+      NTFY_BASE_URL: ${NTFY_BASE_URL:-http://localhost:80}
+    mem_limit: 128m
+    pids_limit: 128
+    !!merge <<: *hardened
+    networks: [default, backend]
+
 # Loki, Prometheus and Tempo have no authentication of their own, so only
 # Grafana and the collector can reach them. cloudflared stays on `default`: its
 # ingress list comes from Cloudflare at runtime, so an edited ingress cannot
@@ -223,5 +255,6 @@ volumes:
   tempo_data:
   prometheus_data:
   grafana_data:
+  ntfy_data:
   # Not in `just backup`: seconds of in-flight telemetry, worthless by restore time.
   otel_queue:
diff --git a/config/blackbox.yaml b/config/blackbox.yaml
@@ -0,0 +1,14 @@
+# Blackbox exporter configuration for outside-in uptime monitoring
+modules:
+  http_2xx:
+    prober: http
+    timeout: 10s
+    http:
+      valid_status_codes: [] # Defaults to 2xx
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      method: GET
+      follow_redirects: true
+      fail_if_ssl: false
+      fail_if_not_ssl: false
+      tls_config:
+        insecure_skip_verify: false
diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml
@@ -328,3 +328,73 @@ groups:
             The slowest 1% of requests to {{ $labels.job }}
             ({{ $labels.project }}/{{ $labels.env }}) took over 2 seconds for 5 minutes.
             The Latency p95 by route panel on Service Health shows which route.
+      - uid: blackbox-probe-failed
+        title: BlackboxProbeFailed
+        !!merge <<: *rule_defaults
+        for: 2m
+        data:
+          - !!merge <<: *query_node
+            model:
+              !!merge <<: *query_model
+              expr: probe_success{job="blackbox"} == bool 0
+          - *firing
+        labels:
+          severity: critical
+        annotations:
+          summary: "Outside-in HTTP probe failed for {{ $labels.instance }}"
+          description: "Blackbox prober failed to reach {{ $labels.instance }} for 2 minutes."
+      # Google SRE multi-window burn rate alerts for a 99.5% availability SLO
+      # (0.5% error budget over 30 days). A 14.4x burn consumes 2% of budget in 1 hour;
+      # 5m short and 1h long windows must both exceed the threshold to prevent flapping.
+      - uid: error-budget-burn-fast
+        title: ErrorBudgetBurnFast
+        !!merge <<: *rule_defaults
+        for: 2m
+        data:
+          - !!merge <<: *query_node
+            model:
+              !!merge <<: *query_model
+              expr: >
+                (sum by (job, project, env)
+                (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[1h]))
+                / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[1h])) > (14.4 * 0.005))
+                and
+                (sum by (job, project, env)
+                (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[5m]))
+                / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[5m])) > (14.4 * 0.005))
+                and sum by (job, project, env) (rate(http_server_request_duration_seconds_count[5m])) > 0.1
+          - *firing
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} 14.4x error budget burn (fast burn)"
+          description: >-
+            Consuming 2% of the 30-day 99.5% error budget within 1 hour.
+            Investigate immediately to protect the availability SLO.
+      # 6x burn consumes 5% of budget in 6 hours. Both 30m short and 6h long
+      # windows must exceed 6x (3% error rate) with minimum traffic floor.
+      - uid: error-budget-burn-slow
+        title: ErrorBudgetBurnSlow
+        !!merge <<: *rule_defaults
+        for: 15m
+        data:
+          - !!merge <<: *query_node
+            model:
+              !!merge <<: *query_model
+              expr: >
+                (sum by (job, project, env)
+                (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[6h]))
+                / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[6h])) > (6 * 0.005))
+                and
+                (sum by (job, project, env)
+                (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[30m]))
+                / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[30m])) > (6 * 0.005))
+                and sum by (job, project, env) (rate(http_server_request_duration_seconds_count[30m])) > 0.1
+          - *firing
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} 6x error budget burn (slow burn)"
+          description: >-
+            Consuming 5% of the 30-day 99.5% error budget within 6 hours.
+            Investigate to prevent error budget exhaustion.
diff --git a/config/ntfy/server.yml b/config/ntfy/server.yml
@@ -0,0 +1,18 @@
+# ntfy configuration for CML monitoring alert delivery (Issue #19)
+listen-http: ":80"
+behind-proxy: true
+
+# Zero-trust auth: anonymous access is completely disabled
+auth-file: "/var/lib/ntfy/auth.db"
+auth-default-access: "deny-all"
+
+# Message cache for subscriber catch-up
+cache-file: "/var/lib/ntfy/cache.db"
+cache-duration: "12h"
+
+# Disable public registration; logins allowed via web app / API
+enable-signup: false
+enable-login: true
+
+# Prometheus metrics exposed on internal port 9090 (isolated from public ingress on port 80)
+metrics-listen-http: ":9090"
diff --git a/config/prometheus.yaml b/config/prometheus.yaml
@@ -48,3 +48,25 @@ scrape_configs:
     static_configs:
       - targets: ["tempo:3200"]
     metric_relabel_configs: *drop_buckets
+
+  - job_name: ntfy
+    static_configs:
+      - targets: ["ntfy:9090"]
+    metric_relabel_configs: *drop_buckets
+
+  - job_name: blackbox
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - https://app.r9lab.io
+          - https://docs.r9lab.io
+          - https://api.r9lab.io/health
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md
@@ -128,6 +128,7 @@ backups and treat all three the same way.
 | `ALERT_WEBHOOK_URL`, `HEARTBEAT_URL` | `.env` | the notification provider |
 | `HEALTHCHECKS_API_KEY` | `.env` | healthchecks.io project settings |
 | `CLOUDFLARE_TUNNEL_TOKEN` | `.env` | `tofu output -raw tunnel_token` |
+| `NTFY_BASE_URL` | `.env` (optional) | public ntfy hostname, e.g. `https://ntfy.example.com` |
 | `CF_ACCESS_AUD` | `.env` (not secret, but paired) | `tofu output -raw grafana_access_aud` |
 | Tunnel secret, API responses | `infra/terraform.tfstate` | written by every `tofu apply` |
 | `CLOUDFLARE_API_TOKEN` | your shell, per session | Cloudflare dashboard |
@@ -136,6 +137,33 @@ backups and treat all three the same way.
 Access email list). It is gitignored for privacy, not because it holds a
 credential.
 
+## Alert catalog
+
+Eighteen alert rules guard the stack and reporting spokes across six folders.
+The PromQL threshold evaluates instant expressions; a rule fires when its
+comparison stays true for the duration of `for`:
+
+| Rule | Group | Severity | For | Trigger & Intent |
+| --- | --- | --- | --- | --- |
+| `Watchdog` | meta | none | 0m | Heartbeat pinging dead man's switch (`vector(1)`); always firing. |
+| `AlertDeliveryFailing` | meta | critical | 10m | Grafana failing to deliver webhook notifications. |
+| `TargetDown` | stack-health | critical | 2m | Prometheus cannot scrape an internal job target (`up == 0`). |
+| `OtelExportFailures` | stack-health | warning | 5m | OTel Collector failing to export spans, metrics, or logs. |
+| `BlackboxProbeFailed` | service-red | critical | 2m | Outside-in HTTP probe failed for an endpoint (`probe_success == 0`). |
+| `ErrorBudgetBurnFast` | service-red | critical | 2m | 14.4x burn rate (2% of 30d 99.5% SLO consumed in 1h; short 5m + long 1h). |
+| `ErrorBudgetBurnSlow` | service-red | warning | 15m | 6x burn rate (5% of 30d 99.5% SLO consumed in 6h; short 30m + long 6h). |
+| `HighErrorRate` | service-red | warning | 5m | Over 5% 5xx HTTP response codes on a service over 5 minutes. |
+| `HighLatencyP99` | service-red | warning | 5m | Slowest 1% of requests exceed 2 seconds over 5 minutes. |
+| `HostDiskSpaceLow` | capacity | warning | 15m | Storage volume under 20% free space. |
+| `HostDiskFilling` | capacity | warning | 30m | Linear fit predicts volume exhaust within 3 days. |
+| `HostMemoryHigh` | capacity | warning | 15m | Host RAM over 90% utilized for 15 minutes. |
+| `PrometheusCardinalityHigh` | capacity | warning | 30m | Active series exceed 30,000 threshold (~18 spokes baseline). |
+| `PrometheusCardinalitySpike` | capacity | warning | 15m | Head series spike by >5,000 in 30 minutes. |
+| `ContainerRestarting` | container-lifecycle | warning | 10m | Container restarts >3 times within 1 hour. |
+| `ContainerOOMKilled` | container-lifecycle | warning | 0m | Container terminated by host kernel OOM killer. |
+| `ProjectTelemetrySilent` | spoke | warning | 15m | Spoke service stopped shipping OTel telemetry. |
+| `ProjectUnscheduledRestart` | spoke | warning | 5m | Spoke container restarted unexpectedly. |
+
 ## Alert delivery
 
 Grafana evaluates and delivers the rules. Rules, contact points, and the
@@ -155,6 +183,73 @@ After changing either variable, run `docker compose up -d grafana`. A plain
 `restart` keeps the old environment. Compose rebuilds a container's
 environment only on `up`.
 
+### Self-hosted alert delivery with ntfy
+
+Alert bodies carry infrastructure detail — hostnames, mount points, job names,
+and environment tags. A public hosted push service where topic names are the
+only protection allows anyone who discovers the topic to eavesdrop or forge
+alerts.
+
+The monitoring stack includes a self-hosted `ntfy` service running behind
+Cloudflare Tunnel with strict zero-trust authentication (`auth-default-access: "deny-all"`):
+
+```text
++------------------------------------------------------------------+
+| Monitoring Host                                                  |
+|                                                                  |
+|   Grafana ------(http://ntfy:80/alerts?tpl=1...)------> ntfy     |
+|   (publisher)                                            |       |
+|                                                     [auth.db]    |
+|                                                          ^       |
+|                                                          |       |
+|   cloudflared <---(https://ntfy.example.com)-------------+       |
+|   (tunnel)                                                       |
++-------^----------------------------------------------------------+
+        |
+   Mobile / Web client (authenticated subscriber)
+```
+
+#### 1. Create the Grafana publisher user
+
+Generate a password and add the user inside the container:
+
+```sh
+docker compose exec -e NTFY_PASSWORD="<grafana-secret>" ntfy ntfy user add grafana
+docker compose exec ntfy ntfy access grafana alerts write-only
+```
+
+#### 2. Configure Grafana contact point
+
+Point `ALERT_WEBHOOK_URL` in `.env` to the local ntfy instance using native
+message templating (`?tpl=1`):
+
+```sh
+ALERT_WEBHOOK_URL=http://grafana:<grafana-secret>@ntfy:80/alerts?tpl=1&title={{.CommonAnnotations.summary}}&message={{.CommonAnnotations.description}}
+```
+
+Apply the change to Grafana:
+
+```sh
+docker compose up -d grafana
+```
+
+#### 3. Create subscriber users for operators
+
+Each team member gets their own account with read-only access to the alerts topic:
+
+```sh
+docker compose exec -e NTFY_PASSWORD="<user-secret>" ntfy ntfy user add simon
+docker compose exec ntfy ntfy access simon alerts read-only
+```
+
+#### 4. Subscribe from phone or desktop
+
+Operators subscribe to `https://ntfy.<domain>/alerts` using the ntfy mobile app
+(iOS / Android) or web UI, authenticating with their username and password.
+Do NOT put the ntfy domain behind Cloudflare Access SSO: mobile push clients
+cannot complete the browser-based OAuth flow, breaking notifications silently.
+ntfy's internal credentials over TLS provide the access boundary.
+
 ## Changing the Cloudflare edge
 
 The tunnel, its ingress rules, both DNS records, and the Access policy in

diff --git a/infra/generate-imports.sh b/infra/generate-imports.sh
@@ -79,6 +79,8 @@ otel_record="$(lookup_dns_record "otel.$domain")" || {
     die "no CNAME found for otel.$domain"
 }
 
+ntfy_record="$(lookup_dns_record "ntfy.$domain" || true)"
+
 # The Access app may legitimately not exist; then the apply should create it.
 access_apps="$(api "accounts/$account/access/apps?per_page=100")"
 access_app_id="$(jq -r --arg d "grafana.$domain" '.result[] | select(.domain == $d) | .id' <<<"$access_apps" | head -1)"
@@ -118,6 +120,16 @@ import {
 }
 EOF
 
+if [[ -n "$ntfy_record" ]]; then
+    cat <<EOF
+
+import {
+  to = cloudflare_dns_record.ntfy
+  id = "$zone/$ntfy_record"
+}
+EOF
+fi
+
 if [[ -n "$access_app_id" ]]; then
     cat <<EOF
 

diff --git a/infra/main.tf b/infra/main.tf
@@ -71,6 +71,11 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "monitoring" {
         hostname = "otel.${var.domain}"
         service  = "http://otel-collector:4318"
       },
+      {
+        # Self-hosted alert delivery (Issue #19); authenticates with its own auth-file.
+        hostname = "ntfy.${var.domain}"
+        service  = "http://ntfy:80"
+      },
       {
         service = "http_status:404"
       },
@@ -99,6 +104,15 @@ resource "cloudflare_dns_record" "otel" {
   ttl     = 1
 }
 
+resource "cloudflare_dns_record" "ntfy" {
+  zone_id = var.zone_id
+  name    = "ntfy.${var.domain}"
+  type    = "CNAME"
+  content = "${cloudflare_zero_trust_tunnel_cloudflared.monitoring.id}.cfargotunnel.com"
+  proxied = true
+  ttl     = 1
+}
+
 # Email one-time PIN in front of Grafana. The ingestion hostname is not behind
 # Access; machines authenticate with the bearer token.
 resource "cloudflare_zero_trust_access_application" "grafana" {