From 4fef4d1fbff7674738488a2127e142a0ac6ba8ce Mon Sep 17 00:00:00 2001 From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:39:51 +0100 Subject: [PATCH 1/4] feat(telemetry): add blackbox prober, multi-window SLO alerts, and self-hosted ntfy - Add Prometheus Blackbox Exporter service for outside-in uptime probing of research endpoints - Implement Google SRE multi-window burn rate alerts (fast and slow burn) for 99.5% availability targets - Add BlackboxProbeFailed critical alert for probed endpoint outages - Add self-hosted ntfy service with zero-trust ACLs, persistent cache, and Cloudflare tunnel ingress --- compose.yml | 33 ++++++++++++++ config/blackbox.yaml | 14 ++++++ config/grafana/alerting/rules.yaml | 70 ++++++++++++++++++++++++++++++ config/ntfy/server.yml | 18 ++++++++ config/prometheus.yaml | 22 ++++++++++ infra/generate-imports.sh | 12 +++++ infra/main.tf | 14 ++++++ justfile | 2 +- scripts/smoke.sh | 5 ++- 9 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 config/blackbox.yaml create mode 100644 config/ntfy/server.yml diff --git a/compose.yml b/compose.yml index d646a40..740fb9d 100644 --- a/compose.yml +++ b/compose.yml @@ -210,6 +210,38 @@ services: !!merge <<: *hardened networks: [default, backend] + blackbox-exporter: + image: prom/blackbox-exporter:v0.26.0@sha256:92e05d5fe0df01d3980518dc42f07b778a8997048b57392ebc7f7391ebd7bb06 + restart: unless-stopped + volumes: + - ./config/blackbox.yaml:/etc/blackbox_exporter/config.yml:ro + command: ["--config.file=/etc/blackbox_exporter/config.yml"] + healthcheck: + !!merge <<: *healthcheck + test: ["CMD", "wget", "-q", "--spider", "http://localhost:9115/-/healthy"] + mem_limit: 64m + pids_limit: 64 + read_only: true + !!merge <<: *hardened + networks: [default, backend] + + ntfy: + image: binwiederhier/ntfy:v2.11.0@sha256:4a7d0f0adc6d5d9fc36e64ab55ef676e76e124a2bdd50ce115b6d9c1c7430294 + restart: unless-stopped + volumes: + - ./config/ntfy/server.yml:/etc/ntfy/server.yml:ro + - ntfy_data:/var/lib/ntfy + command: ["serve", "--config=/etc/ntfy/server.yml"] + healthcheck: + !!merge <<: *healthcheck + test: ["CMD", "wget", "-q", "--spider", "http://localhost:80/v1/health"] + environment: + NTFY_BASE_URL: ${NTFY_BASE_URL:-http://localhost:80} + mem_limit: 128m + pids_limit: 128 + !!merge <<: *hardened + networks: [default, backend] + # Loki, Prometheus and Tempo have no authentication of their own, so only # Grafana and the collector can reach them. cloudflared stays on `default`: its # ingress list comes from Cloudflare at runtime, so an edited ingress cannot @@ -223,5 +255,6 @@ volumes: tempo_data: prometheus_data: grafana_data: + ntfy_data: # Not in `just backup`: seconds of in-flight telemetry, worthless by restore time. otel_queue: diff --git a/config/blackbox.yaml b/config/blackbox.yaml new file mode 100644 index 0000000..a25b2ae --- /dev/null +++ b/config/blackbox.yaml @@ -0,0 +1,14 @@ +# Blackbox exporter configuration for outside-in uptime monitoring +modules: + http_2xx: + prober: http + timeout: 10s + http: + valid_status_codes: [] # Defaults to 2xx + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + method: GET + no_follow_redirects: false + fail_if_ssl: false + fail_if_not_ssl: false + tls_config: + insecure_skip_verify: false diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml index 99a9e8b..52c3269 100644 --- a/config/grafana/alerting/rules.yaml +++ b/config/grafana/alerting/rules.yaml @@ -328,3 +328,73 @@ groups: The slowest 1% of requests to {{ $labels.job }} ({{ $labels.project }}/{{ $labels.env }}) took over 2 seconds for 5 minutes. The Latency p95 by route panel on Service Health shows which route. + - uid: blackbox-probe-failed + title: BlackboxProbeFailed + !!merge <<: *rule_defaults + for: 2m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: probe_success{job="blackbox"} == bool 0 + - *firing + labels: + severity: critical + annotations: + summary: "Outside-in HTTP probe failed for {{ $labels.instance }}" + description: "Blackbox prober failed to reach {{ $labels.instance }} for 2 minutes." + # Google SRE multi-window burn rate alerts for a 99.5% availability SLO + # (0.5% error budget over 30 days). A 14.4x burn consumes 2% of budget in 1 hour; + # 5m short and 1h long windows must both exceed the threshold to prevent flapping. + - uid: error-budget-burn-fast + title: ErrorBudgetBurnFast + !!merge <<: *rule_defaults + for: 2m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: > + (sum by (job, project, env) + (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[1h])) + / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[1h])) > (14.4 * 0.005)) + and + (sum by (job, project, env) + (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[5m])) + / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[5m])) > (14.4 * 0.005)) + and sum by (job, project, env) (rate(http_server_request_duration_seconds_count[5m])) > 0.1 + - *firing + labels: + severity: critical + annotations: + summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} 14.4x error budget burn (fast burn)" + description: >- + Consuming 2% of the 30-day 99.5% error budget within 1 hour. + Investigate immediately to protect the availability SLO. + # 6x burn consumes 5% of budget in 6 hours. Both 30m short and 6h long + # windows must exceed 6x (3% error rate) with minimum traffic floor. + - uid: error-budget-burn-slow + title: ErrorBudgetBurnSlow + !!merge <<: *rule_defaults + for: 15m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: > + (sum by (job, project, env) + (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[6h])) + / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[6h])) > (6 * 0.005)) + and + (sum by (job, project, env) + (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[30m])) + / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[30m])) > (6 * 0.005)) + and sum by (job, project, env) (rate(http_server_request_duration_seconds_count[30m])) > 0.1 + - *firing + labels: + severity: warning + annotations: + summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} 6x error budget burn (slow burn)" + description: >- + Consuming 5% of the 30-day 99.5% error budget within 6 hours. + Investigate to prevent error budget exhaustion. diff --git a/config/ntfy/server.yml b/config/ntfy/server.yml new file mode 100644 index 0000000..03f3d5b --- /dev/null +++ b/config/ntfy/server.yml @@ -0,0 +1,18 @@ +# ntfy configuration for CML monitoring alert delivery (Issue #19) +listen-http: ":80" +behind-proxy: true + +# Zero-trust auth: anonymous access is completely disabled +auth-file: "/var/lib/ntfy/auth.db" +auth-default-access: "deny-all" + +# Message cache for subscriber catch-up +cache-file: "/var/lib/ntfy/cache.db" +cache-duration: "12h" + +# Disable public registration; logins allowed via web app / API +enable-signup: false +enable-login: true + +# Prometheus metrics exposed at /metrics +enable-metrics: true diff --git a/config/prometheus.yaml b/config/prometheus.yaml index 99d0015..0872269 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -48,3 +48,25 @@ scrape_configs: static_configs: - targets: ["tempo:3200"] metric_relabel_configs: *drop_buckets + + - job_name: ntfy + static_configs: + - targets: ["ntfy:80"] + metric_relabel_configs: *drop_buckets + + - job_name: blackbox + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - https://app.r9lab.io + - https://docs.r9lab.io + - https://api.r9lab.io/health + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 diff --git a/infra/generate-imports.sh b/infra/generate-imports.sh index 45c61b8..f375aaa 100755 --- a/infra/generate-imports.sh +++ b/infra/generate-imports.sh @@ -79,6 +79,8 @@ otel_record="$(lookup_dns_record "otel.$domain")" || { die "no CNAME found for otel.$domain" } +ntfy_record="$(lookup_dns_record "ntfy.$domain" || true)" + # The Access app may legitimately not exist; then the apply should create it. access_apps="$(api "accounts/$account/access/apps?per_page=100")" access_app_id="$(jq -r --arg d "grafana.$domain" '.result[] | select(.domain == $d) | .id' <<<"$access_apps" | head -1)" @@ -118,6 +120,16 @@ import { } EOF +if [[ -n "$ntfy_record" ]]; then + cat <__data. -stateful := "grafana prometheus loki tempo" +stateful := "grafana prometheus loki tempo ntfy" # Helper and lint images, pinned once. jq := "ghcr.io/jqlang/jq:1.8.1@sha256:4f34c6d23f4b1372ac789752cc955dc67c2ae177eb1b5860b75cdc5091ce6f91" diff --git a/scripts/smoke.sh b/scripts/smoke.sh index f895d0a..0a5ae90 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -60,7 +60,10 @@ code="$(curl -s -o /dev/null -w '%{http_code}' -H 'Cf-Access-Jwt-Assertion: not- # A renamed service leaves its scrape job silently empty, and TargetDown then # matches nothing. Every job in prometheus.yaml must have scraped its target. n_jobs="$(grep -c '^ *- job_name:' config/prometheus.yaml)" -all_up() { [[ "$(promq 'count(up == 1)' | jq -r '.data.result[0].value[1] // 0')" == "$n_jobs" ]]; } +all_up() { + [[ "$(promq 'count(count by (job) (up == 1))' | jq -r '.data.result[0].value[1] // 0')" == "$n_jobs" ]] \ + && [[ "$(promq 'count(up == 0)' | jq -r '.data.result[0].value[1] // 0')" == "0" ]] +} poll all_up || die "not all $n_jobs scrape targets are up: $(promq up | jq -r '.data.result[] | "\(.metric.job)=\(.value[1])"' | tr '\n' ' ')" # The alert-pipeline check below waits out TargetDown's `for`. Stopping its target From 4acad8a5bd4bf8fa9ad2d9cf687068bcadb454d7 Mon Sep 17 00:00:00 2001 From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:39:59 +0100 Subject: [PATCH 2/4] docs: update runbook alert catalog, ntfy guide, env example, and changelog - Add 18-rule alert catalog reference table in RUNBOOK.md - Document self-hosted ntfy user provisioning and Grafana contact point setup - Add NTFY_BASE_URL and ntfy webhook documentation to .env.example - Record changes under [Unreleased] in CHANGELOG.md --- .env.example | 8 ++++- CHANGELOG.md | 10 ++++++ docs/RUNBOOK.md | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 35e7199..9e288ba 100644 --- a/.env.example +++ b/.env.example @@ -22,12 +22,18 @@ GRAFANA_COOKIE_SECURE=false # openssl rand -hex 32 OTLP_AUTH_TOKEN=local-dev-token -# Where Grafana delivers alert notifications (any webhook: ntfy, Slack, …). +# Where Grafana delivers alert notifications (self-hosted ntfy, hosted ntfy, Slack, …). +# For self-hosted ntfy with basic auth publisher: +# ALERT_WEBHOOK_URL=http://grafana:password@ntfy:80/alerts?tpl=1&title={{.CommonAnnotations.summary}}&message={{.CommonAnnotations.description}} # Not optional: an empty value drops every alert silently while the heartbeat # below keeps reporting healthy. With the tunnel overlay active, `just up` # refuses to start without it. ALERT_WEBHOOK_URL= +# Public URL Cloudflare Tunnel exposes ntfy on, e.g. https://ntfy.example.com +# Defaults to http://localhost:80 +NTFY_BASE_URL= + # Dead man's switch ping target (e.g. https://hc-ping.com/). The Watchdog # alert posts here every 5m; alert externally when pings stop. HEARTBEAT_URL= diff --git a/CHANGELOG.md b/CHANGELOG.md index c14953d..1d5949e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ Notable changes to this stack. Format follows ### Added +- Prometheus Blackbox Exporter service for outside-in uptime probing of HTTP/S + endpoints (`prom/blackbox-exporter:v0.26.0`), reporting `probe_success` and latency + telemetry for research applications. +- Multi-window SLO burn rate alerting (`ErrorBudgetBurnFast` and `ErrorBudgetBurnSlow`) + in `config/grafana/alerting/rules.yaml`, tracking 99.5% availability error budgets + across dual short and long evaluation windows. +- `BlackboxProbeFailed` critical alert firing when an outside-in HTTP probe fails. +- Self-hosted notification channel via `ntfy` (`binwiederhier/ntfy:v2.11.0`) with + zero-trust ACLs (`deny-all`), message caching, Prometheus metrics, and Cloudflare + Tunnel ingress to replace public push topics. - `CITATION.cff`, so GitHub offers a citation and a Zenodo archive of a release has the metadata it needs. - `SECURITY.md` says how to report a vulnerability and what is in scope; diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 23c25d0..b440124 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -128,6 +128,7 @@ backups and treat all three the same way. | `ALERT_WEBHOOK_URL`, `HEARTBEAT_URL` | `.env` | the notification provider | | `HEALTHCHECKS_API_KEY` | `.env` | healthchecks.io project settings | | `CLOUDFLARE_TUNNEL_TOKEN` | `.env` | `tofu output -raw tunnel_token` | +| `NTFY_BASE_URL` | `.env` (optional) | public ntfy hostname, e.g. `https://ntfy.example.com` | | `CF_ACCESS_AUD` | `.env` (not secret, but paired) | `tofu output -raw grafana_access_aud` | | Tunnel secret, API responses | `infra/terraform.tfstate` | written by every `tofu apply` | | `CLOUDFLARE_API_TOKEN` | your shell, per session | Cloudflare dashboard | @@ -136,6 +137,33 @@ backups and treat all three the same way. Access email list). It is gitignored for privacy, not because it holds a credential. +## Alert catalog + +Eighteen alert rules guard the stack and reporting spokes across six folders. +The PromQL threshold evaluates instant expressions; a rule fires when its +comparison stays true for the duration of `for`: + +| Rule | Group | Severity | For | Trigger & Intent | +| --- | --- | --- | --- | --- | +| `Watchdog` | meta | none | 0m | Heartbeat pinging dead man's switch (`vector(1)`); always firing. | +| `AlertDeliveryFailing` | meta | critical | 10m | Grafana failing to deliver webhook notifications. | +| `TargetDown` | stack-health | critical | 2m | Prometheus cannot scrape an internal job target (`up == 0`). | +| `OtelExportFailures` | stack-health | warning | 5m | OTel Collector failing to export spans, metrics, or logs. | +| `BlackboxProbeFailed` | service-red | critical | 2m | Outside-in HTTP probe failed for an endpoint (`probe_success == 0`). | +| `ErrorBudgetBurnFast` | service-red | critical | 2m | 14.4x burn rate (2% of 30d 99.5% SLO consumed in 1h; short 5m + long 1h). | +| `ErrorBudgetBurnSlow` | service-red | warning | 15m | 6x burn rate (5% of 30d 99.5% SLO consumed in 6h; short 30m + long 6h). | +| `HighErrorRate` | service-red | warning | 5m | Over 5% 5xx HTTP response codes on a service over 5 minutes. | +| `HighLatencyP99` | service-red | warning | 5m | Slowest 1% of requests exceed 2 seconds over 5 minutes. | +| `HostDiskSpaceLow` | capacity | warning | 15m | Storage volume under 20% free space. | +| `HostDiskFilling` | capacity | warning | 30m | Linear fit predicts volume exhaust within 3 days. | +| `HostMemoryHigh` | capacity | warning | 15m | Host RAM over 90% utilized for 15 minutes. | +| `PrometheusCardinalityHigh` | capacity | warning | 30m | Active series exceed 30,000 threshold (~18 spokes baseline). | +| `PrometheusCardinalitySpike` | capacity | warning | 15m | Head series spike by >5,000 in 30 minutes. | +| `ContainerRestarting` | container-lifecycle | warning | 10m | Container restarts >3 times within 1 hour. | +| `ContainerOOMKilled` | container-lifecycle | warning | 0m | Container terminated by host kernel OOM killer. | +| `ProjectTelemetrySilent` | spoke | warning | 15m | Spoke service stopped shipping OTel telemetry. | +| `ProjectUnscheduledRestart` | spoke | warning | 5m | Spoke container restarted unexpectedly. | + ## Alert delivery Grafana evaluates and delivers the rules. Rules, contact points, and the @@ -155,6 +183,73 @@ After changing either variable, run `docker compose up -d grafana`. A plain `restart` keeps the old environment. Compose rebuilds a container's environment only on `up`. +### Self-hosted alert delivery with ntfy + +Alert bodies carry infrastructure detail — hostnames, mount points, job names, +and environment tags. A public hosted push service where topic names are the +only protection allows anyone who discovers the topic to eavesdrop or forge +alerts. + +The monitoring stack includes a self-hosted `ntfy` service running behind +Cloudflare Tunnel with strict zero-trust authentication (`auth-default-access: "deny-all"`): + +```text ++------------------------------------------------------------------+ +| Monitoring Host | +| | +| Grafana ------(http://ntfy:80/alerts?tpl=1...)------> ntfy | +| (publisher) | | +| [auth.db] | +| ^ | +| | | +| cloudflared <---(https://ntfy.example.com)-------------+ | +| (tunnel) | ++-------^----------------------------------------------------------+ + | + Mobile / Web client (authenticated subscriber) +``` + +#### 1. Create the Grafana publisher user + +Generate a password and add the user inside the container: + +```sh +docker compose exec -e NTFY_PASSWORD="" ntfy ntfy user add grafana +docker compose exec ntfy ntfy access grafana alerts write-only +``` + +#### 2. Configure Grafana contact point + +Point `ALERT_WEBHOOK_URL` in `.env` to the local ntfy instance using native +message templating (`?tpl=1`): + +```sh +ALERT_WEBHOOK_URL=http://grafana:@ntfy:80/alerts?tpl=1&title={{.CommonAnnotations.summary}}&message={{.CommonAnnotations.description}} +``` + +Apply the change to Grafana: + +```sh +docker compose up -d grafana +``` + +#### 3. Create subscriber users for operators + +Each team member gets their own account with read-only access to the alerts topic: + +```sh +docker compose exec -e NTFY_PASSWORD="" ntfy ntfy user add simon +docker compose exec ntfy ntfy access simon alerts read-only +``` + +#### 4. Subscribe from phone or desktop + +Operators subscribe to `https://ntfy./alerts` using the ntfy mobile app +(iOS / Android) or web UI, authenticating with their username and password. +Do NOT put the ntfy domain behind Cloudflare Access SSO: mobile push clients +cannot complete the browser-based OAuth flow, breaking notifications silently. +ntfy's internal credentials over TLS provide the access boundary. + ## Changing the Cloudflare edge The tunnel, its ingress rules, both DNS records, and the Access policy in From c0129129c462ea22878d10907a5fe5790ee1b448 Mon Sep 17 00:00:00 2001 From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:52:02 +0100 Subject: [PATCH 3/4] sec(ntfy): isolate prometheus metrics on internal port 9090 - Bind ntfy metrics endpoint to internal port 9090 instead of public port 80 - Prevents unauthenticated internal telemetry exposure via the public Cloudflare tunnel --- config/ntfy/server.yml | 4 ++-- config/prometheus.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/ntfy/server.yml b/config/ntfy/server.yml index 03f3d5b..e1ee972 100644 --- a/config/ntfy/server.yml +++ b/config/ntfy/server.yml @@ -14,5 +14,5 @@ cache-duration: "12h" enable-signup: false enable-login: true -# Prometheus metrics exposed at /metrics -enable-metrics: true +# Prometheus metrics exposed on internal port 9090 (isolated from public ingress on port 80) +metrics-listen-http: ":9090" diff --git a/config/prometheus.yaml b/config/prometheus.yaml index 0872269..3392e11 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -51,7 +51,7 @@ scrape_configs: - job_name: ntfy static_configs: - - targets: ["ntfy:80"] + - targets: ["ntfy:9090"] metric_relabel_configs: *drop_buckets - job_name: blackbox From 2bb7867491ef4896cf845ee4823431a0d2c548bb Mon Sep 17 00:00:00 2001 From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:05:47 +0100 Subject: [PATCH 4/4] chore(smoke): strengthen blackbox and ntfy validation in smoke suite --- compose.tunnel.yml | 2 +- config/blackbox.yaml | 2 +- justfile | 4 +++- scripts/smoke.sh | 13 +++++++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/compose.tunnel.yml b/compose.tunnel.yml index 55b5d38..c850446 100644 --- a/compose.tunnel.yml +++ b/compose.tunnel.yml @@ -8,7 +8,7 @@ services: cloudflared: image: cloudflare/cloudflared:2026.10.3@sha256:faa8e0a1475f137c910be6025f43c8e4092fdd35461b31e19fd1390657a1ff39 restart: unless-stopped - depends_on: [grafana, otel-collector] + depends_on: [grafana, otel-collector, ntfy] # Token via env, not --token: argv is readable from the host and from any # container with pid:host (node-exporter has it). command: tunnel --no-autoupdate run diff --git a/config/blackbox.yaml b/config/blackbox.yaml index a25b2ae..a0653ea 100644 --- a/config/blackbox.yaml +++ b/config/blackbox.yaml @@ -7,7 +7,7 @@ modules: valid_status_codes: [] # Defaults to 2xx valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] method: GET - no_follow_redirects: false + follow_redirects: true fail_if_ssl: false fail_if_not_ssl: false tls_config: diff --git a/justfile b/justfile index 09950bb..d07997e 100644 --- a/justfile +++ b/justfile @@ -67,6 +67,7 @@ _expose-guards: @[ -n "${HEARTBEAT_URL:-}" ] || echo "WARNING: HEARTBEAT_URL is empty; the stack goes live without a dead-man's switch" >&2 @for f in .env infra/terraform.tfvars infra/terraform.tfstate infra/terraform.tfstate.backup; do [ ! -e "$f" ] || case "$(stat -c %a "$f")" in *00) ;; *) echo "error: $f is readable by other users (mode $(stat -c %a "$f")); it holds live secrets, run: chmod 600 $f" >&2; exit 1;; esac; done @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; every alert would fire into an empty webhook URL and be dropped. The heartbeat keeps pinging either way, so this failure looks healthy from the outside. Set it, or comment out this guard" >&2; exit 1; } + @case "${NTFY_BASE_URL:-}" in ""|https://*) ;; *) echo "error: NTFY_BASE_URL must be an https:// URL when exposing via tunnel (got '${NTFY_BASE_URL:-}')" >&2; exit 1;; esac # Core stack plus a demo telemetry source, isolated from any running stack (:3002). demo: @@ -107,7 +108,7 @@ lint: # documented default is refused on its own. @good="OTLP_AUTH_TOKEN=t GRAFANA_ADMIN_PASSWORD=p GRAFANA_ROOT_URL=https://g.example GRAFANA_COOKIE_SECURE=true GRAFANA_JWT_AUTH=true CF_ACCESS_TEAM_DOMAIN=d CF_ACCESS_AUD=a HEARTBEAT_URL=https://h ALERT_WEBHOOK_URL=https://w"; \ env $good just _expose-guards || { echo "error: exposure guards rejected a fully set environment" >&2; exit 1; }; \ - for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL=; do \ + for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL= NTFY_BASE_URL=http://insecure; do \ ! env $good $bad just _expose-guards 2>/dev/null || { echo "error: exposure guards accepted $bad" >&2; exit 1; }; \ done # run_scheduled.sh, interrupted mid-job, must take the job down with it, @@ -158,6 +159,7 @@ _image file service: # Each config goes through the binary that will load it. validate: docker run --rm --network none -v ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --entrypoint promtool $(just _image compose.yml prometheus) check config /etc/prometheus/prometheus.yaml + docker run --rm --network none -v ./config/blackbox.yaml:/etc/blackbox_exporter/config.yml:ro $(just _image compose.yml blackbox-exporter) --config.check --config.file=/etc/blackbox_exporter/config.yml docker run --rm --network none -e OTLP_AUTH_TOKEN=dummy -v ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro $(just _image compose.yml otel-collector) validate --config=/etc/otelcol/config.yaml docker run --rm --network none -v ./config/loki.yaml:/etc/loki/loki.yaml:ro $(just _image compose.yml loki) -config.file=/etc/loki/loki.yaml -verify-config docker run --rm --network none -v ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro $(just _image compose.yml tempo) -config.file=/etc/tempo/tempo.yaml -config.verify=true diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 0a5ae90..f777f3b 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -103,6 +103,19 @@ poll smoke_log || die "the smoke log never reached Loki with its department labe counted() { promq "count by (project, env) ({__name__=~\"telemetry_.+_total\", project=\"smoke\", env=\"ci\"})" | jq -e '.data.result | length > 0' >/dev/null; } poll counted || die "the count connector never produced telemetry_*_total{project=smoke,env=ci}; the alert rules that key on it would never fire" +# ------------------------------------------------------------- ntfy & blackbox +# Verify ntfy health endpoint answers over the docker network. +ntfy_healthy() { + docker compose -p "$project" exec -T grafana curl -sf "http://ntfy:80/v1/health" >/dev/null +} +poll ntfy_healthy || die "ntfy health endpoint did not respond on internal port 80" + +# Verify blackbox prober recorded at least one probe duration metric. +blackbox_probed() { + promq 'probe_duration_seconds{job="blackbox"}' | jq -e '.data.result | length > 0' >/dev/null +} +poll blackbox_probed || die "blackbox probe metrics never reached Prometheus; check blackbox-exporter" + # ------------------------------------------------------------- alert pipeline # Opt-in (SMOKE_ALERTS=1): node-exporter was stopped after the scrape-target # check; wait for TargetDown to reach firing. Costs the rule's `for` (2m) plus an