Add the telemetry backend: pinned releases, configs and unit templates

docs/source/api/index.md

@@ -105,6 +105,12 @@ the docs build does not install the optional `influxdb` extra.
    :show-inheritance:
 ```
 
+```{eval-rst}
+.. automodule:: libby.telemetry_backend
+   :members:
+   :show-inheritance:
+```
+
 ## Responses and errors
 
 ```{eval-rst}

docs/source/telemetry.md

@@ -56,3 +56,21 @@ can link it to its trace.
 
 Off, each hook costs about 0.1 µs. On, a traced request adds roughly 15 µs on
 the sender and 12 µs on the handler.
+
+## Backend
+
+`libby.telemetry_backend` describes the backend libby exports to, so an
+installer can set it up without containers or package repositories:
+
+- an OpenTelemetry Collector receiving OTLP on ports 4317 (gRPC) and 4318
+  (HTTP), and fanning out to
+- Prometheus (metrics, port 9090), Tempo (traces, port 3200) and Loki (logs,
+  port 3100), which Grafana reads.
+
+It holds each component's pinned release with its SHA-256 (`RELEASES`), and a
+config and a systemd unit template per service. `render()` fills a template
+from the values in `PLACEHOLDERS`: directories, the account to run as, bind
+addresses and retention. instrumentctl's `install` uses it on the host an
+instrument names as its telemetry backend.
+
+Upgrading a component means changing its `Release` version, URL and checksum.

libby/telemetry_backend/__init__.py

@@ -0,0 +1,96 @@
+"""The telemetry backend libby exports to: pinned binaries, configs and units.
+
+Nothing here is instrument-specific. An installer such as instrumentctl
+downloads each of :data:`RELEASES`, checks it against its SHA-256, and renders
+each of :data:`SERVICES` with :func:`render`, supplying every name in
+:data:`PLACEHOLDERS`. The backend is an OpenTelemetry Collector receiving OTLP
+and fanning out to Prometheus (metrics), Tempo (traces) and Loki (logs), each a
+plain systemd service.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from importlib import resources
+from typing import Mapping, Tuple
+
+_GITHUB = "https://github.com"
+
+
+@dataclass(frozen=True)
+class Release:
+    """One upstream binary, pinned to a version and the archive's SHA-256."""
+
+    binary: str
+    version: str
+    url: str
+    sha256: str
+    # Path of the binary inside the archive
+    member: str
+
+
+# linux/amd64 builds, checked against each project's published checksums
+RELEASES: Tuple[Release, ...] = (
+    Release(
+        binary="otelcol-contrib",
+        version="0.164.0",
+        url=f"{_GITHUB}/open-telemetry/opentelemetry-collector-releases/releases/download/"
+            "v0.164.0/otelcol-contrib_0.164.0_linux_amd64.tar.gz",
+        sha256="f0c1ce9f661399beb4a58b711924716e08ea70c3802db0715fcfde3cbbada2d4",
+        member="otelcol-contrib",
+    ),
+    Release(
+        binary="prometheus",
+        version="3.13.5",
+        url=f"{_GITHUB}/prometheus/prometheus/releases/download/"
+            "v3.13.5/prometheus-3.13.5.linux-amd64.tar.gz",
+        sha256="0388822ace519c5a3a8ef556a541ce1b536e21d872f82deded465bd27a5a0699",
+        member="prometheus-3.13.5.linux-amd64/prometheus",
+    ),
+    Release(
+        binary="tempo",
+        version="3.0.0",
+        url=f"{_GITHUB}/grafana/tempo/releases/download/v3.0.0/tempo_3.0.0_linux_amd64.tar.gz",
+        sha256="f4150470a5f6bf57e94dac5392c552b488932ea9e88a6ee2a75d7b969b7cb6a4",
+        member="tempo",
+    ),
+    Release(
+        binary="loki",
+        version="3.7.9",
+        url=f"{_GITHUB}/grafana/loki/releases/download/v3.7.9/loki-linux-amd64.zip",
+        sha256="17a5fae28ce733b3560c27c76620920b169bd2aea9473947b733b511f92e81dd",
+        member="loki-linux-amd64",
+    ),
+)
+
+# Each has a <service>.yaml config and a <service>.service unit template
+SERVICES: Tuple[str, ...] = ("otelcol", "prometheus", "tempo", "loki")
+
+PLACEHOLDERS = frozenset({
+    "name",               # prefix of each unit, e.g. hispec for hispec-prometheus
+    "display_name",       # as the units read in systemctl status
+    "user",               # account the services run as
+    "bin",                # directory holding the RELEASES binaries
+    "etc",                # directory holding the rendered configs
+    "data",               # directory each service keeps its data under
+    "otlp_bind",          # address the Collector receives OTLP on
+    "query_bind",         # address Prometheus, Tempo and Loki answer Grafana on
+    "metrics_retention",  # Prometheus retention, e.g. 5y
+    "trace_retention",    # Tempo retention, e.g. 336h
+})
+
+_PLACEHOLDER = re.compile(r"@([a-z_]+)@")
+
+
+def render(template: str, values: Mapping[str, str]) -> str:
+    """Fill ``template``'s ``@placeholder@`` values.
+
+    Raise ``KeyError`` naming any placeholder ``values`` lacks, so a missing
+    one fails the install rather than reaching a host as a literal ``@data@``.
+    """
+    text = resources.files(__package__).joinpath("templates", template).read_text(
+        encoding="utf-8")
+    missing = sorted(set(_PLACEHOLDER.findall(text)) - set(values))
+    if missing:
+        raise KeyError(f"{template} needs {', '.join(missing)}")
+    return _PLACEHOLDER.sub(lambda match: str(values[match.group(1)]), text)

libby/telemetry_backend/templates/loki.service

@@ -0,0 +1,23 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+[Unit]
+Description=@display_name@ telemetry: Loki
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=@user@
+Group=@user@
+ExecStart=@bin@/loki -config.file=@etc@/loki.yaml
+Restart=on-failure
+RestartSec=5
+SyslogIdentifier=@name@-loki
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths=@data@/loki
+
+[Install]
+WantedBy=multi-user.target

libby/telemetry_backend/templates/loki.yaml

@@ -0,0 +1,31 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+auth_enabled: false
+server:
+  http_listen_address: @query_bind@
+  http_listen_port: 3100
+  grpc_listen_address: 127.0.0.1
+  # 9095 is Tempo's
+  grpc_listen_port: 9096
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: @data@/loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+  storage:
+    filesystem:
+      chunks_directory: @data@/loki/chunks
+      rules_directory: @data@/loki/rules
+schema_config:
+  configs:
+    - from: 2026-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+limits_config:
+  allow_structured_metadata: true

libby/telemetry_backend/templates/otelcol.service

@@ -0,0 +1,22 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+[Unit]
+Description=@display_name@ telemetry: OpenTelemetry Collector
+After=network-online.target @name@-prometheus.service @name@-tempo.service @name@-loki.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=@user@
+Group=@user@
+ExecStart=@bin@/otelcol-contrib --config=@etc@/otelcol.yaml
+Restart=on-failure
+RestartSec=5
+SyslogIdentifier=@name@-otelcol
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

libby/telemetry_backend/templates/otelcol.yaml

@@ -0,0 +1,40 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+#
+# Receives OTLP from daemons and agents, and fans it out to the services below
+# on this host.
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: @otlp_bind@:4317
+      http:
+        endpoint: @otlp_bind@:4318
+
+processors:
+  batch: {}
+
+exporters:
+  otlp_http/prometheus:
+    metrics_endpoint: http://127.0.0.1:9090/api/v1/otlp/v1/metrics
+  otlp_grpc/tempo:
+    endpoint: 127.0.0.1:4417
+    tls:
+      insecure: true
+  otlp_http/loki:
+    endpoint: http://127.0.0.1:3100/otlp
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp_http/prometheus]
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp_grpc/tempo]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp_http/loki]

libby/telemetry_backend/templates/prometheus.service

@@ -0,0 +1,26 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+[Unit]
+Description=@display_name@ telemetry: Prometheus
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=@user@
+Group=@user@
+ExecStart=@bin@/prometheus --config.file=@etc@/prometheus.yaml \
+  --storage.tsdb.path=@data@/prometheus --storage.tsdb.retention.time=@metrics_retention@ \
+  --web.listen-address=@query_bind@:9090 \
+  --web.enable-otlp-receiver --web.enable-remote-write-receiver
+Restart=on-failure
+RestartSec=5
+SyslogIdentifier=@name@-prometheus
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths=@data@/prometheus
+
+[Install]
+WantedBy=multi-user.target

libby/telemetry_backend/templates/prometheus.yaml

@@ -0,0 +1,10 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+storage:
+  tsdb:
+    # Accept late samples, e.g. a batch resent after an outage
+    out_of_order_time_window: 2h
+otlp:
+  promote_resource_attributes:
+    - service.namespace
+    - host.name

libby/telemetry_backend/templates/tempo.service

@@ -0,0 +1,23 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+[Unit]
+Description=@display_name@ telemetry: Tempo
+After=network-online.target @name@-prometheus.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=@user@
+Group=@user@
+ExecStart=@bin@/tempo -config.file=@etc@/tempo.yaml
+Restart=on-failure
+RestartSec=5
+SyslogIdentifier=@name@-tempo
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths=@data@/tempo
+
+[Install]
+WantedBy=multi-user.target

libby/telemetry_backend/templates/tempo.yaml

@@ -0,0 +1,50 @@
+# Rendered from libby's telemetry backend; edits here are lost on the next install.
+
+stream_over_http_enabled: true
+server:
+  http_listen_address: @query_bind@
+  http_listen_port: 3200
+  grpc_listen_address: 127.0.0.1
+  grpc_listen_port: 9095
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          # 4317 is the Collector's; only it sends here
+          endpoint: 127.0.0.1:4417
+storage:
+  trace:
+    backend: local
+    local:
+      path: @data@/tempo/traces
+    wal:
+      path: @data@/tempo/wal
+# Tempo 3 applies retention in the scheduler and compacts in the worker; both
+# read block_retention, so it is set in each
+backend_scheduler:
+  local_work_path: @data@/tempo/work
+  provider:
+    compaction:
+      compaction:
+        block_retention: @trace_retention@
+backend_worker:
+  compaction:
+    block_retention: @trace_retention@
+live_store:
+  wal:
+    path: @data@/tempo/live-store/traces
+  shutdown_marker_dir: @data@/tempo/live-store/shutdown-marker
+block_builder:
+  wal:
+    path: @data@/tempo/block-builder/traces
+metrics_generator:
+  storage:
+    path: @data@/tempo/generator/wal
+    remote_write:
+      - url: http://127.0.0.1:9090/api/v1/write
+        send_exemplars: true
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics]

pyproject.toml

@@ -50,6 +50,7 @@ exclude = ["peers*", "package*"]
 
 [tool.setuptools.package-data]
 "libby.cli" = ["*.yaml"]
+"libby.telemetry_backend" = ["templates/*"]
 
 [project.scripts]
 libby = "libby.cli.libby_cli:main"