diff --git a/docs/source/api/index.md b/docs/source/api/index.md index 7503ceb..4277973 100644 --- a/docs/source/api/index.md +++ b/docs/source/api/index.md @@ -105,6 +105,12 @@ the docs build does not install the optional `influxdb` extra. :show-inheritance: ``` +```{eval-rst} +.. automodule:: libby.telemetry_backend + :members: + :show-inheritance: +``` + ## Responses and errors ```{eval-rst} diff --git a/docs/source/telemetry.md b/docs/source/telemetry.md index 06e2279..56a3dd5 100644 --- a/docs/source/telemetry.md +++ b/docs/source/telemetry.md @@ -56,3 +56,21 @@ can link it to its trace. Off, each hook costs about 0.1 µs. On, a traced request adds roughly 15 µs on the sender and 12 µs on the handler. + +## Backend + +`libby.telemetry_backend` describes the backend libby exports to, so an +installer can set it up without containers or package repositories: + +- an OpenTelemetry Collector receiving OTLP on ports 4317 (gRPC) and 4318 + (HTTP), and fanning out to +- Prometheus (metrics, port 9090), Tempo (traces, port 3200) and Loki (logs, + port 3100), which Grafana reads. + +It holds each component's pinned release with its SHA-256 (`RELEASES`), and a +config and a systemd unit template per service. `render()` fills a template +from the values in `PLACEHOLDERS`: directories, the account to run as, bind +addresses and retention. instrumentctl's `install` uses it on the host an +instrument names as its telemetry backend. + +Upgrading a component means changing its `Release` version, URL and checksum. diff --git a/libby/telemetry_backend/__init__.py b/libby/telemetry_backend/__init__.py new file mode 100644 index 0000000..dea2305 --- /dev/null +++ b/libby/telemetry_backend/__init__.py @@ -0,0 +1,96 @@ +"""The telemetry backend libby exports to: pinned binaries, configs and units. + +Nothing here is instrument-specific. An installer such as instrumentctl +downloads each of :data:`RELEASES`, checks it against its SHA-256, and renders +each of :data:`SERVICES` with :func:`render`, supplying every name in +:data:`PLACEHOLDERS`. The backend is an OpenTelemetry Collector receiving OTLP +and fanning out to Prometheus (metrics), Tempo (traces) and Loki (logs), each a +plain systemd service. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from importlib import resources +from typing import Mapping, Tuple + +_GITHUB = "https://github.com" + + +@dataclass(frozen=True) +class Release: + """One upstream binary, pinned to a version and the archive's SHA-256.""" + + binary: str + version: str + url: str + sha256: str + # Path of the binary inside the archive + member: str + + +# linux/amd64 builds, checked against each project's published checksums +RELEASES: Tuple[Release, ...] = ( + Release( + binary="otelcol-contrib", + version="0.164.0", + url=f"{_GITHUB}/open-telemetry/opentelemetry-collector-releases/releases/download/" + "v0.164.0/otelcol-contrib_0.164.0_linux_amd64.tar.gz", + sha256="f0c1ce9f661399beb4a58b711924716e08ea70c3802db0715fcfde3cbbada2d4", + member="otelcol-contrib", + ), + Release( + binary="prometheus", + version="3.13.5", + url=f"{_GITHUB}/prometheus/prometheus/releases/download/" + "v3.13.5/prometheus-3.13.5.linux-amd64.tar.gz", + sha256="0388822ace519c5a3a8ef556a541ce1b536e21d872f82deded465bd27a5a0699", + member="prometheus-3.13.5.linux-amd64/prometheus", + ), + Release( + binary="tempo", + version="3.0.0", + url=f"{_GITHUB}/grafana/tempo/releases/download/v3.0.0/tempo_3.0.0_linux_amd64.tar.gz", + sha256="f4150470a5f6bf57e94dac5392c552b488932ea9e88a6ee2a75d7b969b7cb6a4", + member="tempo", + ), + Release( + binary="loki", + version="3.7.9", + url=f"{_GITHUB}/grafana/loki/releases/download/v3.7.9/loki-linux-amd64.zip", + sha256="17a5fae28ce733b3560c27c76620920b169bd2aea9473947b733b511f92e81dd", + member="loki-linux-amd64", + ), +) + +# Each has a .yaml config and a .service unit template +SERVICES: Tuple[str, ...] = ("otelcol", "prometheus", "tempo", "loki") + +PLACEHOLDERS = frozenset({ + "name", # prefix of each unit, e.g. hispec for hispec-prometheus + "display_name", # as the units read in systemctl status + "user", # account the services run as + "bin", # directory holding the RELEASES binaries + "etc", # directory holding the rendered configs + "data", # directory each service keeps its data under + "otlp_bind", # address the Collector receives OTLP on + "query_bind", # address Prometheus, Tempo and Loki answer Grafana on + "metrics_retention", # Prometheus retention, e.g. 5y + "trace_retention", # Tempo retention, e.g. 336h +}) + +_PLACEHOLDER = re.compile(r"@([a-z_]+)@") + + +def render(template: str, values: Mapping[str, str]) -> str: + """Fill ``template``'s ``@placeholder@`` values. + + Raise ``KeyError`` naming any placeholder ``values`` lacks, so a missing + one fails the install rather than reaching a host as a literal ``@data@``. + """ + text = resources.files(__package__).joinpath("templates", template).read_text( + encoding="utf-8") + missing = sorted(set(_PLACEHOLDER.findall(text)) - set(values)) + if missing: + raise KeyError(f"{template} needs {', '.join(missing)}") + return _PLACEHOLDER.sub(lambda match: str(values[match.group(1)]), text) diff --git a/libby/telemetry_backend/templates/loki.service b/libby/telemetry_backend/templates/loki.service new file mode 100644 index 0000000..55ad41d --- /dev/null +++ b/libby/telemetry_backend/templates/loki.service @@ -0,0 +1,23 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +[Unit] +Description=@display_name@ telemetry: Loki +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=@user@ +Group=@user@ +ExecStart=@bin@/loki -config.file=@etc@/loki.yaml +Restart=on-failure +RestartSec=5 +SyslogIdentifier=@name@-loki +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=@data@/loki + +[Install] +WantedBy=multi-user.target diff --git a/libby/telemetry_backend/templates/loki.yaml b/libby/telemetry_backend/templates/loki.yaml new file mode 100644 index 0000000..6f730b5 --- /dev/null +++ b/libby/telemetry_backend/templates/loki.yaml @@ -0,0 +1,31 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +auth_enabled: false +server: + http_listen_address: @query_bind@ + http_listen_port: 3100 + grpc_listen_address: 127.0.0.1 + # 9095 is Tempo's + grpc_listen_port: 9096 +common: + instance_addr: 127.0.0.1 + path_prefix: @data@/loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: @data@/loki/chunks + rules_directory: @data@/loki/rules +schema_config: + configs: + - from: 2026-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true diff --git a/libby/telemetry_backend/templates/otelcol.service b/libby/telemetry_backend/templates/otelcol.service new file mode 100644 index 0000000..7cf7e81 --- /dev/null +++ b/libby/telemetry_backend/templates/otelcol.service @@ -0,0 +1,22 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +[Unit] +Description=@display_name@ telemetry: OpenTelemetry Collector +After=network-online.target @name@-prometheus.service @name@-tempo.service @name@-loki.service +Wants=network-online.target + +[Service] +Type=simple +User=@user@ +Group=@user@ +ExecStart=@bin@/otelcol-contrib --config=@etc@/otelcol.yaml +Restart=on-failure +RestartSec=5 +SyslogIdentifier=@name@-otelcol +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/libby/telemetry_backend/templates/otelcol.yaml b/libby/telemetry_backend/templates/otelcol.yaml new file mode 100644 index 0000000..98d5bbc --- /dev/null +++ b/libby/telemetry_backend/templates/otelcol.yaml @@ -0,0 +1,40 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. +# +# Receives OTLP from daemons and agents, and fans it out to the services below +# on this host. + +receivers: + otlp: + protocols: + grpc: + endpoint: @otlp_bind@:4317 + http: + endpoint: @otlp_bind@:4318 + +processors: + batch: {} + +exporters: + otlp_http/prometheus: + metrics_endpoint: http://127.0.0.1:9090/api/v1/otlp/v1/metrics + otlp_grpc/tempo: + endpoint: 127.0.0.1:4417 + tls: + insecure: true + otlp_http/loki: + endpoint: http://127.0.0.1:3100/otlp + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [otlp_http/prometheus] + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp_grpc/tempo] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlp_http/loki] diff --git a/libby/telemetry_backend/templates/prometheus.service b/libby/telemetry_backend/templates/prometheus.service new file mode 100644 index 0000000..7f6bce4 --- /dev/null +++ b/libby/telemetry_backend/templates/prometheus.service @@ -0,0 +1,26 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +[Unit] +Description=@display_name@ telemetry: Prometheus +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=@user@ +Group=@user@ +ExecStart=@bin@/prometheus --config.file=@etc@/prometheus.yaml \ + --storage.tsdb.path=@data@/prometheus --storage.tsdb.retention.time=@metrics_retention@ \ + --web.listen-address=@query_bind@:9090 \ + --web.enable-otlp-receiver --web.enable-remote-write-receiver +Restart=on-failure +RestartSec=5 +SyslogIdentifier=@name@-prometheus +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=@data@/prometheus + +[Install] +WantedBy=multi-user.target diff --git a/libby/telemetry_backend/templates/prometheus.yaml b/libby/telemetry_backend/templates/prometheus.yaml new file mode 100644 index 0000000..9db9330 --- /dev/null +++ b/libby/telemetry_backend/templates/prometheus.yaml @@ -0,0 +1,10 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +storage: + tsdb: + # Accept late samples, e.g. a batch resent after an outage + out_of_order_time_window: 2h +otlp: + promote_resource_attributes: + - service.namespace + - host.name diff --git a/libby/telemetry_backend/templates/tempo.service b/libby/telemetry_backend/templates/tempo.service new file mode 100644 index 0000000..b3a2942 --- /dev/null +++ b/libby/telemetry_backend/templates/tempo.service @@ -0,0 +1,23 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +[Unit] +Description=@display_name@ telemetry: Tempo +After=network-online.target @name@-prometheus.service +Wants=network-online.target + +[Service] +Type=simple +User=@user@ +Group=@user@ +ExecStart=@bin@/tempo -config.file=@etc@/tempo.yaml +Restart=on-failure +RestartSec=5 +SyslogIdentifier=@name@-tempo +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=@data@/tempo + +[Install] +WantedBy=multi-user.target diff --git a/libby/telemetry_backend/templates/tempo.yaml b/libby/telemetry_backend/templates/tempo.yaml new file mode 100644 index 0000000..571584a --- /dev/null +++ b/libby/telemetry_backend/templates/tempo.yaml @@ -0,0 +1,50 @@ +# Rendered from libby's telemetry backend; edits here are lost on the next install. + +stream_over_http_enabled: true +server: + http_listen_address: @query_bind@ + http_listen_port: 3200 + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9095 +distributor: + receivers: + otlp: + protocols: + grpc: + # 4317 is the Collector's; only it sends here + endpoint: 127.0.0.1:4417 +storage: + trace: + backend: local + local: + path: @data@/tempo/traces + wal: + path: @data@/tempo/wal +# Tempo 3 applies retention in the scheduler and compacts in the worker; both +# read block_retention, so it is set in each +backend_scheduler: + local_work_path: @data@/tempo/work + provider: + compaction: + compaction: + block_retention: @trace_retention@ +backend_worker: + compaction: + block_retention: @trace_retention@ +live_store: + wal: + path: @data@/tempo/live-store/traces + shutdown_marker_dir: @data@/tempo/live-store/shutdown-marker +block_builder: + wal: + path: @data@/tempo/block-builder/traces +metrics_generator: + storage: + path: @data@/tempo/generator/wal + remote_write: + - url: http://127.0.0.1:9090/api/v1/write + send_exemplars: true +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] diff --git a/pyproject.toml b/pyproject.toml index bd9d07a..39deb23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ exclude = ["peers*", "package*"] [tool.setuptools.package-data] "libby.cli" = ["*.yaml"] +"libby.telemetry_backend" = ["templates/*"] [project.scripts] libby = "libby.cli.libby_cli:main" diff --git a/tests/test_telemetry_backend.py b/tests/test_telemetry_backend.py new file mode 100644 index 0000000..fb619f5 --- /dev/null +++ b/tests/test_telemetry_backend.py @@ -0,0 +1,84 @@ +"""Tests for the telemetry backend's pinned releases and templates.""" +from __future__ import annotations + +import re +import unittest +from importlib import resources + +import yaml + +from libby import telemetry_backend as backend + +VALUES = { + "name": "demo", + "display_name": "Demo", + "user": "demo", + "bin": "/opt/demo/telemetry/bin", + "etc": "/etc/demo/telemetry", + "data": "/var/lib/demo/telemetry", + "otlp_bind": "0.0.0.0", + "query_bind": "127.0.0.1", + "metrics_retention": "5y", + "trace_retention": "336h", +} +PLACEHOLDER = re.compile(r"@([a-z_]+)@") + + +def _templates(): + return [f"{service}{suffix}" for service in backend.SERVICES + for suffix in (".yaml", ".service")] + + +class TemplateTests(unittest.TestCase): + """Every template renders completely from PLACEHOLDERS.""" + + def test_placeholders_lists_exactly_what_the_templates_use(self): + """Keep the documented placeholder set honest.""" + folder = resources.files(backend.__name__).joinpath("templates") + used = {name for template in _templates() + for name in PLACEHOLDER.findall(folder.joinpath(template).read_text())} + self.assertEqual(used, set(backend.PLACEHOLDERS)) + + def test_every_template_renders_without_leftovers(self): + """Leave no @placeholder@ behind for a host to read literally.""" + for template in _templates(): + with self.subTest(template): + self.assertIsNone(PLACEHOLDER.search(backend.render(template, VALUES))) + + def test_configs_are_valid_yaml(self): + """Parse each rendered config.""" + for service in backend.SERVICES: + with self.subTest(service): + self.assertIsInstance(yaml.safe_load(backend.render(f"{service}.yaml", VALUES)), + dict) + + def test_a_missing_value_is_named(self): + """Fail the render rather than write a half-filled file.""" + partial = {key: value for key, value in VALUES.items() if key != "data"} + with self.assertRaisesRegex(KeyError, "data"): + backend.render("tempo.yaml", partial) + + def test_units_run_the_pinned_binaries(self): + """Point each unit at a binary RELEASES provides.""" + binaries = {release.binary for release in backend.RELEASES} + for service in backend.SERVICES: + with self.subTest(service): + unit = backend.render(f"{service}.service", VALUES) + started = re.search(r"^ExecStart=/opt/demo/telemetry/bin/(\S+)", unit, re.M) + self.assertIn(started.group(1), binaries) + + +class ReleaseTests(unittest.TestCase): + """Pinned releases are fetched over HTTPS and checked by SHA-256.""" + + def test_each_release_is_https_with_a_sha256(self): + """Refuse anything an installer could not verify.""" + for release in backend.RELEASES: + with self.subTest(release.binary): + self.assertTrue(release.url.startswith("https://github.com/")) + self.assertRegex(release.sha256, r"^[0-9a-f]{64}$") + self.assertIn(release.version, release.url) + + +if __name__ == "__main__": + unittest.main()