From 383cb8eea52ab1833efbf0044461b476b756d837 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Fri, 30 Oct 2026 08:58:40 +0100 Subject: [PATCH 1/7] Add direct Custom DB output alongside TimescaleDB --- FORK.md | 2 + ai-tests/custom-db/smoke.py | 168 ++++++++ internal/cmd/builtin_output_gen.go | 14 +- internal/cmd/outputs.go | 1 + internal/cmd/outputs_full.go | 1 + internal/cmd/outputs_lean.go | 1 + internal/cmd/outputs_test.go | 10 +- internal/output/breakingit/CUSTOM_DB.md | 99 +++++ internal/output/breakingit/README.md | 6 + internal/output/breakingit/breakingit.go | 133 +++--- internal/output/breakingit/custom.go | 424 ++++++++++++++++++++ internal/output/breakingit/custom_binary.go | 92 +++++ internal/output/breakingit/custom_test.go | 315 +++++++++++++++ 13 files changed, 1193 insertions(+), 73 deletions(-) create mode 100644 ai-tests/custom-db/smoke.py create mode 100644 internal/output/breakingit/CUSTOM_DB.md create mode 100644 internal/output/breakingit/custom.go create mode 100644 internal/output/breakingit/custom_binary.go create mode 100644 internal/output/breakingit/custom_test.go diff --git a/FORK.md b/FORK.md index 785c2894e..eeeb2d2a0 100644 --- a/FORK.md +++ b/FORK.md @@ -22,6 +22,8 @@ code to the k6 executable. ## Fork-specific changes - Adds the `timescaledb-breakingit` output for writing load-test data through a PG-proxy. +- Adds the opt-in `custom-db-breakingit` direct output alongside TimescaleDB; see + [configuration and current limits](internal/output/breakingit/CUSTOM_DB.md). - Adds optional detailed request and response data for failed HTTP requests. - Extends browser network and error metrics, including optional error screenshots. - Adds the `lean` build tag used for smaller cloud-free release binaries. diff --git a/ai-tests/custom-db/smoke.py b/ai-tests/custom-db/smoke.py new file mode 100644 index 000000000..da7d0bb84 --- /dev/null +++ b/ai-tests/custom-db/smoke.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Isolated k6 -> Rust protocol test with the existing CSV writer as an oracle. + +Requires Docker, Python 3, a built k6 fork and a built Custom DB service image. +Creates a temporary service/container and directory; never connects to customer DBs. +""" +import argparse +import csv +import email.parser +import gzip +import http.server +import io +import json +import os +from pathlib import Path +import secrets +import subprocess +import tempfile +import threading +import time +import urllib.request +import uuid +from datetime import datetime + + +def run(args): + name = "k6-custom-writer-smoke-" + uuid.uuid4().hex[:12] + admin, producer = secrets.token_hex(32), secrets.token_hex(32) + oracle = {} + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + self.send_response(503 if self.path == "/failure" else 200) + self.end_headers() + self.wfile.write(b"fixture response") + + def do_POST(self): + if self.path != "/ingest/batch": + self.send_error(404) + return + data = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers.get("Content-Encoding") == "gzip": + data = gzip.decompress(data) + msg = email.parser.BytesParser().parsebytes( + b"MIME-Version: 1.0\r\nContent-Type: " + + self.headers["Content-Type"].encode() + b"\r\n\r\n" + data) + for part in msg.get_payload(): + table = part["X-Table"].removeprefix("public.") + rows = csv.reader(io.StringIO(part.get_payload(decode=True).decode())) + oracle.setdefault(table, []).extend(rows) + self.send_response(200) + self.end_headers() + self.wfile.write(b"OK") + + target = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=target.serve_forever, daemon=True) + thread.start() + with tempfile.TemporaryDirectory(prefix=name) as temp: + root = Path(temp) + data_dir = root / "data" + data_dir.mkdir() + env = {**os.environ, "CUSTOM_STORAGE_ADMIN_TOKEN": admin} + try: + subprocess.run(["docker", "run", "-d", "--name", name, + "--label", "breaktest.k6-writer-smoke=true", "--memory", "512m", + "-e", "CUSTOM_STORAGE_ADMIN_TOKEN", "-p", "127.0.0.1::8099", + "-v", f"{data_dir}:/data", args.image, + "v3", "service", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True) + port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1] + service = f"http://127.0.0.1:{port}" + + def post(path, payload): + request = urllib.request.Request(service + path, data=json.dumps(payload).encode(), + headers={"Authorization": "Bearer " + admin, "Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + for _ in range(100): + try: + post("/health", {}) + break + except OSError: + time.sleep(.05) + origin = int(time.time() - 946684800 - 10) * 1_000_000 + post("/runs/create", {"run_id": "k6-smoke", "source_tokens": [producer], + "origin_us": origin, "customer_id": "isolated-k6-smoke"}) + script = root / "smoke.js" + script.write_text('''import http from 'k6/http'; +import { group, check, sleep } from 'k6'; +export const options = { vus: 2, iterations: 20 }; +export default function () { + group('Checkout', function () { + http.get(__ENV.TARGET + '/ok'); + const r = http.get(__ENV.TARGET + '/failure'); + check(r, { 'expected failure': x => x.status === 200 }); + }); + sleep(0.15); +} +''') + target_url = f"http://127.0.0.1:{target.server_port}" + writer_env = {**os.environ, "CUSTOM_STORAGE_TOKEN": producer, "runId": "k6-smoke", + "nodeName": "k6-probe", "location": "Local", "PG_PROXY_TOKEN": "isolated-oracle", + "TARGET": target_url, "K6_NO_USAGE_REPORT": "true", "K6_HTTP_ERROR_DATA": "true"} + for key in ("syntheticMonitoring", "K6_BROWSER_SCREENSHOT_ON_ERROR", "CUSTOM_URL_PATTERNS", "BREAKINGIT_TRANSACTION_SOURCE"): + writer_env.pop(key, None) + result = subprocess.run([str(Path(args.k6).resolve()), "run", "--quiet", + "--out", "custom-db-breakingit=" + service, + "--out", "timescaledb-breakingit=" + target_url, str(script)], + env=writer_env, text=True, capture_output=True, timeout=90) + assert result.returncode == 0, result.stdout + result.stderr + end = int(time.time() - 946684800 + 10) * 1_000_000 + final = post("/runs/k6-smoke/finalize", {}) + assert final["sealed"] and final["run_complete"], final + expected_counts = {"requests_raw": 40, "transactions": 20, "requests_error": 40} + counts = {} + for table in ("requests_raw", "transactions", "requests_error", "virtual_users"): + actual = post("/runs/k6-smoke/query", {"table": table, "start_us": origin, "end_us": end}) + counts[table] = len(actual["rows"]) + assert counts[table] == len(oracle[table]), (table, counts[table], len(oracle[table])) + if table in expected_counts: + assert counts[table] == expected_counts[table], (table, counts) + else: + assert counts[table] > 0 + # Match all fields including timestamp, except independently-generated UUIDs. + def normalized(row, csv_row=False): + values = [] + for i, value in enumerate(row): + if table == "requests_error" and i == 15: + continue + if csv_row and i == 0: + value = round((datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() - 946684800) * 1_000_000) + if value is None or value == "": + value = None + elif isinstance(value, bool): + value = "true" if value else "false" + else: + value = str(value) + values.append(value) + return json.dumps(values, ensure_ascii=False) + left = sorted(normalized(row) for row in actual["rows"]) + right = sorted(normalized(row, True) for row in oracle[table]) + assert left == right, (table, "row mismatch", left[:1], right[:1], actual.get("columns")) + subprocess.run(["docker", "restart", name], check=True, capture_output=True) + port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1] + service = f"http://127.0.0.1:{port}" + for _ in range(100): + try: + post("/health", {}) + break + except OSError: + time.sleep(.05) + for table, count in counts.items(): + actual = post("/runs/k6-smoke/query", {"table": table, "start_us": origin, "end_us": end}) + assert len(actual["rows"]) == count + print(json.dumps({"status": "passed", "counts": counts, "parity": "all columns except generated error UUIDs", "sealed": True, "restart_reads": True}, indent=2)) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + target.shutdown() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--k6", required=True) + parser.add_argument("--image", default="breaktest-storage-lab:local") + run(parser.parse_args()) diff --git a/internal/cmd/builtin_output_gen.go b/internal/cmd/builtin_output_gen.go index 97ce2795c..b3f562fcd 100644 --- a/internal/cmd/builtin_output_gen.go +++ b/internal/cmd/builtin_output_gen.go @@ -1,4 +1,4 @@ -// Code generated by "enumer -type=builtinOutput -trimprefix builtinOutput -transform=kebab -output builtin_output_gen.go"; DO NOT EDIT. +// Code generated by "enumer -type=builtinOutput -trimprefix builtinOutput -transform=kebab -output builtin_output_gen.go enum.go"; DO NOT EDIT. package cmd @@ -7,11 +7,11 @@ import ( "strings" ) -const _builtinOutputName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingit" +const _builtinOutputName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingitcustom-db-breakingit" -var _builtinOutputIndex = [...]uint8{0, 5, 8, 15, 41, 49, 53, 58, 64, 90, 103, 110, 132} +var _builtinOutputIndex = [...]uint8{0, 5, 8, 15, 41, 49, 53, 58, 64, 90, 103, 110, 132, 152} -const _builtinOutputLowerName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingit" +const _builtinOutputLowerName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingitcustom-db-breakingit" func (i builtinOutput) String() string { if i >= builtinOutput(len(_builtinOutputIndex)-1) { @@ -36,9 +36,10 @@ func _builtinOutputNoOp() { _ = x[builtinOutputOpentelemetry-(9)] _ = x[builtinOutputSummary-(10)] _ = x[builtinOutputTimescaledbBreakingit-(11)] + _ = x[builtinOutputCustomDBBreakingit-(12)] } -var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit} +var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit, builtinOutputCustomDBBreakingit} var _builtinOutputNameToValueMap = map[string]builtinOutput{ _builtinOutputName[0:5]: builtinOutputCloud, @@ -65,6 +66,8 @@ var _builtinOutputNameToValueMap = map[string]builtinOutput{ _builtinOutputLowerName[103:110]: builtinOutputSummary, _builtinOutputName[110:132]: builtinOutputTimescaledbBreakingit, _builtinOutputLowerName[110:132]: builtinOutputTimescaledbBreakingit, + _builtinOutputName[132:152]: builtinOutputCustomDBBreakingit, + _builtinOutputLowerName[132:152]: builtinOutputCustomDBBreakingit, } var _builtinOutputNames = []string{ @@ -80,6 +83,7 @@ var _builtinOutputNames = []string{ _builtinOutputName[90:103], _builtinOutputName[103:110], _builtinOutputName[110:132], + _builtinOutputName[132:152], } // builtinOutputString retrieves an enum value from the enum constants string name. diff --git a/internal/cmd/outputs.go b/internal/cmd/outputs.go index a641fc2ac..f619f0f63 100644 --- a/internal/cmd/outputs.go +++ b/internal/cmd/outputs.go @@ -35,6 +35,7 @@ const ( builtinOutputOpentelemetry builtinOutputSummary builtinOutputTimescaledbBreakingit + builtinOutputCustomDBBreakingit ) func getPossibleIDList(constrs map[string]output.Constructor) string { diff --git a/internal/cmd/outputs_full.go b/internal/cmd/outputs_full.go index ae701b283..0703fc3db 100644 --- a/internal/cmd/outputs_full.go +++ b/internal/cmd/outputs_full.go @@ -48,6 +48,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { }, "web-dashboard": dashboard.New, builtinOutputTimescaledbBreakingit.String(): breakingit.New, + builtinOutputCustomDBBreakingit.String(): breakingit.NewCustom, builtinOutputExperimentalOpentelemetry.String(): func(params output.Params) (output.Output, error) { params.Logger.Warnf("OpenTelemetry output has been graduated as a stable output."+ "You can now use just %q instead of %q. The experimental version will be removed in future versions.", diff --git a/internal/cmd/outputs_lean.go b/internal/cmd/outputs_lean.go index c58dc328a..61b73e8bc 100644 --- a/internal/cmd/outputs_lean.go +++ b/internal/cmd/outputs_lean.go @@ -56,6 +56,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { return nil, errors.New("the web-dashboard output is not available in this lean build of k6") }, builtinOutputTimescaledbBreakingit.String(): breakingit.New, + builtinOutputCustomDBBreakingit.String(): breakingit.NewCustom, } exts := ext.Get(ext.OutputExtension) diff --git a/internal/cmd/outputs_test.go b/internal/cmd/outputs_test.go index 0501f2b64..019df536b 100644 --- a/internal/cmd/outputs_test.go +++ b/internal/cmd/outputs_test.go @@ -12,7 +12,15 @@ func TestBuiltinOutputString(t *testing.T) { "cloud", "csv", "datadog", "experimental-prometheus-rw", "influxdb", "json", "kafka", "statsd", "experimental-opentelemetry", "opentelemetry", - "summary", "timescaledb-breakingit", + "summary", "timescaledb-breakingit", "custom-db-breakingit", } assert.Equal(t, exp, builtinOutputStrings()) } + +func TestCustomDBAndTimescaleConstructors(t *testing.T) { + t.Parallel() + constructors, err := getAllOutputConstructors() + assert.NoError(t, err) + assert.Contains(t, constructors, "timescaledb-breakingit") + assert.Contains(t, constructors, "custom-db-breakingit") +} diff --git a/internal/output/breakingit/CUSTOM_DB.md b/internal/output/breakingit/CUSTOM_DB.md new file mode 100644 index 000000000..b4fdc3a96 --- /dev/null +++ b/internal/output/breakingit/CUSTOM_DB.md @@ -0,0 +1,99 @@ +# Direct Custom DB output + +The fork has two independent built-in outputs, in both normal and `lean` builds: + +- `timescaledb-breakingit`: the existing CSV/PG-proxy writer. +- `custom-db-breakingit`: direct zstd-compressed PostgreSQL binary COPY batches + to the BreakTest Custom DB service. It does not contact PostgreSQL or PG-proxy. + +The implementation lives in this fork, alongside the active built-in TimescaleDB +writer. It does not use the older xk6 extension in the BreakTest repository. + +## Invocation and application contract + +The application must first provision a Custom DB run and one producer token per +load generator. Pass the run ID and that generator's token when invoking k6: + +```sh +export runId='' +export CUSTOM_STORAGE_TOKEN='' +export location='Amsterdam' +export nodeName='generator-1' +k6 run --out custom-db-breakingit=https://storage.example.test script.js +``` + +The URL is the service base URL (an optional routing prefix is retained). The +writer appends `/runs//hello`, `/ingest/batch` and `/finish` to it. TLS +certificates are verified, connections reused, and redirects disabled so tokens +cannot be redirected to another origin. Never pass the service admin token. + +`BREAKINGIT_TRANSACTION_SOURCE` and `CUSTOM_URL_PATTERNS` retain their existing +mapping behavior. Native groups, legacy `pages`, requests, WebSocket/custom trend +request rows, failed checks, browser errors and VUs use the existing mapping code. +Unknown CSV fields remain NULL in the binary encoding, and negative/unavailable +connect times become NULL. Error UUIDs are generated independently per output. + +The application must select the output from the frozen test backend, provide the +producer credentials, and finalize/publish only after all producers finish. +This fork change does **not** yet update the BreakTest launch configuration, +remove its k6 preview restriction, rebuild its deployed generator image, or change +any customer's storage selection. Those changes must accompany deployment. +Generator machine statistics remain the responsibility of the existing agent. + +## Delivery and limits + +- Flush every 200 ms or at a 2 MiB encoded-row budget. Four queued batches bound + outgoing buffering; a full queue applies backpressure rather than dropping rows. + The building batch, one waiting enqueue, sender/compression buffers, and k6's own + sample containers consume additional memory; 8 MiB is not a total RSS limit. +- One ordered sender per generator. Multiple generators have independent sources. + This first version does not claim the throughput of the parallel Java writer. +- Incomplete HTTP joins are limited to 4,096 groups / an estimated 2 MiB payload + budget. Oversized rows or exhausted join budgets stop the run explicitly. +- HTTP/browser joins include timestamp and tag-set identity. All required request + metrics must arrive before a row is emitted; delayed browser failure payloads + are not treated as success. Ambiguous duplicate metrics or incomplete groups at + shutdown fail the output instead of publishing an apparently complete run. +- Retry transient failures and throttling with the exact same compressed bytes, + boundary, and sequence, for at most 30 seconds per operation. Validate ACKs. + Rejection, invalid ACK, or exhausted retries stops k6 and forbids `/finish`. +- Drain accepted sample containers and all queued requests before `/finish`. + The producer does not seal the run itself. A final barrier proves delivery of + this producer's data; it is not a durable-on-disk/fsync acknowledgement. +- A fresh producer source is required. A server incarnation change is fatal in + this first implementation. Automatic relaxed-durability reconnect requires a + source clock contract that k6 does not yet supply. + +## Watermarks and unsupported data + +This writer **does not send live watermarks**. k6 output callbacks can be delayed +and out of event-time order, particularly for browser failure bodies. Neither +wall-clock time nor the latest observed sample proves that older samples cannot +arrive. Live reads therefore use the service's pending/raw and grace-period paths; +the final producer barrier permits complete aggregation after drain. Proven live +source barriers are a separate k6 core integration, not a timer setting. + +Synthetic monitoring and browser screenshots are explicitly unsupported and +rejected. Ordinary browser request timings, failure response data, native groups +and browser error messages are supported. An unexpected screenshot-bearing error +also fails the output, rather than storing a dangling screenshot reference. + +## Validation + +```sh +go test -race ./internal/output/breakingit +go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' +go test ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' +go build -tags lean -o /tmp/k6-custom-db-writer . +python3 ai-tests/custom-db/smoke.py --k6 /tmp/k6-custom-db-writer \ + --image breaktest-storage-lab:local +``` + +The smoke test creates its own disposable Docker service and temporary directory, +then runs both writers on the **same** metric stream. The TimescaleDB writer goes +to an isolated CSV-capture server; it does not write an actual TimescaleDB database. +The test compares every stored column against that CSV oracle, except independently +generated error UUIDs. It checks request, transaction, error and VU counts, complete +sealing and reads after storage-service restart. It never connects to existing +customer databases. This establishes protocol/mapping compatibility, not a +TimescaleDB performance benchmark or browser end-to-end acceptance. diff --git a/internal/output/breakingit/README.md b/internal/output/breakingit/README.md index c30219386..d85bcef9b 100644 --- a/internal/output/breakingit/README.md +++ b/internal/output/breakingit/README.md @@ -106,3 +106,9 @@ an intentional nested transaction from two measurements of the same step. For a full migration, remove `pages.add()` and use `await group(...)` for timing and context, while keeping any business checks and throwing on failures. + +## Custom DB alternative + +The fork also includes `custom-db-breakingit` for direct binary delivery to the +BreakTest Custom DB service. See [configuration, delivery semantics and current +limits](CUSTOM_DB.md). The existing TimescaleDB output remains available. diff --git a/internal/output/breakingit/breakingit.go b/internal/output/breakingit/breakingit.go index 9dcc29d98..3546a67b8 100644 --- a/internal/output/breakingit/breakingit.go +++ b/internal/output/breakingit/breakingit.go @@ -6,8 +6,8 @@ import ( "crypto/rand" "crypto/sha256" "crypto/tls" - "encoding/hex" "encoding/csv" + "encoding/hex" "encoding/json" "fmt" "io" @@ -62,6 +62,9 @@ const ( // Logger writes k6 metric samples to pg-proxy using COPY via multipart batch type Logger struct { + // rowSink is used by the direct Custom DB output; nil preserves CSV/PG-proxy. + rowSink func(string, []string) error + out io.Writer proxyURL string database string @@ -69,9 +72,9 @@ type Logger struct { httpClient *http.Client // batch buffers - mu sync.Mutex - bufHTTP *bytes.Buffer - bufVUs *bytes.Buffer + mu sync.Mutex + bufHTTP *bytes.Buffer + bufVUs *bytes.Buffer bufTrans *bytes.Buffer bufErrors *bytes.Buffer batchTimer *time.Timer @@ -625,9 +628,7 @@ func (l *Logger) AddMetricSamples(samples []metrics.SampleContainer) { } } l.mu.Lock() - wt := csv.NewWriter(l.bufTrans) - _ = wt.Write(row) - wt.Flush() + l.writeRow(l.bufTrans, row) l.mu.Unlock() continue } @@ -674,9 +675,7 @@ func (l *Logger) AddMetricSamples(samples []metrics.SampleContainer) { } } l.mu.Lock() - wt := csv.NewWriter(l.bufTrans) - _ = wt.Write(row) - wt.Flush() + l.writeRow(l.bufTrans, row) l.mu.Unlock() continue } @@ -692,9 +691,7 @@ func (l *Logger) AddMetricSamples(samples []metrics.SampleContainer) { "", // finished_threads (unknown) } l.mu.Lock() - wv := csv.NewWriter(l.bufVUs) - _ = wv.Write(row) - wv.Flush() + l.writeRow(l.bufVUs, row) l.mu.Unlock() continue } @@ -733,25 +730,23 @@ func (l *Logger) AddMetricSamples(samples []metrics.SampleContainer) { fmt.Sprintf("%v", l.envTags["scenario_name"]), // scenario_name fmt.Sprintf("%v", l.envTags["location"]), // location fmt.Sprintf("%v", l.envTags["node_name"]), // node_name - m["scenario"], // thread_group_name - transactionName, // transaction_name - samplerName, // sampler_name - responseCode, // response_code - responseTime, // response_time - "", // connection_time - url, // url - "", // assertions - checkMsg, // response_message - requestData, // request_headers (used for request data) - "", // response_headers - responseData, // response_data - id, // id (UUIDv7 generated from measurement timestamp) - fmt.Sprintf("%v", l.envTags["run_id"]), // run_id last + m["scenario"], // thread_group_name + transactionName, // transaction_name + samplerName, // sampler_name + responseCode, // response_code + responseTime, // response_time + "", // connection_time + url, // url + "", // assertions + checkMsg, // response_message + requestData, // request_headers (used for request data) + "", // response_headers + responseData, // response_data + id, // id (UUIDv7 generated from measurement timestamp) + fmt.Sprintf("%v", l.envTags["run_id"]), // run_id last } l.mu.Lock() - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeRow(l.bufErrors, row) l.mu.Unlock() } else { row := []string{ @@ -773,9 +768,7 @@ func (l *Logger) AddMetricSamples(samples []metrics.SampleContainer) { id, // id (UUIDv7 generated from measurement timestamp) } l.mu.Lock() - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeRow(l.bufErrors, row) l.mu.Unlock() } continue @@ -911,9 +904,7 @@ func (l *Logger) processHttpGroup(g *MetricGroup) { } } l.mu.Lock() - w := csv.NewWriter(l.bufHTTP) - _ = w.Write(row) - w.Flush() + l.writeRow(l.bufHTTP, row) // If not successful, also emit into requests_error(_sm) if success == "false" { // Build response_message from tags @@ -981,9 +972,7 @@ func (l *Logger) processHttpGroup(g *MetricGroup) { id, // id (UUIDv7) fmt.Sprintf("%v", l.envTags["run_id"]), } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(erow) - we.Flush() + l.writeRow(l.bufErrors, erow) } else { erow := []string{ time.Unix(0, g.Timestamp).UTC().Format(time.RFC3339Nano), @@ -1003,9 +992,7 @@ func (l *Logger) processHttpGroup(g *MetricGroup) { g.ErrorResBody, // response_data id, // id (UUIDv7) } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(erow) - we.Flush() + l.writeRow(l.bufErrors, erow) } } l.mu.Unlock() @@ -1106,9 +1093,7 @@ func (l *Logger) processWebSocketMetric(s metrics.Sample, ts int64) { } } l.mu.Lock() - w := csv.NewWriter(l.bufHTTP) - _ = w.Write(row) - w.Flush() + l.writeRow(l.bufHTTP, row) l.mu.Unlock() } @@ -1190,9 +1175,7 @@ func (l *Logger) processCustomTrendMetric(s metrics.Sample, ts int64) { } } l.mu.Lock() - w := csv.NewWriter(l.bufHTTP) - _ = w.Write(row) - w.Flush() + l.writeRow(l.bufHTTP, row) l.mu.Unlock() } @@ -1241,19 +1224,19 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { fmt.Sprintf("%v", l.envTags["scenario_name"]), // scenario_name fmt.Sprintf("%v", l.envTags["location"]), // location fmt.Sprintf("%v", l.envTags["node_name"]), // node_name - m["scenario"], // thread_group_name - transactionName, // transaction_name - samplerName, // sampler_name - "", // response_code - "", // response_time - "", // connection_time - "", // url - "", // assertions - errMsg, // response_message - "", // request_headers - "", // response_headers - screenshotHash, // response_data (screenshot hash reference) - id, // id (UUIDv7) + m["scenario"], // thread_group_name + transactionName, // transaction_name + samplerName, // sampler_name + "", // response_code + "", // response_time + "", // connection_time + "", // url + "", // assertions + errMsg, // response_message + "", // request_headers + "", // response_headers + screenshotHash, // response_data (screenshot hash reference) + id, // id (UUIDv7) fmt.Sprintf("%v", l.envTags["run_id"]), // run_id last } l.mu.Lock() @@ -1263,9 +1246,7 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { }) fmt.Fprintf(l.out, "[screenshot] queued for upsert, pendingScreenshots=%d\n", len(l.pendingScreenshots)) } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeRow(l.bufErrors, row) l.mu.Unlock() } else { row := []string{ @@ -1292,9 +1273,7 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { screenshotHash, screenshot, "image/png", len(screenshot), }) } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeRow(l.bufErrors, row) l.mu.Unlock() } } @@ -1478,3 +1457,23 @@ func normalizeRegex(p string) string { ) return replacer.Replace(p) } + +// writeRow preserves one mapping for both storage backends. Call with mu held. +func (l *Logger) writeRow(buf *bytes.Buffer, row []string) { + if l.rowSink != nil { + table := "requests_raw" + switch buf { + case l.bufTrans: + table = "transactions" + case l.bufVUs: + table = "virtual_users" + case l.bufErrors: + table = "requests_error" + } + _ = l.rowSink(table, row) // Sink records failures and stops the run. + return + } + w := csv.NewWriter(buf) + _ = w.Write(row) + w.Flush() +} diff --git a/internal/output/breakingit/custom.go b/internal/output/breakingit/custom.go new file mode 100644 index 000000000..7c57bc8bf --- /dev/null +++ b/internal/output/breakingit/custom.go @@ -0,0 +1,424 @@ +package breakingit + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "github.com/klauspost/compress/zstd" + "go.k6.io/k6/v2/metrics" + "go.k6.io/k6/v2/output" +) + +// CustomOutput reuses BreakTest's metric mappings, with bounded direct binary delivery. +// Only the sender owns incarnation/sequence. An ACK is acceptance, not an fsync guarantee. +type customGroupKey struct { + time int64 + tags *metrics.TagSet +} + +type CustomOutput struct { + groups map[customGroupKey]*MetricGroup + + mapper *Logger + client *http.Client + base, token, runID string + incarnation string + sequence uint64 + retryTimeout time.Duration + encoder *zstd.Encoder + queue chan customBatch + done, tickStop, tickDone, failed chan struct{} + batchMu sync.Mutex + parts map[string]*bytes.Buffer + batchBytes int + addMu sync.Mutex // Serializes sample mapping and pending-group completion. + failureMu sync.Mutex + failure error + stopCallback func(error) + started bool + stopOnce sync.Once + stopErr error +} +type customBatch struct { + body []byte + contentType string +} + +// NewCustom is registered separately from timescaledb-breakingit in full and lean builds. +func NewCustom(params output.Params) (output.Output, error) { + env := params.Environment + u, err := url.Parse(params.ConfigArgument) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("Custom DB requires an HTTP(S) base URL without credentials, query or fragment") + } + runID, token := env["runId"], env["CUSTOM_STORAGE_TOKEN"] + if !regexp.MustCompile(`^[A-Za-z0-9_-]+$`).MatchString(runID) || token == "" { + return nil, errors.New("Custom DB requires runId and a run-scoped CUSTOM_STORAGE_TOKEN") + } + if strings.EqualFold(env["syntheticMonitoring"], "true") { + return nil, errors.New("Custom DB k6 output does not support synthetic monitoring") + } + if strings.EqualFold(env["K6_BROWSER_SCREENSHOT_ON_ERROR"], "true") { + return nil, errors.New("Custom DB k6 output does not yet support browser screenshot storage") + } + source, err := parseTransactionSource(env["BREAKINGIT_TRANSACTION_SOURCE"]) + if err != nil { + return nil, err + } + patterns := map[string]string{} + for _, pair := range strings.Split(env["CUSTOM_URL_PATTERNS"], ",") { + if pair == "" { + continue + } + parts := strings.Split(pair, "==") + if len(parts) != 2 { + return nil, errors.New("invalid CUSTOM_URL_PATTERNS entry") + } + patterns[parts[0]] = parts[1] + } + tags := map[string]string{"run_id": runID, "location": env["location"], "node_name": env["nodeName"], "scenario_name": env["scenarioName"]} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxConnsPerHost = 1 + c := &CustomOutput{ + base: strings.TrimRight(u.String(), "/") + "/runs/" + runID, token: token, runID: runID, + client: &http.Client{Transport: transport, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, + retryTimeout: 30 * time.Second, queue: make(chan customBatch, 4), done: make(chan struct{}), + tickStop: make(chan struct{}), tickDone: make(chan struct{}), failed: make(chan struct{}), parts: map[string]*bytes.Buffer{}, + } + c.groups = map[customGroupKey]*MetricGroup{} + c.mapper = &Logger{out: params.StdOut, envTags: tags, customPatterns: patterns, transactionSource: source, + bufHTTP: &bytes.Buffer{}, bufVUs: &bytes.Buffer{}, bufTrans: &bytes.Buffer{}, bufErrors: &bytes.Buffer{}, + pendingHttpGroups: map[int64]*MetricGroup{}, rowSink: c.addRow} + return c, nil +} + +func (c *CustomOutput) Description() string { return "custom-db-breakingit" } +func (c *CustomOutput) SetTestRunStopCallback(callback func(error)) { + c.failureMu.Lock() + c.stopCallback = callback + c.failureMu.Unlock() +} +func (c *CustomOutput) err() error { c.failureMu.Lock(); defer c.failureMu.Unlock(); return c.failure } +func (c *CustomOutput) fail(err error) { + if err == nil { + return + } + c.failureMu.Lock() + if c.failure != nil { + c.failureMu.Unlock() + return + } + c.failure = fmt.Errorf("CUSTOM_STORAGE_FAILED: final publication forbidden: %w", err) + failure, callback := c.failure, c.stopCallback + close(c.failed) + c.failureMu.Unlock() + // Never invoke the engine callback under a mapping/buffer lock. + if callback != nil { + go callback(failure) + } +} +func (c *CustomOutput) Start() error { + // An empty HELLO is sufficient for a fresh source. Do not claim the Java + // monotonic sample-clock contract: k6 output callbacks cannot prove it. + var hello struct { + Incarnation string `json:"incarnation"` + Accepted *uint64 `json:"accepted_sequence"` + Sealed bool `json:"sealed"` + } + if err := c.request("/hello", []byte(`{}`), "application/json", 0, &hello); err != nil { + return err + } + if hello.Sealed || hello.Incarnation == "" || hello.Accepted == nil || *hello.Accepted != 0 { + return errors.New("Custom DB requires a fresh unfinished producer source") + } + c.incarnation = hello.Incarnation + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1)) + if err != nil { + return err + } + c.encoder = encoder + c.started = true + go c.sender() + go func() { + defer close(c.tickDone) + tick := time.NewTicker(200 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-c.tickStop: + return + case <-c.failed: + return + case <-tick.C: + c.addMu.Lock() + c.completeGroups(false) + c.flush() + c.addMu.Unlock() + } + } + }() + return nil +} + +func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { + c.addMu.Lock() + defer c.addMu.Unlock() + if c.err() != nil { + return + } + for _, sc := range samples { + for _, sample := range sc.GetSamples() { + if sample.Metric.Name == "browser_errors" && sample.Tags.Map()["screenshot"] != "" { + c.fail(errors.New("browser screenshots require a Custom DB payload integration")) + return + } + } + for _, sample := range sc.GetSamples() { + key := httpMetricCanonicalKey(sample.Metric.Name) + if key == "" { + c.mapper.AddMetricSamples([]metrics.SampleContainer{sample}) + continue + } + id := customGroupKey{sample.Time.UnixNano(), sample.Tags} + g := c.groups[id] + if g == nil { + g = &MetricGroup{Timestamp: id.time, Tags: map[string]string{}, Values: map[string]float64{}} + copyTagsForHTTP(g.Tags, sample.Tags.Map(), c.mapper.envTags, c.mapper.customPatterns) + c.groups[id] = g + } + if _, duplicate := g.Values[key]; duplicate { + c.fail(errors.New("ambiguous HTTP metric identity: duplicate metric before request completion")) + return + } + g.Values[key] = sample.Value + if key == "http_req_failed" { + g.ErrorReqHeaders = sample.HTTPErrorReqHeaders + g.ErrorReqBody = sample.HTTPErrorReqBody + g.ErrorResHeaders = sample.HTTPErrorResHeaders + g.ErrorResBody = sample.HTTPErrorResBody + } + } + c.completeGroups(false) + + if c.err() != nil { + return + } + // Bound incomplete metric joins as well as outgoing payloads. Complete HTTP + // containers usually leave this map empty; browser fragments may remain. + n := 0 + for _, g := range c.groups { + n += 256 + len(g.ErrorReqHeaders) + len(g.ErrorReqBody) + len(g.ErrorResHeaders) + len(g.ErrorResBody) + for k, v := range g.Tags { + n += len(k) + len(v) + 32 + } + n += len(g.Values) * 64 + } + if len(c.groups) > 4096 || n > customBatchBytes { + c.fail(errors.New("incomplete metric join budget exceeded")) + return + } + } +} + +// Complete only after every request metric has arrived. Browser failure payloads +// may arrive much later than duration metrics; a timer is not proof of success. +func (c *CustomOutput) completeGroups(final bool) { + for key, g := range c.groups { + complete := true + for _, name := range []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving", "http_req_failed"} { + if _, ok := g.Values[name]; !ok { + complete = false + break + } + } + if complete { + delete(c.groups, key) + c.mapper.processHttpGroup(g) + } else if final { + c.fail(errors.New("incomplete HTTP metric group at final drain")) + return + } + } +} +func (c *CustomOutput) addRow(table string, row []string) error { + if err := c.err(); err != nil { + return err + } + data, err := encodeCustomRow(table, row) + if err != nil { + c.fail(err) + return err + } + c.batchMu.Lock() + defer c.batchMu.Unlock() + if c.batchBytes+len(data) > customBatchBytes { + if err = c.enqueueLocked(); err != nil { + return err + } + } + if c.parts[table] == nil { + c.parts[table] = &bytes.Buffer{} + } + _, _ = c.parts[table].Write(data) + c.batchBytes += len(data) + return nil +} +func (c *CustomOutput) flush() { + c.batchMu.Lock() + defer c.batchMu.Unlock() + if err := c.enqueueLocked(); err != nil { + c.fail(err) + } +} +func (c *CustomOutput) enqueueLocked() error { + if c.batchBytes == 0 { + return nil + } + var body bytes.Buffer + writer := multipart.NewWriter(&body) + for _, table := range []string{"requests_raw", "transactions", "requests_error", "virtual_users"} { + data := c.parts[table] + if data == nil { + continue + } + header := textproto.MIMEHeader{} + header.Set("X-Table", "public."+table) + header.Set("X-Format", "binary") + part, err := writer.CreatePart(header) + if err != nil { + return err + } + _, _ = part.Write([]byte("PGCOPY\n\xff\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00")) + _, _ = part.Write(data.Bytes()) + _, _ = part.Write([]byte{255, 255}) + } + if err := writer.Close(); err != nil { + return err + } + batch := customBatch{body.Bytes(), "multipart/mixed; boundary=" + writer.Boundary()} + select { + case <-c.failed: + return c.err() + case c.queue <- batch: + c.parts = map[string]*bytes.Buffer{} + c.batchBytes = 0 + return nil + } +} +func (c *CustomOutput) sender() { + defer close(c.done) + for batch := range c.queue { + if c.err() != nil { + continue + } + compressed := c.encoder.EncodeAll(batch.body, nil) + var ack struct { + Accepted *uint64 `json:"accepted"` + } + next := c.sequence + 1 + if err := c.request("/ingest/batch", compressed, batch.contentType, next, &ack); err != nil { + c.fail(err) + continue + } + if ack.Accepted == nil || *ack.Accepted != next { + c.fail(errors.New("invalid Custom DB batch acknowledgement")) + continue + } + c.sequence = next + } +} + +func (c *CustomOutput) Stop() error { + c.stopOnce.Do(func() { + if !c.started { + return + } + close(c.tickStop) + <-c.tickDone + c.addMu.Lock() + if c.err() == nil { + c.completeGroups(true) + c.flush() + } + c.addMu.Unlock() + close(c.queue) + <-c.done + if c.err() == nil { + var ack struct { + Accepted *uint64 `json:"accepted_sequence"` + Finished bool `json:"finished"` + } + err := c.request("/finish", nil, "application/octet-stream", c.sequence, &ack) + if err == nil && (ack.Accepted == nil || *ack.Accepted != c.sequence || !ack.Finished) { + err = errors.New("invalid Custom DB final acknowledgement") + } + c.fail(err) + } + c.encoder.Close() + c.client.CloseIdleConnections() + c.stopErr = c.err() + }) + return c.stopErr +} + +// Retries keep the same sequence, multipart boundary and compressed bytes. +// A changed incarnation is deliberately fatal until k6 has a proven recovery clock. +func (c *CustomOutput) request(path string, body []byte, contentType string, seq uint64, result any) error { + ctx, cancel := context.WithTimeout(context.Background(), c.retryTimeout) + defer cancel() + for { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Content-Type", contentType) + req.Header.Set("X-Run-Id", c.runID) + if path != "/hello" { + req.Header.Set("X-Incarnation", c.incarnation) + req.Header.Set("X-Sequence", fmt.Sprint(seq)) + } + if path == "/ingest/batch" { + req.Header.Set("Content-Encoding", "zstd") + } + response, err := c.client.Do(req) + if err == nil { + data, readErr := io.ReadAll(io.LimitReader(response.Body, 64*1024+1)) + _ = response.Body.Close() + if len(data) > 64*1024 { + return errors.New("oversized Custom DB acknowledgement") + } + if response.StatusCode == http.StatusOK && readErr == nil { + if err = json.Unmarshal(data, result); err != nil { + return errors.New("invalid Custom DB acknowledgement JSON") + } + return nil + } + if readErr == nil && response.StatusCode != 429 && response.StatusCode < 500 { + return fmt.Errorf("Custom DB %s rejected request (HTTP %d)", path, response.StatusCode) + } + } + timer := time.NewTimer(100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("Custom DB %s retry deadline: %w", path, ctx.Err()) + case <-c.failed: + timer.Stop() + return c.err() + case <-timer.C: + } + } +} diff --git a/internal/output/breakingit/custom_binary.go b/internal/output/breakingit/custom_binary.go new file mode 100644 index 000000000..2f5924e8f --- /dev/null +++ b/internal/output/breakingit/custom_binary.go @@ -0,0 +1,92 @@ +package breakingit + +import ( + "bytes" + "encoding/binary" + "fmt" + "strconv" + "time" + + "github.com/gofrs/uuid/v5" +) + +const pgEpochSeconds int64 = 946684800 +const customBatchBytes = 2 * 1024 * 1024 + +// PostgreSQL binary COPY types in the existing non-SM BreakTest table order. +// t = timestamp, s = text, i = int32, b = boolean, u = UUID. +var customSchemas = map[string]string{ + "requests_raw": "tssssbiisiii", + "transactions": "tsssbiii", + "requests_error": "tssssssiissssssu", + "virtual_users": "tsssiii", +} + +func encodeCustomRow(table string, row []string) ([]byte, error) { + schema := customSchemas[table] + if schema == "" || len(row) != len(schema) { + return nil, fmt.Errorf("invalid %s row width: %d", table, len(row)) + } + // Reject before allocating a second copy of a potentially enormous error response. + size := 2 + 4*len(row) + for _, s := range row { + size += len(s) + } + if size > customBatchBytes-1024 { + return nil, fmt.Errorf("%s row exceeds Custom DB batch budget", table) + } + b := bytes.NewBuffer(make([]byte, 0, size)) + _ = binary.Write(b, binary.BigEndian, int16(len(row))) + for i, kind := range []byte(schema) { + value := row[i] + // Match the existing CSV writer's empty-field NULL semantics. + if value == "" { + _ = binary.Write(b, binary.BigEndian, int32(-1)) + continue + } + var data []byte + switch kind { + case 's': + data = []byte(value) + case 't': + t, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, fmt.Errorf("%s timestamp: %w", table, err) + } + n := (t.Unix()-pgEpochSeconds)*1_000_000 + int64(t.Nanosecond()/1000) + if n < 0 { + return nil, fmt.Errorf("timestamp before Custom DB epoch") + } + data = binary.BigEndian.AppendUint64(nil, uint64(n)) + case 'i': + n, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return nil, fmt.Errorf("%s integer column %d: %w", table, i, err) + } + // Unavailable connect times must never become negative averages. + if n < 0 && ((table == "requests_raw" && i == 9) || (table == "requests_error" && i == 8)) { + _ = binary.Write(b, binary.BigEndian, int32(-1)) + continue + } + data = binary.BigEndian.AppendUint32(nil, uint32(n)) + case 'b': + v, err := strconv.ParseBool(value) + if err != nil { + return nil, err + } + data = []byte{0} + if v { + data[0] = 1 + } + case 'u': + id, err := uuid.FromString(value) + if err != nil { + return nil, err + } + data = id[:] + } + _ = binary.Write(b, binary.BigEndian, int32(len(data))) + _, _ = b.Write(data) + } + return b.Bytes(), nil +} diff --git a/internal/output/breakingit/custom_test.go b/internal/output/breakingit/custom_test.go new file mode 100644 index 000000000..5181fbe73 --- /dev/null +++ b/internal/output/breakingit/custom_test.go @@ -0,0 +1,315 @@ +package breakingit + +import ( + "bytes" + "encoding/binary" + "encoding/csv" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/metrics" + "go.k6.io/k6/v2/output" +) + +func newCustomTest(t *testing.T, endpoint string) *CustomOutput { + t.Helper() + out, err := NewCustom(output.Params{ConfigArgument: endpoint, StdOut: io.Discard, Environment: map[string]string{ + "runId": "run-123", "CUSTOM_STORAGE_TOKEN": "producer-token", "location": "Amsterdam", "nodeName": "node-1", + }}) + require.NoError(t, err) + return out.(*CustomOutput) +} +func decodeCustomBatch(t *testing.T, data []byte, content string) map[string][][][]byte { + t.Helper() + decoder, err := zstd.NewReader(nil) + require.NoError(t, err) + defer decoder.Close() + plain, err := decoder.DecodeAll(data, nil) + require.NoError(t, err) + _, params, err := mime.ParseMediaType(content) + require.NoError(t, err) + reader := multipart.NewReader(bytes.NewReader(plain), params["boundary"]) + result := map[string][][][]byte{} + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err) + require.Equal(t, "binary", part.Header.Get("X-Format")) + data, err := io.ReadAll(part) + require.NoError(t, err) + require.Equal(t, []byte("PGCOPY\n\xff\r\n\x00"), data[:11]) + require.Equal(t, make([]byte, 8), data[11:19]) + r := bytes.NewReader(data[19:]) + table := strings.TrimPrefix(part.Header.Get("X-Table"), "public.") + for { + var n int16 + require.NoError(t, binary.Read(r, binary.BigEndian, &n)) + if n == -1 { + break + } + row := make([][]byte, n) + for i := range row { + var length int32 + require.NoError(t, binary.Read(r, binary.BigEndian, &length)) + if length == -1 { + continue + } + require.GreaterOrEqual(t, length, int32(0)) + row[i] = make([]byte, length) + _, err = io.ReadFull(r, row[i]) + require.NoError(t, err) + } + result[table] = append(result[table], row) + } + require.Zero(t, r.Len()) + } + return result +} +func TestCustomBinaryMatchesCSVMapping(t *testing.T) { + t.Parallel() + rows := []struct { + table string + row []string + }{ + {"requests_raw", []string{"2026-10-30T12:00:00.123456Z", "run-123", "NL", "Checkout", "/pay", "false", "", "123", "503", "-1", "25", "100"}}, + {"transactions", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "120"}}, + {"requests_error", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "Checkout", "/pay", "503", "100", "", "https://example.test/", "", "bad\nreply", "POST,\"a\"", "Header: value", "雪", "019a3524-3890-7000-8000-000000000001"}}, + {"virtual_users", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "10", "", ""}}, + } + c := newCustomTest(t, "http://localhost") + for _, r := range rows { + require.NoError(t, c.addRow(r.table, r.row)) + } + c.flush() + batch := <-c.queue + encoder, err := zstd.NewWriter(nil) + require.NoError(t, err) + defer encoder.Close() + decoded := decodeCustomBatch(t, encoder.EncodeAll(batch.body, nil), batch.contentType) + require.Len(t, decoded, 4) + require.Nil(t, decoded["requests_raw"][0][9]) + require.Nil(t, decoded["requests_raw"][0][6]) + require.Equal(t, uint32(123), binary.BigEndian.Uint32(decoded["requests_raw"][0][7])) + require.Equal(t, []byte("雪"), decoded["requests_error"][0][14]) + require.Len(t, decoded["requests_error"][0][15], 16) + require.Equal(t, []byte{1}, decoded["transactions"][0][4]) + require.Equal(t, int64(846676800123456), int64(binary.BigEndian.Uint64(decoded["requests_raw"][0][0]))) + // Transaction mapping itself is shared and byte-for-byte equivalent before encoding. + legacy := newTransactionTestLogger(false) + sample := transactionTestSample("group_duration", map[string]string{"group": "::Checkout"}, true) + legacy.AddMetricSamples([]metrics.SampleContainer{sample}) + expected, err := csv.NewReader(legacy.bufTrans).Read() + require.NoError(t, err) + var actual []string + c.mapper.rowSink = func(table string, row []string) error { + require.Equal(t, "transactions", table) + actual = row + return nil + } + c.AddMetricSamples([]metrics.SampleContainer{sample}) + require.Equal(t, expected, actual) +} + +func TestCustomRetriesExactBytesAndFinalDrain(t *testing.T) { + t.Parallel() + var first []byte + var content string + var calls, finishes int + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + require.Equal(t, "Bearer producer-token", r.Header.Get("Authorization")) + switch r.URL.Path { + case "/runs/run-123/hello": + fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0,"sealed":false}`) + case "/runs/run-123/ingest/batch": + require.Equal(t, "one", r.Header.Get("X-Incarnation")) + require.Equal(t, "1", r.Header.Get("X-Sequence")) + require.Equal(t, "zstd", r.Header.Get("Content-Encoding")) + data, err := io.ReadAll(r.Body) + require.NoError(t, err) + calls++ + if calls == 1 { + first = data + content = r.Header.Get("Content-Type") + w.WriteHeader(503) + return + } + require.Equal(t, first, data) + require.Equal(t, content, r.Header.Get("Content-Type")) + fmt.Fprint(w, `{"accepted":1,"duplicate":true}`) + case "/runs/run-123/finish": + require.Equal(t, 2, calls) + require.Equal(t, "1", r.Header.Get("X-Sequence")) + finishes++ + fmt.Fprint(w, `{"accepted_sequence":1,"finished":true}`) + default: + t.Errorf("unexpected route %s", r.URL.Path) + w.WriteHeader(404) + } + })) + defer server.Close() + c := newCustomTest(t, server.URL) + require.NoError(t, c.Start()) + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)}) + require.NoError(t, c.Stop()) + require.NoError(t, c.Stop()) + mu.Lock() + defer mu.Unlock() + require.Equal(t, 1, finishes) + decoded := decodeCustomBatch(t, first, content) + require.Len(t, decoded["transactions"], 1) +} + +func TestCustomFailureForbidsFinish(t *testing.T) { + for _, status := range []int{400, 401, 403, 409, 413, 429, 503, 200} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + var finished bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/hello") { + fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0}`) + return + } + if strings.HasSuffix(r.URL.Path, "/finish") { + finished = true + } + w.WriteHeader(status) + fmt.Fprint(w, `{"accepted":999}`) + })) + defer server.Close() + c := newCustomTest(t, server.URL) + c.retryTimeout = 50 * time.Millisecond + stopped := make(chan error, 1) + c.SetTestRunStopCallback(func(err error) { stopped <- err }) + require.NoError(t, c.Start()) + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("pages", nil, false)}) + require.ErrorContains(t, c.Stop(), "final publication forbidden") + select { + case err := <-stopped: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("run not stopped") + } + require.False(t, finished) + }) + } +} + +func TestCustomConfigurationAndBounds(t *testing.T) { + t.Parallel() + for _, entry := range []struct{ k, v string }{{"syntheticMonitoring", "true"}, {"K6_BROWSER_SCREENSHOT_ON_ERROR", "true"}, {"runId", "../wrong"}, {"CUSTOM_STORAGE_TOKEN", ""}, {"BREAKINGIT_TRANSACTION_SOURCE", "wrong"}} { + env := map[string]string{"runId": "test", "CUSTOM_STORAGE_TOKEN": "token"} + env[entry.k] = entry.v + _, err := NewCustom(output.Params{ConfigArgument: "http://localhost", Environment: env}) + require.Error(t, err) + } + _, err := NewCustom(output.Params{ConfigArgument: "http://user:password@localhost"}) + require.Error(t, err) + c := newCustomTest(t, "http://localhost") + row := []string{"2026-10-30T00:00:00Z", "run-123", "NL", strings.Repeat("x", customBatchBytes), "true", "", "", "1"} + require.Error(t, c.addRow("transactions", row)) + require.Error(t, c.err()) + require.Empty(t, c.queue) +} + +func TestCustomHTTPJoinsAndLateFailures(t *testing.T) { + t.Parallel() + c := newCustomTest(t, "http://localhost") + var rows [][]string + c.mapper.rowSink = func(table string, row []string) error { + if table == "requests_raw" { + rows = append(rows, row) + } + return nil + } + reg := metrics.NewRegistry() + ts := time.Now() + names := []string{"browser_http_req_duration", "browser_http_req_blocked", "browser_http_req_connecting", "browser_http_req_tls_handshaking", "browser_http_req_sending", "browser_http_req_waiting", "browser_http_req_receiving", "browser_data_received"} + tagsA := reg.RootTagSet().With("name", "/a").With("resource_type", "fetch") + tagsB := reg.RootTagSet().With("name", "/b").With("resource_type", "fetch") + for _, tags := range []*metrics.TagSet{tagsA, tagsB} { + samples := metrics.Samples{} + for _, name := range names { + samples = append(samples, metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric(name, metrics.Trend), Tags: tags}, Time: ts, Value: 12}) + } + c.AddMetricSamples([]metrics.SampleContainer{samples}) + } + require.Empty(t, rows) + c.completeGroups(false) + require.Empty(t, rows) // A flush timer cannot infer request success. + for _, tags := range []*metrics.TagSet{tagsA, tagsB} { + sample := metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric("browser_http_req_failed", metrics.Rate), Tags: tags}, Time: ts, Value: 1, HTTPErrorResBody: "failure"} + c.AddMetricSamples([]metrics.SampleContainer{sample}) + } + require.Len(t, rows, 2) + require.Empty(t, c.groups) + for _, row := range rows { + require.Equal(t, "false", row[5]) + require.Equal(t, "12", row[7]) + } +} + +func TestCustomIncompleteDrainFails(t *testing.T) { + t.Parallel() + c := newCustomTest(t, "http://localhost") + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("http_req_duration", nil, false)}) + c.completeGroups(true) + require.ErrorContains(t, c.err(), "incomplete HTTP metric group") +} + +func TestCustomQueueBackpressure(t *testing.T) { + t.Parallel() + c := newCustomTest(t, "http://localhost") + row := []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "1"} + for range cap(c.queue) { + require.NoError(t, c.addRow("transactions", row)) + c.flush() + } + blocked := make(chan struct{}) + go func() { defer close(blocked); _ = c.addRow("transactions", row); c.flush() }() + select { + case <-blocked: + t.Fatal("full queue did not apply backpressure") + case <-time.After(20 * time.Millisecond): + } + c.fail(fmt.Errorf("sender unavailable")) + select { + case <-blocked: + case <-time.After(time.Second): + t.Fatal("failure did not unblock producer") + } + require.Len(t, c.queue, cap(c.queue)) +} + +func TestCustomStartupRejectsNonFreshAndMissingAck(t *testing.T) { + t.Parallel() + for _, body := range []string{`{}`, `{"incarnation":"a"}`, `{"incarnation":"a","accepted_sequence":1}`, `{"incarnation":"a","accepted_sequence":0,"sealed":true}`} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + c := newCustomTest(t, server.URL) + require.Error(t, c.Start()) + require.False(t, c.started) + server.Close() + } +} + +func TestCustomHTTPDuplicateIdentityFails(t *testing.T) { + t.Parallel() + c := newCustomTest(t, "http://localhost") + sample := transactionTestSample("http_req_duration", nil, false) + c.AddMetricSamples([]metrics.SampleContainer{sample, sample}) + require.ErrorContains(t, c.err(), "ambiguous HTTP metric identity") +} From b42c6eb09180558566e277e0860e7610777f0090 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Fri, 30 Oct 2026 10:09:07 +0100 Subject: [PATCH 2/7] Add native request watermarks, parallel lanes and Custom DB recovery --- ai-tests/custom-db/smoke.py | 74 ++++++-- internal/js/modules/k6/metrics/metrics.go | 5 +- .../js/modules/k6/websockets/websockets.go | 18 +- internal/js/modules/k6/ws/ws.go | 18 +- internal/output/breakingit/CUSTOM_DB.md | 71 +++++--- internal/output/breakingit/custom.go | 169 ++++++++++++------ internal/output/breakingit/custom_recovery.go | 106 +++++++++++ internal/output/breakingit/custom_test.go | 148 +++++++++++++++ .../output/breakingit/custom_watermark.go | 38 ++++ lib/netext/httpext/tracer.go | 5 +- lib/netext/httpext/transport.go | 9 +- metrics/request_tracker.go | 110 ++++++++++++ metrics/request_tracker_test.go | 44 +++++ output/manager.go | 22 +++ 14 files changed, 733 insertions(+), 104 deletions(-) create mode 100644 internal/output/breakingit/custom_recovery.go create mode 100644 internal/output/breakingit/custom_watermark.go create mode 100644 metrics/request_tracker.go create mode 100644 metrics/request_tracker_test.go diff --git a/ai-tests/custom-db/smoke.py b/ai-tests/custom-db/smoke.py index da7d0bb84..97fee9598 100644 --- a/ai-tests/custom-db/smoke.py +++ b/ai-tests/custom-db/smoke.py @@ -14,6 +14,7 @@ import os from pathlib import Path import secrets +import socket import subprocess import tempfile import threading @@ -25,7 +26,9 @@ def run(args): name = "k6-custom-writer-smoke-" + uuid.uuid4().hex[:12] - admin, producer = secrets.token_hex(32), secrets.token_hex(32) + admin = secrets.token_hex(32) + producers = [secrets.token_hex(32) for _ in range(args.lanes)] + producer = ",".join(producers) oracle = {} class Handler(http.server.BaseHTTPRequestHandler): @@ -63,10 +66,13 @@ def do_POST(self): data_dir = root / "data" data_dir.mkdir() env = {**os.environ, "CUSTOM_STORAGE_ADMIN_TOKEN": admin} + with socket.socket() as port_socket: + port_socket.bind(('127.0.0.1',0)) + fixed_port=port_socket.getsockname()[1] try: subprocess.run(["docker", "run", "-d", "--name", name, "--label", "breaktest.k6-writer-smoke=true", "--memory", "512m", - "-e", "CUSTOM_STORAGE_ADMIN_TOKEN", "-p", "127.0.0.1::8099", + "-e", "CUSTOM_STORAGE_ADMIN_TOKEN", "-p", f"127.0.0.1:{fixed_port}:8099", "-v", f"{data_dir}:/data", args.image, "v3", "service", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True) port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1] @@ -85,42 +91,72 @@ def post(path, payload): except OSError: time.sleep(.05) origin = int(time.time() - 946684800 - 10) * 1_000_000 - post("/runs/create", {"run_id": "k6-smoke", "source_tokens": [producer], + post("/runs/create", {"run_id": "k6-smoke", "source_tokens": producers, "origin_us": origin, "customer_id": "isolated-k6-smoke"}) script = root / "smoke.js" script.write_text('''import http from 'k6/http'; import { group, check, sleep } from 'k6'; -export const options = { vus: 2, iterations: 20 }; +import { Trend } from 'k6/metrics'; +const customLatency = new Trend('custom_latency'); +export const options = { vus: 2, iterations: Number(__ENV.FIXTURE_ITERATIONS) }; export default function () { group('Checkout', function () { http.get(__ENV.TARGET + '/ok'); const r = http.get(__ENV.TARGET + '/failure'); check(r, { 'expected failure': x => x.status === 200 }); }); + if (__ENV.FIXTURE_CUSTOM === 'true') customLatency.add(37); sleep(0.15); } ''') target_url = f"http://127.0.0.1:{target.server_port}" writer_env = {**os.environ, "CUSTOM_STORAGE_TOKEN": producer, "runId": "k6-smoke", "nodeName": "k6-probe", "location": "Local", "PG_PROXY_TOKEN": "isolated-oracle", - "TARGET": target_url, "K6_NO_USAGE_REPORT": "true", "K6_HTTP_ERROR_DATA": "true"} + "TARGET": target_url, "FIXTURE_ITERATIONS": str(args.iterations), "FIXTURE_CUSTOM": str(args.custom_metrics).lower(), "K6_NO_USAGE_REPORT": "true", "K6_HTTP_ERROR_DATA": "true"} for key in ("syntheticMonitoring", "K6_BROWSER_SCREENSHOT_ON_ERROR", "CUSTOM_URL_PATTERNS", "BREAKINGIT_TRANSACTION_SOURCE"): writer_env.pop(key, None) - result = subprocess.run([str(Path(args.k6).resolve()), "run", "--quiet", + command = [str(Path(args.k6).resolve()), "run", "--quiet", "--out", "custom-db-breakingit=" + service, - "--out", "timescaledb-breakingit=" + target_url, str(script)], - env=writer_env, text=True, capture_output=True, timeout=90) - assert result.returncode == 0, result.stdout + result.stderr + "--out", "timescaledb-breakingit=" + target_url, str(script)] + observations=[] + with (root/'k6.log').open('w+') as log: + process=subprocess.Popen(command,env=writer_env,text=True,stdout=log,stderr=log) + began=time.monotonic(); restarted=False + try: + while process.poll() is None: + if time.monotonic()-began>90:raise TimeoutError('k6 smoke deadline') + try: + observation=post('/runs/k6-smoke/snapshot',{}) + if not observation['sealed']: observations.append(observation) + except OSError: pass + if args.restart_live and not restarted and time.monotonic()-began>=2: + subprocess.run(['docker','kill','--signal=KILL',name],check=True,capture_output=True) + subprocess.run(['docker','start',name],check=True,capture_output=True) + restarted=True + time.sleep(.1) + log.seek(0) + text=log.read() + assert process.returncode==0 and 'CUSTOM_STORAGE_FAILED' not in text,text + finally: + if process.poll() is None: process.kill(); process.wait() + live=[s for s in observations if all(not x['finished'] for x in s['sources']) and min(x['watermark'] for x in s['sources'])>origin and any(r['complete'] for r in s['rows'])] + assert live, 'No live watermark-completed aggregates observed' end = int(time.time() - 946684800 + 10) * 1_000_000 final = post("/runs/k6-smoke/finalize", {}) - assert final["sealed"] and final["run_complete"], final - expected_counts = {"requests_raw": 40, "transactions": 20, "requests_error": 40} + assert final["sealed"], final + if args.restart_live: + assert restarted and final['recovered'],final + assert all(g.get('max_second') is not None for g in final['gaps']),final + assert final['run_complete'] == (not final['gaps'] and not final['unknown_coverage']),final + else: assert final['run_complete'] and not final['gaps'],final + expected_counts = {"requests_raw": args.iterations*(3 if args.custom_metrics else 2), "transactions": args.iterations, "requests_error": args.iterations*2} counts = {} for table in ("requests_raw", "transactions", "requests_error", "virtual_users"): actual = post("/runs/k6-smoke/query", {"table": table, "start_us": origin, "end_us": end}) counts[table] = len(actual["rows"]) - assert counts[table] == len(oracle[table]), (table, counts[table], len(oracle[table])) - if table in expected_counts: + if not args.restart_live: assert counts[table] == len(oracle[table]), (table, counts[table], len(oracle[table])) + else: assert counts[table] <= len(oracle[table]), (table, counts) + if table in expected_counts and not args.restart_live: assert counts[table] == expected_counts[table], (table, counts) else: assert counts[table] > 0 @@ -142,7 +178,11 @@ def normalized(row, csv_row=False): return json.dumps(values, ensure_ascii=False) left = sorted(normalized(row) for row in actual["rows"]) right = sorted(normalized(row, True) for row in oracle[table]) - assert left == right, (table, "row mismatch", left[:1], right[:1], actual.get("columns")) + if args.restart_live: + from collections import Counter + assert not (Counter(left)-Counter(right)), (table,'duplicate or corrupted recovered rows') + if len(left) 0 { + ticket = tickets[0] + } state := w.vu.State() - metrics.PushIfNotDone(ctx, state.Samples, metrics.ConnectedSamples{ + metrics.PushTrackedRequest(ctx, state.Samples, ticket, metrics.ConnectedSamples{ Samples: []metrics.Sample{ { TimeSeries: metrics.TimeSeries{Metric: state.BuiltinMetrics.WSSessions, Tags: w.tagsAndMeta.Tags}, @@ -827,7 +832,6 @@ func (w *webSocket) ping() { } func (w *webSocket) trackPong(pingID string) { - pongTimestamp := time.Now() pingTimestamp, ok := w.sendPings.timestamps[pingID] if !ok { @@ -838,7 +842,9 @@ func (w *webSocket) trackPong(pingID string) { return } - metrics.PushIfNotDone(w.vu.Context(), w.vu.State().Samples, metrics.Sample{ + ticket := metrics.BeginRequest(w.vu.State().Samples) + pongTimestamp := metrics.RequestTime(ticket, time.Now()) + metrics.PushTrackedRequest(w.vu.Context(), w.vu.State().Samples, ticket, metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: w.builtinMetrics.WSPing, Tags: w.tagsAndMeta.Tags, diff --git a/internal/js/modules/k6/ws/ws.go b/internal/js/modules/k6/ws/ws.go index 5389fc788..dfe186a44 100644 --- a/internal/js/modules/k6/ws/ws.go +++ b/internal/js/modules/k6/ws/ws.go @@ -268,7 +268,8 @@ func (mi *WS) dial( wsd.Jar = args.cookieJar } - connStart := time.Now() + ticket := metrics.BeginRequest(state.Samples) + connStart := metrics.RequestTime(ticket, time.Now()) conn, httpResponse, dialErr := wsd.DialContext(ctx, url, args.headers) connEnd := time.Now() @@ -303,7 +304,7 @@ func (mi *WS) dial( builtinMetrics: state.BuiltinMetrics, } - connEndHook := socket.pushSessionMetrics(connStart, connEnd) + connEndHook := socket.pushSessionMetrics(connStart, connEnd, ticket) return &socket, httpResponse, connEndHook, dialErr } @@ -392,7 +393,6 @@ func (s *Socket) Ping() { } func (s *Socket) trackPong(pingID string) { - pongTimestamp := time.Now() if _, ok := s.pingSendTimestamps[pingID]; !ok { // We received a pong for a ping we didn't send; ignore @@ -400,8 +400,10 @@ func (s *Socket) trackPong(pingID string) { return } pingTimestamp := s.pingSendTimestamps[pingID] + ticket := metrics.BeginRequest(s.samplesOutput) + pongTimestamp := metrics.RequestTime(ticket, time.Now()) - metrics.PushIfNotDone(s.ctx, s.samplesOutput, metrics.Sample{ + metrics.PushTrackedRequest(s.ctx, s.samplesOutput, ticket, metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: s.builtinMetrics.WSPing, Tags: s.tagsAndMeta.Tags, @@ -515,10 +517,14 @@ func (s *Socket) closeConnection(code int) error { return err } -func (s *Socket) pushSessionMetrics(connStart, connEnd time.Time) func() { +func (s *Socket) pushSessionMetrics(connStart, connEnd time.Time, tickets ...*metrics.RequestTicket) func() { + var ticket *metrics.RequestTicket + if len(tickets) > 0 { + ticket = tickets[0] + } connDuration := metrics.D(connEnd.Sub(connStart)) - metrics.PushIfNotDone(s.ctx, s.samplesOutput, metrics.ConnectedSamples{ + metrics.PushTrackedRequest(s.ctx, s.samplesOutput, ticket, metrics.ConnectedSamples{ Samples: []metrics.Sample{ { TimeSeries: metrics.TimeSeries{ diff --git a/internal/output/breakingit/CUSTOM_DB.md b/internal/output/breakingit/CUSTOM_DB.md index b4fdc3a96..3c8b1b261 100644 --- a/internal/output/breakingit/CUSTOM_DB.md +++ b/internal/output/breakingit/CUSTOM_DB.md @@ -11,19 +11,19 @@ writer. It does not use the older xk6 extension in the BreakTest repository. ## Invocation and application contract -The application must first provision a Custom DB run and one producer token per +The application must first provision a Custom DB run and one to four producer tokens per load generator. Pass the run ID and that generator's token when invoking k6: ```sh export runId='' -export CUSTOM_STORAGE_TOKEN='' +export CUSTOM_STORAGE_TOKEN=',' export location='Amsterdam' export nodeName='generator-1' k6 run --out custom-db-breakingit=https://storage.example.test script.js ``` The URL is the service base URL (an optional routing prefix is retained). The -writer appends `/runs//hello`, `/ingest/batch` and `/finish` to it. TLS +writer appends `/runs//hello`, `/ingest/batch`, `/watermark` and `/finish` to it. TLS certificates are verified, connections reused, and redirects disabled so tokens cannot be redirected to another origin. Never pass the service admin token. @@ -33,21 +33,21 @@ request rows, failed checks, browser errors and VUs use the existing mapping cod Unknown CSV fields remain NULL in the binary encoding, and negative/unavailable connect times become NULL. Error UUIDs are generated independently per output. -The application must select the output from the frozen test backend, provide the -producer credentials, and finalize/publish only after all producers finish. -This fork change does **not** yet update the BreakTest launch configuration, -remove its k6 preview restriction, rebuild its deployed generator image, or change -any customer's storage selection. Those changes must accompany deployment. +The application selects the output from the frozen test backend, provisions all +lane tokens, and finalizes only after every lane finishes. BreakTest's integration +requires generator capability `k6-custom-v2`, verified against the installed binary +checksum; this prevents routing multi-lane jobs to the older single-source writer. Generator machine statistics remain the responsibility of the existing agent. ## Delivery and limits -- Flush every 200 ms or at a 2 MiB encoded-row budget. Four queued batches bound +- Flush every 200 ms or at a 2 MiB encoded-row budget. Four queued batches per lane bound outgoing buffering; a full queue applies backpressure rather than dropping rows. The building batch, one waiting enqueue, sender/compression buffers, and k6's own - sample containers consume additional memory; 8 MiB is not a total RSS limit. -- One ordered sender per generator. Multiple generators have independent sources. - This first version does not claim the throughput of the parallel Java writer. + sample containers consume additional memory; 8 MiB per lane is not a total RSS limit. +- One to four ordered compression/sender workers per generator, with independent + sequences and shared keep-alive connections. Batches are distributed round-robin. + Watermark-only messages reach idle lanes too. No high-load capacity claim yet. - Incomplete HTTP joins are limited to 4,096 groups / an estimated 2 MiB payload budget. Oversized rows or exhausted join budgets stop the run explicitly. - HTTP/browser joins include timestamp and tag-set identity. All required request @@ -60,18 +60,37 @@ Generator machine statistics remain the responsibility of the existing agent. - Drain accepted sample containers and all queued requests before `/finish`. The producer does not seal the run itself. A final barrier proves delivery of this producer's data; it is not a durable-on-disk/fsync acknowledgement. -- A fresh producer source is required. A server incarnation change is fatal in - this first implementation. Automatic relaxed-durability reconnect requires a - source clock contract that k6 does not yet supply. +- A fresh producer source is required at startup. Native runs recover after a + server incarnation change using the anchored source clock and checkpoint sequence: + skip a held batch proven checkpointed; otherwise renumber and replay only that + unacknowledged batch. Previously acknowledged, lost rows are not replayed; the + service exposes its relaxed-durability coverage gaps. Same-incarnation conflicts + remain fatal. Browser runs without a proven source clock fail closed on restart. ## Watermarks and unsupported data -This writer **does not send live watermarks**. k6 output callbacks can be delayed -and out of event-time order, particularly for browser failure bodies. Neither -wall-clock time nor the latest observed sample proves that older samples cannot -arrive. Live reads therefore use the service's pending/raw and grace-period paths; -the final producer barrier permits complete aggregation after drain. Proven live -source barriers are a separate k6 core integration, not a timer setting. +Native k6 runs send live request watermarks, using the same sequence-backed, +all-source barrier as the Java writer. The output manager installs a per-run +tracker before producers start. HTTP completion assigns its timestamp and pin +atomically; WebSocket connecting/ping and custom metrics are similarly tracked. +Pins survive the engine sample channel and pending metric joins until rows are +serialized. A sharded tracker computes the oldest unresolved timestamp. It uses +an anchored monotonic clock, so a wall-clock step cannot invalidate a sent barrier. + +After detaching all preceding rows, the writer queues the same whole-second +barrier on every lane. A lane sends it only after acknowledging its earlier +batches. The service closes the common prefix into immediate 1s aggregates; open +seconds remain queryable through the pending/raw path. Event-time completeness +still does not imply fsync durability. Transactions and monitoring are separate +streams, just as in the Java request-watermark contract. + +**Browser scenarios are not yet at watermark parity.** Their request timestamps +come from asynchronous Chromium/CDP events in another process. They retain the +previous grace/pending/raw path, including mixed browser/native scenarios; a host +wall-clock timer cannot prove their event-time completeness. Browser timestamps +are not rewritten to manufacture progress. Browser live barriers require a +separate CDP lifecycle contract. Uninstrumented native request metric producers +fail closed rather than silently violating a watermark. Synthetic monitoring and browser screenshots are explicitly unsupported and rejected. Ordinary browser request timings, failure response data, native groups @@ -81,12 +100,13 @@ also fails the output, rather than storing a dangling screenshot reference. ## Validation ```sh -go test -race ./internal/output/breakingit +go test -race ./metrics ./output ./lib/netext/httpext ./internal/output/breakingit go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' go test ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' go build -tags lean -o /tmp/k6-custom-db-writer . python3 ai-tests/custom-db/smoke.py --k6 /tmp/k6-custom-db-writer \ - --image breaktest-storage-lab:local + --image breaktest-storage-lab:local --lanes 4 +python3 ai-tests/custom-db/smoke.py --k6 /tmp/k6-custom-db-writer --lanes 4 --restart-live ``` The smoke test creates its own disposable Docker service and temporary directory, @@ -97,3 +117,8 @@ generated error UUIDs. It checks request, transaction, error and VU counts, comp sealing and reads after storage-service restart. It never connects to existing customer databases. This establishes protocol/mapping compatibility, not a TimescaleDB performance benchmark or browser end-to-end acceptance. + +The live smoke test also requires unfinished sources with advancing watermarks and +complete aggregate buckets before finalization. Crash mode kills only its own +throwaway service, verifies retained rows against the oracle without duplicates, +and requires explicit incomplete coverage whenever rows were lost. diff --git a/internal/output/breakingit/custom.go b/internal/output/breakingit/custom.go index 7c57bc8bf..8d805e462 100644 --- a/internal/output/breakingit/custom.go +++ b/internal/output/breakingit/custom.go @@ -29,7 +29,16 @@ type customGroupKey struct { } type CustomOutput struct { - groups map[customGroupKey]*MetricGroup + browserMode bool + groups map[customGroupKey]*MetricGroup + groupTickets map[customGroupKey]*metrics.RequestTicket + tracker *metrics.RequestTracker + lanes []*CustomOutput + owner *CustomOutput + nextLane int + watermark int64 + queuedWatermark int64 + tokens []string mapper *Logger client *http.Client @@ -54,6 +63,7 @@ type CustomOutput struct { type customBatch struct { body []byte contentType string + beforeUS int64 } // NewCustom is registered separately from timescaledb-breakingit in full and lean builds. @@ -73,6 +83,17 @@ func NewCustom(params output.Params) (output.Output, error) { if strings.EqualFold(env["K6_BROWSER_SCREENSHOT_ON_ERROR"], "true") { return nil, errors.New("Custom DB k6 output does not yet support browser screenshot storage") } + tokens := strings.Split(token, ",") + seen := map[string]bool{} + if len(tokens) > 4 { + return nil, errors.New("Custom DB supports 1-4 ingestion lanes") + } + for _, t := range tokens { + if t == "" || seen[t] { + return nil, errors.New("invalid Custom DB lane tokens") + } + seen[t] = true + } source, err := parseTransactionSource(env["BREAKINGIT_TRANSACTION_SOURCE"]) if err != nil { return nil, err @@ -90,28 +111,49 @@ func NewCustom(params output.Params) (output.Output, error) { } tags := map[string]string{"run_id": runID, "location": env["location"], "node_name": env["nodeName"], "scenario_name": env["scenarioName"]} transport := http.DefaultTransport.(*http.Transport).Clone() - transport.MaxConnsPerHost = 1 + transport.MaxConnsPerHost = len(tokens) c := &CustomOutput{ - base: strings.TrimRight(u.String(), "/") + "/runs/" + runID, token: token, runID: runID, + base: strings.TrimRight(u.String(), "/") + "/runs/" + runID, token: tokens[0], tokens: tokens, runID: runID, client: &http.Client{Transport: transport, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, retryTimeout: 30 * time.Second, queue: make(chan customBatch, 4), done: make(chan struct{}), tickStop: make(chan struct{}), tickDone: make(chan struct{}), failed: make(chan struct{}), parts: map[string]*bytes.Buffer{}, } + for _, scenario := range params.ScriptOptions.Scenarios { + if opts := scenario.GetScenarioOptions(); opts != nil && len(opts.Browser) > 0 { + c.browserMode = true + } + } c.groups = map[customGroupKey]*MetricGroup{} + c.groupTickets = map[customGroupKey]*metrics.RequestTicket{} c.mapper = &Logger{out: params.StdOut, envTags: tags, customPatterns: patterns, transactionSource: source, bufHTTP: &bytes.Buffer{}, bufVUs: &bytes.Buffer{}, bufTrans: &bytes.Buffer{}, bufErrors: &bytes.Buffer{}, pendingHttpGroups: map[int64]*MetricGroup{}, rowSink: c.addRow} return c, nil } +func (c *CustomOutput) UsesRequestTracking() bool { return !c.browserMode } + +func (c *CustomOutput) SetRequestTracker(t *metrics.RequestTracker) { c.tracker = t } + func (c *CustomOutput) Description() string { return "custom-db-breakingit" } func (c *CustomOutput) SetTestRunStopCallback(callback func(error)) { c.failureMu.Lock() c.stopCallback = callback c.failureMu.Unlock() } -func (c *CustomOutput) err() error { c.failureMu.Lock(); defer c.failureMu.Unlock(); return c.failure } +func (c *CustomOutput) err() error { + if c.owner != nil { + return c.owner.err() + } + c.failureMu.Lock() + defer c.failureMu.Unlock() + return c.failure +} func (c *CustomOutput) fail(err error) { + if c.owner != nil { + c.owner.fail(err) + return + } if err == nil { return } @@ -130,27 +172,32 @@ func (c *CustomOutput) fail(err error) { } } func (c *CustomOutput) Start() error { - // An empty HELLO is sufficient for a fresh source. Do not claim the Java - // monotonic sample-clock contract: k6 output callbacks cannot prove it. - var hello struct { - Incarnation string `json:"incarnation"` - Accepted *uint64 `json:"accepted_sequence"` - Sealed bool `json:"sealed"` - } - if err := c.request("/hello", []byte(`{}`), "application/json", 0, &hello); err != nil { - return err - } - if hello.Sealed || hello.Incarnation == "" || hello.Accepted == nil || *hello.Accepted != 0 { - return errors.New("Custom DB requires a fresh unfinished producer source") + c.lanes = []*CustomOutput{c} + for _, token := range c.tokens[1:] { + c.lanes = append(c.lanes, &CustomOutput{owner: c, tracker: c.tracker, base: c.base, runID: c.runID, token: token, + client: c.client, retryTimeout: c.retryTimeout, queue: make(chan customBatch, 4), done: make(chan struct{}), failed: c.failed}) } - c.incarnation = hello.Incarnation - encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1)) - if err != nil { - return err + for _, lane := range c.lanes { + if err := lane.hello(); err != nil { + for _, opened := range c.lanes { + if opened.encoder != nil { + opened.encoder.Close() + } + } + return err + } + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1)) + if err != nil { + return err + } + lane.encoder = encoder } - c.encoder = encoder + // One keep-alive connection and compression worker per ordered source. + c.started = true - go c.sender() + for _, lane := range c.lanes { + go lane.sender() + } go func() { defer close(c.tickDone) tick := time.NewTicker(200 * time.Millisecond) @@ -165,6 +212,7 @@ func (c *CustomOutput) Start() error { c.addMu.Lock() c.completeGroups(false) c.flush() + c.publishWatermark() c.addMu.Unlock() } } @@ -179,6 +227,11 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { return } for _, sc := range samples { + httpContainer := false + var ticket *metrics.RequestTicket + if tracked, ok := sc.(interface{ RequestTicket() *metrics.RequestTicket }); ok { + ticket = tracked.RequestTicket() + } for _, sample := range sc.GetSamples() { if sample.Metric.Name == "browser_errors" && sample.Tags.Map()["screenshot"] != "" { c.fail(errors.New("browser screenshots require a Custom DB payload integration")) @@ -188,10 +241,23 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { for _, sample := range sc.GetSamples() { key := httpMetricCanonicalKey(sample.Metric.Name) if key == "" { + requestMetric := strings.HasPrefix(sample.Metric.Name, "custom_") || sample.Metric.Name == "ws_connecting" || sample.Metric.Name == "ws_ping" + if c.tracker != nil && requestMetric && ticket == nil { + c.fail(errors.New("request metric lacks producer completion tracking")) + return + } c.mapper.AddMetricSamples([]metrics.SampleContainer{sample}) continue } + httpContainer = true + if c.tracker != nil && ticket == nil { + c.fail(errors.New("HTTP metrics lack producer completion tracking; live watermark would be unsafe")) + return + } id := customGroupKey{sample.Time.UnixNano(), sample.Tags} + if ticket != nil { + c.groupTickets[id] = ticket + } g := c.groups[id] if g == nil { g = &MetricGroup{Timestamp: id.time, Tags: map[string]string{}, Values: map[string]float64{}} @@ -211,6 +277,9 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { } } c.completeGroups(false) + if !httpContainer { + ticket.Release() + } if c.err() != nil { return @@ -246,6 +315,8 @@ func (c *CustomOutput) completeGroups(final bool) { if complete { delete(c.groups, key) c.mapper.processHttpGroup(g) + c.groupTickets[key].Release() + delete(c.groupTickets, key) } else if final { c.fail(errors.New("incomplete HTTP metric group at final drain")) return @@ -307,11 +378,16 @@ func (c *CustomOutput) enqueueLocked() error { if err := writer.Close(); err != nil { return err } - batch := customBatch{body.Bytes(), "multipart/mixed; boundary=" + writer.Boundary()} + batch := customBatch{body: body.Bytes(), contentType: "multipart/mixed; boundary=" + writer.Boundary()} + lane := c + if len(c.lanes) > 0 { + lane = c.lanes[c.nextLane] + c.nextLane = (c.nextLane + 1) % len(c.lanes) + } select { case <-c.failed: return c.err() - case c.queue <- batch: + case lane.queue <- batch: c.parts = map[string]*bytes.Buffer{} c.batchBytes = 0 return nil @@ -323,20 +399,12 @@ func (c *CustomOutput) sender() { if c.err() != nil { continue } - compressed := c.encoder.EncodeAll(batch.body, nil) - var ack struct { - Accepted *uint64 `json:"accepted"` - } - next := c.sequence + 1 - if err := c.request("/ingest/batch", compressed, batch.contentType, next, &ack); err != nil { - c.fail(err) - continue - } - if ack.Accepted == nil || *ack.Accepted != next { - c.fail(errors.New("invalid Custom DB batch acknowledgement")) + if batch.beforeUS != 0 { + c.fail(c.sendControl("/watermark", batch.beforeUS)) continue } - c.sequence = next + compressed := c.encoder.EncodeAll(batch.body, nil) + c.fail(c.sendBatch(compressed, batch.contentType)) } } @@ -353,20 +421,21 @@ func (c *CustomOutput) Stop() error { c.flush() } c.addMu.Unlock() - close(c.queue) - <-c.done - if c.err() == nil { - var ack struct { - Accepted *uint64 `json:"accepted_sequence"` - Finished bool `json:"finished"` - } - err := c.request("/finish", nil, "application/octet-stream", c.sequence, &ack) - if err == nil && (ack.Accepted == nil || *ack.Accepted != c.sequence || !ack.Finished) { - err = errors.New("invalid Custom DB final acknowledgement") + for _, lane := range c.lanes { + close(lane.queue) + } + for _, lane := range c.lanes { + <-lane.done + } + for _, lane := range c.lanes { + if c.err() != nil { + break } - c.fail(err) + c.fail(lane.sendControl("/finish", 0)) + } + for _, lane := range c.lanes { + lane.encoder.Close() } - c.encoder.Close() c.client.CloseIdleConnections() c.stopErr = c.err() }) @@ -374,7 +443,7 @@ func (c *CustomOutput) Stop() error { } // Retries keep the same sequence, multipart boundary and compressed bytes. -// A changed incarnation is deliberately fatal until k6 has a proven recovery clock. +// Incarnation recovery is handled by the lane, after this request returns 409. func (c *CustomOutput) request(path string, body []byte, contentType string, seq uint64, result any) error { ctx, cancel := context.WithTimeout(context.Background(), c.retryTimeout) defer cancel() @@ -407,7 +476,7 @@ func (c *CustomOutput) request(path string, body []byte, contentType string, seq return nil } if readErr == nil && response.StatusCode != 429 && response.StatusCode < 500 { - return fmt.Errorf("Custom DB %s rejected request (HTTP %d)", path, response.StatusCode) + return customRejection(response.StatusCode) } } timer := time.NewTimer(100 * time.Millisecond) diff --git a/internal/output/breakingit/custom_recovery.go b/internal/output/breakingit/custom_recovery.go new file mode 100644 index 000000000..6856a7f90 --- /dev/null +++ b/internal/output/breakingit/custom_recovery.go @@ -0,0 +1,106 @@ +package breakingit + +import ( + "encoding/json" + "errors" + "fmt" +) + +type customRejection int + +func (s customRejection) Error() string { + return fmt.Sprintf("Custom DB rejected request (HTTP %d)", int(s)) +} + +type sourceHello struct { + Incarnation string `json:"incarnation"` + Accepted *uint64 `json:"accepted_sequence"` + Checkpoint *uint64 `json:"checkpoint_sequence"` + Sealed bool `json:"sealed"` +} + +func (c *CustomOutput) register() (sourceHello, error) { + body := []byte(`{}`) + if c.tracker != nil { + body, _ = json.Marshal(map[string]any{"source_clock_us": c.tracker.Clock().UnixMicro() - pgEpochSeconds*1_000_000, "clock_contract": "sample-end-monotonic-v1"}) + } + var hello sourceHello + err := c.request("/hello", body, "application/json", 0, &hello) + return hello, err +} + +// Acknowledged, uncheckpointed rows may be lost under relaxed durability. The +// server exposes that loss as coverage gaps. Replay only the held unacknowledged +// batch, never already acknowledged history. Each ordered lane recovers itself. +func (c *CustomOutput) recoverSource(err error) (uint64, error) { + var status customRejection + if c.tracker == nil || !errors.As(err, &status) || status != 409 { + return 0, err + } + hello, e := c.register() + if e != nil { + return 0, e + } + if hello.Sealed || hello.Incarnation == "" || hello.Incarnation == c.incarnation || hello.Accepted == nil || hello.Checkpoint == nil || *hello.Accepted != *hello.Checkpoint { + return 0, err + } + c.incarnation = hello.Incarnation + c.sequence = *hello.Accepted + return *hello.Checkpoint, nil +} + +func (c *CustomOutput) sendBatch(body []byte, content string) error { + for attempt := 0; attempt < 4; attempt++ { + next := c.sequence + 1 + var ack struct { + Accepted *uint64 `json:"accepted"` + } + err := c.request("/ingest/batch", body, content, next, &ack) + if err == nil { + if ack.Accepted == nil || *ack.Accepted != next { + return errors.New("invalid Custom DB batch acknowledgement") + } + c.sequence = next + return nil + } + checkpoint, e := c.recoverSource(err) + if e != nil { + return e + } + if checkpoint > next { + return errors.New("checkpoint exceeds outstanding batch sequence") + } + if next <= checkpoint { + return nil + } // Lost ACK, but checkpoint proves this batch persisted. + } + return errors.New("Custom DB restarted repeatedly during batch delivery") +} + +func (c *CustomOutput) sendControl(path string, before int64) error { + body := []byte(nil) + if path == "/watermark" { + body, _ = json.Marshal(map[string]int64{"before_us": before}) + } + for attempt := 0; attempt < 4; attempt++ { + var ack struct { + Accepted *uint64 `json:"accepted_sequence"` + Watermark int64 `json:"watermark_us"` + Finished bool `json:"finished"` + } + previous := c.sequence + err := c.request(path, body, "application/json", c.sequence, &ack) + if err == nil { + if ack.Accepted == nil || *ack.Accepted != c.sequence || (path == "/watermark" && ack.Watermark != before) || (path == "/finish" && !ack.Finished) { + return errors.New("invalid Custom DB control acknowledgement") + } + return nil + } + if checkpoint, e := c.recoverSource(err); e != nil { + return e + } else if checkpoint > previous { + return errors.New("checkpoint exceeds acknowledged sequence") + } + } + return errors.New("Custom DB restarted repeatedly during control delivery") +} diff --git a/internal/output/breakingit/custom_test.go b/internal/output/breakingit/custom_test.go index 5181fbe73..5923f0660 100644 --- a/internal/output/breakingit/custom_test.go +++ b/internal/output/breakingit/custom_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "encoding/csv" + "encoding/json" "fmt" "io" "mime" @@ -313,3 +314,150 @@ func TestCustomHTTPDuplicateIdentityFails(t *testing.T) { c.AddMetricSamples([]metrics.SampleContainer{sample, sample}) require.ErrorContains(t, c.err(), "ambiguous HTTP metric identity") } + +func TestCustomWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) { + var mu sync.Mutex + sequences := map[string]uint64{} + watermarks := map[string]int64{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + token := r.Header.Get("Authorization") + seq, _ := strconv.ParseUint(r.Header.Get("X-Sequence"), 10, 64) + switch { + case strings.HasSuffix(r.URL.Path, "/hello"): + fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0}`) + case strings.HasSuffix(r.URL.Path, "/ingest/batch"): + require.Equal(t, sequences[token]+1, seq) + sequences[token] = seq + fmt.Fprintf(w, `{"accepted":%d}`, seq) + case strings.HasSuffix(r.URL.Path, "/watermark"): + var body struct { + Before int64 `json:"before_us"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + require.Equal(t, sequences[token], seq) + watermarks[token] = body.Before + fmt.Fprintf(w, `{"accepted_sequence":%d,"watermark_us":%d}`, seq, body.Before) + case strings.HasSuffix(r.URL.Path, "/finish"): + require.Equal(t, sequences[token], seq) + fmt.Fprintf(w, `{"accepted_sequence":%d,"finished":true}`, seq) + default: + w.WriteHeader(404) + } + })) + defer server.Close() + c := newCustomTest(t, server.URL) + c.tokens = []string{"producer-token", "second"} + ch := make(chan metrics.SampleContainer, 1) + tracker, release := metrics.BindRequestTracker(ch) + defer release() + c.SetRequestTracker(tracker) + require.NoError(t, c.Start()) + ticket := metrics.BeginRequest(ch) // Request not delivered to the output yet. + pin := ticket.Time.Truncate(time.Second).UnixMicro() - pgEpochSeconds*1_000_000 + for range 4 { + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)}) + c.flush() + } + require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(watermarks) == 2 }, 2*time.Second, 10*time.Millisecond) + mu.Lock() + for _, watermark := range watermarks { + require.LessOrEqual(t, watermark, pin) + } + mu.Unlock() + // Cancellation before enqueue may release its pin; no request row exists. + ticket.Release() + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return watermarks["Bearer producer-token"] > pin && watermarks["Bearer second"] > pin + }, 2*time.Second, 10*time.Millisecond) + require.NoError(t, c.Stop()) + mu.Lock() + defer mu.Unlock() + require.Equal(t, uint64(2), sequences["Bearer producer-token"]) + require.Equal(t, uint64(2), sequences["Bearer second"]) +} + +func TestCustomRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) { + for _, checkpoint := range []uint64{0, 2} { + t.Run(strconv.Itoa(int(checkpoint)), func(t *testing.T) { + calls := 0 + var first []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/hello") { + fmt.Fprintf(w, `{"incarnation":"new","accepted_sequence":%d,"checkpoint_sequence":%d}`, checkpoint, checkpoint) + return + } + calls++ + body, _ := io.ReadAll(r.Body) + if calls == 1 { + first = body + require.Equal(t, "old", r.Header.Get("X-Incarnation")) + require.Equal(t, "2", r.Header.Get("X-Sequence")) + w.WriteHeader(409) + return + } + require.Equal(t, first, body) + require.Equal(t, "new", r.Header.Get("X-Incarnation")) + require.Equal(t, "1", r.Header.Get("X-Sequence")) + fmt.Fprint(w, `{"accepted":1}`) + })) + defer server.Close() + c := newCustomTest(t, server.URL) + ch := make(chan metrics.SampleContainer) + tracker, release := metrics.BindRequestTracker(ch) + defer release() + c.tracker = tracker + c.sequence = 1 + c.incarnation = "old" + require.NoError(t, c.sendBatch([]byte("identical-compressed-body"), "multipart/mixed")) + if checkpoint == 0 { + require.Equal(t, 2, calls) + require.Equal(t, uint64(1), c.sequence) + } else { + require.Equal(t, 1, calls) + require.Equal(t, uint64(2), c.sequence) + } + }) + } +} + +func TestTrackedHTTPJoinKeepsPinUntilFailureMetric(t *testing.T) { + c := newCustomTest(t, "http://localhost") + channel := make(chan metrics.SampleContainer) + tracker, release := metrics.BindRequestTracker(channel) + defer release() + c.tracker = tracker + ticket := metrics.BeginRequest(channel) + reg := metrics.NewRegistry() + tags := reg.RootTagSet().With("name", "/tracked") + names := []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving"} + samples := metrics.Samples{} + for _, name := range names { + samples = append(samples, metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric(name, metrics.Trend), Tags: tags}, Time: ticket.Time, Value: 12}) + } + c.AddMetricSamples([]metrics.SampleContainer{metrics.RequestSamples{SampleContainer: samples, Ticket: ticket}}) + require.Len(t, c.groups, 1) + require.Len(t, c.groupTickets, 1) + require.False(t, tracker.Before().After(ticket.Time)) + failed := metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric("http_req_failed", metrics.Rate), Tags: tags}, Time: ticket.Time, Value: 1, HTTPErrorResBody: "delayed failure"} + c.AddMetricSamples([]metrics.SampleContainer{metrics.RequestSamples{SampleContainer: failed, Ticket: ticket}}) + require.Empty(t, c.groups) + require.Empty(t, c.groupTickets) + require.NoError(t, c.err()) + require.NotNil(t, c.parts["requests_raw"]) + require.NotNil(t, c.parts["requests_error"]) +} + +func TestUntrackedRequestCannotCrossNativeWatermark(t *testing.T) { + c := newCustomTest(t, "http://localhost") + channel := make(chan metrics.SampleContainer) + tracker, release := metrics.BindRequestTracker(channel) + defer release() + c.tracker = tracker + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("custom_missing_hook", nil, false)}) + require.ErrorContains(t, c.err(), "lacks producer completion tracking") + require.Empty(t, c.parts) +} diff --git a/internal/output/breakingit/custom_watermark.go b/internal/output/breakingit/custom_watermark.go new file mode 100644 index 000000000..b9c728ca5 --- /dev/null +++ b/internal/output/breakingit/custom_watermark.go @@ -0,0 +1,38 @@ +package breakingit + +import ( + "errors" +) + +func (c *CustomOutput) hello() error { + hello, err := c.register() + if err != nil { + return err + } + if hello.Sealed || hello.Incarnation == "" || hello.Accepted == nil || *hello.Accepted != 0 { + return errors.New("Custom DB requires a fresh unfinished producer source") + } + c.incarnation = hello.Incarnation + return nil +} + +// Called under addMu, after detaching every serialized row. Tickets in the engine +// queue and incomplete joins still pin Before(). Each FIFO carries its own barrier: +// the database takes the minimum across all lanes and generators. +func (c *CustomOutput) publishWatermark() { + if c.tracker == nil || c.err() != nil { + return + } + before := c.tracker.Before().UnixMicro() - pgEpochSeconds*1_000_000 + if before <= c.queuedWatermark { + return + } + for _, lane := range c.lanes { + select { + case <-c.failed: + return + case lane.queue <- customBatch{beforeUS: before}: + } + } + c.queuedWatermark = before +} diff --git a/lib/netext/httpext/tracer.go b/lib/netext/httpext/tracer.go index 673770d8a..1b8cc58f2 100644 --- a/lib/netext/httpext/tracer.go +++ b/lib/netext/httpext/tracer.go @@ -14,7 +14,8 @@ import ( // A Trail represents detailed information about an HTTP request. // You'd typically get one from a Tracer. type Trail struct { - EndTime time.Time + requestTicket *metrics.RequestTicket + EndTime time.Time // Total connect time (Connecting + TLSHandshaking) ConnDuration time.Duration @@ -121,6 +122,8 @@ func (tr *Trail) SaveSamples(builtinMetrics *metrics.BuiltinMetrics, ctm *metric }...) } +func (tr *Trail) RequestTicket() *metrics.RequestTicket { return tr.requestTicket } + // GetSamples implements the metrics.SampleContainer interface. func (tr *Trail) GetSamples() []metrics.Sample { return tr.Samples diff --git a/lib/netext/httpext/transport.go b/lib/netext/httpext/transport.go index 0ab937224..f5b328347 100644 --- a/lib/netext/httpext/transport.go +++ b/lib/netext/httpext/transport.go @@ -79,7 +79,12 @@ func newTransport( // Helper method to finish the tracer trail, assemble the tag values and emits // the metric samples for the supplied unfinished request. func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRequest { + ticket := metrics.BeginRequest(t.state.Samples) trail := unfReq.tracer.Done() + if ticket != nil { + trail.EndTime = ticket.Time + } + trail.requestTicket = ticket result := &finishedRequest{ unfinishedRequest: unfReq, @@ -167,7 +172,9 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe } trail.Samples = append(trail.Samples, failedSample) } - metrics.PushIfNotDone(t.ctx, t.state.Samples, trail) + if !metrics.PushIfNotDone(t.ctx, t.state.Samples, trail) { + ticket.Release() + } return result } diff --git a/metrics/request_tracker.go b/metrics/request_tracker.go new file mode 100644 index 000000000..8ddc29927 --- /dev/null +++ b/metrics/request_tracker.go @@ -0,0 +1,110 @@ +package metrics + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +// RequestTracker pins request-end timestamps until the output has serialized the +// entire request. Registration happens before assigning the timestamp, not when +// the metrics eventually arrive at the output. Only opted-in outputs pay this cost. +type requestShard struct { + mu sync.Mutex + active map[*RequestTicket]time.Time +} +type RequestTracker struct { + origin time.Time + next atomic.Uint64 + shards [64]requestShard +} + +type RequestTicket struct { + tracker *RequestTracker + shard uint64 + Time time.Time +} + +var requestTrackers sync.Map // chan<- SampleContainer -> *RequestTracker + +func BindRequestTracker(ch chan<- SampleContainer) (*RequestTracker, func()) { + t := &RequestTracker{origin: time.Now()} + for i := range t.shards { + t.shards[i].active = make(map[*RequestTicket]time.Time) + } + requestTrackers.Store(ch, t) + return t, func() { requestTrackers.CompareAndDelete(ch, t) } +} + +// BeginRequest assigns an end timestamp using an anchored monotonic clock. +// The pin and the timestamp are atomic with respect to Before(). +func BeginRequest(ch chan<- SampleContainer) *RequestTicket { + v, ok := requestTrackers.Load(ch) + if !ok { + return nil + } + return v.(*RequestTracker).Begin() +} +func (t *RequestTracker) Begin() *RequestTicket { + index := t.next.Add(1) % uint64(len(t.shards)) + shard := &t.shards[index] + shard.mu.Lock() + defer shard.mu.Unlock() + stamp := t.Clock() + ticket := &RequestTicket{tracker: t, shard: index, Time: stamp} + shard.active[ticket] = stamp + return ticket +} +func (t *RequestTicket) Release() { + if t == nil { + return + } + shard := &t.tracker.shards[t.shard] + shard.mu.Lock() + delete(shard.active, t) + shard.mu.Unlock() +} +func (t *RequestTracker) Before() time.Time { + // New pins created during the scan have timestamps >= this captured clock. + before := t.Clock() + for i := range t.shards { + shard := &t.shards[i] + shard.mu.Lock() + for _, stamp := range shard.active { + if stamp.Before(before) { + before = stamp + } + } + shard.mu.Unlock() + } + return before.Truncate(time.Second) +} +func (t *RequestTracker) Clock() time.Time { return t.origin.Add(time.Since(t.origin)) } + +// RequestSamples preserves the normal payload while identifying its lifecycle pin. +type RequestSamples struct { + SampleContainer + Ticket *RequestTicket +} + +func (s RequestSamples) RequestTicket() *RequestTicket { return s.Ticket } + +// PushTrackedRequest keeps the lifecycle pin across both the engine queue and +// the output's mapping queue. A cancelled producer releases a never-enqueued pin. +func PushTrackedRequest(ctx context.Context, ch chan<- SampleContainer, ticket *RequestTicket, sample SampleContainer) bool { + if ticket != nil { + sample = RequestSamples{SampleContainer: sample, Ticket: ticket} + } + if PushIfNotDone(ctx, ch, sample) { + return true + } + ticket.Release() + return false +} +func RequestTime(ticket *RequestTicket, fallback time.Time) time.Time { + if ticket != nil { + return ticket.Time + } + return fallback +} diff --git a/metrics/request_tracker_test.go b/metrics/request_tracker_test.go new file mode 100644 index 000000000..55fff1897 --- /dev/null +++ b/metrics/request_tracker_test.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRequestTrackerPinsBeforeTimestampAssignment(t *testing.T) { + ch := make(chan SampleContainer, 1) + tracker, unbind := BindRequestTracker(ch) + defer unbind() + ticket := BeginRequest(ch) + require.NotNil(t, ticket) + require.False(t, tracker.Before().After(ticket.Time)) + ticket.Release() + ticket.Release() + for i := range tracker.shards { + require.Empty(t, tracker.shards[i].active) + } + unbind() + require.Nil(t, BeginRequest(ch)) +} + +func TestRequestTrackerConcurrentProducers(t *testing.T) { + ch := make(chan SampleContainer) + tracker, unbind := BindRequestTracker(ch) + defer unbind() + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + for range 1000 { + ticket := BeginRequest(ch) + require.False(t, tracker.Before().After(ticket.Time)) + ticket.Release() + } + }) + } + wg.Wait() + for i := range tracker.shards { + require.Empty(t, tracker.shards[i].active) + } +} diff --git a/output/manager.go b/output/manager.go index 0a65d67c1..6c5b5397c 100644 --- a/output/manager.go +++ b/output/manager.go @@ -1,6 +1,7 @@ package output import ( + "fmt" "sync" "time" @@ -40,7 +41,25 @@ func NewManager(outputs []Output, logger logrus.FieldLogger, testStopCallback fu // all of its buffered metrics have been sent to all outputs. The second // callback will call the Stop() or StopWithTestError() method of every output. func (om *Manager) Start(samplesChan chan metrics.SampleContainer) (wait func(), finish func(error), err error) { + var unbind func() + for _, out := range om.outputs { + if tracked, ok := out.(interface{ SetRequestTracker(*metrics.RequestTracker) }); ok { + if mode, ok := out.(interface{ UsesRequestTracking() bool }); ok && !mode.UsesRequestTracking() { + continue + } + if unbind != nil { + unbind() + return nil, nil, fmt.Errorf("only one request-tracked output is supported") + } + tracker, release := metrics.BindRequestTracker(samplesChan) + unbind = release + tracked.SetRequestTracker(tracker) + } + } if err := om.startOutputs(); err != nil { + if unbind != nil { + unbind() + } return nil, nil, err } @@ -78,6 +97,9 @@ func (om *Manager) Start(samplesChan chan metrics.SampleContainer) (wait func(), finish = func(testErr error) { wait() // just in case, though it shouldn't be needed if API is used correctly om.stopOutputs(testErr, len(om.outputs)) + if unbind != nil { + unbind() + } } return wait, finish, nil } From 3b6d549c8ab77c34a190e819075c570104255284 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Fri, 30 Oct 2026 11:16:21 +0100 Subject: [PATCH 3/7] Track browser completion watermarks and bound full-batch zstd windows --- ai-tests/custom-db/smoke.py | 48 ++++++++++- .../k6/browser/common/network_manager.go | 79 +++++++++++-------- .../k6/browser/common/network_manager_test.go | 12 +-- .../k6/browser/common/request_completion.go | 24 ++++++ .../browser/common/request_completion_test.go | 57 +++++++++++++ internal/output/breakingit/CUSTOM_DB.md | 32 +++++--- internal/output/breakingit/custom.go | 16 ++-- internal/output/breakingit/custom_test.go | 17 ++++ 8 files changed, 225 insertions(+), 60 deletions(-) create mode 100644 internal/js/modules/k6/browser/common/request_completion.go create mode 100644 internal/js/modules/k6/browser/common/request_completion_test.go diff --git a/ai-tests/custom-db/smoke.py b/ai-tests/custom-db/smoke.py index 97fee9598..838306797 100644 --- a/ai-tests/custom-db/smoke.py +++ b/ai-tests/custom-db/smoke.py @@ -15,6 +15,7 @@ from pathlib import Path import secrets import socket +import html import subprocess import tempfile import threading @@ -36,9 +37,19 @@ def log_message(self, *_): pass def do_GET(self): - self.send_response(503 if self.path == "/failure" else 200) + if self.path == '/redirect': + self.send_response(302);self.send_header('Location','/ok');self.end_headers();return + if self.path == '/reset': + self.connection.shutdown(socket.SHUT_RDWR);self.connection.close();return + body=b'fixture' if self.path=='/page' else b'fixture response' + self.send_response(503 if self.path in ('/failure','/slowfailure') else 200) + self.send_header('Content-Type','text/html') + self.send_header('Content-Length',str(len(body))) + self.send_header('Cache-Control','max-age=3600' if self.path=='/cached' else 'no-store') self.end_headers() - self.wfile.write(b"fixture response") + if self.path=='/slowfailure': + self.wfile.write(body[:3]);self.wfile.flush();time.sleep(1.2);body=body[3:] + self.wfile.write(body) def do_POST(self): if self.path != "/ingest/batch": @@ -109,6 +120,23 @@ def post(path, payload): sleep(0.15); } ''') + if args.browser: + script.write_text("""import { browser } from 'k6/browser'; +import { group, check, sleep } from 'k6'; +export const options = { scenarios: { browser: {executor:'per-vu-iterations',vus:1,iterations:Number(__ENV.FIXTURE_ITERATIONS),options:{browser:{type:'chromium'}}} } }; +export default async function(){ + const page=await browser.newPage(); + try { await group('Checkout',async function(){ + await page.goto(__ENV.TARGET+'/page'); + const status=await page.evaluate(async()=>{ + await fetch('/redirect');await fetch('/cached');await fetch('/cached'); + try {await fetch('/reset')}catch(e){} + return (await fetch('/slowfailure').then(async r=>{await r.text();return r.status})); + }); + check(status,{'expected failure': s=>s===200}); + }); sleep(.3); } finally {await page.close();} +} +""") target_url = f"http://127.0.0.1:{target.server_port}" writer_env = {**os.environ, "CUSTOM_STORAGE_TOKEN": producer, "runId": "k6-smoke", "nodeName": "k6-probe", "location": "Local", "PG_PROXY_TOKEN": "isolated-oracle", @@ -118,6 +146,16 @@ def post(path, payload): command = [str(Path(args.k6).resolve()), "run", "--quiet", "--out", "custom-db-breakingit=" + service, "--out", "timescaledb-breakingit=" + target_url, str(script)] + if args.container_image: + root.chmod(0o755); script.chmod(0o644) + for key in ('TARGET',): writer_env[key]=writer_env[key].replace('127.0.0.1','host.docker.internal') + command=['docker','run','--rm','--shm-size','512m','--entrypoint','/usr/bin/k6', + '-v',str(Path(args.k6).resolve())+':/usr/bin/k6:ro','-v',str(script)+':/fixture.js:ro', + '-e','K6_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium','-e','K6_BROWSER_ARGS=no-sandbox', + '-e','K6_BROWSER_HEADLESS=true', + *[item for key in ('CUSTOM_STORAGE_TOKEN','runId','nodeName','location','PG_PROXY_TOKEN','TARGET','K6_NO_USAGE_REPORT','K6_HTTP_ERROR_DATA','FIXTURE_ITERATIONS','FIXTURE_CUSTOM') for item in ('-e',key)], + args.container_image,'run','--quiet','--out','custom-db-breakingit='+service.replace('127.0.0.1','host.docker.internal'), + '--out','timescaledb-breakingit='+target_url.replace('127.0.0.1','host.docker.internal'),'/fixture.js'] observations=[] with (root/'k6.log').open('w+') as log: process=subprocess.Popen(command,env=writer_env,text=True,stdout=log,stderr=log) @@ -156,7 +194,7 @@ def post(path, payload): counts[table] = len(actual["rows"]) if not args.restart_live: assert counts[table] == len(oracle[table]), (table, counts[table], len(oracle[table])) else: assert counts[table] <= len(oracle[table]), (table, counts) - if table in expected_counts and not args.restart_live: + if table in expected_counts and not args.restart_live and not args.browser: assert counts[table] == expected_counts[table], (table, counts) else: assert counts[table] > 0 @@ -195,7 +233,7 @@ def normalized(row, csv_row=False): for table, count in counts.items(): actual = post("/runs/k6-smoke/query", {"table": table, "start_us": origin, "end_us": end}) assert len(actual["rows"]) == count - print(json.dumps({"status": "passed", "counts": counts, "lanes":args.lanes, "live_watermark_observations":len(live), "crash_recovery":args.restart_live, "coverage_gaps":final["gaps"], "parity": "retained rows are a duplicate-free subset of oracle" if args.restart_live else "all columns except generated error UUIDs", "sealed": True, "restart_reads": True}, indent=2)) + print(json.dumps({"status": "passed", "counts": counts, "lanes":args.lanes, "live_watermark_observations":len(live), "crash_recovery":args.restart_live, "browser":args.browser, "coverage_gaps":final["gaps"], "parity": "retained rows are a duplicate-free subset of oracle" if args.restart_live else "all columns except generated error UUIDs", "sealed": True, "restart_reads": True}, indent=2)) finally: subprocess.run(["docker", "rm", "-f", name], capture_output=True) target.shutdown() @@ -209,4 +247,6 @@ def normalized(row, csv_row=False): parser.add_argument('--iterations',type=int,default=80) parser.add_argument('--restart-live',action='store_true') parser.add_argument('--custom-metrics',action='store_true') + parser.add_argument('--browser',action='store_true') + parser.add_argument('--container-image') run(parser.parse_args()) diff --git a/internal/js/modules/k6/browser/common/network_manager.go b/internal/js/modules/k6/browser/common/network_manager.go index 5b10c9314..8f90c99ab 100644 --- a/internal/js/modules/k6/browser/common/network_manager.go +++ b/internal/js/modules/k6/browser/common/network_manager.go @@ -286,7 +286,13 @@ func (m *NetworkManager) emitRequestMetrics(req *Request) { }) } -func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //nolint:funlen +func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request, completion browserCompletion) { //nolint:funlen + transferred := false + defer func() { + if !transferred { + completion.ticket.Release() + } + }() state := m.vu.State() if state == nil { return // CDP events can arrive after iteration teardown. @@ -368,67 +374,70 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //n return } - pushIfNotDone(m.vu.Context(), m.logger, state.Samples, k6metrics.ConnectedSamples{ + bundle := k6metrics.ConnectedSamples{ Samples: []k6metrics.Sample{ { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqDuration, Tags: tags}, Value: timings.durationMs, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqBlocked, Tags: tags}, Value: timings.blocked, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqConnecting, Tags: tags}, Value: timings.connecting, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqTLSHandshaking, Tags: tags}, Value: timings.tlsHandshaking, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqSending, Tags: tags}, Value: timings.sending, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqWaiting, Tags: tags}, Value: timings.waiting, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqReceiving, Tags: tags}, Value: timings.receiving, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserDataReceived, Tags: tags}, Value: float64(bodySize), - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, }, - }) + } - if resp != nil { + { + if resp == nil { + failed = 1 + } sample := k6metrics.Sample{ TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqFailed, Tags: tags}, Value: failed, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, } - if failed == 1 { + if failed == 1 && resp != nil { reqHeaders, reqBody := req.formatErrorRequestData() sample.HTTPErrorReqHeaders = reqHeaders sample.HTTPErrorReqBody = reqBody @@ -438,10 +447,9 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //n sample.HTTPErrorResHeaders = resHeaders sample.HTTPErrorResBody = resBody } - pushIfNotDone(m.vu.Context(), m.logger, state.Samples, k6metrics.ConnectedSamples{ - Samples: []k6metrics.Sample{sample}, - }) + bundle.Samples = append(bundle.Samples, sample) } + transferred = k6metrics.PushTrackedRequest(m.vu.Context(), state.Samples, completion.ticket, bundle) } type browserResponseTimings struct { @@ -542,7 +550,13 @@ func (m *NetworkManager) calcReceiving(req *Request, t *network.ResourceTiming, // ERR_CONNECTION_REFUSED). This ensures outputs like breakingit receive metrics for // failed navigations/requests. The failure reason (e.g. net::ERR_NAME_NOT_RESOLVED) is // added as the "error" tag so CSV and other outputs can show it. -func (m *NetworkManager) emitRequestFailedMetrics(req *Request) { +func (m *NetworkManager) emitRequestFailedMetrics(req *Request, completion browserCompletion) { + transferred := false + defer func() { + if !transferred { + completion.ticket.Release() + } + }() state := m.vu.State() if state == nil { return @@ -586,60 +600,60 @@ func (m *NetworkManager) emitRequestFailedMetrics(req *Request) { durationMs := k6metrics.D(wallTime.Sub(req.wallTime)) reqHeaders, reqBody := req.formatErrorRequestData() - k6metrics.PushIfNotDone(m.vu.Context(), state.Samples, k6metrics.ConnectedSamples{ + transferred = k6metrics.PushTrackedRequest(m.vu.Context(), state.Samples, completion.ticket, k6metrics.ConnectedSamples{ Samples: []k6metrics.Sample{ { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqDuration, Tags: tags}, Value: durationMs, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqBlocked, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqConnecting, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqTLSHandshaking, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqSending, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqWaiting, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqReceiving, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserDataReceived, Tags: tags}, Value: 0, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqFailed, Tags: tags}, Value: 1, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, HTTPErrorReqHeaders: reqHeaders, HTTPErrorReqBody: reqBody, @@ -669,6 +683,7 @@ func handleURLTag(mi eventInterceptor, url string, method string, tags *k6metric func (m *NetworkManager) handleRequestRedirect( req *Request, redirectResponse *network.Response, hasExtraInfo bool, timestamp *cdp.MonotonicTime, ) { + completion := m.beginBrowserCompletion() resp := NewHTTPResponse(m.ctx, req, redirectResponse, timestamp) m.extraInfoTracker.processResponse(req.requestID, resp, hasExtraInfo) req.responseMu.Lock() @@ -681,7 +696,7 @@ func (m *NetworkManager) handleRequestRedirect( // them (the redirect's ExtraInfo may still be in flight here). m.wg.Go(func() { resp.WaitForRawHeaders() - m.emitResponseMetrics(resp, req) + m.emitResponseMetrics(resp, req, completion) }) m.deleteRequestByID(req.requestID) @@ -783,7 +798,7 @@ func (m *NetworkManager) onLoadingFailed(event *network.EventLoadingFailed) { // Emit browser metrics for failed requests so outputs (e.g. breakingit) can store them. if !isInternalURL(req.url) { - m.emitRequestFailedMetrics(req) + m.emitRequestFailedMetrics(req, m.beginBrowserCompletion()) } m.eventInterceptor.onRequestFailed(req) @@ -800,6 +815,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished) return } + completion := m.beginBrowserCompletion() req.responseEndWall = event.Timestamp.Time() req.responseEndTiming = float64(event.Timestamp.Time().Sub(req.timestamp).Milliseconds()) req.encodedDataLength = int64(event.EncodedDataLength) @@ -811,6 +827,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished) // Skip data and blob URLs when emitting metrics, since they're internal to the browser. if isInternalURL(req.url) { + completion.ticket.Release() return } // Emit the response metrics in a separate goroutine, once the raw headers @@ -827,7 +844,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished) if resp != nil { resp.WaitForRawHeaders() } - m.emitResponseMetrics(resp, req) + m.emitResponseMetrics(resp, req, completion) }) } diff --git a/internal/js/modules/k6/browser/common/network_manager_test.go b/internal/js/modules/k6/browser/common/network_manager_test.go index d1338a4a0..7fe402094 100644 --- a/internal/js/modules/k6/browser/common/network_manager_test.go +++ b/internal/js/modules/k6/browser/common/network_manager_test.go @@ -311,7 +311,7 @@ func TestNetworkManagerEmitRequestResponseMetricsTimingSkew(t *testing.T) { &network.Response{Timing: &network.ResourceTiming{}}, (*cdp.MonotonicTime)(&tt.res.ts), ) - nm.emitResponseMetrics(res, req) + nm.emitResponseMetrics(res, req, browserCompletion{at: tt.wantRes.wt}) n = vu.AssertSamples(func(s k6metrics.Sample) { assert.Equalf(t, tt.wantRes.wt, s.Time, "timing skew in %s", s.Metric.Name) }) @@ -363,7 +363,7 @@ func TestNetworkManagerMetricsRetainRequestTagsAndMetadata(t *testing.T) { URL: req.URL(), Timing: &network.ResourceTiming{}, }, (*cdp.MonotonicTime)(&now)) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) n := vu.AssertSamples(func(sample k6metrics.Sample) { group, ok := sample.Tags.Get(k6metrics.TagGroup.String()) @@ -379,7 +379,7 @@ func TestNetworkManagerMetricsRetainRequestTagsAndMetadata(t *testing.T) { // The fork also emits samples when navigation fails before any response. // Those must use the request's context, not the VU's restored context. req.setErrorText("net::ERR_CONNECTION_REFUSED") - nm.emitRequestFailedMetrics(req) + nm.emitRequestFailedMetrics(req, nm.beginBrowserCompletion()) n = vu.AssertSamples(func(sample k6metrics.Sample) { group, _ := sample.Tags.Get(k6metrics.TagGroup.String()) assert.Equal(t, "::request", group) @@ -621,7 +621,7 @@ func TestNetworkManagerReceivingMetric(t *testing.T) { }, (*cdp.MonotonicTime)(&reqTimestamp), ) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) var receivingVal float64 vu.AssertSamples(func(s k6metrics.Sample) { @@ -687,7 +687,7 @@ func TestNetworkManagerReceivingMetricNoHeadersEndWall(t *testing.T) { }, (*cdp.MonotonicTime)(&reqTimestamp), ) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) var receivingVal float64 vu.AssertSamples(func(s k6metrics.Sample) { @@ -751,7 +751,7 @@ func TestNetworkManagerReceivingMetricCachedResponse(t *testing.T) { }, (*cdp.MonotonicTime)(&reqTimestamp), ) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) var receivingVal float64 receivingFound := false diff --git a/internal/js/modules/k6/browser/common/request_completion.go b/internal/js/modules/k6/browser/common/request_completion.go new file mode 100644 index 000000000..aaf7570ab --- /dev/null +++ b/internal/js/modules/k6/browser/common/request_completion.go @@ -0,0 +1,24 @@ +package common + +import ( + "time" + + k6metrics "go.k6.io/k6/v2/metrics" +) + +// Browser completion is observed at terminal CDP-event dispatch, before async +// header/body retrieval. Chromium phase durations remain in their CDP time base; +// the result's timestamp is the generator-observed sampler completion, for ALL +// outputs. Retimestamping at output delivery would hide backlog and is forbidden. +type browserCompletion struct { + at time.Time + ticket *k6metrics.RequestTicket +} + +func (m *NetworkManager) beginBrowserCompletion() browserCompletion { + var ticket *k6metrics.RequestTicket + if state := m.vu.State(); state != nil { + ticket = k6metrics.BeginRequest(state.Samples) + } + return browserCompletion{at: k6metrics.RequestTime(ticket, time.Now()), ticket: ticket} +} diff --git a/internal/js/modules/k6/browser/common/request_completion_test.go b/internal/js/modules/k6/browser/common/request_completion_test.go new file mode 100644 index 000000000..1f275dd3f --- /dev/null +++ b/internal/js/modules/k6/browser/common/request_completion_test.go @@ -0,0 +1,57 @@ +package common + +import ( + "net/url" + "testing" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/internal/js/modules/k6/browser/k6ext" + "go.k6.io/k6/v2/internal/js/modules/k6/browser/k6ext/k6test" + "go.k6.io/k6/v2/internal/js/modules/k6/browser/log" + k6metrics "go.k6.io/k6/v2/metrics" +) + +func TestBrowserCompletionPinsHeaderDelayAndOutputQueue(t *testing.T) { + samples := make(chan k6metrics.SampleContainer, 10) + tracker, unbind := k6metrics.BindRequestTracker(samples) + defer unbind() + vu := k6test.NewVU(t, k6test.WithSamples(samples)) + vu.ActivateVU() + nm := &NetworkManager{vu: vu, customMetrics: k6ext.RegisterCustomMetrics(k6metrics.NewRegistry()), eventInterceptor: &EventInterceptorMock{}, logger: log.NewNullLogger()} + u, _ := url.Parse("https://example.test/error") + req := &Request{url: u, wallTime: time.Now().Add(-time.Second), tagsAndMeta: vu.State().Tags.GetCurrentValues()} + completion := nm.beginBrowserCompletion() + // A captured terminal event waits through delayed headers/body extraction. + require.NotNil(t, completion.ticket) + time.Sleep(1100 * time.Millisecond) + require.False(t, tracker.Before().After(completion.at)) + resp := &Response{url: u.String(), status: 503, remoteAddress: &RemoteAddress{}, wallTime: completion.at.Add(-500 * time.Millisecond), timing: &network.ResourceTiming{}, body: []byte("error detail"), request: req} + nm.emitResponseMetrics(resp, req, completion) + require.False(t, tracker.Before().After(completion.at)) // buffered output still pins it + frame := (<-samples).(k6metrics.RequestSamples) + require.Len(t, frame.GetSamples(), 9) + for _, sample := range frame.GetSamples() { + require.Equal(t, completion.at, sample.Time) + } + require.Equal(t, float64(1), frame.GetSamples()[8].Value) + frame.Ticket.Release() + require.True(t, tracker.Before().After(completion.at)) +} + +func TestBrowserCacheSkipReleasesCompletionPin(t *testing.T) { + samples := make(chan k6metrics.SampleContainer, 10) + tracker, unbind := k6metrics.BindRequestTracker(samples) + defer unbind() + vu := k6test.NewVU(t, k6test.WithSamples(samples)) + vu.ActivateVU() + nm := &NetworkManager{vu: vu, customMetrics: k6ext.RegisterCustomMetrics(k6metrics.NewRegistry()), eventInterceptor: &EventInterceptorMock{}, logger: log.NewNullLogger()} + u, _ := url.Parse("https://example.test/cached") + req := &Request{url: u, wallTime: time.Now(), tagsAndMeta: vu.State().Tags.GetCurrentValues(), fromMemoryCache: true} + completion := nm.beginBrowserCompletion() + nm.emitResponseMetrics(&Response{url: u.String(), remoteAddress: &RemoteAddress{}, wallTime: time.Now(), request: req, body: []byte("cached")}, req, completion) + require.Empty(t, samples) + time.Sleep(1100 * time.Millisecond) + require.True(t, tracker.Before().After(completion.at)) +} diff --git a/internal/output/breakingit/CUSTOM_DB.md b/internal/output/breakingit/CUSTOM_DB.md index 3c8b1b261..a51880c7b 100644 --- a/internal/output/breakingit/CUSTOM_DB.md +++ b/internal/output/breakingit/CUSTOM_DB.md @@ -60,16 +60,16 @@ Generator machine statistics remain the responsibility of the existing agent. - Drain accepted sample containers and all queued requests before `/finish`. The producer does not seal the run itself. A final barrier proves delivery of this producer's data; it is not a durable-on-disk/fsync acknowledgement. -- A fresh producer source is required at startup. Native runs recover after a +- A fresh producer source is required at startup. Tracked native and browser runs recover after a server incarnation change using the anchored source clock and checkpoint sequence: skip a held batch proven checkpointed; otherwise renumber and replay only that unacknowledged batch. Previously acknowledged, lost rows are not replayed; the service exposes its relaxed-durability coverage gaps. Same-incarnation conflicts - remain fatal. Browser runs without a proven source clock fail closed on restart. + remain fatal. ## Watermarks and unsupported data -Native k6 runs send live request watermarks, using the same sequence-backed, +Native and browser k6 runs send live request watermarks, using the same sequence-backed, all-source barrier as the Java writer. The output manager installs a per-run tracker before producers start. HTTP completion assigns its timestamp and pin atomically; WebSocket connecting/ping and custom metrics are similarly tracked. @@ -84,13 +84,19 @@ seconds remain queryable through the pending/raw path. Event-time completeness still does not imply fsync durability. Transactions and monitoring are separate streams, just as in the Java request-watermark contract. -**Browser scenarios are not yet at watermark parity.** Their request timestamps -come from asynchronous Chromium/CDP events in another process. They retain the -previous grace/pending/raw path, including mixed browser/native scenarios; a host -wall-clock timer cannot prove their event-time completeness. Browser timestamps -are not rewritten to manufacture progress. Browser live barriers require a -separate CDP lifecycle contract. Uninstrumented native request metric producers -fail closed rather than silently violating a watermark. +Browser requests acquire a completion pin when the generator dispatches the +terminal CDP event (including redirects and loading failures), before deferred +header/body retrieval. They emit one complete timing/failure bundle. The pin +survives deferred work and the output queue; skipped cached requests release it. +Mixed browser/native scenarios use the same tracker and lane barriers. + +**Timestamp contract:** browser sampler timestamps now mean generator-observed +terminal CDP completion, for both Custom DB and TimescaleDB outputs. Previously, +successful rows used the response-header timestamp. Chromium phase/duration +measurements remain in their original time base. CDP transport delay precedes +this observation; this is not a claim that Chromium's original event clock is +watermarked. Nothing is retimestamped at output delivery to hide writer backlog. +Uninstrumented request metric producers fail closed. Synthetic monitoring and browser screenshots are explicitly unsupported and rejected. Ordinary browser request timings, failure response data, native groups @@ -122,3 +128,9 @@ The live smoke test also requires unfinished sources with advancing watermarks a complete aggregate buckets before finalization. Crash mode kills only its own throwaway service, verifies retained rows against the oracle without duplicates, and requires explicit incomplete coverage whenever rows were lost. + +Browser validation also runs real Chromium with redirects, iframe requests, +connection failure, caching and a delayed 503 response, comparing both writers +on the same stream. Use `--browser --container-image IMAGE` with a Linux binary +and an image containing Chromium. This mapping oracle is distinct from the +actual-database high-load benchmark in the BreakTest integration repository. diff --git a/internal/output/breakingit/custom.go b/internal/output/breakingit/custom.go index 8d805e462..0f44c9a86 100644 --- a/internal/output/breakingit/custom.go +++ b/internal/output/breakingit/custom.go @@ -29,7 +29,6 @@ type customGroupKey struct { } type CustomOutput struct { - browserMode bool groups map[customGroupKey]*MetricGroup groupTickets map[customGroupKey]*metrics.RequestTicket tracker *metrics.RequestTracker @@ -118,11 +117,6 @@ func NewCustom(params output.Params) (output.Output, error) { retryTimeout: 30 * time.Second, queue: make(chan customBatch, 4), done: make(chan struct{}), tickStop: make(chan struct{}), tickDone: make(chan struct{}), failed: make(chan struct{}), parts: map[string]*bytes.Buffer{}, } - for _, scenario := range params.ScriptOptions.Scenarios { - if opts := scenario.GetScenarioOptions(); opts != nil && len(opts.Browser) > 0 { - c.browserMode = true - } - } c.groups = map[customGroupKey]*MetricGroup{} c.groupTickets = map[customGroupKey]*metrics.RequestTicket{} c.mapper = &Logger{out: params.StdOut, envTags: tags, customPatterns: patterns, transactionSource: source, @@ -131,8 +125,6 @@ func NewCustom(params output.Params) (output.Output, error) { return c, nil } -func (c *CustomOutput) UsesRequestTracking() bool { return !c.browserMode } - func (c *CustomOutput) SetRequestTracker(t *metrics.RequestTracker) { c.tracker = t } func (c *CustomOutput) Description() string { return "custom-db-breakingit" } @@ -186,7 +178,7 @@ func (c *CustomOutput) Start() error { } return err } - encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1)) + encoder, err := newCustomEncoder() if err != nil { return err } @@ -491,3 +483,9 @@ func (c *CustomOutput) request(path string, body []byte, contentType string, seq } } } + +// The storage ingress caps zstd decoder windows at 2^18 bytes, independently +// of the larger decoded batch budget. Enforce this even on full 2 MiB batches. +func newCustomEncoder() (*zstd.Encoder, error) { + return zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), zstd.WithWindowSize(1<<18)) +} diff --git a/internal/output/breakingit/custom_test.go b/internal/output/breakingit/custom_test.go index 5923f0660..061a5474a 100644 --- a/internal/output/breakingit/custom_test.go +++ b/internal/output/breakingit/custom_test.go @@ -461,3 +461,20 @@ func TestUntrackedRequestCannotCrossNativeWatermark(t *testing.T) { require.ErrorContains(t, c.err(), "lacks producer completion tracking") require.Empty(t, c.parts) } + +func TestCustomFullBatchRespectsStorageDecoderWindow(t *testing.T) { + encoder, err := newCustomEncoder() + require.NoError(t, err) + defer encoder.Close() + plain := bytes.Repeat([]byte("large-request-metric-row\n"), 100000) + encoded := encoder.EncodeAll(plain, nil) + var header zstd.Header + require.NoError(t, header.Decode(encoded)) + require.LessOrEqual(t, header.WindowSize, uint64(1<<18)) + decoder, err := zstd.NewReader(nil, zstd.WithDecoderMaxWindow(1<<18)) + require.NoError(t, err) + defer decoder.Close() + decoded, err := decoder.DecodeAll(encoded, nil) + require.NoError(t, err) + require.Equal(t, plain, decoded) +} From df6ec19023bc07ee41a480afd2a1daf6b6eac388 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Fri, 30 Oct 2026 16:44:14 +0100 Subject: [PATCH 4/7] Wait for all TimescaleDB tail batches and metric cleanup at shutdown --- internal/output/breakingit/breakingit.go | 54 ++++++--- internal/output/breakingit/drain_test.go | 133 +++++++++++++++++++++++ 2 files changed, 169 insertions(+), 18 deletions(-) create mode 100644 internal/output/breakingit/drain_test.go diff --git a/internal/output/breakingit/breakingit.go b/internal/output/breakingit/breakingit.go index 3546a67b8..72a84922b 100644 --- a/internal/output/breakingit/breakingit.go +++ b/internal/output/breakingit/breakingit.go @@ -73,6 +73,8 @@ type Logger struct { // batch buffers mu sync.Mutex + flushMu sync.Mutex // owns the entire drain, including follow-up batches + flushErr error bufHTTP *bytes.Buffer bufVUs *bytes.Buffer bufTrans *bytes.Buffer @@ -86,6 +88,7 @@ type Logger struct { // merging state pendingHttpGroups map[int64]*MetricGroup // http and browser (unified) cleanupStop chan struct{} + cleanupDone chan struct{} batchDone chan struct{} // closed when batchLoop exits envTags map[string]string customPatterns map[string]string @@ -179,6 +182,7 @@ func New(params output.Params) (output.Output, error) { bufErrors: &bytes.Buffer{}, pendingHttpGroups: make(map[int64]*MetricGroup), cleanupStop: make(chan struct{}), + cleanupDone: make(chan struct{}), batchDone: make(chan struct{}), envTags: envTags, customPatterns: customPatterns, @@ -195,16 +199,22 @@ func (l *Logger) Description() string { return "breakingit" } func (l *Logger) Start() error { return nil } func (l *Logger) Stop() error { - fmt.Fprintf(l.out, "[screenshot] Stop() called, waiting for batchLoop to finish\n") l.batchTimer.Stop() close(l.cleanupStop) - // Wait for any in-progress batchLoop flush to complete <-l.batchDone - fmt.Fprintf(l.out, "[screenshot] batchLoop done, running final flush (pendingScreenshots=%d)\n", len(l.pendingScreenshots)) - // Final flush to send any remaining data (including screenshots) + <-l.cleanupDone + // Cleanup can own a detached metric group; join it before draining the tail. + l.mu.Lock() + groups := l.pendingHttpGroups + l.pendingHttpGroups = make(map[int64]*MetricGroup) + l.mu.Unlock() + for _, g := range groups { + l.processHttpGroup(g) + } l.flush() - fmt.Fprintf(l.out, "[screenshot] Stop() complete\n") - return nil + l.mu.Lock() + defer l.mu.Unlock() + return l.flushErr } func (l *Logger) batchLoop() { @@ -228,6 +238,7 @@ func (l *Logger) batchLoop() { } func (l *Logger) cleanupProcessor() { + defer close(l.cleanupDone) t := time.NewTicker(100 * time.Millisecond) defer t.Stop() for { @@ -252,7 +263,23 @@ func (l *Logger) cleanupProcessor() { } } +// A single owner drains every accepted tail batch. Stop must join that owner; +// spawning a follow-up goroutine lets Stop return while rows are still in flight. func (l *Logger) flush() { + l.flushMu.Lock() + defer l.flushMu.Unlock() + for { + l.flushOnce() + l.mu.Lock() + more := l.flushErr == nil && (l.bufHTTP.Len() > 0 || l.bufVUs.Len() > 0 || l.bufTrans.Len() > 0 || l.bufErrors.Len() > 0 || len(l.pendingScreenshots) > 0) + l.mu.Unlock() + if !more { + return + } + } +} + +func (l *Logger) flushOnce() { l.mu.Lock() // Prevent recursive flush calls if l.flushing { @@ -418,29 +445,24 @@ func (l *Logger) flush() { // Reset buffers and check for new data that accumulated during retries l.mu.Lock() - hasNewData := false var newHTTPData, newVUsData, newTransData, newErrorsData []byte if success { // Check if any buffers grew during retries (new data accumulated) // If so, preserve the new data before clearing if l.bufHTTP.Len() > bufHTTPLen { - hasNewData = true newHTTPData = make([]byte, l.bufHTTP.Len()-bufHTTPLen) copy(newHTTPData, l.bufHTTP.Bytes()[bufHTTPLen:]) } if l.bufVUs.Len() > bufVUsLen { - hasNewData = true newVUsData = make([]byte, l.bufVUs.Len()-bufVUsLen) copy(newVUsData, l.bufVUs.Bytes()[bufVUsLen:]) } if l.bufTrans.Len() > bufTransLen { - hasNewData = true newTransData = make([]byte, l.bufTrans.Len()-bufTransLen) copy(newTransData, l.bufTrans.Bytes()[bufTransLen:]) } if l.bufErrors.Len() > bufErrorsLen { - hasNewData = true newErrorsData = make([]byte, l.bufErrors.Len()-bufErrorsLen) copy(newErrorsData, l.bufErrors.Bytes()[bufErrorsLen:]) } @@ -465,7 +487,9 @@ func (l *Logger) flush() { l.bufErrors.Write(newErrorsData) } } else { - // Retry loop exited without success (either timeout or early break), clear buffers to prevent unbounded growth + // Report delivery failure at Stop rather than claiming a successful drain. + l.flushErr = fmt.Errorf("pg-proxy delivery failed after %d attempts", attempts) + // Clear buffers to prevent unbounded growth after failed retries. l.bufHTTP.Reset() l.bufVUs.Reset() l.bufTrans.Reset() @@ -474,12 +498,6 @@ func (l *Logger) flush() { l.flushing = false l.mu.Unlock() - // If new data accumulated during retries and we succeeded, flush immediately to send it - // This ensures no gap in data delivery - if success && hasNewData { - // Use a goroutine to avoid blocking and prevent potential deadlock - go l.flush() - } } // upsertScreenshots sends screenshots to the pg-proxy upsert endpoint. diff --git a/internal/output/breakingit/drain_test.go b/internal/output/breakingit/drain_test.go new file mode 100644 index 000000000..c49a6b5d2 --- /dev/null +++ b/internal/output/breakingit/drain_test.go @@ -0,0 +1,133 @@ +package breakingit + +import ( + "compress/gzip" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/output" +) + +// The second request is the follow-up tail of an already running flush. +// Stop must wait for it and for the third row appended while it is in flight. +func TestTimescaleStopJoinsFollowupFlush(t *testing.T) { + t.Setenv("PG_PROXY_TOKEN", "test-token") + first, second := make(chan struct{}), make(chan struct{}) + releaseFirst, releaseSecond := make(chan struct{}), make(chan struct{}) + var onceFirst, onceSecond sync.Once + var mu sync.Mutex + var received []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gz, err := gzip.NewReader(r.Body) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + defer gz.Close() + _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + reader := multipart.NewReader(gz, params["boundary"]) + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Error(err) + return + } + body, _ := io.ReadAll(part) + mu.Lock() + received = append(received, strings.TrimSpace(string(body))) + n := len(received) + mu.Unlock() + if n == 1 { + close(first) + <-releaseFirst + } + if n == 2 { + close(second) + <-releaseSecond + } + } + w.WriteHeader(200) + })) + defer server.Close() + defer onceFirst.Do(func() { close(releaseFirst) }) + defer onceSecond.Do(func() { close(releaseSecond) }) + out, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard}) + require.NoError(t, err) + l := out.(*Logger) + appendRow := func(row string) { l.mu.Lock(); l.bufHTTP.WriteString(row + "\n"); l.mu.Unlock() } + await := func(ch <-chan struct{}) { + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("flush did not reach handler") + } + } + appendRow("first") + go l.flush() + await(first) + appendRow("second") + onceFirst.Do(func() { close(releaseFirst) }) + await(second) + appendRow("third") + stopped := make(chan error, 1) + go func() { stopped <- l.Stop() }() + select { + case err := <-stopped: + onceSecond.Do(func() { close(releaseSecond) }) + t.Fatalf("Stop returned before the in-flight tail was acknowledged: %v", err) + case <-time.After(100 * time.Millisecond): + } + onceSecond.Do(func() { close(releaseSecond) }) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Stop failed to drain") + } + mu.Lock() + defer mu.Unlock() + require.Equal(t, []string{"first", "second", "third"}, received) +} + +func TestTimescaleStopDrainsUnexpiredMetricGroup(t *testing.T) { + t.Setenv("PG_PROXY_TOKEN", "test-token") + var mu sync.Mutex + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + requests++ + mu.Unlock() + w.WriteHeader(200) + })) + defer server.Close() + out, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard}) + require.NoError(t, err) + l := out.(*Logger) + l.mu.Lock() + now := time.Now() + l.pendingHttpGroups[now.UnixNano()] = &MetricGroup{Timestamp: now.UnixNano(), LastUpdate: now, Tags: map[string]string{"name": "tail-request"}, Values: map[string]float64{"http_req_duration": 12}} + l.mu.Unlock() + require.NoError(t, l.Stop()) + mu.Lock() + defer mu.Unlock() + require.Equal(t, 1, requests) + require.Empty(t, l.pendingHttpGroups) +} From c8f1dcc64958ac9380c3a2a1bb3c58077b5b44e7 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Sun, 1 Nov 2026 18:56:08 +0100 Subject: [PATCH 5/7] Name the direct writer BreakTest DB without pre-release aliases --- FORK.md | 4 +- ai-tests/{custom-db => breaktest-db}/smoke.py | 16 +-- internal/cmd/builtin_output_gen.go | 16 +-- internal/cmd/outputs.go | 2 +- internal/cmd/outputs_full.go | 2 +- internal/cmd/outputs_lean.go | 2 +- internal/cmd/outputs_test.go | 6 +- .../{CUSTOM_DB.md => BREAKTEST_DB.md} | 18 +-- internal/output/breakingit/README.md | 8 +- internal/output/breakingit/breakingit.go | 2 +- .../breakingit/{custom.go => breaktest_db.go} | 104 +++++++++--------- ...ustom_binary.go => breaktest_db_binary.go} | 10 +- ...m_recovery.go => breaktest_db_recovery.go} | 24 ++-- .../{custom_test.go => breaktest_db_test.go} | 78 ++++++------- ...watermark.go => breaktest_db_watermark.go} | 8 +- 15 files changed, 150 insertions(+), 150 deletions(-) rename ai-tests/{custom-db => breaktest-db}/smoke.py (94%) rename internal/output/breakingit/{CUSTOM_DB.md => BREAKTEST_DB.md} (91%) rename internal/output/breakingit/{custom.go => breaktest_db.go} (78%) rename internal/output/breakingit/{custom_binary.go => breaktest_db_binary.go} (89%) rename internal/output/breakingit/{custom_recovery.go => breaktest_db_recovery.go} (78%) rename internal/output/breakingit/{custom_test.go => breaktest_db_test.go} (86%) rename internal/output/breakingit/{custom_watermark.go => breaktest_db_watermark.go} (77%) diff --git a/FORK.md b/FORK.md index eeeb2d2a0..fd506140e 100644 --- a/FORK.md +++ b/FORK.md @@ -22,8 +22,8 @@ code to the k6 executable. ## Fork-specific changes - Adds the `timescaledb-breakingit` output for writing load-test data through a PG-proxy. -- Adds the opt-in `custom-db-breakingit` direct output alongside TimescaleDB; see - [configuration and current limits](internal/output/breakingit/CUSTOM_DB.md). +- Adds the opt-in `breaktest-db-breakingit` direct output alongside TimescaleDB; see + [configuration and current limits](internal/output/breakingit/BREAKTEST_DB.md). - Adds optional detailed request and response data for failed HTTP requests. - Extends browser network and error metrics, including optional error screenshots. - Adds the `lean` build tag used for smaller cloud-free release binaries. diff --git a/ai-tests/custom-db/smoke.py b/ai-tests/breaktest-db/smoke.py similarity index 94% rename from ai-tests/custom-db/smoke.py rename to ai-tests/breaktest-db/smoke.py index 838306797..391ea1468 100644 --- a/ai-tests/custom-db/smoke.py +++ b/ai-tests/breaktest-db/smoke.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Isolated k6 -> Rust protocol test with the existing CSV writer as an oracle. -Requires Docker, Python 3, a built k6 fork and a built Custom DB service image. +Requires Docker, Python 3, a built k6 fork and a built BreakTest DB service image. Creates a temporary service/container and directory; never connects to customer DBs. """ import argparse @@ -76,14 +76,14 @@ def do_POST(self): root = Path(temp) data_dir = root / "data" data_dir.mkdir() - env = {**os.environ, "CUSTOM_STORAGE_ADMIN_TOKEN": admin} + env = {**os.environ, "BREAKTEST_DB_ADMIN_TOKEN": admin} with socket.socket() as port_socket: port_socket.bind(('127.0.0.1',0)) fixed_port=port_socket.getsockname()[1] try: subprocess.run(["docker", "run", "-d", "--name", name, "--label", "breaktest.k6-writer-smoke=true", "--memory", "512m", - "-e", "CUSTOM_STORAGE_ADMIN_TOKEN", "-p", f"127.0.0.1:{fixed_port}:8099", + "-e", "BREAKTEST_DB_ADMIN_TOKEN", "-p", f"127.0.0.1:{fixed_port}:8099", "-v", f"{data_dir}:/data", args.image, "v3", "service", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True) port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1] @@ -138,13 +138,13 @@ def post(path, payload): } """) target_url = f"http://127.0.0.1:{target.server_port}" - writer_env = {**os.environ, "CUSTOM_STORAGE_TOKEN": producer, "runId": "k6-smoke", + writer_env = {**os.environ, "BREAKTEST_DB_TOKEN": producer, "runId": "k6-smoke", "nodeName": "k6-probe", "location": "Local", "PG_PROXY_TOKEN": "isolated-oracle", "TARGET": target_url, "FIXTURE_ITERATIONS": str(args.iterations), "FIXTURE_CUSTOM": str(args.custom_metrics).lower(), "K6_NO_USAGE_REPORT": "true", "K6_HTTP_ERROR_DATA": "true"} for key in ("syntheticMonitoring", "K6_BROWSER_SCREENSHOT_ON_ERROR", "CUSTOM_URL_PATTERNS", "BREAKINGIT_TRANSACTION_SOURCE"): writer_env.pop(key, None) command = [str(Path(args.k6).resolve()), "run", "--quiet", - "--out", "custom-db-breakingit=" + service, + "--out", "breaktest-db-breakingit=" + service, "--out", "timescaledb-breakingit=" + target_url, str(script)] if args.container_image: root.chmod(0o755); script.chmod(0o644) @@ -153,8 +153,8 @@ def post(path, payload): '-v',str(Path(args.k6).resolve())+':/usr/bin/k6:ro','-v',str(script)+':/fixture.js:ro', '-e','K6_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium','-e','K6_BROWSER_ARGS=no-sandbox', '-e','K6_BROWSER_HEADLESS=true', - *[item for key in ('CUSTOM_STORAGE_TOKEN','runId','nodeName','location','PG_PROXY_TOKEN','TARGET','K6_NO_USAGE_REPORT','K6_HTTP_ERROR_DATA','FIXTURE_ITERATIONS','FIXTURE_CUSTOM') for item in ('-e',key)], - args.container_image,'run','--quiet','--out','custom-db-breakingit='+service.replace('127.0.0.1','host.docker.internal'), + *[item for key in ('BREAKTEST_DB_TOKEN','runId','nodeName','location','PG_PROXY_TOKEN','TARGET','K6_NO_USAGE_REPORT','K6_HTTP_ERROR_DATA','FIXTURE_ITERATIONS','FIXTURE_CUSTOM') for item in ('-e',key)], + args.container_image,'run','--quiet','--out','breaktest-db-breakingit='+service.replace('127.0.0.1','host.docker.internal'), '--out','timescaledb-breakingit='+target_url.replace('127.0.0.1','host.docker.internal'),'/fixture.js'] observations=[] with (root/'k6.log').open('w+') as log: @@ -174,7 +174,7 @@ def post(path, payload): time.sleep(.1) log.seek(0) text=log.read() - assert process.returncode==0 and 'CUSTOM_STORAGE_FAILED' not in text,text + assert process.returncode==0 and 'BREAKTEST_DB_FAILED' not in text,text finally: if process.poll() is None: process.kill(); process.wait() live=[s for s in observations if all(not x['finished'] for x in s['sources']) and min(x['watermark'] for x in s['sources'])>origin and any(r['complete'] for r in s['rows'])] diff --git a/internal/cmd/builtin_output_gen.go b/internal/cmd/builtin_output_gen.go index b3f562fcd..61f83800e 100644 --- a/internal/cmd/builtin_output_gen.go +++ b/internal/cmd/builtin_output_gen.go @@ -7,11 +7,11 @@ import ( "strings" ) -const _builtinOutputName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingitcustom-db-breakingit" +const _builtinOutputName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingitbreaktest-db-breakingit" -var _builtinOutputIndex = [...]uint8{0, 5, 8, 15, 41, 49, 53, 58, 64, 90, 103, 110, 132, 152} +var _builtinOutputIndex = [...]uint8{0, 5, 8, 15, 41, 49, 53, 58, 64, 90, 103, 110, 132, 155} -const _builtinOutputLowerName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingitcustom-db-breakingit" +const _builtinOutputLowerName = "cloudcsvdatadogexperimental-prometheus-rwinfluxdbjsonkafkastatsdexperimental-opentelemetryopentelemetrysummarytimescaledb-breakingitbreaktest-db-breakingit" func (i builtinOutput) String() string { if i >= builtinOutput(len(_builtinOutputIndex)-1) { @@ -36,10 +36,10 @@ func _builtinOutputNoOp() { _ = x[builtinOutputOpentelemetry-(9)] _ = x[builtinOutputSummary-(10)] _ = x[builtinOutputTimescaledbBreakingit-(11)] - _ = x[builtinOutputCustomDBBreakingit-(12)] + _ = x[builtinOutputBreakTestDBBreakingit-(12)] } -var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit, builtinOutputCustomDBBreakingit} +var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit, builtinOutputBreakTestDBBreakingit} var _builtinOutputNameToValueMap = map[string]builtinOutput{ _builtinOutputName[0:5]: builtinOutputCloud, @@ -66,8 +66,8 @@ var _builtinOutputNameToValueMap = map[string]builtinOutput{ _builtinOutputLowerName[103:110]: builtinOutputSummary, _builtinOutputName[110:132]: builtinOutputTimescaledbBreakingit, _builtinOutputLowerName[110:132]: builtinOutputTimescaledbBreakingit, - _builtinOutputName[132:152]: builtinOutputCustomDBBreakingit, - _builtinOutputLowerName[132:152]: builtinOutputCustomDBBreakingit, + _builtinOutputName[132:155]: builtinOutputBreakTestDBBreakingit, + _builtinOutputLowerName[132:155]: builtinOutputBreakTestDBBreakingit, } var _builtinOutputNames = []string{ @@ -83,7 +83,7 @@ var _builtinOutputNames = []string{ _builtinOutputName[90:103], _builtinOutputName[103:110], _builtinOutputName[110:132], - _builtinOutputName[132:152], + _builtinOutputName[132:155], } // builtinOutputString retrieves an enum value from the enum constants string name. diff --git a/internal/cmd/outputs.go b/internal/cmd/outputs.go index f619f0f63..c95a11882 100644 --- a/internal/cmd/outputs.go +++ b/internal/cmd/outputs.go @@ -35,7 +35,7 @@ const ( builtinOutputOpentelemetry builtinOutputSummary builtinOutputTimescaledbBreakingit - builtinOutputCustomDBBreakingit + builtinOutputBreakTestDBBreakingit ) func getPossibleIDList(constrs map[string]output.Constructor) string { diff --git a/internal/cmd/outputs_full.go b/internal/cmd/outputs_full.go index 0703fc3db..45e1fd671 100644 --- a/internal/cmd/outputs_full.go +++ b/internal/cmd/outputs_full.go @@ -48,7 +48,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { }, "web-dashboard": dashboard.New, builtinOutputTimescaledbBreakingit.String(): breakingit.New, - builtinOutputCustomDBBreakingit.String(): breakingit.NewCustom, + builtinOutputBreakTestDBBreakingit.String(): breakingit.NewBreakTestDB, builtinOutputExperimentalOpentelemetry.String(): func(params output.Params) (output.Output, error) { params.Logger.Warnf("OpenTelemetry output has been graduated as a stable output."+ "You can now use just %q instead of %q. The experimental version will be removed in future versions.", diff --git a/internal/cmd/outputs_lean.go b/internal/cmd/outputs_lean.go index 61b73e8bc..4527636e7 100644 --- a/internal/cmd/outputs_lean.go +++ b/internal/cmd/outputs_lean.go @@ -56,7 +56,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { return nil, errors.New("the web-dashboard output is not available in this lean build of k6") }, builtinOutputTimescaledbBreakingit.String(): breakingit.New, - builtinOutputCustomDBBreakingit.String(): breakingit.NewCustom, + builtinOutputBreakTestDBBreakingit.String(): breakingit.NewBreakTestDB, } exts := ext.Get(ext.OutputExtension) diff --git a/internal/cmd/outputs_test.go b/internal/cmd/outputs_test.go index 019df536b..572dc72f1 100644 --- a/internal/cmd/outputs_test.go +++ b/internal/cmd/outputs_test.go @@ -12,15 +12,15 @@ func TestBuiltinOutputString(t *testing.T) { "cloud", "csv", "datadog", "experimental-prometheus-rw", "influxdb", "json", "kafka", "statsd", "experimental-opentelemetry", "opentelemetry", - "summary", "timescaledb-breakingit", "custom-db-breakingit", + "summary", "timescaledb-breakingit", "breaktest-db-breakingit", } assert.Equal(t, exp, builtinOutputStrings()) } -func TestCustomDBAndTimescaleConstructors(t *testing.T) { +func TestBreakTestDBAndTimescaleConstructors(t *testing.T) { t.Parallel() constructors, err := getAllOutputConstructors() assert.NoError(t, err) assert.Contains(t, constructors, "timescaledb-breakingit") - assert.Contains(t, constructors, "custom-db-breakingit") + assert.Contains(t, constructors, "breaktest-db-breakingit") } diff --git a/internal/output/breakingit/CUSTOM_DB.md b/internal/output/breakingit/BREAKTEST_DB.md similarity index 91% rename from internal/output/breakingit/CUSTOM_DB.md rename to internal/output/breakingit/BREAKTEST_DB.md index a51880c7b..f9004dda3 100644 --- a/internal/output/breakingit/CUSTOM_DB.md +++ b/internal/output/breakingit/BREAKTEST_DB.md @@ -1,25 +1,25 @@ -# Direct Custom DB output +# Direct BreakTest DB output The fork has two independent built-in outputs, in both normal and `lean` builds: - `timescaledb-breakingit`: the existing CSV/PG-proxy writer. -- `custom-db-breakingit`: direct zstd-compressed PostgreSQL binary COPY batches - to the BreakTest Custom DB service. It does not contact PostgreSQL or PG-proxy. +- `breaktest-db-breakingit`: direct zstd-compressed PostgreSQL binary COPY batches + to the BreakTest BreakTest DB service. It does not contact PostgreSQL or PG-proxy. The implementation lives in this fork, alongside the active built-in TimescaleDB writer. It does not use the older xk6 extension in the BreakTest repository. ## Invocation and application contract -The application must first provision a Custom DB run and one to four producer tokens per +The application must first provision a BreakTest DB run and one to four producer tokens per load generator. Pass the run ID and that generator's token when invoking k6: ```sh export runId='' -export CUSTOM_STORAGE_TOKEN=',' +export BREAKTEST_DB_TOKEN=',' export location='Amsterdam' export nodeName='generator-1' -k6 run --out custom-db-breakingit=https://storage.example.test script.js +k6 run --out breaktest-db-breakingit=https://storage.example.test script.js ``` The URL is the service base URL (an optional routing prefix is retained). The @@ -91,7 +91,7 @@ survives deferred work and the output queue; skipped cached requests release it. Mixed browser/native scenarios use the same tracker and lane barriers. **Timestamp contract:** browser sampler timestamps now mean generator-observed -terminal CDP completion, for both Custom DB and TimescaleDB outputs. Previously, +terminal CDP completion, for both BreakTest DB and TimescaleDB outputs. Previously, successful rows used the response-header timestamp. Chromium phase/duration measurements remain in their original time base. CDP transport delay precedes this observation; this is not a claim that Chromium's original event clock is @@ -110,9 +110,9 @@ go test -race ./metrics ./output ./lib/netext/httpext ./internal/output/breaking go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' go test ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' go build -tags lean -o /tmp/k6-custom-db-writer . -python3 ai-tests/custom-db/smoke.py --k6 /tmp/k6-custom-db-writer \ +python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-custom-db-writer \ --image breaktest-storage-lab:local --lanes 4 -python3 ai-tests/custom-db/smoke.py --k6 /tmp/k6-custom-db-writer --lanes 4 --restart-live +python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-custom-db-writer --lanes 4 --restart-live ``` The smoke test creates its own disposable Docker service and temporary directory, diff --git a/internal/output/breakingit/README.md b/internal/output/breakingit/README.md index d85bcef9b..714c1de33 100644 --- a/internal/output/breakingit/README.md +++ b/internal/output/breakingit/README.md @@ -107,8 +107,8 @@ an intentional nested transaction from two measurements of the same step. For a full migration, remove `pages.add()` and use `await group(...)` for timing and context, while keeping any business checks and throwing on failures. -## Custom DB alternative +## BreakTest DB alternative -The fork also includes `custom-db-breakingit` for direct binary delivery to the -BreakTest Custom DB service. See [configuration, delivery semantics and current -limits](CUSTOM_DB.md). The existing TimescaleDB output remains available. +The fork also includes `breaktest-db-breakingit` for direct binary delivery to the +BreakTest BreakTest DB service. See [configuration, delivery semantics and current +limits](BREAKTEST_DB.md). The existing TimescaleDB output remains available. diff --git a/internal/output/breakingit/breakingit.go b/internal/output/breakingit/breakingit.go index 72a84922b..dd3fb254f 100644 --- a/internal/output/breakingit/breakingit.go +++ b/internal/output/breakingit/breakingit.go @@ -62,7 +62,7 @@ const ( // Logger writes k6 metric samples to pg-proxy using COPY via multipart batch type Logger struct { - // rowSink is used by the direct Custom DB output; nil preserves CSV/PG-proxy. + // rowSink is used by the direct BreakTest DB output; nil preserves CSV/PG-proxy. rowSink func(string, []string) error out io.Writer diff --git a/internal/output/breakingit/custom.go b/internal/output/breakingit/breaktest_db.go similarity index 78% rename from internal/output/breakingit/custom.go rename to internal/output/breakingit/breaktest_db.go index 0f44c9a86..66d83b0f7 100644 --- a/internal/output/breakingit/custom.go +++ b/internal/output/breakingit/breaktest_db.go @@ -21,19 +21,19 @@ import ( "go.k6.io/k6/v2/output" ) -// CustomOutput reuses BreakTest's metric mappings, with bounded direct binary delivery. +// BreakTestDBOutput reuses BreakTest's metric mappings, with bounded direct binary delivery. // Only the sender owns incarnation/sequence. An ACK is acceptance, not an fsync guarantee. -type customGroupKey struct { +type dbGroupKey struct { time int64 tags *metrics.TagSet } -type CustomOutput struct { - groups map[customGroupKey]*MetricGroup - groupTickets map[customGroupKey]*metrics.RequestTicket +type BreakTestDBOutput struct { + groups map[dbGroupKey]*MetricGroup + groupTickets map[dbGroupKey]*metrics.RequestTicket tracker *metrics.RequestTracker - lanes []*CustomOutput - owner *CustomOutput + lanes []*BreakTestDBOutput + owner *BreakTestDBOutput nextLane int watermark int64 queuedWatermark int64 @@ -46,7 +46,7 @@ type CustomOutput struct { sequence uint64 retryTimeout time.Duration encoder *zstd.Encoder - queue chan customBatch + queue chan dbBatch done, tickStop, tickDone, failed chan struct{} batchMu sync.Mutex parts map[string]*bytes.Buffer @@ -59,37 +59,37 @@ type CustomOutput struct { stopOnce sync.Once stopErr error } -type customBatch struct { +type dbBatch struct { body []byte contentType string beforeUS int64 } -// NewCustom is registered separately from timescaledb-breakingit in full and lean builds. -func NewCustom(params output.Params) (output.Output, error) { +// NewBreakTestDB is registered separately from timescaledb-breakingit in full and lean builds. +func NewBreakTestDB(params output.Params) (output.Output, error) { env := params.Environment u, err := url.Parse(params.ConfigArgument) if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" { - return nil, errors.New("Custom DB requires an HTTP(S) base URL without credentials, query or fragment") + return nil, errors.New("BreakTest DB requires an HTTP(S) base URL without credentials, query or fragment") } - runID, token := env["runId"], env["CUSTOM_STORAGE_TOKEN"] + runID, token := env["runId"], env["BREAKTEST_DB_TOKEN"] if !regexp.MustCompile(`^[A-Za-z0-9_-]+$`).MatchString(runID) || token == "" { - return nil, errors.New("Custom DB requires runId and a run-scoped CUSTOM_STORAGE_TOKEN") + return nil, errors.New("BreakTest DB requires runId and a run-scoped BREAKTEST_DB_TOKEN") } if strings.EqualFold(env["syntheticMonitoring"], "true") { - return nil, errors.New("Custom DB k6 output does not support synthetic monitoring") + return nil, errors.New("BreakTest DB k6 output does not support synthetic monitoring") } if strings.EqualFold(env["K6_BROWSER_SCREENSHOT_ON_ERROR"], "true") { - return nil, errors.New("Custom DB k6 output does not yet support browser screenshot storage") + return nil, errors.New("BreakTest DB k6 output does not yet support browser screenshot storage") } tokens := strings.Split(token, ",") seen := map[string]bool{} if len(tokens) > 4 { - return nil, errors.New("Custom DB supports 1-4 ingestion lanes") + return nil, errors.New("BreakTest DB supports 1-4 ingestion lanes") } for _, t := range tokens { if t == "" || seen[t] { - return nil, errors.New("invalid Custom DB lane tokens") + return nil, errors.New("invalid BreakTest DB lane tokens") } seen[t] = true } @@ -111,29 +111,29 @@ func NewCustom(params output.Params) (output.Output, error) { tags := map[string]string{"run_id": runID, "location": env["location"], "node_name": env["nodeName"], "scenario_name": env["scenarioName"]} transport := http.DefaultTransport.(*http.Transport).Clone() transport.MaxConnsPerHost = len(tokens) - c := &CustomOutput{ + c := &BreakTestDBOutput{ base: strings.TrimRight(u.String(), "/") + "/runs/" + runID, token: tokens[0], tokens: tokens, runID: runID, client: &http.Client{Transport: transport, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, - retryTimeout: 30 * time.Second, queue: make(chan customBatch, 4), done: make(chan struct{}), + retryTimeout: 30 * time.Second, queue: make(chan dbBatch, 4), done: make(chan struct{}), tickStop: make(chan struct{}), tickDone: make(chan struct{}), failed: make(chan struct{}), parts: map[string]*bytes.Buffer{}, } - c.groups = map[customGroupKey]*MetricGroup{} - c.groupTickets = map[customGroupKey]*metrics.RequestTicket{} + c.groups = map[dbGroupKey]*MetricGroup{} + c.groupTickets = map[dbGroupKey]*metrics.RequestTicket{} c.mapper = &Logger{out: params.StdOut, envTags: tags, customPatterns: patterns, transactionSource: source, bufHTTP: &bytes.Buffer{}, bufVUs: &bytes.Buffer{}, bufTrans: &bytes.Buffer{}, bufErrors: &bytes.Buffer{}, pendingHttpGroups: map[int64]*MetricGroup{}, rowSink: c.addRow} return c, nil } -func (c *CustomOutput) SetRequestTracker(t *metrics.RequestTracker) { c.tracker = t } +func (c *BreakTestDBOutput) SetRequestTracker(t *metrics.RequestTracker) { c.tracker = t } -func (c *CustomOutput) Description() string { return "custom-db-breakingit" } -func (c *CustomOutput) SetTestRunStopCallback(callback func(error)) { +func (c *BreakTestDBOutput) Description() string { return "breaktest-db-breakingit" } +func (c *BreakTestDBOutput) SetTestRunStopCallback(callback func(error)) { c.failureMu.Lock() c.stopCallback = callback c.failureMu.Unlock() } -func (c *CustomOutput) err() error { +func (c *BreakTestDBOutput) err() error { if c.owner != nil { return c.owner.err() } @@ -141,7 +141,7 @@ func (c *CustomOutput) err() error { defer c.failureMu.Unlock() return c.failure } -func (c *CustomOutput) fail(err error) { +func (c *BreakTestDBOutput) fail(err error) { if c.owner != nil { c.owner.fail(err) return @@ -154,7 +154,7 @@ func (c *CustomOutput) fail(err error) { c.failureMu.Unlock() return } - c.failure = fmt.Errorf("CUSTOM_STORAGE_FAILED: final publication forbidden: %w", err) + c.failure = fmt.Errorf("BREAKTEST_DB_FAILED: final publication forbidden: %w", err) failure, callback := c.failure, c.stopCallback close(c.failed) c.failureMu.Unlock() @@ -163,11 +163,11 @@ func (c *CustomOutput) fail(err error) { go callback(failure) } } -func (c *CustomOutput) Start() error { - c.lanes = []*CustomOutput{c} +func (c *BreakTestDBOutput) Start() error { + c.lanes = []*BreakTestDBOutput{c} for _, token := range c.tokens[1:] { - c.lanes = append(c.lanes, &CustomOutput{owner: c, tracker: c.tracker, base: c.base, runID: c.runID, token: token, - client: c.client, retryTimeout: c.retryTimeout, queue: make(chan customBatch, 4), done: make(chan struct{}), failed: c.failed}) + c.lanes = append(c.lanes, &BreakTestDBOutput{owner: c, tracker: c.tracker, base: c.base, runID: c.runID, token: token, + client: c.client, retryTimeout: c.retryTimeout, queue: make(chan dbBatch, 4), done: make(chan struct{}), failed: c.failed}) } for _, lane := range c.lanes { if err := lane.hello(); err != nil { @@ -178,7 +178,7 @@ func (c *CustomOutput) Start() error { } return err } - encoder, err := newCustomEncoder() + encoder, err := newDBEncoder() if err != nil { return err } @@ -212,7 +212,7 @@ func (c *CustomOutput) Start() error { return nil } -func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { +func (c *BreakTestDBOutput) AddMetricSamples(samples []metrics.SampleContainer) { c.addMu.Lock() defer c.addMu.Unlock() if c.err() != nil { @@ -226,7 +226,7 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { } for _, sample := range sc.GetSamples() { if sample.Metric.Name == "browser_errors" && sample.Tags.Map()["screenshot"] != "" { - c.fail(errors.New("browser screenshots require a Custom DB payload integration")) + c.fail(errors.New("browser screenshots require a BreakTest DB payload integration")) return } } @@ -246,7 +246,7 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { c.fail(errors.New("HTTP metrics lack producer completion tracking; live watermark would be unsafe")) return } - id := customGroupKey{sample.Time.UnixNano(), sample.Tags} + id := dbGroupKey{sample.Time.UnixNano(), sample.Tags} if ticket != nil { c.groupTickets[id] = ticket } @@ -286,7 +286,7 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { } n += len(g.Values) * 64 } - if len(c.groups) > 4096 || n > customBatchBytes { + if len(c.groups) > 4096 || n > dbBatchBytes { c.fail(errors.New("incomplete metric join budget exceeded")) return } @@ -295,7 +295,7 @@ func (c *CustomOutput) AddMetricSamples(samples []metrics.SampleContainer) { // Complete only after every request metric has arrived. Browser failure payloads // may arrive much later than duration metrics; a timer is not proof of success. -func (c *CustomOutput) completeGroups(final bool) { +func (c *BreakTestDBOutput) completeGroups(final bool) { for key, g := range c.groups { complete := true for _, name := range []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving", "http_req_failed"} { @@ -315,18 +315,18 @@ func (c *CustomOutput) completeGroups(final bool) { } } } -func (c *CustomOutput) addRow(table string, row []string) error { +func (c *BreakTestDBOutput) addRow(table string, row []string) error { if err := c.err(); err != nil { return err } - data, err := encodeCustomRow(table, row) + data, err := encodeDBRow(table, row) if err != nil { c.fail(err) return err } c.batchMu.Lock() defer c.batchMu.Unlock() - if c.batchBytes+len(data) > customBatchBytes { + if c.batchBytes+len(data) > dbBatchBytes { if err = c.enqueueLocked(); err != nil { return err } @@ -338,14 +338,14 @@ func (c *CustomOutput) addRow(table string, row []string) error { c.batchBytes += len(data) return nil } -func (c *CustomOutput) flush() { +func (c *BreakTestDBOutput) flush() { c.batchMu.Lock() defer c.batchMu.Unlock() if err := c.enqueueLocked(); err != nil { c.fail(err) } } -func (c *CustomOutput) enqueueLocked() error { +func (c *BreakTestDBOutput) enqueueLocked() error { if c.batchBytes == 0 { return nil } @@ -370,7 +370,7 @@ func (c *CustomOutput) enqueueLocked() error { if err := writer.Close(); err != nil { return err } - batch := customBatch{body: body.Bytes(), contentType: "multipart/mixed; boundary=" + writer.Boundary()} + batch := dbBatch{body: body.Bytes(), contentType: "multipart/mixed; boundary=" + writer.Boundary()} lane := c if len(c.lanes) > 0 { lane = c.lanes[c.nextLane] @@ -385,7 +385,7 @@ func (c *CustomOutput) enqueueLocked() error { return nil } } -func (c *CustomOutput) sender() { +func (c *BreakTestDBOutput) sender() { defer close(c.done) for batch := range c.queue { if c.err() != nil { @@ -400,7 +400,7 @@ func (c *CustomOutput) sender() { } } -func (c *CustomOutput) Stop() error { +func (c *BreakTestDBOutput) Stop() error { c.stopOnce.Do(func() { if !c.started { return @@ -436,7 +436,7 @@ func (c *CustomOutput) Stop() error { // Retries keep the same sequence, multipart boundary and compressed bytes. // Incarnation recovery is handled by the lane, after this request returns 409. -func (c *CustomOutput) request(path string, body []byte, contentType string, seq uint64, result any) error { +func (c *BreakTestDBOutput) request(path string, body []byte, contentType string, seq uint64, result any) error { ctx, cancel := context.WithTimeout(context.Background(), c.retryTimeout) defer cancel() for { @@ -459,23 +459,23 @@ func (c *CustomOutput) request(path string, body []byte, contentType string, seq data, readErr := io.ReadAll(io.LimitReader(response.Body, 64*1024+1)) _ = response.Body.Close() if len(data) > 64*1024 { - return errors.New("oversized Custom DB acknowledgement") + return errors.New("oversized BreakTest DB acknowledgement") } if response.StatusCode == http.StatusOK && readErr == nil { if err = json.Unmarshal(data, result); err != nil { - return errors.New("invalid Custom DB acknowledgement JSON") + return errors.New("invalid BreakTest DB acknowledgement JSON") } return nil } if readErr == nil && response.StatusCode != 429 && response.StatusCode < 500 { - return customRejection(response.StatusCode) + return dbRejection(response.StatusCode) } } timer := time.NewTimer(100 * time.Millisecond) select { case <-ctx.Done(): timer.Stop() - return fmt.Errorf("Custom DB %s retry deadline: %w", path, ctx.Err()) + return fmt.Errorf("BreakTest DB %s retry deadline: %w", path, ctx.Err()) case <-c.failed: timer.Stop() return c.err() @@ -486,6 +486,6 @@ func (c *CustomOutput) request(path string, body []byte, contentType string, seq // The storage ingress caps zstd decoder windows at 2^18 bytes, independently // of the larger decoded batch budget. Enforce this even on full 2 MiB batches. -func newCustomEncoder() (*zstd.Encoder, error) { +func newDBEncoder() (*zstd.Encoder, error) { return zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), zstd.WithWindowSize(1<<18)) } diff --git a/internal/output/breakingit/custom_binary.go b/internal/output/breakingit/breaktest_db_binary.go similarity index 89% rename from internal/output/breakingit/custom_binary.go rename to internal/output/breakingit/breaktest_db_binary.go index 2f5924e8f..64bfe1545 100644 --- a/internal/output/breakingit/custom_binary.go +++ b/internal/output/breakingit/breaktest_db_binary.go @@ -11,7 +11,7 @@ import ( ) const pgEpochSeconds int64 = 946684800 -const customBatchBytes = 2 * 1024 * 1024 +const dbBatchBytes = 2 * 1024 * 1024 // PostgreSQL binary COPY types in the existing non-SM BreakTest table order. // t = timestamp, s = text, i = int32, b = boolean, u = UUID. @@ -22,7 +22,7 @@ var customSchemas = map[string]string{ "virtual_users": "tsssiii", } -func encodeCustomRow(table string, row []string) ([]byte, error) { +func encodeDBRow(table string, row []string) ([]byte, error) { schema := customSchemas[table] if schema == "" || len(row) != len(schema) { return nil, fmt.Errorf("invalid %s row width: %d", table, len(row)) @@ -32,8 +32,8 @@ func encodeCustomRow(table string, row []string) ([]byte, error) { for _, s := range row { size += len(s) } - if size > customBatchBytes-1024 { - return nil, fmt.Errorf("%s row exceeds Custom DB batch budget", table) + if size > dbBatchBytes-1024 { + return nil, fmt.Errorf("%s row exceeds BreakTest DB batch budget", table) } b := bytes.NewBuffer(make([]byte, 0, size)) _ = binary.Write(b, binary.BigEndian, int16(len(row))) @@ -55,7 +55,7 @@ func encodeCustomRow(table string, row []string) ([]byte, error) { } n := (t.Unix()-pgEpochSeconds)*1_000_000 + int64(t.Nanosecond()/1000) if n < 0 { - return nil, fmt.Errorf("timestamp before Custom DB epoch") + return nil, fmt.Errorf("timestamp before BreakTest DB epoch") } data = binary.BigEndian.AppendUint64(nil, uint64(n)) case 'i': diff --git a/internal/output/breakingit/custom_recovery.go b/internal/output/breakingit/breaktest_db_recovery.go similarity index 78% rename from internal/output/breakingit/custom_recovery.go rename to internal/output/breakingit/breaktest_db_recovery.go index 6856a7f90..dd78d6c16 100644 --- a/internal/output/breakingit/custom_recovery.go +++ b/internal/output/breakingit/breaktest_db_recovery.go @@ -6,10 +6,10 @@ import ( "fmt" ) -type customRejection int +type dbRejection int -func (s customRejection) Error() string { - return fmt.Sprintf("Custom DB rejected request (HTTP %d)", int(s)) +func (s dbRejection) Error() string { + return fmt.Sprintf("BreakTest DB rejected request (HTTP %d)", int(s)) } type sourceHello struct { @@ -19,7 +19,7 @@ type sourceHello struct { Sealed bool `json:"sealed"` } -func (c *CustomOutput) register() (sourceHello, error) { +func (c *BreakTestDBOutput) register() (sourceHello, error) { body := []byte(`{}`) if c.tracker != nil { body, _ = json.Marshal(map[string]any{"source_clock_us": c.tracker.Clock().UnixMicro() - pgEpochSeconds*1_000_000, "clock_contract": "sample-end-monotonic-v1"}) @@ -32,8 +32,8 @@ func (c *CustomOutput) register() (sourceHello, error) { // Acknowledged, uncheckpointed rows may be lost under relaxed durability. The // server exposes that loss as coverage gaps. Replay only the held unacknowledged // batch, never already acknowledged history. Each ordered lane recovers itself. -func (c *CustomOutput) recoverSource(err error) (uint64, error) { - var status customRejection +func (c *BreakTestDBOutput) recoverSource(err error) (uint64, error) { + var status dbRejection if c.tracker == nil || !errors.As(err, &status) || status != 409 { return 0, err } @@ -49,7 +49,7 @@ func (c *CustomOutput) recoverSource(err error) (uint64, error) { return *hello.Checkpoint, nil } -func (c *CustomOutput) sendBatch(body []byte, content string) error { +func (c *BreakTestDBOutput) sendBatch(body []byte, content string) error { for attempt := 0; attempt < 4; attempt++ { next := c.sequence + 1 var ack struct { @@ -58,7 +58,7 @@ func (c *CustomOutput) sendBatch(body []byte, content string) error { err := c.request("/ingest/batch", body, content, next, &ack) if err == nil { if ack.Accepted == nil || *ack.Accepted != next { - return errors.New("invalid Custom DB batch acknowledgement") + return errors.New("invalid BreakTest DB batch acknowledgement") } c.sequence = next return nil @@ -74,10 +74,10 @@ func (c *CustomOutput) sendBatch(body []byte, content string) error { return nil } // Lost ACK, but checkpoint proves this batch persisted. } - return errors.New("Custom DB restarted repeatedly during batch delivery") + return errors.New("BreakTest DB restarted repeatedly during batch delivery") } -func (c *CustomOutput) sendControl(path string, before int64) error { +func (c *BreakTestDBOutput) sendControl(path string, before int64) error { body := []byte(nil) if path == "/watermark" { body, _ = json.Marshal(map[string]int64{"before_us": before}) @@ -92,7 +92,7 @@ func (c *CustomOutput) sendControl(path string, before int64) error { err := c.request(path, body, "application/json", c.sequence, &ack) if err == nil { if ack.Accepted == nil || *ack.Accepted != c.sequence || (path == "/watermark" && ack.Watermark != before) || (path == "/finish" && !ack.Finished) { - return errors.New("invalid Custom DB control acknowledgement") + return errors.New("invalid BreakTest DB control acknowledgement") } return nil } @@ -102,5 +102,5 @@ func (c *CustomOutput) sendControl(path string, before int64) error { return errors.New("checkpoint exceeds acknowledged sequence") } } - return errors.New("Custom DB restarted repeatedly during control delivery") + return errors.New("BreakTest DB restarted repeatedly during control delivery") } diff --git a/internal/output/breakingit/custom_test.go b/internal/output/breakingit/breaktest_db_test.go similarity index 86% rename from internal/output/breakingit/custom_test.go rename to internal/output/breakingit/breaktest_db_test.go index 061a5474a..3890e6ea4 100644 --- a/internal/output/breakingit/custom_test.go +++ b/internal/output/breakingit/breaktest_db_test.go @@ -23,15 +23,15 @@ import ( "go.k6.io/k6/v2/output" ) -func newCustomTest(t *testing.T, endpoint string) *CustomOutput { +func newBreakTestDBTest(t *testing.T, endpoint string) *BreakTestDBOutput { t.Helper() - out, err := NewCustom(output.Params{ConfigArgument: endpoint, StdOut: io.Discard, Environment: map[string]string{ - "runId": "run-123", "CUSTOM_STORAGE_TOKEN": "producer-token", "location": "Amsterdam", "nodeName": "node-1", + out, err := NewBreakTestDB(output.Params{ConfigArgument: endpoint, StdOut: io.Discard, Environment: map[string]string{ + "runId": "run-123", "BREAKTEST_DB_TOKEN": "producer-token", "location": "Amsterdam", "nodeName": "node-1", }}) require.NoError(t, err) - return out.(*CustomOutput) + return out.(*BreakTestDBOutput) } -func decodeCustomBatch(t *testing.T, data []byte, content string) map[string][][][]byte { +func decodeDBBatch(t *testing.T, data []byte, content string) map[string][][][]byte { t.Helper() decoder, err := zstd.NewReader(nil) require.NoError(t, err) @@ -79,7 +79,7 @@ func decodeCustomBatch(t *testing.T, data []byte, content string) map[string][][ } return result } -func TestCustomBinaryMatchesCSVMapping(t *testing.T) { +func TestBreakTestDBBinaryMatchesCSVMapping(t *testing.T) { t.Parallel() rows := []struct { table string @@ -90,7 +90,7 @@ func TestCustomBinaryMatchesCSVMapping(t *testing.T) { {"requests_error", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "Checkout", "/pay", "503", "100", "", "https://example.test/", "", "bad\nreply", "POST,\"a\"", "Header: value", "雪", "019a3524-3890-7000-8000-000000000001"}}, {"virtual_users", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "10", "", ""}}, } - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") for _, r := range rows { require.NoError(t, c.addRow(r.table, r.row)) } @@ -99,7 +99,7 @@ func TestCustomBinaryMatchesCSVMapping(t *testing.T) { encoder, err := zstd.NewWriter(nil) require.NoError(t, err) defer encoder.Close() - decoded := decodeCustomBatch(t, encoder.EncodeAll(batch.body, nil), batch.contentType) + decoded := decodeDBBatch(t, encoder.EncodeAll(batch.body, nil), batch.contentType) require.Len(t, decoded, 4) require.Nil(t, decoded["requests_raw"][0][9]) require.Nil(t, decoded["requests_raw"][0][6]) @@ -124,7 +124,7 @@ func TestCustomBinaryMatchesCSVMapping(t *testing.T) { require.Equal(t, expected, actual) } -func TestCustomRetriesExactBytesAndFinalDrain(t *testing.T) { +func TestBreakTestDBRetriesExactBytesAndFinalDrain(t *testing.T) { t.Parallel() var first []byte var content string @@ -164,7 +164,7 @@ func TestCustomRetriesExactBytesAndFinalDrain(t *testing.T) { } })) defer server.Close() - c := newCustomTest(t, server.URL) + c := newBreakTestDBTest(t, server.URL) require.NoError(t, c.Start()) c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)}) require.NoError(t, c.Stop()) @@ -172,11 +172,11 @@ func TestCustomRetriesExactBytesAndFinalDrain(t *testing.T) { mu.Lock() defer mu.Unlock() require.Equal(t, 1, finishes) - decoded := decodeCustomBatch(t, first, content) + decoded := decodeDBBatch(t, first, content) require.Len(t, decoded["transactions"], 1) } -func TestCustomFailureForbidsFinish(t *testing.T) { +func TestBreakTestDBFailureForbidsFinish(t *testing.T) { for _, status := range []int{400, 401, 403, 409, 413, 429, 503, 200} { t.Run(strconv.Itoa(status), func(t *testing.T) { var finished bool @@ -192,7 +192,7 @@ func TestCustomFailureForbidsFinish(t *testing.T) { fmt.Fprint(w, `{"accepted":999}`) })) defer server.Close() - c := newCustomTest(t, server.URL) + c := newBreakTestDBTest(t, server.URL) c.retryTimeout = 50 * time.Millisecond stopped := make(chan error, 1) c.SetTestRunStopCallback(func(err error) { stopped <- err }) @@ -210,26 +210,26 @@ func TestCustomFailureForbidsFinish(t *testing.T) { } } -func TestCustomConfigurationAndBounds(t *testing.T) { +func TestBreakTestDBConfigurationAndBounds(t *testing.T) { t.Parallel() - for _, entry := range []struct{ k, v string }{{"syntheticMonitoring", "true"}, {"K6_BROWSER_SCREENSHOT_ON_ERROR", "true"}, {"runId", "../wrong"}, {"CUSTOM_STORAGE_TOKEN", ""}, {"BREAKINGIT_TRANSACTION_SOURCE", "wrong"}} { - env := map[string]string{"runId": "test", "CUSTOM_STORAGE_TOKEN": "token"} + for _, entry := range []struct{ k, v string }{{"syntheticMonitoring", "true"}, {"K6_BROWSER_SCREENSHOT_ON_ERROR", "true"}, {"runId", "../wrong"}, {"BREAKTEST_DB_TOKEN", ""}, {"BREAKINGIT_TRANSACTION_SOURCE", "wrong"}} { + env := map[string]string{"runId": "test", "BREAKTEST_DB_TOKEN": "token"} env[entry.k] = entry.v - _, err := NewCustom(output.Params{ConfigArgument: "http://localhost", Environment: env}) + _, err := NewBreakTestDB(output.Params{ConfigArgument: "http://localhost", Environment: env}) require.Error(t, err) } - _, err := NewCustom(output.Params{ConfigArgument: "http://user:password@localhost"}) + _, err := NewBreakTestDB(output.Params{ConfigArgument: "http://user:password@localhost"}) require.Error(t, err) - c := newCustomTest(t, "http://localhost") - row := []string{"2026-10-30T00:00:00Z", "run-123", "NL", strings.Repeat("x", customBatchBytes), "true", "", "", "1"} + c := newBreakTestDBTest(t, "http://localhost") + row := []string{"2026-10-30T00:00:00Z", "run-123", "NL", strings.Repeat("x", dbBatchBytes), "true", "", "", "1"} require.Error(t, c.addRow("transactions", row)) require.Error(t, c.err()) require.Empty(t, c.queue) } -func TestCustomHTTPJoinsAndLateFailures(t *testing.T) { +func TestBreakTestDBHTTPJoinsAndLateFailures(t *testing.T) { t.Parallel() - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") var rows [][]string c.mapper.rowSink = func(table string, row []string) error { if table == "requests_raw" { @@ -264,17 +264,17 @@ func TestCustomHTTPJoinsAndLateFailures(t *testing.T) { } } -func TestCustomIncompleteDrainFails(t *testing.T) { +func TestBreakTestDBIncompleteDrainFails(t *testing.T) { t.Parallel() - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("http_req_duration", nil, false)}) c.completeGroups(true) require.ErrorContains(t, c.err(), "incomplete HTTP metric group") } -func TestCustomQueueBackpressure(t *testing.T) { +func TestBreakTestDBQueueBackpressure(t *testing.T) { t.Parallel() - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") row := []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "1"} for range cap(c.queue) { require.NoError(t, c.addRow("transactions", row)) @@ -296,26 +296,26 @@ func TestCustomQueueBackpressure(t *testing.T) { require.Len(t, c.queue, cap(c.queue)) } -func TestCustomStartupRejectsNonFreshAndMissingAck(t *testing.T) { +func TestBreakTestDBStartupRejectsNonFreshAndMissingAck(t *testing.T) { t.Parallel() for _, body := range []string{`{}`, `{"incarnation":"a"}`, `{"incarnation":"a","accepted_sequence":1}`, `{"incarnation":"a","accepted_sequence":0,"sealed":true}`} { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) - c := newCustomTest(t, server.URL) + c := newBreakTestDBTest(t, server.URL) require.Error(t, c.Start()) require.False(t, c.started) server.Close() } } -func TestCustomHTTPDuplicateIdentityFails(t *testing.T) { +func TestBreakTestDBHTTPDuplicateIdentityFails(t *testing.T) { t.Parallel() - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") sample := transactionTestSample("http_req_duration", nil, false) c.AddMetricSamples([]metrics.SampleContainer{sample, sample}) require.ErrorContains(t, c.err(), "ambiguous HTTP metric identity") } -func TestCustomWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) { +func TestBreakTestDBWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) { var mu sync.Mutex sequences := map[string]uint64{} watermarks := map[string]int64{} @@ -347,7 +347,7 @@ func TestCustomWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) { } })) defer server.Close() - c := newCustomTest(t, server.URL) + c := newBreakTestDBTest(t, server.URL) c.tokens = []string{"producer-token", "second"} ch := make(chan metrics.SampleContainer, 1) tracker, release := metrics.BindRequestTracker(ch) @@ -380,7 +380,7 @@ func TestCustomWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) { require.Equal(t, uint64(2), sequences["Bearer second"]) } -func TestCustomRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) { +func TestBreakTestDBRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) { for _, checkpoint := range []uint64{0, 2} { t.Run(strconv.Itoa(int(checkpoint)), func(t *testing.T) { calls := 0 @@ -405,7 +405,7 @@ func TestCustomRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) { fmt.Fprint(w, `{"accepted":1}`) })) defer server.Close() - c := newCustomTest(t, server.URL) + c := newBreakTestDBTest(t, server.URL) ch := make(chan metrics.SampleContainer) tracker, release := metrics.BindRequestTracker(ch) defer release() @@ -425,7 +425,7 @@ func TestCustomRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) { } func TestTrackedHTTPJoinKeepsPinUntilFailureMetric(t *testing.T) { - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") channel := make(chan metrics.SampleContainer) tracker, release := metrics.BindRequestTracker(channel) defer release() @@ -452,18 +452,18 @@ func TestTrackedHTTPJoinKeepsPinUntilFailureMetric(t *testing.T) { } func TestUntrackedRequestCannotCrossNativeWatermark(t *testing.T) { - c := newCustomTest(t, "http://localhost") + c := newBreakTestDBTest(t, "http://localhost") channel := make(chan metrics.SampleContainer) tracker, release := metrics.BindRequestTracker(channel) defer release() c.tracker = tracker - c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("custom_missing_hook", nil, false)}) + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("db_missing_hook", nil, false)}) require.ErrorContains(t, c.err(), "lacks producer completion tracking") require.Empty(t, c.parts) } -func TestCustomFullBatchRespectsStorageDecoderWindow(t *testing.T) { - encoder, err := newCustomEncoder() +func TestBreakTestDBFullBatchRespectsStorageDecoderWindow(t *testing.T) { + encoder, err := newDBEncoder() require.NoError(t, err) defer encoder.Close() plain := bytes.Repeat([]byte("large-request-metric-row\n"), 100000) diff --git a/internal/output/breakingit/custom_watermark.go b/internal/output/breakingit/breaktest_db_watermark.go similarity index 77% rename from internal/output/breakingit/custom_watermark.go rename to internal/output/breakingit/breaktest_db_watermark.go index b9c728ca5..453fe81bd 100644 --- a/internal/output/breakingit/custom_watermark.go +++ b/internal/output/breakingit/breaktest_db_watermark.go @@ -4,13 +4,13 @@ import ( "errors" ) -func (c *CustomOutput) hello() error { +func (c *BreakTestDBOutput) hello() error { hello, err := c.register() if err != nil { return err } if hello.Sealed || hello.Incarnation == "" || hello.Accepted == nil || *hello.Accepted != 0 { - return errors.New("Custom DB requires a fresh unfinished producer source") + return errors.New("BreakTest DB requires a fresh unfinished producer source") } c.incarnation = hello.Incarnation return nil @@ -19,7 +19,7 @@ func (c *CustomOutput) hello() error { // Called under addMu, after detaching every serialized row. Tickets in the engine // queue and incomplete joins still pin Before(). Each FIFO carries its own barrier: // the database takes the minimum across all lanes and generators. -func (c *CustomOutput) publishWatermark() { +func (c *BreakTestDBOutput) publishWatermark() { if c.tracker == nil || c.err() != nil { return } @@ -31,7 +31,7 @@ func (c *CustomOutput) publishWatermark() { select { case <-c.failed: return - case lane.queue <- customBatch{beforeUS: before}: + case lane.queue <- dbBatch{beforeUS: before}: } } c.queuedWatermark = before From 4f7913b529566f7cd96237b5da7a469ac524d14e Mon Sep 17 00:00:00 2001 From: jgaalen Date: Sun, 1 Nov 2026 18:57:01 +0100 Subject: [PATCH 6/7] Keep the custom metric prefix in missing-watermark test --- internal/output/breakingit/breaktest_db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/output/breakingit/breaktest_db_test.go b/internal/output/breakingit/breaktest_db_test.go index 3890e6ea4..3e4463d43 100644 --- a/internal/output/breakingit/breaktest_db_test.go +++ b/internal/output/breakingit/breaktest_db_test.go @@ -457,7 +457,7 @@ func TestUntrackedRequestCannotCrossNativeWatermark(t *testing.T) { tracker, release := metrics.BindRequestTracker(channel) defer release() c.tracker = tracker - c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("db_missing_hook", nil, false)}) + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("custom_missing_hook", nil, false)}) require.ErrorContains(t, c.err(), "lacks producer completion tracking") require.Empty(t, c.parts) } From 86d57eb2598b0ae3ce8e9f711b25edda58040b3e Mon Sep 17 00:00:00 2001 From: jgaalen Date: Sun, 1 Nov 2026 19:02:34 +0100 Subject: [PATCH 7/7] Run writer smoke checks against the production BreakTest DB command --- ai-tests/breaktest-db/smoke.py | 6 +++--- internal/output/breakingit/BREAKTEST_DB.md | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ai-tests/breaktest-db/smoke.py b/ai-tests/breaktest-db/smoke.py index 391ea1468..b3d37bccd 100644 --- a/ai-tests/breaktest-db/smoke.py +++ b/ai-tests/breaktest-db/smoke.py @@ -26,7 +26,7 @@ def run(args): - name = "k6-custom-writer-smoke-" + uuid.uuid4().hex[:12] + name = "k6-breaktest-db-smoke-" + uuid.uuid4().hex[:12] admin = secrets.token_hex(32) producers = [secrets.token_hex(32) for _ in range(args.lanes)] producer = ",".join(producers) @@ -85,7 +85,7 @@ def do_POST(self): "--label", "breaktest.k6-writer-smoke=true", "--memory", "512m", "-e", "BREAKTEST_DB_ADMIN_TOKEN", "-p", f"127.0.0.1:{fixed_port}:8099", "-v", f"{data_dir}:/data", args.image, - "v3", "service", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True) + "serve", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True) port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1] service = f"http://127.0.0.1:{port}" @@ -242,7 +242,7 @@ def normalized(row, csv_row=False): if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--k6", required=True) - parser.add_argument("--image", default="breaktest-storage-lab:local") + parser.add_argument("--image", default="breaktest-db:local") parser.add_argument('--lanes',type=int,choices=range(1,5),default=1) parser.add_argument('--iterations',type=int,default=80) parser.add_argument('--restart-live',action='store_true') diff --git a/internal/output/breakingit/BREAKTEST_DB.md b/internal/output/breakingit/BREAKTEST_DB.md index f9004dda3..9764e8094 100644 --- a/internal/output/breakingit/BREAKTEST_DB.md +++ b/internal/output/breakingit/BREAKTEST_DB.md @@ -35,7 +35,7 @@ connect times become NULL. Error UUIDs are generated independently per output. The application selects the output from the frozen test backend, provisions all lane tokens, and finalizes only after every lane finishes. BreakTest's integration -requires generator capability `k6-custom-v2`, verified against the installed binary +requires generator capability `k6-breaktest-db-v1`, verified against the installed binary checksum; this prevents routing multi-lane jobs to the older single-source writer. Generator machine statistics remain the responsibility of the existing agent. @@ -107,12 +107,12 @@ also fails the output, rather than storing a dangling screenshot reference. ```sh go test -race ./metrics ./output ./lib/netext/httpext ./internal/output/breakingit -go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' -go test ./internal/cmd -run 'TestBuiltinOutputString|TestCustomDBAndTimescaleConstructors' -go build -tags lean -o /tmp/k6-custom-db-writer . -python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-custom-db-writer \ +go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestBreakTestDBAndTimescaleConstructors' +go test ./internal/cmd -run 'TestBuiltinOutputString|TestBreakTestDBAndTimescaleConstructors' +go build -tags lean -o /tmp/k6-breaktest-db-writer . +python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-breaktest-db-writer \ --image breaktest-storage-lab:local --lanes 4 -python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-custom-db-writer --lanes 4 --restart-live +python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-breaktest-db-writer --lanes 4 --restart-live ``` The smoke test creates its own disposable Docker service and temporary directory,