diff --git a/FORK.md b/FORK.md index f0108a00e..5d3980111 100644 --- a/FORK.md +++ b/FORK.md @@ -23,6 +23,8 @@ code to the k6 executable. ## Fork-specific changes - Adds the `timescaledb-breakingit` output for writing load-test data through a PG-proxy. +- Adds the opt-in `breaktest-db-breakingit` direct output alongside TimescaleDB; see + [configuration and current limits](internal/output/breakingit/BREAKTEST_DB.md). - Adds optional detailed request and response data for failed HTTP requests. - Adds the optional `httpRequestUrl` option to keep a named request's real URL in sample metadata. - Extends browser network and error metrics, including optional error screenshots. diff --git a/ai-tests/breaktest-db/smoke.py b/ai-tests/breaktest-db/smoke.py new file mode 100644 index 000000000..33db2f3bc --- /dev/null +++ b/ai-tests/breaktest-db/smoke.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Isolated k6 -> Rust protocol test with the pg-proxy binary COPY writer as an oracle. + +Requires Docker, Python 3, a built k6 fork and a built BreakTest DB service image. +Creates a temporary service/container and directory; never connects to customer DBs. +""" +import argparse +import email.parser +import gzip +import http.server +import io +import json +import os +from pathlib import Path +import secrets +import struct +import socket +import html +import subprocess +import tempfile +import threading +import time +import urllib.request +import uuid + + + +def copy_rows(table, payload): + schemas = { + "requests_raw": "tssssbiisiii", + "transactions": "tsssbiii", + "requests_error": "tssssssiissssssu", + "virtual_users": "tsssiii", + } + schema = schemas[table] + stream = io.BytesIO(payload) + if stream.read(19) != b"PGCOPY\n\xff\r\n\x00" + bytes(8): + raise ValueError("Invalid binary COPY header") + while True: + count, = struct.unpack(">h", stream.read(2)) + if count == -1: + if stream.read(1): + raise ValueError("Trailing bytes after COPY data") + return + if count != len(schema): + raise ValueError(f"Unexpected {table} row width {count}") + row = [] + for kind in schema: + size, = struct.unpack(">i", stream.read(4)) + if size == -1: + row.append(None) + continue + if size < 0: + raise ValueError("Invalid COPY field length") + field = stream.read(size) + if len(field) != size: + raise ValueError("Truncated COPY field") + if kind == "s": + value = field.decode() + elif kind == "u": + value = str(uuid.UUID(bytes=field)) + else: + value, = struct.unpack({"i": ">i", "t": ">q", "b": ">?"}[kind], field) + row.append(value) + yield row + + +def run(args): + name = "k6-breaktest-db-smoke-" + uuid.uuid4().hex[:12] + admin = secrets.token_hex(32) + producers = [secrets.token_hex(32) for _ in range(args.lanes)] + producer = ",".join(producers) + oracle = {} + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + if self.path == '/redirect': + self.send_response(302);self.send_header('Location','/ok');self.end_headers();return + if self.path == '/reset': + self.connection.shutdown(socket.SHUT_RDWR);self.connection.close();return + body=b'fixture' if self.path=='/page' else b'fixture response' + self.send_response(503 if self.path in ('/failure','/slowfailure') else 200) + self.send_header('Content-Type','text/html') + self.send_header('Content-Length',str(len(body))) + self.send_header('Cache-Control','max-age=3600' if self.path=='/cached' else 'no-store') + self.end_headers() + if self.path=='/slowfailure': + self.wfile.write(body[:3]);self.wfile.flush();time.sleep(1.2);body=body[3:] + self.wfile.write(body) + + def do_POST(self): + if self.path != "/ingest/batch": + self.send_error(404) + return + data = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers.get("Content-Encoding") == "gzip": + data = gzip.decompress(data) + msg = email.parser.BytesParser().parsebytes( + b"MIME-Version: 1.0\r\nContent-Type: " + + self.headers["Content-Type"].encode() + b"\r\n\r\n" + data) + for part in msg.get_payload(): + table = part["X-Table"].removeprefix("public.") + if part["X-Format"] != "binary": + raise ValueError(f"Expected binary COPY for {table}") + rows = copy_rows(table, part.get_payload(decode=True)) + oracle.setdefault(table, []).extend(rows) + self.send_response(200) + self.end_headers() + self.wfile.write(b"OK") + + target = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=target.serve_forever, daemon=True) + thread.start() + with tempfile.TemporaryDirectory(prefix=name) as temp: + root = Path(temp) + data_dir = root / "data" + data_dir.mkdir() + env = {**os.environ, "BREAKTEST_DB_ADMIN_TOKEN": admin} + with socket.socket() as port_socket: + port_socket.bind(('127.0.0.1',0)) + fixed_port=port_socket.getsockname()[1] + try: + subprocess.run(["docker", "run", "-d", "--name", name, + "--label", "breaktest.k6-writer-smoke=true", "--memory", "512m", + "-e", "BREAKTEST_DB_ADMIN_TOKEN", "-p", f"127.0.0.1:{fixed_port}:8099", + "-v", f"{data_dir}:/data", args.image, + "serve", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True) + port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1] + service = f"http://127.0.0.1:{port}" + + def post(path, payload): + request = urllib.request.Request(service + path, data=json.dumps(payload).encode(), + headers={"Authorization": "Bearer " + admin, "Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + for _ in range(100): + try: + post("/health", {}) + break + except OSError: + time.sleep(.05) + origin = int(time.time() - 946684800 - 10) * 1_000_000 + post("/runs/create", {"run_id": "k6-smoke", "source_tokens": producers, + "origin_us": origin, "customer_id": "isolated-k6-smoke"}) + script = root / "smoke.js" + script.write_text('''import http from 'k6/http'; +import { group, check, sleep } from 'k6'; +import { Trend } from 'k6/metrics'; +const customLatency = new Trend('custom_latency'); +export const options = { vus: 2, iterations: Number(__ENV.FIXTURE_ITERATIONS) }; +export default function () { + group('Checkout', function () { + http.get(__ENV.TARGET + '/ok'); + const r = http.get(__ENV.TARGET + '/failure'); + check(r, { 'expected failure': x => x.status === 200 }); + }); + if (__ENV.FIXTURE_CUSTOM === 'true') customLatency.add(37); + sleep(0.15); +} +''') + if args.browser: + script.write_text("""import { browser } from 'k6/browser'; +import { group, check, sleep } from 'k6'; +export const options = { scenarios: { browser: {executor:'per-vu-iterations',vus:1,iterations:Number(__ENV.FIXTURE_ITERATIONS),options:{browser:{type:'chromium'}}} } }; +export default async function(){ + const page=await browser.newPage(); + try { await group('Checkout',async function(){ + await page.goto(__ENV.TARGET+'/page'); + const status=await page.evaluate(async()=>{ + await fetch('/redirect');await fetch('/cached');await fetch('/cached'); + try {await fetch('/reset')}catch(e){} + return (await fetch('/slowfailure').then(async r=>{await r.text();return r.status})); + }); + check(status,{'expected failure': s=>s===200}); + }); sleep(.3); } finally {await page.close();} +} +""") + target_url = f"http://127.0.0.1:{target.server_port}" + writer_env = {**os.environ, "BREAKTEST_DB_TOKEN": producer, "runId": "k6-smoke", + "nodeName": "k6-probe", "location": "Local", "PG_PROXY_TOKEN": "isolated-oracle", + "TARGET": target_url, "FIXTURE_ITERATIONS": str(args.iterations), "FIXTURE_CUSTOM": str(args.custom_metrics).lower(), "K6_NO_USAGE_REPORT": "true", "K6_HTTP_ERROR_DATA": "true"} + for key in ("syntheticMonitoring", "K6_BROWSER_SCREENSHOT_ON_ERROR", "CUSTOM_URL_PATTERNS", "BREAKINGIT_TRANSACTION_SOURCE"): + writer_env.pop(key, None) + command = [str(Path(args.k6).resolve()), "run", "--quiet", + "--out", "breaktest-db-breakingit=" + service, + "--out", "timescaledb-breakingit=" + target_url, str(script)] + if args.container_image: + root.chmod(0o755); script.chmod(0o644) + for key in ('TARGET',): writer_env[key]=writer_env[key].replace('127.0.0.1','host.docker.internal') + command=['docker','run','--rm','--shm-size','512m','--entrypoint','/usr/bin/k6', + '-v',str(Path(args.k6).resolve())+':/usr/bin/k6:ro','-v',str(script)+':/fixture.js:ro', + '-e','K6_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium','-e','K6_BROWSER_ARGS=no-sandbox', + '-e','K6_BROWSER_HEADLESS=true', + *[item for key in ('BREAKTEST_DB_TOKEN','runId','nodeName','location','PG_PROXY_TOKEN','TARGET','K6_NO_USAGE_REPORT','K6_HTTP_ERROR_DATA','FIXTURE_ITERATIONS','FIXTURE_CUSTOM') for item in ('-e',key)], + args.container_image,'run','--quiet','--out','breaktest-db-breakingit='+service.replace('127.0.0.1','host.docker.internal'), + '--out','timescaledb-breakingit='+target_url.replace('127.0.0.1','host.docker.internal'),'/fixture.js'] + observations=[] + with (root/'k6.log').open('w+') as log: + process=subprocess.Popen(command,env=writer_env,text=True,stdout=log,stderr=log) + began=time.monotonic(); restarted=False + try: + while process.poll() is None: + if time.monotonic()-began>90:raise TimeoutError('k6 smoke deadline') + try: + observation=post('/runs/k6-smoke/snapshot',{}) + if not observation['sealed']: observations.append(observation) + except OSError: pass + if args.restart_live and not restarted and time.monotonic()-began>=2: + subprocess.run(['docker','kill','--signal=KILL',name],check=True,capture_output=True) + subprocess.run(['docker','start',name],check=True,capture_output=True) + restarted=True + time.sleep(.1) + log.seek(0) + text=log.read() + assert process.returncode==0 and 'BREAKTEST_DB_FAILED' not in text,text + finally: + if process.poll() is None: process.kill(); process.wait() + live=[s for s in observations if all(not x['finished'] for x in s['sources']) and min(x['watermark'] for x in s['sources'])>origin and any(r['complete'] for r in s['rows'])] + assert live, 'No live watermark-completed aggregates observed' + end = int(time.time() - 946684800 + 10) * 1_000_000 + final = post("/runs/k6-smoke/finalize", {}) + assert final["sealed"], final + if args.restart_live: + assert restarted and final['recovered'],final + assert all(g.get('max_second') is not None for g in final['gaps']),final + assert final['run_complete'] == (not final['gaps'] and not final['unknown_coverage']),final + else: assert final['run_complete'] and not final['gaps'],final + expected_counts = {"requests_raw": args.iterations*(3 if args.custom_metrics else 2), "transactions": args.iterations, "requests_error": args.iterations*2} + counts = {} + for table in ("requests_raw", "transactions", "requests_error", "virtual_users"): + actual = post("/runs/k6-smoke/query", {"table": table, "start_us": origin, "end_us": end}) + counts[table] = len(actual["rows"]) + if not args.restart_live: assert counts[table] == len(oracle[table]), (table, counts[table], len(oracle[table])) + else: assert counts[table] <= len(oracle[table]), (table, counts) + if table in expected_counts and not args.restart_live and not args.browser: + assert counts[table] == expected_counts[table], (table, counts) + else: + assert counts[table] > 0 + # Match all fields including timestamp, except independently-generated UUIDs. + def normalized(row): + values = [] + for i, value in enumerate(row): + if table == "requests_error" and i == 15: + continue + if value is None or value == "": + value = None + elif isinstance(value, bool): + value = "true" if value else "false" + else: + value = str(value) + values.append(value) + return json.dumps(values, ensure_ascii=False) + left = sorted(normalized(row) for row in actual["rows"]) + right = sorted(normalized(row) for row in oracle[table]) + if args.restart_live: + from collections import Counter + assert not (Counter(left)-Counter(right)), (table,'duplicate or corrupted recovered rows') + if len(left)= builtinOutput(len(_builtinOutputIndex)-1) { @@ -36,9 +36,10 @@ func _builtinOutputNoOp() { _ = x[builtinOutputOpentelemetry-(9)] _ = x[builtinOutputSummary-(10)] _ = x[builtinOutputTimescaledbBreakingit-(11)] + _ = x[builtinOutputBreakTestDBBreakingit-(12)] } -var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit} +var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit, builtinOutputBreakTestDBBreakingit} var _builtinOutputNameToValueMap = map[string]builtinOutput{ _builtinOutputName[0:5]: builtinOutputCloud, @@ -65,6 +66,8 @@ var _builtinOutputNameToValueMap = map[string]builtinOutput{ _builtinOutputLowerName[103:110]: builtinOutputSummary, _builtinOutputName[110:132]: builtinOutputTimescaledbBreakingit, _builtinOutputLowerName[110:132]: builtinOutputTimescaledbBreakingit, + _builtinOutputName[132:155]: builtinOutputBreakTestDBBreakingit, + _builtinOutputLowerName[132:155]: builtinOutputBreakTestDBBreakingit, } var _builtinOutputNames = []string{ @@ -80,6 +83,7 @@ var _builtinOutputNames = []string{ _builtinOutputName[90:103], _builtinOutputName[103:110], _builtinOutputName[110:132], + _builtinOutputName[132:155], } // builtinOutputString retrieves an enum value from the enum constants string name. diff --git a/internal/cmd/outputs.go b/internal/cmd/outputs.go index a641fc2ac..c95a11882 100644 --- a/internal/cmd/outputs.go +++ b/internal/cmd/outputs.go @@ -35,6 +35,7 @@ const ( builtinOutputOpentelemetry builtinOutputSummary builtinOutputTimescaledbBreakingit + builtinOutputBreakTestDBBreakingit ) func getPossibleIDList(constrs map[string]output.Constructor) string { diff --git a/internal/cmd/outputs_full.go b/internal/cmd/outputs_full.go index ae701b283..45e1fd671 100644 --- a/internal/cmd/outputs_full.go +++ b/internal/cmd/outputs_full.go @@ -48,6 +48,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { }, "web-dashboard": dashboard.New, builtinOutputTimescaledbBreakingit.String(): breakingit.New, + builtinOutputBreakTestDBBreakingit.String(): breakingit.NewBreakTestDB, builtinOutputExperimentalOpentelemetry.String(): func(params output.Params) (output.Output, error) { params.Logger.Warnf("OpenTelemetry output has been graduated as a stable output."+ "You can now use just %q instead of %q. The experimental version will be removed in future versions.", diff --git a/internal/cmd/outputs_lean.go b/internal/cmd/outputs_lean.go index c58dc328a..4527636e7 100644 --- a/internal/cmd/outputs_lean.go +++ b/internal/cmd/outputs_lean.go @@ -56,6 +56,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { return nil, errors.New("the web-dashboard output is not available in this lean build of k6") }, builtinOutputTimescaledbBreakingit.String(): breakingit.New, + builtinOutputBreakTestDBBreakingit.String(): breakingit.NewBreakTestDB, } exts := ext.Get(ext.OutputExtension) diff --git a/internal/cmd/outputs_test.go b/internal/cmd/outputs_test.go index 0501f2b64..572dc72f1 100644 --- a/internal/cmd/outputs_test.go +++ b/internal/cmd/outputs_test.go @@ -12,7 +12,15 @@ func TestBuiltinOutputString(t *testing.T) { "cloud", "csv", "datadog", "experimental-prometheus-rw", "influxdb", "json", "kafka", "statsd", "experimental-opentelemetry", "opentelemetry", - "summary", "timescaledb-breakingit", + "summary", "timescaledb-breakingit", "breaktest-db-breakingit", } assert.Equal(t, exp, builtinOutputStrings()) } + +func TestBreakTestDBAndTimescaleConstructors(t *testing.T) { + t.Parallel() + constructors, err := getAllOutputConstructors() + assert.NoError(t, err) + assert.Contains(t, constructors, "timescaledb-breakingit") + assert.Contains(t, constructors, "breaktest-db-breakingit") +} diff --git a/internal/js/modules/k6/browser/common/network_manager.go b/internal/js/modules/k6/browser/common/network_manager.go index 7dbc58ac3..77cdc3d09 100644 --- a/internal/js/modules/k6/browser/common/network_manager.go +++ b/internal/js/modules/k6/browser/common/network_manager.go @@ -286,7 +286,13 @@ func (m *NetworkManager) emitRequestMetrics(req *Request) { }) } -func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //nolint:funlen +func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request, completion browserCompletion) { //nolint:funlen + transferred := false + defer func() { + if !transferred { + completion.ticket.Release() + } + }() state := m.vu.State() if state == nil { return // CDP events can arrive after iteration teardown. @@ -368,67 +374,70 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //n return } - pushIfNotDone(m.vu.Context(), m.logger, state.Samples, k6metrics.ConnectedSamples{ + bundle := k6metrics.ConnectedSamples{ Samples: []k6metrics.Sample{ { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqDuration, Tags: tags}, Value: timings.durationMs, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqBlocked, Tags: tags}, Value: timings.blocked, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqConnecting, Tags: tags}, Value: timings.connecting, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqTLSHandshaking, Tags: tags}, Value: timings.tlsHandshaking, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqSending, Tags: tags}, Value: timings.sending, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqWaiting, Tags: tags}, Value: timings.waiting, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqReceiving, Tags: tags}, Value: timings.receiving, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, { TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserDataReceived, Tags: tags}, Value: float64(bodySize), - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }, }, - }) + } - if resp != nil { + { + if resp == nil { + failed = 1 + } sample := k6metrics.Sample{ TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqFailed, Tags: tags}, Value: failed, - Time: wallTime, + Time: completion.at, Metadata: tagsAndMeta.Metadata, } - if failed == 1 { + if failed == 1 && resp != nil { reqHeaders, reqBody := req.formatErrorRequestData() sample.HTTPErrorReqHeaders = reqHeaders sample.HTTPErrorReqBody = reqBody @@ -438,10 +447,9 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //n sample.HTTPErrorResHeaders = resHeaders sample.HTTPErrorResBody = resBody } - pushIfNotDone(m.vu.Context(), m.logger, state.Samples, k6metrics.ConnectedSamples{ - Samples: []k6metrics.Sample{sample}, - }) + bundle.Samples = append(bundle.Samples, sample) } + transferred = k6metrics.PushTrackedRequest(m.vu.Context(), state.Samples, completion.ticket, bundle) } type browserResponseTimings struct { @@ -542,7 +550,13 @@ func (m *NetworkManager) calcReceiving(req *Request, t *network.ResourceTiming, // ERR_CONNECTION_REFUSED). This ensures outputs like breakingit receive metrics for // failed navigations/requests. The failure reason (e.g. net::ERR_NAME_NOT_RESOLVED) is // added as the "error" tag so CSV and other outputs can show it. -func (m *NetworkManager) emitRequestFailedMetrics(req *Request) { +func (m *NetworkManager) emitRequestFailedMetrics(req *Request, completion browserCompletion) { + transferred := false + defer func() { + if !transferred { + completion.ticket.Release() + } + }() state := m.vu.State() if state == nil { return @@ -596,7 +610,7 @@ func (m *NetworkManager) emitRequestFailedMetrics(req *Request) { } { samples = append(samples, k6metrics.Sample{ TimeSeries: k6metrics.TimeSeries{Metric: metric, Tags: tags}, - Time: wallTime, Metadata: tagsAndMeta.Metadata, + Time: completion.at, Metadata: tagsAndMeta.Metadata, }) } samples[0].Value = durationMs @@ -604,7 +618,7 @@ func (m *NetworkManager) emitRequestFailedMetrics(req *Request) { failed.Value = 1 failed.HTTPErrorReqHeaders = reqHeaders failed.HTTPErrorReqBody = reqBody - k6metrics.PushIfNotDone(m.vu.Context(), state.Samples, k6metrics.ConnectedSamples{Samples: samples}) + transferred = k6metrics.PushTrackedRequest(m.vu.Context(), state.Samples, completion.ticket, k6metrics.ConnectedSamples{Samples: samples}) } // handleURLTag will check if the url tag needs to be grouped by testing @@ -626,6 +640,7 @@ func handleURLTag(mi eventInterceptor, url string, method string, tags *k6metric func (m *NetworkManager) handleRequestRedirect( req *Request, redirectResponse *network.Response, hasExtraInfo bool, timestamp *cdp.MonotonicTime, ) { + completion := m.beginBrowserCompletion() resp := NewHTTPResponse(m.ctx, req, redirectResponse, timestamp) m.extraInfoTracker.processResponse(req.requestID, resp, hasExtraInfo) req.responseMu.Lock() @@ -638,7 +653,7 @@ func (m *NetworkManager) handleRequestRedirect( // them (the redirect's ExtraInfo may still be in flight here). m.wg.Go(func() { resp.WaitForRawHeaders() - m.emitResponseMetrics(resp, req) + m.emitResponseMetrics(resp, req, completion) }) m.deleteRequestByID(req.requestID) @@ -740,7 +755,7 @@ func (m *NetworkManager) onLoadingFailed(event *network.EventLoadingFailed) { // Emit browser metrics for failed requests so outputs (e.g. breakingit) can store them. if !isInternalURL(req.url) { - m.emitRequestFailedMetrics(req) + m.emitRequestFailedMetrics(req, m.beginBrowserCompletion()) } m.eventInterceptor.onRequestFailed(req) @@ -757,6 +772,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished) return } + completion := m.beginBrowserCompletion() req.responseEndWall = event.Timestamp.Time() req.responseEndTiming = float64(event.Timestamp.Time().Sub(req.timestamp).Milliseconds()) req.encodedDataLength = int64(event.EncodedDataLength) @@ -768,6 +784,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished) // Skip data and blob URLs when emitting metrics, since they're internal to the browser. if isInternalURL(req.url) { + completion.ticket.Release() return } // Emit the response metrics in a separate goroutine, once the raw headers @@ -784,7 +801,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished) if resp != nil { resp.WaitForRawHeaders() } - m.emitResponseMetrics(resp, req) + m.emitResponseMetrics(resp, req, completion) }) } diff --git a/internal/js/modules/k6/browser/common/network_manager_test.go b/internal/js/modules/k6/browser/common/network_manager_test.go index d1338a4a0..7fe402094 100644 --- a/internal/js/modules/k6/browser/common/network_manager_test.go +++ b/internal/js/modules/k6/browser/common/network_manager_test.go @@ -311,7 +311,7 @@ func TestNetworkManagerEmitRequestResponseMetricsTimingSkew(t *testing.T) { &network.Response{Timing: &network.ResourceTiming{}}, (*cdp.MonotonicTime)(&tt.res.ts), ) - nm.emitResponseMetrics(res, req) + nm.emitResponseMetrics(res, req, browserCompletion{at: tt.wantRes.wt}) n = vu.AssertSamples(func(s k6metrics.Sample) { assert.Equalf(t, tt.wantRes.wt, s.Time, "timing skew in %s", s.Metric.Name) }) @@ -363,7 +363,7 @@ func TestNetworkManagerMetricsRetainRequestTagsAndMetadata(t *testing.T) { URL: req.URL(), Timing: &network.ResourceTiming{}, }, (*cdp.MonotonicTime)(&now)) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) n := vu.AssertSamples(func(sample k6metrics.Sample) { group, ok := sample.Tags.Get(k6metrics.TagGroup.String()) @@ -379,7 +379,7 @@ func TestNetworkManagerMetricsRetainRequestTagsAndMetadata(t *testing.T) { // The fork also emits samples when navigation fails before any response. // Those must use the request's context, not the VU's restored context. req.setErrorText("net::ERR_CONNECTION_REFUSED") - nm.emitRequestFailedMetrics(req) + nm.emitRequestFailedMetrics(req, nm.beginBrowserCompletion()) n = vu.AssertSamples(func(sample k6metrics.Sample) { group, _ := sample.Tags.Get(k6metrics.TagGroup.String()) assert.Equal(t, "::request", group) @@ -621,7 +621,7 @@ func TestNetworkManagerReceivingMetric(t *testing.T) { }, (*cdp.MonotonicTime)(&reqTimestamp), ) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) var receivingVal float64 vu.AssertSamples(func(s k6metrics.Sample) { @@ -687,7 +687,7 @@ func TestNetworkManagerReceivingMetricNoHeadersEndWall(t *testing.T) { }, (*cdp.MonotonicTime)(&reqTimestamp), ) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) var receivingVal float64 vu.AssertSamples(func(s k6metrics.Sample) { @@ -751,7 +751,7 @@ func TestNetworkManagerReceivingMetricCachedResponse(t *testing.T) { }, (*cdp.MonotonicTime)(&reqTimestamp), ) - nm.emitResponseMetrics(resp, req) + nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion()) var receivingVal float64 receivingFound := false diff --git a/internal/js/modules/k6/browser/common/request_completion.go b/internal/js/modules/k6/browser/common/request_completion.go new file mode 100644 index 000000000..aaf7570ab --- /dev/null +++ b/internal/js/modules/k6/browser/common/request_completion.go @@ -0,0 +1,24 @@ +package common + +import ( + "time" + + k6metrics "go.k6.io/k6/v2/metrics" +) + +// Browser completion is observed at terminal CDP-event dispatch, before async +// header/body retrieval. Chromium phase durations remain in their CDP time base; +// the result's timestamp is the generator-observed sampler completion, for ALL +// outputs. Retimestamping at output delivery would hide backlog and is forbidden. +type browserCompletion struct { + at time.Time + ticket *k6metrics.RequestTicket +} + +func (m *NetworkManager) beginBrowserCompletion() browserCompletion { + var ticket *k6metrics.RequestTicket + if state := m.vu.State(); state != nil { + ticket = k6metrics.BeginRequest(state.Samples) + } + return browserCompletion{at: k6metrics.RequestTime(ticket, time.Now()), ticket: ticket} +} diff --git a/internal/js/modules/k6/browser/common/request_completion_test.go b/internal/js/modules/k6/browser/common/request_completion_test.go new file mode 100644 index 000000000..1f275dd3f --- /dev/null +++ b/internal/js/modules/k6/browser/common/request_completion_test.go @@ -0,0 +1,57 @@ +package common + +import ( + "net/url" + "testing" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/internal/js/modules/k6/browser/k6ext" + "go.k6.io/k6/v2/internal/js/modules/k6/browser/k6ext/k6test" + "go.k6.io/k6/v2/internal/js/modules/k6/browser/log" + k6metrics "go.k6.io/k6/v2/metrics" +) + +func TestBrowserCompletionPinsHeaderDelayAndOutputQueue(t *testing.T) { + samples := make(chan k6metrics.SampleContainer, 10) + tracker, unbind := k6metrics.BindRequestTracker(samples) + defer unbind() + vu := k6test.NewVU(t, k6test.WithSamples(samples)) + vu.ActivateVU() + nm := &NetworkManager{vu: vu, customMetrics: k6ext.RegisterCustomMetrics(k6metrics.NewRegistry()), eventInterceptor: &EventInterceptorMock{}, logger: log.NewNullLogger()} + u, _ := url.Parse("https://example.test/error") + req := &Request{url: u, wallTime: time.Now().Add(-time.Second), tagsAndMeta: vu.State().Tags.GetCurrentValues()} + completion := nm.beginBrowserCompletion() + // A captured terminal event waits through delayed headers/body extraction. + require.NotNil(t, completion.ticket) + time.Sleep(1100 * time.Millisecond) + require.False(t, tracker.Before().After(completion.at)) + resp := &Response{url: u.String(), status: 503, remoteAddress: &RemoteAddress{}, wallTime: completion.at.Add(-500 * time.Millisecond), timing: &network.ResourceTiming{}, body: []byte("error detail"), request: req} + nm.emitResponseMetrics(resp, req, completion) + require.False(t, tracker.Before().After(completion.at)) // buffered output still pins it + frame := (<-samples).(k6metrics.RequestSamples) + require.Len(t, frame.GetSamples(), 9) + for _, sample := range frame.GetSamples() { + require.Equal(t, completion.at, sample.Time) + } + require.Equal(t, float64(1), frame.GetSamples()[8].Value) + frame.Ticket.Release() + require.True(t, tracker.Before().After(completion.at)) +} + +func TestBrowserCacheSkipReleasesCompletionPin(t *testing.T) { + samples := make(chan k6metrics.SampleContainer, 10) + tracker, unbind := k6metrics.BindRequestTracker(samples) + defer unbind() + vu := k6test.NewVU(t, k6test.WithSamples(samples)) + vu.ActivateVU() + nm := &NetworkManager{vu: vu, customMetrics: k6ext.RegisterCustomMetrics(k6metrics.NewRegistry()), eventInterceptor: &EventInterceptorMock{}, logger: log.NewNullLogger()} + u, _ := url.Parse("https://example.test/cached") + req := &Request{url: u, wallTime: time.Now(), tagsAndMeta: vu.State().Tags.GetCurrentValues(), fromMemoryCache: true} + completion := nm.beginBrowserCompletion() + nm.emitResponseMetrics(&Response{url: u.String(), remoteAddress: &RemoteAddress{}, wallTime: time.Now(), request: req, body: []byte("cached")}, req, completion) + require.Empty(t, samples) + time.Sleep(1100 * time.Millisecond) + require.True(t, tracker.Before().After(completion.at)) +} diff --git a/internal/js/modules/k6/metrics/metrics.go b/internal/js/modules/k6/metrics/metrics.go index 5e0bc3bdf..bae63e2c0 100644 --- a/internal/js/modules/k6/metrics/metrics.go +++ b/internal/js/modules/k6/metrics/metrics.go @@ -115,16 +115,17 @@ func (m Metric) add(v sobek.Value, addTags sobek.Value) (bool, error) { return false, fmt.Errorf("cannot add tags for the '%s' custom metric: %w", m.metric.Name, err) } + ticket := metrics.BeginRequest(state.Samples) sample := metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: m.metric, Tags: ctm.Tags, }, - Time: time.Now(), + Time: metrics.RequestTime(ticket, time.Now()), Metadata: ctm.Metadata, Value: vfloat, } - metrics.PushIfNotDone(m.vu.Context(), state.Samples, sample) + metrics.PushTrackedRequest(m.vu.Context(), state.Samples, ticket, sample) return true, nil } diff --git a/internal/js/modules/k6/websockets/websockets.go b/internal/js/modules/k6/websockets/websockets.go index 90529fc9f..4938a72c1 100644 --- a/internal/js/modules/k6/websockets/websockets.go +++ b/internal/js/modules/k6/websockets/websockets.go @@ -306,7 +306,8 @@ func (w *webSocket) establishConnection(ctx context.Context, params *wsParams) { w.started = time.Now() handshakeDone := make(chan struct{}) go w.closePendingConnectionOnContextDone(ctx, handshakeDone) - start := time.Now() + ticket := metrics.BeginRequest(w.vu.State().Samples) + start := metrics.RequestTime(ticket, time.Now()) conn, httpResponse, connErr := w.dial(ctx, params) w.connectionMu.Lock() w.pendingConnection = nil @@ -345,7 +346,7 @@ func (w *webSocket) establishConnection(ctx context.Context, params *wsParams) { } //nolint:contextcheck // Handshake cancellation must not suppress connection metrics. - w.emitConnectionMetrics(w.vu.Context(), start, connectionDuration) + w.emitConnectionMetrics(w.vu.Context(), start, connectionDuration, ticket) if connErr != nil { // Pass the error to the user script before exiting immediately w.tq.Queue(func() error { @@ -437,10 +438,14 @@ func (w *webSocket) closePendingConnection() { } // emitConnectionMetrics emits the metrics for a websocket connection. -func (w *webSocket) emitConnectionMetrics(ctx context.Context, start time.Time, duration float64) { +func (w *webSocket) emitConnectionMetrics(ctx context.Context, start time.Time, duration float64, tickets ...*metrics.RequestTicket) { + var ticket *metrics.RequestTicket + if len(tickets) > 0 { + ticket = tickets[0] + } state := w.vu.State() - metrics.PushIfNotDone(ctx, state.Samples, metrics.ConnectedSamples{ + metrics.PushTrackedRequest(ctx, state.Samples, ticket, metrics.ConnectedSamples{ Samples: []metrics.Sample{ { TimeSeries: metrics.TimeSeries{Metric: state.BuiltinMetrics.WSSessions, Tags: w.tagsAndMeta.Tags}, @@ -827,7 +832,6 @@ func (w *webSocket) ping() { } func (w *webSocket) trackPong(pingID string) { - pongTimestamp := time.Now() pingTimestamp, ok := w.sendPings.timestamps[pingID] if !ok { @@ -838,7 +842,9 @@ func (w *webSocket) trackPong(pingID string) { return } - metrics.PushIfNotDone(w.vu.Context(), w.vu.State().Samples, metrics.Sample{ + ticket := metrics.BeginRequest(w.vu.State().Samples) + pongTimestamp := metrics.RequestTime(ticket, time.Now()) + metrics.PushTrackedRequest(w.vu.Context(), w.vu.State().Samples, ticket, metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: w.builtinMetrics.WSPing, Tags: w.tagsAndMeta.Tags, diff --git a/internal/js/modules/k6/ws/ws.go b/internal/js/modules/k6/ws/ws.go index 5389fc788..dfe186a44 100644 --- a/internal/js/modules/k6/ws/ws.go +++ b/internal/js/modules/k6/ws/ws.go @@ -268,7 +268,8 @@ func (mi *WS) dial( wsd.Jar = args.cookieJar } - connStart := time.Now() + ticket := metrics.BeginRequest(state.Samples) + connStart := metrics.RequestTime(ticket, time.Now()) conn, httpResponse, dialErr := wsd.DialContext(ctx, url, args.headers) connEnd := time.Now() @@ -303,7 +304,7 @@ func (mi *WS) dial( builtinMetrics: state.BuiltinMetrics, } - connEndHook := socket.pushSessionMetrics(connStart, connEnd) + connEndHook := socket.pushSessionMetrics(connStart, connEnd, ticket) return &socket, httpResponse, connEndHook, dialErr } @@ -392,7 +393,6 @@ func (s *Socket) Ping() { } func (s *Socket) trackPong(pingID string) { - pongTimestamp := time.Now() if _, ok := s.pingSendTimestamps[pingID]; !ok { // We received a pong for a ping we didn't send; ignore @@ -400,8 +400,10 @@ func (s *Socket) trackPong(pingID string) { return } pingTimestamp := s.pingSendTimestamps[pingID] + ticket := metrics.BeginRequest(s.samplesOutput) + pongTimestamp := metrics.RequestTime(ticket, time.Now()) - metrics.PushIfNotDone(s.ctx, s.samplesOutput, metrics.Sample{ + metrics.PushTrackedRequest(s.ctx, s.samplesOutput, ticket, metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: s.builtinMetrics.WSPing, Tags: s.tagsAndMeta.Tags, @@ -515,10 +517,14 @@ func (s *Socket) closeConnection(code int) error { return err } -func (s *Socket) pushSessionMetrics(connStart, connEnd time.Time) func() { +func (s *Socket) pushSessionMetrics(connStart, connEnd time.Time, tickets ...*metrics.RequestTicket) func() { + var ticket *metrics.RequestTicket + if len(tickets) > 0 { + ticket = tickets[0] + } connDuration := metrics.D(connEnd.Sub(connStart)) - metrics.PushIfNotDone(s.ctx, s.samplesOutput, metrics.ConnectedSamples{ + metrics.PushTrackedRequest(s.ctx, s.samplesOutput, ticket, metrics.ConnectedSamples{ Samples: []metrics.Sample{ { TimeSeries: metrics.TimeSeries{ diff --git a/internal/output/breakingit/BREAKTEST_DB.md b/internal/output/breakingit/BREAKTEST_DB.md new file mode 100644 index 000000000..ecc2f47bd --- /dev/null +++ b/internal/output/breakingit/BREAKTEST_DB.md @@ -0,0 +1,136 @@ +# Direct BreakTest DB output + +The fork has two independent built-in outputs, in both normal and `lean` builds: + +- `timescaledb-breakingit`: the pg-proxy binary COPY writer. +- `breaktest-db-breakingit`: direct zstd-compressed PostgreSQL binary COPY batches + to the BreakTest BreakTest DB service. It does not contact PostgreSQL or PG-proxy. + +The implementation lives in this fork, alongside the active built-in TimescaleDB +writer. It does not use the older xk6 extension in the BreakTest repository. + +## Invocation and application contract + +The application must first provision a BreakTest DB run and one to four producer tokens per +load generator. Pass the run ID and that generator's token when invoking k6: + +```sh +export runId='' +export BREAKTEST_DB_TOKEN=',' +export location='Amsterdam' +export nodeName='generator-1' +k6 run --out breaktest-db-breakingit=https://storage.example.test script.js +``` + +The URL is the service base URL (an optional routing prefix is retained). The +writer appends `/runs//hello`, `/ingest/batch`, `/watermark` and `/finish` to it. TLS +certificates are verified, connections reused, and redirects disabled so tokens +cannot be redirected to another origin. Never pass the service admin token. + +`BREAKINGIT_TRANSACTION_SOURCE` and `CUSTOM_URL_PATTERNS` retain their existing +mapping behavior. Native groups, legacy `pages`, requests, WebSocket/custom trend +request rows, failed checks, browser errors and VUs use the existing mapping code. +Empty fields remain NULL in the binary encoding, and negative/unavailable +connect times become NULL. Error UUIDs are generated independently per output. + +The application selects the output from the frozen test backend, provisions all +lane tokens, and finalizes only after every lane finishes. BreakTest's integration +requires generator capability `k6-breaktest-db-v1`, verified against the installed binary +checksum; this prevents routing multi-lane jobs to the older single-source writer. +Generator machine statistics remain the responsibility of the existing agent. + +## Delivery and limits + +- Flush every 200 ms or at a 2 MiB encoded-row budget. Four queued batches per lane bound + outgoing buffering; a full queue applies backpressure rather than dropping rows. + The building batch, one waiting enqueue, sender/compression buffers, and k6's own + sample containers consume additional memory; 8 MiB per lane is not a total RSS limit. +- One to four ordered compression/sender workers per generator, with independent + sequences and shared keep-alive connections. Batches are distributed round-robin. + Watermark-only messages reach idle lanes too. No high-load capacity claim yet. +- Incomplete HTTP joins are limited to 4,096 groups / an estimated 2 MiB payload + budget. Oversized rows or exhausted join budgets stop the run explicitly. +- HTTP/browser joins include timestamp and tag-set identity. All required request + metrics must arrive before a row is emitted; delayed browser failure payloads + are not treated as success. Ambiguous duplicate metrics or incomplete groups at + shutdown fail the output instead of publishing an apparently complete run. +- Retry transient failures and throttling with the exact same compressed bytes, + boundary, and sequence, for at most 30 seconds per operation. Validate ACKs. + Rejection, invalid ACK, or exhausted retries stops k6 and forbids `/finish`. +- Drain accepted sample containers and all queued requests before `/finish`. + The producer does not seal the run itself. A final barrier proves delivery of + this producer's data; it is not a durable-on-disk/fsync acknowledgement. +- A fresh producer source is required at startup. Tracked native and browser runs recover after a + server incarnation change using the anchored source clock and checkpoint sequence: + skip a held batch proven checkpointed; otherwise renumber and replay only that + unacknowledged batch. Previously acknowledged, lost rows are not replayed; the + service exposes its relaxed-durability coverage gaps. Same-incarnation conflicts + remain fatal. + +## Watermarks and unsupported data + +Native and browser k6 runs send live request watermarks, using the same sequence-backed, +all-source barrier as the Java writer. The output manager installs a per-run +tracker before producers start. HTTP completion assigns its timestamp and pin +atomically; WebSocket connecting/ping and custom metrics are similarly tracked. +Pins survive the engine sample channel and pending metric joins until rows are +serialized. A sharded tracker computes the oldest unresolved timestamp. It uses +an anchored monotonic clock, so a wall-clock step cannot invalidate a sent barrier. + +After detaching all preceding rows, the writer queues the same whole-second +barrier on every lane. A lane sends it only after acknowledging its earlier +batches. The service closes the common prefix into immediate 1s aggregates; open +seconds remain queryable through the pending/raw path. Event-time completeness +still does not imply fsync durability. Transactions and monitoring are separate +streams, just as in the Java request-watermark contract. + +Browser requests acquire a completion pin when the generator dispatches the +terminal CDP event (including redirects and loading failures), before deferred +header/body retrieval. They emit one complete timing/failure bundle. The pin +survives deferred work and the output queue; skipped cached requests release it. +Mixed browser/native scenarios use the same tracker and lane barriers. + +**Timestamp contract:** browser sampler timestamps now mean generator-observed +terminal CDP completion, for both BreakTest DB and TimescaleDB outputs. Previously, +successful rows used the response-header timestamp. Chromium phase/duration +measurements remain in their original time base. CDP transport delay precedes +this observation; this is not a claim that Chromium's original event clock is +watermarked. Nothing is retimestamped at output delivery to hide writer backlog. +Uninstrumented request metric producers fail closed. + +Synthetic monitoring and browser screenshots are explicitly unsupported and +rejected. Ordinary browser request timings, failure response data, native groups +and browser error messages are supported. An unexpected screenshot-bearing error +also fails the output, rather than storing a dangling screenshot reference. + +## Validation + +```sh +go test -race ./metrics ./output ./lib/netext/httpext ./internal/output/breakingit +go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestBreakTestDBAndTimescaleConstructors' +go test ./internal/cmd -run 'TestBuiltinOutputString|TestBreakTestDBAndTimescaleConstructors' +go build -tags lean -o /tmp/k6-breaktest-db-writer . +python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-breaktest-db-writer \ + --image breaktest-db:local --lanes 4 +python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-breaktest-db-writer --lanes 4 --restart-live +``` + +The smoke test creates its own disposable Docker service and temporary directory, +then runs both writers on the **same** metric stream. The TimescaleDB writer goes +to an isolated binary COPY capture server; it does not write an actual TimescaleDB database. +The test compares every stored column against the independently decoded proxy rows, except independently +generated error UUIDs. It checks request, transaction, error and VU counts, complete +sealing and reads after storage-service restart. It never connects to existing +customer databases. This establishes protocol/mapping compatibility, not a +TimescaleDB performance benchmark or browser end-to-end acceptance. + +The live smoke test also requires unfinished sources with advancing watermarks and +complete aggregate buckets before finalization. Crash mode kills only its own +throwaway service, verifies retained rows against the oracle without duplicates, +and requires explicit incomplete coverage whenever rows were lost. + +Browser validation also runs real Chromium with redirects, iframe requests, +connection failure, caching and a delayed 503 response, comparing both writers +on the same stream. Use `--browser --container-image IMAGE` with a Linux binary +and an image containing Chromium. This mapping oracle is distinct from the +actual-database high-load benchmark in the BreakTest integration repository. diff --git a/internal/output/breakingit/README.md b/internal/output/breakingit/README.md index 99605eb87..19bd20587 100644 --- a/internal/output/breakingit/README.md +++ b/internal/output/breakingit/README.md @@ -134,3 +134,9 @@ so producers can fill the next batch during retries. Exhausted retries discard only the failed batch; newly collected rows remain queued for the next send. Synthetic run IDs use UUID binary fields, with the same deterministic fallback as JMeter for legacy identifiers that are not UUIDs. + +## BreakTest DB alternative + +The fork also includes `breaktest-db-breakingit` for direct binary delivery to the +BreakTest DB service. See [configuration, delivery semantics and current +limits](BREAKTEST_DB.md). The existing TimescaleDB output remains available. diff --git a/internal/output/breakingit/batch_test.go b/internal/output/breakingit/batch_test.go index df8360ed9..4f686bdda 100644 --- a/internal/output/breakingit/batch_test.go +++ b/internal/output/breakingit/batch_test.go @@ -64,9 +64,5 @@ func TestBatchRetryPreservesConcurrentSamples(t *testing.T) { t.Fatal("missing batch") } } - require.Eventually(t, func() bool { - logger.mu.Lock() - defer logger.mu.Unlock() - return !logger.flushing && logger.bufTrans.Len() == 0 - }, 5*time.Second, 10*time.Millisecond) + require.Empty(t, logger.bufTrans.Bytes()) } diff --git a/internal/output/breakingit/breakingit.go b/internal/output/breakingit/breakingit.go index 22f597aa4..97f3f4520 100644 --- a/internal/output/breakingit/breakingit.go +++ b/internal/output/breakingit/breakingit.go @@ -62,6 +62,9 @@ const ( // Logger writes k6 metric samples to pg-proxy using COPY via multipart batch type Logger struct { + rowSink func(string, []string) error // direct BreakTest DB delivery, when configured + flushMu sync.Mutex // serialize sends, including the final drain + flushErr error // first exhausted delivery, guarded by flushMu out io.Writer proxyURL string database string @@ -78,7 +81,6 @@ type Logger struct { firstData chan struct{} firstQueued bool cleanupDone chan struct{} - flushing bool // flag to prevent recursive flush calls // pending screenshots to upsert via /upsert endpoint (low-volume) pendingScreenshots [][]any @@ -191,11 +193,19 @@ func (l *Logger) Stop() error { <-l.cleanupDone <-l.batchDone // Finish incomplete groups before the final batch. - for _, g := range l.pendingHTTPGroups { + l.mu.Lock() + groups := l.pendingHTTPGroups + l.pendingHTTPGroups = make(map[int64]*MetricGroup) + l.mu.Unlock() + for _, g := range groups { l.processHTTPGroup(g) } - l.flush() - return nil + l.flushMu.Lock() + defer l.flushMu.Unlock() + for l.flushOnce() { + // Drain rows accepted during the preceding send. + } + return l.flushErr } func (l *Logger) batchLoop() { @@ -250,17 +260,19 @@ func (l *Logger) cleanupProcessor() { } func (l *Logger) flush() { + l.flushMu.Lock() + defer l.flushMu.Unlock() + l.flushOnce() +} + +// flushOnce sends one detached batch. The caller holds flushMu. +func (l *Logger) flushOnce() bool { l.mu.Lock() - // Prevent recursive flush calls - if l.flushing { - l.mu.Unlock() - return - } hasBatchData := l.bufHTTP.Len() > 0 || l.bufVUs.Len() > 0 || l.bufTrans.Len() > 0 || l.bufErrors.Len() > 0 hasScreenshots := len(l.pendingScreenshots) > 0 if !hasBatchData && !hasScreenshots { l.mu.Unlock() - return + return false } // Transfer ownership so producers can fill the next batch during network I/O. @@ -273,7 +285,6 @@ func (l *Logger) flush() { // Snapshot pending screenshots screenshotsSnapshot := l.pendingScreenshots l.pendingScreenshots = nil - l.flushing = true l.mu.Unlock() // Send screenshots via upsert endpoint (separate from batch COPY path) @@ -284,18 +295,13 @@ func (l *Logger) flush() { } // Skip batch if no COPY data - if !hasBatchData { - l.mu.Lock() - l.flushing = false - l.mu.Unlock() - return + if hasBatchData { + body, boundary := l.batchBody(bufHTTPSnapshot, bufVUsSnapshot, bufTransSnapshot, bufErrorsSnapshot) + if err := l.sendBatch(body, boundary); err != nil && l.flushErr == nil { + l.flushErr = err + } } - - body, boundary := l.batchBody(bufHTTPSnapshot, bufVUsSnapshot, bufTransSnapshot, bufErrorsSnapshot) - l.sendBatch(body, boundary) - l.mu.Lock() - l.flushing = false - l.mu.Unlock() + return true } // upsertScreenshots sends screenshots to the pg-proxy upsert endpoint. @@ -480,7 +486,7 @@ func (l *Logger) processHTTPGroup(g *MetricGroup) { } } l.mu.Lock() - l.writeBinaryRow(l.bufHTTP, row) + l.writeRow(l.bufHTTP, row) // If not successful, also emit into requests_error(_sm) if success == "false" { l.writeHTTPError(g, transactionName, samplerName, responseCode, responseTime, connectTime, isBrowser) @@ -561,7 +567,7 @@ func (l *Logger) processWebSocketMetric(s metrics.Sample, ts int64) { } } l.mu.Lock() - l.writeBinaryRow(l.bufHTTP, row) + l.writeRow(l.bufHTTP, row) l.mu.Unlock() } @@ -643,7 +649,7 @@ func (l *Logger) processCustomTrendMetric(s metrics.Sample, ts int64) { } } l.mu.Lock() - l.writeBinaryRow(l.bufHTTP, row) + l.writeRow(l.bufHTTP, row) l.mu.Unlock() } @@ -716,7 +722,7 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { }) _, _ = fmt.Fprintf(l.out, "[screenshot] queued for upsert, pendingScreenshots=%d\n", len(l.pendingScreenshots)) } - l.writeBinaryRow(l.bufErrors, row) + l.writeRow(l.bufErrors, row) l.mu.Unlock() } else { row := []any{ @@ -743,7 +749,7 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { screenshotHash, screenshot, "image/png", len(screenshot), }) } - l.writeBinaryRow(l.bufErrors, row) + l.writeRow(l.bufErrors, row) l.mu.Unlock() } } @@ -987,7 +993,7 @@ func (l *Logger) processPageSample(s metrics.Sample, ts int64) { } } l.mu.Lock() - l.writeBinaryRow(l.bufTrans, row) + l.writeRow(l.bufTrans, row) l.mu.Unlock() } @@ -1033,7 +1039,7 @@ func (l *Logger) processGroupSample(s metrics.Sample, ts int64) { } } l.mu.Lock() - l.writeBinaryRow(l.bufTrans, row) + l.writeRow(l.bufTrans, row) l.mu.Unlock() } @@ -1048,7 +1054,7 @@ func (l *Logger) processVUsSample(s metrics.Sample, ts int64) { "", // finished_threads (unknown) } l.mu.Lock() - l.writeBinaryRow(l.bufVUs, row) + l.writeRow(l.bufVUs, row) l.mu.Unlock() } @@ -1102,7 +1108,7 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) { fmt.Sprintf("%v", l.envTags["run_id"]), // run_id last } l.mu.Lock() - l.writeBinaryRow(l.bufErrors, row) + l.writeRow(l.bufErrors, row) l.mu.Unlock() } else { row := []any{ @@ -1124,7 +1130,7 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) { id, // id (UUIDv7 generated from measurement timestamp) } l.mu.Lock() - l.writeBinaryRow(l.bufErrors, row) + l.writeRow(l.bufErrors, row) l.mu.Unlock() } } @@ -1191,23 +1197,22 @@ func (l *Logger) batchBody( return gzBody.Bytes(), mw.Boundary() } -func (l *Logger) sendBatch(body []byte, boundary string) bool { +func (l *Logger) sendBatch(body []byte, boundary string) error { start := time.Now() delay := time.Duration(initialBackoffMs) * time.Millisecond - for attempts := 1; time.Since(start) < maxRetryTimeMs*time.Millisecond; attempts++ { + for attempts := 1; ; attempts++ { err := l.postBatch(body, boundary) if err == nil { - return true + return nil } elapsed := time.Since(start) _, _ = fmt.Fprintf(l.out, "pg-proxy ingest attempt %d failed (%v elapsed): %v\n", attempts, elapsed, err) if elapsed+delay >= maxRetryTimeMs*time.Millisecond { - break + return fmt.Errorf("pg-proxy delivery failed after %d attempts: %w", attempts, err) } time.Sleep(delay) delay = min(delay*2, maxBackoffMs*time.Millisecond) } - return false } func (l *Logger) postBatch(body []byte, boundary string) error { @@ -1303,7 +1308,7 @@ func (l *Logger) writeHTTPError(g *MetricGroup, id, // id (UUIDv7) fmt.Sprintf("%v", l.envTags["run_id"]), } - l.writeBinaryRow(l.bufErrors, erow) + l.writeRow(l.bufErrors, erow) } else { erow := []any{ time.Unix(0, g.Timestamp), @@ -1323,7 +1328,7 @@ func (l *Logger) writeHTTPError(g *MetricGroup, g.ErrorResBody, // response_data id, // id (UUIDv7) } - l.writeBinaryRow(l.bufErrors, erow) + l.writeRow(l.bufErrors, erow) } } @@ -1423,3 +1428,32 @@ func requestTimings(values map[string]float64) (connectTime, latency, responseTi int64(connect + values["http_req_sending"] + values["http_req_waiting"]), int64(connect + values["http_req_duration"]) } + +// writeRow routes the shared metric mapping to the selected storage writer with mu held. +func (l *Logger) writeRow(buf *bytes.Buffer, row []any) { + if l.rowSink == nil { + l.writeBinaryRow(buf, row) + return + } + table := "requests_raw" + switch buf { + case l.bufTrans: + table = "transactions" + case l.bufVUs: + table = "virtual_users" + case l.bufErrors: + table = "requests_error" + } + fields := make([]string, len(row)) + for i, value := range row { + switch v := value.(type) { + case time.Time: + fields[i] = v.UTC().Format(time.RFC3339Nano) + case string: + fields[i] = v + default: + fields[i] = fmt.Sprint(v) + } + } + _ = l.rowSink(table, fields) // The direct writer records failures and stops the run. +} diff --git a/internal/output/breakingit/breaktest_db.go b/internal/output/breakingit/breaktest_db.go new file mode 100644 index 000000000..c27aace7e --- /dev/null +++ b/internal/output/breakingit/breaktest_db.go @@ -0,0 +1,491 @@ +package breakingit + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "github.com/klauspost/compress/zstd" + "go.k6.io/k6/v2/metrics" + "go.k6.io/k6/v2/output" +) + +// BreakTestDBOutput reuses BreakTest's metric mappings, with bounded direct binary delivery. +// Only the sender owns incarnation/sequence. An ACK is acceptance, not an fsync guarantee. +type dbGroupKey struct { + time int64 + tags *metrics.TagSet +} + +type BreakTestDBOutput struct { + groups map[dbGroupKey]*MetricGroup + groupTickets map[dbGroupKey]*metrics.RequestTicket + tracker *metrics.RequestTracker + lanes []*BreakTestDBOutput + owner *BreakTestDBOutput + nextLane int + watermark int64 + queuedWatermark int64 + tokens []string + + mapper *Logger + client *http.Client + base, token, runID string + incarnation string + sequence uint64 + retryTimeout time.Duration + encoder *zstd.Encoder + queue chan dbBatch + done, tickStop, tickDone, failed chan struct{} + batchMu sync.Mutex + parts map[string]*bytes.Buffer + batchBytes int + addMu sync.Mutex // Serializes sample mapping and pending-group completion. + failureMu sync.Mutex + failure error + stopCallback func(error) + started bool + stopOnce sync.Once + stopErr error +} +type dbBatch struct { + body []byte + contentType string + beforeUS int64 +} + +// NewBreakTestDB is registered separately from timescaledb-breakingit in full and lean builds. +func NewBreakTestDB(params output.Params) (output.Output, error) { + env := params.Environment + u, err := url.Parse(params.ConfigArgument) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("BreakTest DB requires an HTTP(S) base URL without credentials, query or fragment") + } + runID, token := env["runId"], env["BREAKTEST_DB_TOKEN"] + if !regexp.MustCompile(`^[A-Za-z0-9_-]+$`).MatchString(runID) || token == "" { + return nil, errors.New("BreakTest DB requires runId and a run-scoped BREAKTEST_DB_TOKEN") + } + if strings.EqualFold(env["syntheticMonitoring"], "true") { + return nil, errors.New("BreakTest DB k6 output does not support synthetic monitoring") + } + if strings.EqualFold(env["K6_BROWSER_SCREENSHOT_ON_ERROR"], "true") { + return nil, errors.New("BreakTest DB k6 output does not yet support browser screenshot storage") + } + tokens := strings.Split(token, ",") + seen := map[string]bool{} + if len(tokens) > 4 { + return nil, errors.New("BreakTest DB supports 1-4 ingestion lanes") + } + for _, t := range tokens { + if t == "" || seen[t] { + return nil, errors.New("invalid BreakTest DB lane tokens") + } + seen[t] = true + } + source, err := parseTransactionSource(env["BREAKINGIT_TRANSACTION_SOURCE"]) + if err != nil { + return nil, err + } + patterns := map[string]string{} + for _, pair := range strings.Split(env["CUSTOM_URL_PATTERNS"], ",") { + if pair == "" { + continue + } + parts := strings.Split(pair, "==") + if len(parts) != 2 { + return nil, errors.New("invalid CUSTOM_URL_PATTERNS entry") + } + patterns[parts[0]] = parts[1] + } + tags := map[string]string{"run_id": runID, "location": env["location"], "node_name": env["nodeName"], "scenario_name": env["scenarioName"]} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxConnsPerHost = len(tokens) + c := &BreakTestDBOutput{ + base: strings.TrimRight(u.String(), "/") + "/runs/" + runID, token: tokens[0], tokens: tokens, runID: runID, + client: &http.Client{Transport: transport, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, + retryTimeout: 30 * time.Second, queue: make(chan dbBatch, 4), done: make(chan struct{}), + tickStop: make(chan struct{}), tickDone: make(chan struct{}), failed: make(chan struct{}), parts: map[string]*bytes.Buffer{}, + } + c.groups = map[dbGroupKey]*MetricGroup{} + c.groupTickets = map[dbGroupKey]*metrics.RequestTicket{} + c.mapper = &Logger{out: params.StdOut, envTags: tags, customPatterns: patterns, transactionSource: source, + bufHTTP: &bytes.Buffer{}, bufVUs: &bytes.Buffer{}, bufTrans: &bytes.Buffer{}, bufErrors: &bytes.Buffer{}, + pendingHTTPGroups: map[int64]*MetricGroup{}, rowSink: c.addRow} + return c, nil +} + +func (c *BreakTestDBOutput) SetRequestTracker(t *metrics.RequestTracker) { c.tracker = t } + +func (c *BreakTestDBOutput) Description() string { return "breaktest-db-breakingit" } +func (c *BreakTestDBOutput) SetTestRunStopCallback(callback func(error)) { + c.failureMu.Lock() + c.stopCallback = callback + c.failureMu.Unlock() +} +func (c *BreakTestDBOutput) err() error { + if c.owner != nil { + return c.owner.err() + } + c.failureMu.Lock() + defer c.failureMu.Unlock() + return c.failure +} +func (c *BreakTestDBOutput) fail(err error) { + if c.owner != nil { + c.owner.fail(err) + return + } + if err == nil { + return + } + c.failureMu.Lock() + if c.failure != nil { + c.failureMu.Unlock() + return + } + c.failure = fmt.Errorf("BREAKTEST_DB_FAILED: final publication forbidden: %w", err) + failure, callback := c.failure, c.stopCallback + close(c.failed) + c.failureMu.Unlock() + // Never invoke the engine callback under a mapping/buffer lock. + if callback != nil { + go callback(failure) + } +} +func (c *BreakTestDBOutput) Start() error { + c.lanes = []*BreakTestDBOutput{c} + for _, token := range c.tokens[1:] { + c.lanes = append(c.lanes, &BreakTestDBOutput{owner: c, tracker: c.tracker, base: c.base, runID: c.runID, token: token, + client: c.client, retryTimeout: c.retryTimeout, queue: make(chan dbBatch, 4), done: make(chan struct{}), failed: c.failed}) + } + for _, lane := range c.lanes { + if err := lane.hello(); err != nil { + for _, opened := range c.lanes { + if opened.encoder != nil { + opened.encoder.Close() + } + } + return err + } + encoder, err := newDBEncoder() + if err != nil { + return err + } + lane.encoder = encoder + } + // One keep-alive connection and compression worker per ordered source. + + c.started = true + for _, lane := range c.lanes { + go lane.sender() + } + go func() { + defer close(c.tickDone) + tick := time.NewTicker(200 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-c.tickStop: + return + case <-c.failed: + return + case <-tick.C: + c.addMu.Lock() + c.completeGroups(false) + c.flush() + c.publishWatermark() + c.addMu.Unlock() + } + } + }() + return nil +} + +func (c *BreakTestDBOutput) AddMetricSamples(samples []metrics.SampleContainer) { + c.addMu.Lock() + defer c.addMu.Unlock() + if c.err() != nil { + return + } + for _, sc := range samples { + httpContainer := false + var ticket *metrics.RequestTicket + if tracked, ok := sc.(interface{ RequestTicket() *metrics.RequestTicket }); ok { + ticket = tracked.RequestTicket() + } + for _, sample := range sc.GetSamples() { + if sample.Metric.Name == "browser_errors" && sample.Tags.Map()["screenshot"] != "" { + c.fail(errors.New("browser screenshots require a BreakTest DB payload integration")) + return + } + } + for _, sample := range sc.GetSamples() { + key := httpMetricCanonicalKey(sample.Metric.Name) + if key == "" { + requestMetric := strings.HasPrefix(sample.Metric.Name, "custom_") || sample.Metric.Name == "ws_connecting" || sample.Metric.Name == "ws_ping" + if c.tracker != nil && requestMetric && ticket == nil { + c.fail(errors.New("request metric lacks producer completion tracking")) + return + } + c.mapper.AddMetricSamples([]metrics.SampleContainer{sample}) + continue + } + httpContainer = true + if c.tracker != nil && ticket == nil { + c.fail(errors.New("HTTP metrics lack producer completion tracking; live watermark would be unsafe")) + return + } + id := dbGroupKey{sample.Time.UnixNano(), sample.Tags} + if ticket != nil { + c.groupTickets[id] = ticket + } + g := c.groups[id] + if g == nil { + g = &MetricGroup{Timestamp: id.time, Tags: map[string]string{}, Values: map[string]float64{}} + copyTagsForHTTP(g.Tags, sample.Tags.Map(), c.mapper.envTags, c.mapper.customPatterns) + c.groups[id] = g + } + if _, duplicate := g.Values[key]; duplicate { + c.fail(errors.New("ambiguous HTTP metric identity: duplicate metric before request completion")) + return + } + g.Values[key] = sample.Value + if key == "http_req_failed" { + g.ErrorReqHeaders = sample.HTTPErrorReqHeaders + g.ErrorReqBody = sample.HTTPErrorReqBody + g.ErrorResHeaders = sample.HTTPErrorResHeaders + g.ErrorResBody = sample.HTTPErrorResBody + } + } + c.completeGroups(false) + if !httpContainer { + ticket.Release() + } + + if c.err() != nil { + return + } + // Bound incomplete metric joins as well as outgoing payloads. Complete HTTP + // containers usually leave this map empty; browser fragments may remain. + n := 0 + for _, g := range c.groups { + n += 256 + len(g.ErrorReqHeaders) + len(g.ErrorReqBody) + len(g.ErrorResHeaders) + len(g.ErrorResBody) + for k, v := range g.Tags { + n += len(k) + len(v) + 32 + } + n += len(g.Values) * 64 + } + if len(c.groups) > 4096 || n > dbBatchBytes { + c.fail(errors.New("incomplete metric join budget exceeded")) + return + } + } +} + +// Complete only after every request metric has arrived. Browser failure payloads +// may arrive much later than duration metrics; a timer is not proof of success. +func (c *BreakTestDBOutput) completeGroups(final bool) { + for key, g := range c.groups { + complete := true + for _, name := range []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving", "http_req_failed"} { + if _, ok := g.Values[name]; !ok { + complete = false + break + } + } + if complete { + delete(c.groups, key) + c.mapper.processHTTPGroup(g) + c.groupTickets[key].Release() + delete(c.groupTickets, key) + } else if final { + c.fail(errors.New("incomplete HTTP metric group at final drain")) + return + } + } +} +func (c *BreakTestDBOutput) addRow(table string, row []string) error { + if err := c.err(); err != nil { + return err + } + data, err := encodeDBRow(table, row) + if err != nil { + c.fail(err) + return err + } + c.batchMu.Lock() + defer c.batchMu.Unlock() + if c.batchBytes+len(data) > dbBatchBytes { + if err = c.enqueueLocked(); err != nil { + return err + } + } + if c.parts[table] == nil { + c.parts[table] = &bytes.Buffer{} + } + _, _ = c.parts[table].Write(data) + c.batchBytes += len(data) + return nil +} +func (c *BreakTestDBOutput) flush() { + c.batchMu.Lock() + defer c.batchMu.Unlock() + if err := c.enqueueLocked(); err != nil { + c.fail(err) + } +} +func (c *BreakTestDBOutput) enqueueLocked() error { + if c.batchBytes == 0 { + return nil + } + var body bytes.Buffer + writer := multipart.NewWriter(&body) + for _, table := range []string{"requests_raw", "transactions", "requests_error", "virtual_users"} { + data := c.parts[table] + if data == nil { + continue + } + header := textproto.MIMEHeader{} + header.Set("X-Table", "public."+table) + header.Set("X-Format", "binary") + part, err := writer.CreatePart(header) + if err != nil { + return err + } + _, _ = part.Write([]byte("PGCOPY\n\xff\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00")) + _, _ = part.Write(data.Bytes()) + _, _ = part.Write([]byte{255, 255}) + } + if err := writer.Close(); err != nil { + return err + } + batch := dbBatch{body: body.Bytes(), contentType: "multipart/mixed; boundary=" + writer.Boundary()} + lane := c + if len(c.lanes) > 0 { + lane = c.lanes[c.nextLane] + c.nextLane = (c.nextLane + 1) % len(c.lanes) + } + select { + case <-c.failed: + return c.err() + case lane.queue <- batch: + c.parts = map[string]*bytes.Buffer{} + c.batchBytes = 0 + return nil + } +} +func (c *BreakTestDBOutput) sender() { + defer close(c.done) + for batch := range c.queue { + if c.err() != nil { + continue + } + if batch.beforeUS != 0 { + c.fail(c.sendControl("/watermark", batch.beforeUS)) + continue + } + compressed := c.encoder.EncodeAll(batch.body, nil) + c.fail(c.sendBatch(compressed, batch.contentType)) + } +} + +func (c *BreakTestDBOutput) Stop() error { + c.stopOnce.Do(func() { + if !c.started { + return + } + close(c.tickStop) + <-c.tickDone + c.addMu.Lock() + if c.err() == nil { + c.completeGroups(true) + c.flush() + } + c.addMu.Unlock() + for _, lane := range c.lanes { + close(lane.queue) + } + for _, lane := range c.lanes { + <-lane.done + } + for _, lane := range c.lanes { + if c.err() != nil { + break + } + c.fail(lane.sendControl("/finish", 0)) + } + for _, lane := range c.lanes { + lane.encoder.Close() + } + c.client.CloseIdleConnections() + c.stopErr = c.err() + }) + return c.stopErr +} + +// Retries keep the same sequence, multipart boundary and compressed bytes. +// Incarnation recovery is handled by the lane, after this request returns 409. +func (c *BreakTestDBOutput) request(path string, body []byte, contentType string, seq uint64, result any) error { + ctx, cancel := context.WithTimeout(context.Background(), c.retryTimeout) + defer cancel() + for { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Content-Type", contentType) + req.Header.Set("X-Run-Id", c.runID) + if path != "/hello" { + req.Header.Set("X-Incarnation", c.incarnation) + req.Header.Set("X-Sequence", fmt.Sprint(seq)) + } + if path == "/ingest/batch" { + req.Header.Set("Content-Encoding", "zstd") + } + response, err := c.client.Do(req) + if err == nil { + data, readErr := io.ReadAll(io.LimitReader(response.Body, 64*1024+1)) + _ = response.Body.Close() + if len(data) > 64*1024 { + return errors.New("oversized BreakTest DB acknowledgement") + } + if response.StatusCode == http.StatusOK && readErr == nil { + if err = json.Unmarshal(data, result); err != nil { + return errors.New("invalid BreakTest DB acknowledgement JSON") + } + return nil + } + if readErr == nil && response.StatusCode != 429 && response.StatusCode < 500 { + return dbRejection(response.StatusCode) + } + } + timer := time.NewTimer(100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("BreakTest DB %s retry deadline: %w", path, ctx.Err()) + case <-c.failed: + timer.Stop() + return c.err() + case <-timer.C: + } + } +} + +// The storage ingress caps zstd decoder windows at 2^18 bytes, independently +// of the larger decoded batch budget. Enforce this even on full 2 MiB batches. +func newDBEncoder() (*zstd.Encoder, error) { + return zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), zstd.WithWindowSize(1<<18)) +} diff --git a/internal/output/breakingit/breaktest_db_binary.go b/internal/output/breakingit/breaktest_db_binary.go new file mode 100644 index 000000000..64bfe1545 --- /dev/null +++ b/internal/output/breakingit/breaktest_db_binary.go @@ -0,0 +1,92 @@ +package breakingit + +import ( + "bytes" + "encoding/binary" + "fmt" + "strconv" + "time" + + "github.com/gofrs/uuid/v5" +) + +const pgEpochSeconds int64 = 946684800 +const dbBatchBytes = 2 * 1024 * 1024 + +// PostgreSQL binary COPY types in the existing non-SM BreakTest table order. +// t = timestamp, s = text, i = int32, b = boolean, u = UUID. +var customSchemas = map[string]string{ + "requests_raw": "tssssbiisiii", + "transactions": "tsssbiii", + "requests_error": "tssssssiissssssu", + "virtual_users": "tsssiii", +} + +func encodeDBRow(table string, row []string) ([]byte, error) { + schema := customSchemas[table] + if schema == "" || len(row) != len(schema) { + return nil, fmt.Errorf("invalid %s row width: %d", table, len(row)) + } + // Reject before allocating a second copy of a potentially enormous error response. + size := 2 + 4*len(row) + for _, s := range row { + size += len(s) + } + if size > dbBatchBytes-1024 { + return nil, fmt.Errorf("%s row exceeds BreakTest DB batch budget", table) + } + b := bytes.NewBuffer(make([]byte, 0, size)) + _ = binary.Write(b, binary.BigEndian, int16(len(row))) + for i, kind := range []byte(schema) { + value := row[i] + // Match the existing CSV writer's empty-field NULL semantics. + if value == "" { + _ = binary.Write(b, binary.BigEndian, int32(-1)) + continue + } + var data []byte + switch kind { + case 's': + data = []byte(value) + case 't': + t, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, fmt.Errorf("%s timestamp: %w", table, err) + } + n := (t.Unix()-pgEpochSeconds)*1_000_000 + int64(t.Nanosecond()/1000) + if n < 0 { + return nil, fmt.Errorf("timestamp before BreakTest DB epoch") + } + data = binary.BigEndian.AppendUint64(nil, uint64(n)) + case 'i': + n, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return nil, fmt.Errorf("%s integer column %d: %w", table, i, err) + } + // Unavailable connect times must never become negative averages. + if n < 0 && ((table == "requests_raw" && i == 9) || (table == "requests_error" && i == 8)) { + _ = binary.Write(b, binary.BigEndian, int32(-1)) + continue + } + data = binary.BigEndian.AppendUint32(nil, uint32(n)) + case 'b': + v, err := strconv.ParseBool(value) + if err != nil { + return nil, err + } + data = []byte{0} + if v { + data[0] = 1 + } + case 'u': + id, err := uuid.FromString(value) + if err != nil { + return nil, err + } + data = id[:] + } + _ = binary.Write(b, binary.BigEndian, int32(len(data))) + _, _ = b.Write(data) + } + return b.Bytes(), nil +} diff --git a/internal/output/breakingit/breaktest_db_recovery.go b/internal/output/breakingit/breaktest_db_recovery.go new file mode 100644 index 000000000..dd78d6c16 --- /dev/null +++ b/internal/output/breakingit/breaktest_db_recovery.go @@ -0,0 +1,106 @@ +package breakingit + +import ( + "encoding/json" + "errors" + "fmt" +) + +type dbRejection int + +func (s dbRejection) Error() string { + return fmt.Sprintf("BreakTest DB rejected request (HTTP %d)", int(s)) +} + +type sourceHello struct { + Incarnation string `json:"incarnation"` + Accepted *uint64 `json:"accepted_sequence"` + Checkpoint *uint64 `json:"checkpoint_sequence"` + Sealed bool `json:"sealed"` +} + +func (c *BreakTestDBOutput) register() (sourceHello, error) { + body := []byte(`{}`) + if c.tracker != nil { + body, _ = json.Marshal(map[string]any{"source_clock_us": c.tracker.Clock().UnixMicro() - pgEpochSeconds*1_000_000, "clock_contract": "sample-end-monotonic-v1"}) + } + var hello sourceHello + err := c.request("/hello", body, "application/json", 0, &hello) + return hello, err +} + +// Acknowledged, uncheckpointed rows may be lost under relaxed durability. The +// server exposes that loss as coverage gaps. Replay only the held unacknowledged +// batch, never already acknowledged history. Each ordered lane recovers itself. +func (c *BreakTestDBOutput) recoverSource(err error) (uint64, error) { + var status dbRejection + if c.tracker == nil || !errors.As(err, &status) || status != 409 { + return 0, err + } + hello, e := c.register() + if e != nil { + return 0, e + } + if hello.Sealed || hello.Incarnation == "" || hello.Incarnation == c.incarnation || hello.Accepted == nil || hello.Checkpoint == nil || *hello.Accepted != *hello.Checkpoint { + return 0, err + } + c.incarnation = hello.Incarnation + c.sequence = *hello.Accepted + return *hello.Checkpoint, nil +} + +func (c *BreakTestDBOutput) sendBatch(body []byte, content string) error { + for attempt := 0; attempt < 4; attempt++ { + next := c.sequence + 1 + var ack struct { + Accepted *uint64 `json:"accepted"` + } + err := c.request("/ingest/batch", body, content, next, &ack) + if err == nil { + if ack.Accepted == nil || *ack.Accepted != next { + return errors.New("invalid BreakTest DB batch acknowledgement") + } + c.sequence = next + return nil + } + checkpoint, e := c.recoverSource(err) + if e != nil { + return e + } + if checkpoint > next { + return errors.New("checkpoint exceeds outstanding batch sequence") + } + if next <= checkpoint { + return nil + } // Lost ACK, but checkpoint proves this batch persisted. + } + return errors.New("BreakTest DB restarted repeatedly during batch delivery") +} + +func (c *BreakTestDBOutput) sendControl(path string, before int64) error { + body := []byte(nil) + if path == "/watermark" { + body, _ = json.Marshal(map[string]int64{"before_us": before}) + } + for attempt := 0; attempt < 4; attempt++ { + var ack struct { + Accepted *uint64 `json:"accepted_sequence"` + Watermark int64 `json:"watermark_us"` + Finished bool `json:"finished"` + } + previous := c.sequence + err := c.request(path, body, "application/json", c.sequence, &ack) + if err == nil { + if ack.Accepted == nil || *ack.Accepted != c.sequence || (path == "/watermark" && ack.Watermark != before) || (path == "/finish" && !ack.Finished) { + return errors.New("invalid BreakTest DB control acknowledgement") + } + return nil + } + if checkpoint, e := c.recoverSource(err); e != nil { + return e + } else if checkpoint > previous { + return errors.New("checkpoint exceeds acknowledged sequence") + } + } + return errors.New("BreakTest DB restarted repeatedly during control delivery") +} diff --git a/internal/output/breakingit/breaktest_db_test.go b/internal/output/breakingit/breaktest_db_test.go new file mode 100644 index 000000000..76ff9694c --- /dev/null +++ b/internal/output/breakingit/breaktest_db_test.go @@ -0,0 +1,472 @@ +package breakingit + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/metrics" + "go.k6.io/k6/v2/output" +) + +func newBreakTestDBTest(t *testing.T, endpoint string) *BreakTestDBOutput { + t.Helper() + out, err := NewBreakTestDB(output.Params{ConfigArgument: endpoint, StdOut: io.Discard, Environment: map[string]string{ + "runId": "run-123", "BREAKTEST_DB_TOKEN": "producer-token", "location": "Amsterdam", "nodeName": "node-1", + }}) + require.NoError(t, err) + return out.(*BreakTestDBOutput) +} +func decodeDBBatch(t *testing.T, data []byte, content string) map[string][][][]byte { + t.Helper() + decoder, err := zstd.NewReader(nil) + require.NoError(t, err) + defer decoder.Close() + plain, err := decoder.DecodeAll(data, nil) + require.NoError(t, err) + _, params, err := mime.ParseMediaType(content) + require.NoError(t, err) + reader := multipart.NewReader(bytes.NewReader(plain), params["boundary"]) + result := map[string][][][]byte{} + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err) + require.Equal(t, "binary", part.Header.Get("X-Format")) + data, err := io.ReadAll(part) + require.NoError(t, err) + require.Equal(t, []byte("PGCOPY\n\xff\r\n\x00"), data[:11]) + require.Equal(t, make([]byte, 8), data[11:19]) + r := bytes.NewReader(data[19:]) + table := strings.TrimPrefix(part.Header.Get("X-Table"), "public.") + for { + var n int16 + require.NoError(t, binary.Read(r, binary.BigEndian, &n)) + if n == -1 { + break + } + row := make([][]byte, n) + for i := range row { + var length int32 + require.NoError(t, binary.Read(r, binary.BigEndian, &length)) + if length == -1 { + continue + } + require.GreaterOrEqual(t, length, int32(0)) + row[i] = make([]byte, length) + _, err = io.ReadFull(r, row[i]) + require.NoError(t, err) + } + result[table] = append(result[table], row) + } + require.Zero(t, r.Len()) + } + return result +} +func TestBreakTestDBBinaryMatchesProxyMapping(t *testing.T) { + t.Parallel() + rows := []struct { + table string + row []string + }{ + {"requests_raw", []string{"2026-10-30T12:00:00.123456Z", "run-123", "NL", "Checkout", "/pay", "false", "", "123", "503", "-1", "25", "100"}}, + {"transactions", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "120"}}, + {"requests_error", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "Checkout", "/pay", "503", "100", "", "https://example.test/", "", "bad\nreply", "POST,\"a\"", "Header: value", "雪", "019a3524-3890-7000-8000-000000000001"}}, + {"virtual_users", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "10", "", ""}}, + } + c := newBreakTestDBTest(t, "http://localhost") + for _, r := range rows { + require.NoError(t, c.addRow(r.table, r.row)) + } + c.flush() + batch := <-c.queue + encoder, err := zstd.NewWriter(nil) + require.NoError(t, err) + defer encoder.Close() + decoded := decodeDBBatch(t, encoder.EncodeAll(batch.body, nil), batch.contentType) + require.Len(t, decoded, 4) + require.Nil(t, decoded["requests_raw"][0][9]) + require.Nil(t, decoded["requests_raw"][0][6]) + require.Equal(t, uint32(123), binary.BigEndian.Uint32(decoded["requests_raw"][0][7])) + require.Equal(t, []byte("雪"), decoded["requests_error"][0][14]) + require.Len(t, decoded["requests_error"][0][15], 16) + require.Equal(t, []byte{1}, decoded["transactions"][0][4]) + require.Equal(t, int64(846676800123456), int64(binary.BigEndian.Uint64(decoded["requests_raw"][0][0]))) + // The transaction row must be byte-identical on both binary COPY paths. + proxy := newTransactionTestLogger(false) + sample := transactionTestSample("group_duration", map[string]string{"group": "::Checkout"}, true) + proxy.AddMetricSamples([]metrics.SampleContainer{sample}) + c.AddMetricSamples([]metrics.SampleContainer{sample}) + require.NotEmpty(t, proxy.bufTrans.Bytes()) + require.Equal(t, proxy.bufTrans.Bytes(), c.parts["transactions"].Bytes()) +} + +func TestBreakTestDBRetriesExactBytesAndFinalDrain(t *testing.T) { + t.Parallel() + var first []byte + var content string + var calls, finishes int + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + require.Equal(t, "Bearer producer-token", r.Header.Get("Authorization")) + switch r.URL.Path { + case "/runs/run-123/hello": + fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0,"sealed":false}`) + case "/runs/run-123/ingest/batch": + require.Equal(t, "one", r.Header.Get("X-Incarnation")) + require.Equal(t, "1", r.Header.Get("X-Sequence")) + require.Equal(t, "zstd", r.Header.Get("Content-Encoding")) + data, err := io.ReadAll(r.Body) + require.NoError(t, err) + calls++ + if calls == 1 { + first = data + content = r.Header.Get("Content-Type") + w.WriteHeader(503) + return + } + require.Equal(t, first, data) + require.Equal(t, content, r.Header.Get("Content-Type")) + fmt.Fprint(w, `{"accepted":1,"duplicate":true}`) + case "/runs/run-123/finish": + require.Equal(t, 2, calls) + require.Equal(t, "1", r.Header.Get("X-Sequence")) + finishes++ + fmt.Fprint(w, `{"accepted_sequence":1,"finished":true}`) + default: + t.Errorf("unexpected route %s", r.URL.Path) + w.WriteHeader(404) + } + })) + defer server.Close() + c := newBreakTestDBTest(t, server.URL) + require.NoError(t, c.Start()) + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)}) + require.NoError(t, c.Stop()) + require.NoError(t, c.Stop()) + mu.Lock() + defer mu.Unlock() + require.Equal(t, 1, finishes) + decoded := decodeDBBatch(t, first, content) + require.Len(t, decoded["transactions"], 1) +} + +func TestBreakTestDBFailureForbidsFinish(t *testing.T) { + for _, status := range []int{400, 401, 403, 409, 413, 429, 503, 200} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + var finished bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/hello") { + fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0}`) + return + } + if strings.HasSuffix(r.URL.Path, "/finish") { + finished = true + } + w.WriteHeader(status) + fmt.Fprint(w, `{"accepted":999}`) + })) + defer server.Close() + c := newBreakTestDBTest(t, server.URL) + c.retryTimeout = 50 * time.Millisecond + stopped := make(chan error, 1) + c.SetTestRunStopCallback(func(err error) { stopped <- err }) + require.NoError(t, c.Start()) + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("pages", nil, false)}) + require.ErrorContains(t, c.Stop(), "final publication forbidden") + select { + case err := <-stopped: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("run not stopped") + } + require.False(t, finished) + }) + } +} + +func TestBreakTestDBConfigurationAndBounds(t *testing.T) { + t.Parallel() + for _, entry := range []struct{ k, v string }{{"syntheticMonitoring", "true"}, {"K6_BROWSER_SCREENSHOT_ON_ERROR", "true"}, {"runId", "../wrong"}, {"BREAKTEST_DB_TOKEN", ""}, {"BREAKINGIT_TRANSACTION_SOURCE", "wrong"}} { + env := map[string]string{"runId": "test", "BREAKTEST_DB_TOKEN": "token"} + env[entry.k] = entry.v + _, err := NewBreakTestDB(output.Params{ConfigArgument: "http://localhost", Environment: env}) + require.Error(t, err) + } + _, err := NewBreakTestDB(output.Params{ConfigArgument: "http://user:password@localhost"}) + require.Error(t, err) + c := newBreakTestDBTest(t, "http://localhost") + row := []string{"2026-10-30T00:00:00Z", "run-123", "NL", strings.Repeat("x", dbBatchBytes), "true", "", "", "1"} + require.Error(t, c.addRow("transactions", row)) + require.Error(t, c.err()) + require.Empty(t, c.queue) +} + +func TestBreakTestDBHTTPJoinsAndLateFailures(t *testing.T) { + t.Parallel() + c := newBreakTestDBTest(t, "http://localhost") + var rows [][]string + c.mapper.rowSink = func(table string, row []string) error { + if table == "requests_raw" { + rows = append(rows, row) + } + return nil + } + reg := metrics.NewRegistry() + ts := time.Now() + names := []string{"browser_http_req_duration", "browser_http_req_blocked", "browser_http_req_connecting", "browser_http_req_tls_handshaking", "browser_http_req_sending", "browser_http_req_waiting", "browser_http_req_receiving", "browser_data_received"} + tagsA := reg.RootTagSet().With("name", "/a").With("resource_type", "fetch") + tagsB := reg.RootTagSet().With("name", "/b").With("resource_type", "fetch") + for _, tags := range []*metrics.TagSet{tagsA, tagsB} { + samples := metrics.Samples{} + for _, name := range names { + samples = append(samples, metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric(name, metrics.Trend), Tags: tags}, Time: ts, Value: 12}) + } + c.AddMetricSamples([]metrics.SampleContainer{samples}) + } + require.Empty(t, rows) + c.completeGroups(false) + require.Empty(t, rows) // A flush timer cannot infer request success. + for _, tags := range []*metrics.TagSet{tagsA, tagsB} { + sample := metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric("browser_http_req_failed", metrics.Rate), Tags: tags}, Time: ts, Value: 1, HTTPErrorResBody: "failure"} + c.AddMetricSamples([]metrics.SampleContainer{sample}) + } + require.Len(t, rows, 2) + require.Empty(t, c.groups) + for _, row := range rows { + require.Equal(t, "false", row[5]) + require.Equal(t, "12", row[7]) + } +} + +func TestBreakTestDBIncompleteDrainFails(t *testing.T) { + t.Parallel() + c := newBreakTestDBTest(t, "http://localhost") + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("http_req_duration", nil, false)}) + c.completeGroups(true) + require.ErrorContains(t, c.err(), "incomplete HTTP metric group") +} + +func TestBreakTestDBQueueBackpressure(t *testing.T) { + t.Parallel() + c := newBreakTestDBTest(t, "http://localhost") + row := []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "1"} + for range cap(c.queue) { + require.NoError(t, c.addRow("transactions", row)) + c.flush() + } + blocked := make(chan struct{}) + go func() { defer close(blocked); _ = c.addRow("transactions", row); c.flush() }() + select { + case <-blocked: + t.Fatal("full queue did not apply backpressure") + case <-time.After(20 * time.Millisecond): + } + c.fail(fmt.Errorf("sender unavailable")) + select { + case <-blocked: + case <-time.After(time.Second): + t.Fatal("failure did not unblock producer") + } + require.Len(t, c.queue, cap(c.queue)) +} + +func TestBreakTestDBStartupRejectsNonFreshAndMissingAck(t *testing.T) { + t.Parallel() + for _, body := range []string{`{}`, `{"incarnation":"a"}`, `{"incarnation":"a","accepted_sequence":1}`, `{"incarnation":"a","accepted_sequence":0,"sealed":true}`} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + c := newBreakTestDBTest(t, server.URL) + require.Error(t, c.Start()) + require.False(t, c.started) + server.Close() + } +} + +func TestBreakTestDBHTTPDuplicateIdentityFails(t *testing.T) { + t.Parallel() + c := newBreakTestDBTest(t, "http://localhost") + sample := transactionTestSample("http_req_duration", nil, false) + c.AddMetricSamples([]metrics.SampleContainer{sample, sample}) + require.ErrorContains(t, c.err(), "ambiguous HTTP metric identity") +} + +func TestBreakTestDBWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) { + var mu sync.Mutex + sequences := map[string]uint64{} + watermarks := map[string]int64{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + token := r.Header.Get("Authorization") + seq, _ := strconv.ParseUint(r.Header.Get("X-Sequence"), 10, 64) + switch { + case strings.HasSuffix(r.URL.Path, "/hello"): + fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0}`) + case strings.HasSuffix(r.URL.Path, "/ingest/batch"): + require.Equal(t, sequences[token]+1, seq) + sequences[token] = seq + fmt.Fprintf(w, `{"accepted":%d}`, seq) + case strings.HasSuffix(r.URL.Path, "/watermark"): + var body struct { + Before int64 `json:"before_us"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + require.Equal(t, sequences[token], seq) + watermarks[token] = body.Before + fmt.Fprintf(w, `{"accepted_sequence":%d,"watermark_us":%d}`, seq, body.Before) + case strings.HasSuffix(r.URL.Path, "/finish"): + require.Equal(t, sequences[token], seq) + fmt.Fprintf(w, `{"accepted_sequence":%d,"finished":true}`, seq) + default: + w.WriteHeader(404) + } + })) + defer server.Close() + c := newBreakTestDBTest(t, server.URL) + c.tokens = []string{"producer-token", "second"} + ch := make(chan metrics.SampleContainer, 1) + tracker, release := metrics.BindRequestTracker(ch) + defer release() + c.SetRequestTracker(tracker) + require.NoError(t, c.Start()) + ticket := metrics.BeginRequest(ch) // Request not delivered to the output yet. + pin := ticket.Time.Truncate(time.Second).UnixMicro() - pgEpochSeconds*1_000_000 + for range 4 { + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)}) + c.flush() + } + require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(watermarks) == 2 }, 2*time.Second, 10*time.Millisecond) + mu.Lock() + for _, watermark := range watermarks { + require.LessOrEqual(t, watermark, pin) + } + mu.Unlock() + // Cancellation before enqueue may release its pin; no request row exists. + ticket.Release() + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return watermarks["Bearer producer-token"] > pin && watermarks["Bearer second"] > pin + }, 2*time.Second, 10*time.Millisecond) + require.NoError(t, c.Stop()) + mu.Lock() + defer mu.Unlock() + require.Equal(t, uint64(2), sequences["Bearer producer-token"]) + require.Equal(t, uint64(2), sequences["Bearer second"]) +} + +func TestBreakTestDBRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) { + for _, checkpoint := range []uint64{0, 2} { + t.Run(strconv.Itoa(int(checkpoint)), func(t *testing.T) { + calls := 0 + var first []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/hello") { + fmt.Fprintf(w, `{"incarnation":"new","accepted_sequence":%d,"checkpoint_sequence":%d}`, checkpoint, checkpoint) + return + } + calls++ + body, _ := io.ReadAll(r.Body) + if calls == 1 { + first = body + require.Equal(t, "old", r.Header.Get("X-Incarnation")) + require.Equal(t, "2", r.Header.Get("X-Sequence")) + w.WriteHeader(409) + return + } + require.Equal(t, first, body) + require.Equal(t, "new", r.Header.Get("X-Incarnation")) + require.Equal(t, "1", r.Header.Get("X-Sequence")) + fmt.Fprint(w, `{"accepted":1}`) + })) + defer server.Close() + c := newBreakTestDBTest(t, server.URL) + ch := make(chan metrics.SampleContainer) + tracker, release := metrics.BindRequestTracker(ch) + defer release() + c.tracker = tracker + c.sequence = 1 + c.incarnation = "old" + require.NoError(t, c.sendBatch([]byte("identical-compressed-body"), "multipart/mixed")) + if checkpoint == 0 { + require.Equal(t, 2, calls) + require.Equal(t, uint64(1), c.sequence) + } else { + require.Equal(t, 1, calls) + require.Equal(t, uint64(2), c.sequence) + } + }) + } +} + +func TestTrackedHTTPJoinKeepsPinUntilFailureMetric(t *testing.T) { + c := newBreakTestDBTest(t, "http://localhost") + channel := make(chan metrics.SampleContainer) + tracker, release := metrics.BindRequestTracker(channel) + defer release() + c.tracker = tracker + ticket := metrics.BeginRequest(channel) + reg := metrics.NewRegistry() + tags := reg.RootTagSet().With("name", "/tracked") + names := []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving"} + samples := metrics.Samples{} + for _, name := range names { + samples = append(samples, metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric(name, metrics.Trend), Tags: tags}, Time: ticket.Time, Value: 12}) + } + c.AddMetricSamples([]metrics.SampleContainer{metrics.RequestSamples{SampleContainer: samples, Ticket: ticket}}) + require.Len(t, c.groups, 1) + require.Len(t, c.groupTickets, 1) + require.False(t, tracker.Before().After(ticket.Time)) + failed := metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric("http_req_failed", metrics.Rate), Tags: tags}, Time: ticket.Time, Value: 1, HTTPErrorResBody: "delayed failure"} + c.AddMetricSamples([]metrics.SampleContainer{metrics.RequestSamples{SampleContainer: failed, Ticket: ticket}}) + require.Empty(t, c.groups) + require.Empty(t, c.groupTickets) + require.NoError(t, c.err()) + require.NotNil(t, c.parts["requests_raw"]) + require.NotNil(t, c.parts["requests_error"]) +} + +func TestUntrackedRequestCannotCrossNativeWatermark(t *testing.T) { + c := newBreakTestDBTest(t, "http://localhost") + channel := make(chan metrics.SampleContainer) + tracker, release := metrics.BindRequestTracker(channel) + defer release() + c.tracker = tracker + c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("custom_missing_hook", nil, false)}) + require.ErrorContains(t, c.err(), "lacks producer completion tracking") + require.Empty(t, c.parts) +} + +func TestBreakTestDBFullBatchRespectsStorageDecoderWindow(t *testing.T) { + encoder, err := newDBEncoder() + require.NoError(t, err) + defer encoder.Close() + plain := bytes.Repeat([]byte("large-request-metric-row\n"), 100000) + encoded := encoder.EncodeAll(plain, nil) + var header zstd.Header + require.NoError(t, header.Decode(encoded)) + require.LessOrEqual(t, header.WindowSize, uint64(1<<18)) + decoder, err := zstd.NewReader(nil, zstd.WithDecoderMaxWindow(1<<18)) + require.NoError(t, err) + defer decoder.Close() + decoded, err := decoder.DecodeAll(encoded, nil) + require.NoError(t, err) + require.Equal(t, plain, decoded) +} diff --git a/internal/output/breakingit/breaktest_db_watermark.go b/internal/output/breakingit/breaktest_db_watermark.go new file mode 100644 index 000000000..453fe81bd --- /dev/null +++ b/internal/output/breakingit/breaktest_db_watermark.go @@ -0,0 +1,38 @@ +package breakingit + +import ( + "errors" +) + +func (c *BreakTestDBOutput) hello() error { + hello, err := c.register() + if err != nil { + return err + } + if hello.Sealed || hello.Incarnation == "" || hello.Accepted == nil || *hello.Accepted != 0 { + return errors.New("BreakTest DB requires a fresh unfinished producer source") + } + c.incarnation = hello.Incarnation + return nil +} + +// Called under addMu, after detaching every serialized row. Tickets in the engine +// queue and incomplete joins still pin Before(). Each FIFO carries its own barrier: +// the database takes the minimum across all lanes and generators. +func (c *BreakTestDBOutput) publishWatermark() { + if c.tracker == nil || c.err() != nil { + return + } + before := c.tracker.Before().UnixMicro() - pgEpochSeconds*1_000_000 + if before <= c.queuedWatermark { + return + } + for _, lane := range c.lanes { + select { + case <-c.failed: + return + case lane.queue <- dbBatch{beforeUS: before}: + } + } + c.queuedWatermark = before +} diff --git a/internal/output/breakingit/cadence_test.go b/internal/output/breakingit/cadence_test.go index 076e17e65..85fa598f6 100644 --- a/internal/output/breakingit/cadence_test.go +++ b/internal/output/breakingit/cadence_test.go @@ -25,7 +25,7 @@ func TestFirstBatchImmediateThenOneSecond(t *testing.T) { w.WriteHeader(http.StatusOK) })) defer server.Close() - exporter, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test", "isSynthetic": fmt.Sprint(synthetic)}}) + exporter, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test", "syntheticMonitoring": fmt.Sprint(synthetic)}}) require.NoError(t, err) stopped := false defer func() { diff --git a/internal/output/breakingit/drain_test.go b/internal/output/breakingit/drain_test.go new file mode 100644 index 000000000..62786fb7b --- /dev/null +++ b/internal/output/breakingit/drain_test.go @@ -0,0 +1,161 @@ +package breakingit + +import ( + "compress/gzip" + "errors" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/metrics" + "go.k6.io/k6/v2/output" +) + +// The second request is the follow-up tail of an already running flush. +// Stop must wait for it and for the third row appended while it is in flight. +func TestTimescaleStopJoinsFollowupFlush(t *testing.T) { + first, second := make(chan struct{}), make(chan struct{}) + releaseFirst, releaseSecond := make(chan struct{}), make(chan struct{}) + var onceFirst, onceSecond sync.Once + var mu sync.Mutex + var received []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gz, err := gzip.NewReader(r.Body) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + defer gz.Close() + _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + reader := multipart.NewReader(gz, params["boundary"]) + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Error(err) + return + } + assert.Equal(t, "binary", part.Header.Get("X-Format")) + assert.Equal(t, "public.transactions", part.Header.Get("X-Table")) + rows, err := readBinaryRows(part, "tsssbiii", true) + if !assert.NoError(t, err) || !assert.Len(t, rows, 1) { + w.WriteHeader(http.StatusBadRequest) + return + } + mu.Lock() + received = append(received, rows[0][3]) + n := len(received) + mu.Unlock() + if n == 1 { + close(first) + <-releaseFirst + } + if n == 2 { + close(second) + <-releaseSecond + } + } + w.WriteHeader(200) + })) + defer server.Close() + defer onceFirst.Do(func() { close(releaseFirst) }) + defer onceSecond.Do(func() { close(releaseSecond) }) + out, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test-token"}}) + require.NoError(t, err) + l := out.(*Logger) + appendRow := func(name string) { + sample := transactionTestSample("group_duration", map[string]string{"group": "::" + name}, false) + l.AddMetricSamples([]metrics.SampleContainer{sample}) + } + await := func(ch <-chan struct{}) { + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("flush did not reach handler") + } + } + appendRow("first") + await(first) + appendRow("second") + onceFirst.Do(func() { close(releaseFirst) }) + await(second) + appendRow("third") + stopped := make(chan error, 1) + go func() { stopped <- l.Stop() }() + select { + case err := <-stopped: + onceSecond.Do(func() { close(releaseSecond) }) + t.Fatalf("Stop returned before the in-flight tail was acknowledged: %v", err) + case <-time.After(100 * time.Millisecond): + } + onceSecond.Do(func() { close(releaseSecond) }) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Stop failed to drain") + } + mu.Lock() + defer mu.Unlock() + require.Equal(t, []string{"first", "second", "third"}, received) +} + +func TestTimescaleStopDrainsUnexpiredMetricGroup(t *testing.T) { + var mu sync.Mutex + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + requests++ + mu.Unlock() + w.WriteHeader(200) + })) + defer server.Close() + out, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test-token"}}) + require.NoError(t, err) + l := out.(*Logger) + l.mu.Lock() + now := time.Now() + l.pendingHTTPGroups[now.UnixNano()] = &MetricGroup{Timestamp: now.UnixNano(), LastUpdate: now, Tags: map[string]string{"name": "tail-request"}, Values: map[string]float64{"http_req_duration": 12}} + l.mu.Unlock() + require.NoError(t, l.Stop()) + mu.Lock() + defer mu.Unlock() + require.Equal(t, 1, requests) + require.Empty(t, l.pendingHTTPGroups) +} + +type unavailableProxy struct{} + +func (unavailableProxy) RoundTrip(*http.Request) (*http.Response, error) { + return nil, errors.New("proxy offline") +} + +func TestTimescaleStopReportsExhaustedDelivery(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + out, err := New(output.Params{ConfigArgument: "http://proxy.test", StdOut: io.Discard, + Environment: map[string]string{"PG_PROXY_TOKEN": "test-token"}}) + require.NoError(t, err) + logger := out.(*Logger) + logger.httpClient.Transport = unavailableProxy{} + sample := transactionTestSample("group_duration", map[string]string{"group": "::tail"}, false) + logger.AddMetricSamples([]metrics.SampleContainer{sample}) + require.ErrorContains(t, logger.Stop(), "proxy offline") + }) +} diff --git a/lib/netext/httpext/tracer.go b/lib/netext/httpext/tracer.go index 673770d8a..1b8cc58f2 100644 --- a/lib/netext/httpext/tracer.go +++ b/lib/netext/httpext/tracer.go @@ -14,7 +14,8 @@ import ( // A Trail represents detailed information about an HTTP request. // You'd typically get one from a Tracer. type Trail struct { - EndTime time.Time + requestTicket *metrics.RequestTicket + EndTime time.Time // Total connect time (Connecting + TLSHandshaking) ConnDuration time.Duration @@ -121,6 +122,8 @@ func (tr *Trail) SaveSamples(builtinMetrics *metrics.BuiltinMetrics, ctm *metric }...) } +func (tr *Trail) RequestTicket() *metrics.RequestTicket { return tr.requestTicket } + // GetSamples implements the metrics.SampleContainer interface. func (tr *Trail) GetSamples() []metrics.Sample { return tr.Samples diff --git a/lib/netext/httpext/transport.go b/lib/netext/httpext/transport.go index 8e293525f..e15e55931 100644 --- a/lib/netext/httpext/transport.go +++ b/lib/netext/httpext/transport.go @@ -84,7 +84,12 @@ func newTransport( // Helper method to finish the tracer trail, assemble the tag values and emits // the metric samples for the supplied unfinished request. func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRequest { + ticket := metrics.BeginRequest(t.state.Samples) trail := unfReq.tracer.Done() + if ticket != nil { + trail.EndTime = ticket.Time + } + trail.requestTicket = ticket result := &finishedRequest{ unfinishedRequest: unfReq, @@ -166,7 +171,9 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe t.addFailedSample(trail, &tagsAndMeta, failed) } - metrics.PushIfNotDone(t.ctx, t.state.Samples, trail) + if !metrics.PushIfNotDone(t.ctx, t.state.Samples, trail) { + ticket.Release() + } return result } diff --git a/metrics/request_tracker.go b/metrics/request_tracker.go new file mode 100644 index 000000000..8ddc29927 --- /dev/null +++ b/metrics/request_tracker.go @@ -0,0 +1,110 @@ +package metrics + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +// RequestTracker pins request-end timestamps until the output has serialized the +// entire request. Registration happens before assigning the timestamp, not when +// the metrics eventually arrive at the output. Only opted-in outputs pay this cost. +type requestShard struct { + mu sync.Mutex + active map[*RequestTicket]time.Time +} +type RequestTracker struct { + origin time.Time + next atomic.Uint64 + shards [64]requestShard +} + +type RequestTicket struct { + tracker *RequestTracker + shard uint64 + Time time.Time +} + +var requestTrackers sync.Map // chan<- SampleContainer -> *RequestTracker + +func BindRequestTracker(ch chan<- SampleContainer) (*RequestTracker, func()) { + t := &RequestTracker{origin: time.Now()} + for i := range t.shards { + t.shards[i].active = make(map[*RequestTicket]time.Time) + } + requestTrackers.Store(ch, t) + return t, func() { requestTrackers.CompareAndDelete(ch, t) } +} + +// BeginRequest assigns an end timestamp using an anchored monotonic clock. +// The pin and the timestamp are atomic with respect to Before(). +func BeginRequest(ch chan<- SampleContainer) *RequestTicket { + v, ok := requestTrackers.Load(ch) + if !ok { + return nil + } + return v.(*RequestTracker).Begin() +} +func (t *RequestTracker) Begin() *RequestTicket { + index := t.next.Add(1) % uint64(len(t.shards)) + shard := &t.shards[index] + shard.mu.Lock() + defer shard.mu.Unlock() + stamp := t.Clock() + ticket := &RequestTicket{tracker: t, shard: index, Time: stamp} + shard.active[ticket] = stamp + return ticket +} +func (t *RequestTicket) Release() { + if t == nil { + return + } + shard := &t.tracker.shards[t.shard] + shard.mu.Lock() + delete(shard.active, t) + shard.mu.Unlock() +} +func (t *RequestTracker) Before() time.Time { + // New pins created during the scan have timestamps >= this captured clock. + before := t.Clock() + for i := range t.shards { + shard := &t.shards[i] + shard.mu.Lock() + for _, stamp := range shard.active { + if stamp.Before(before) { + before = stamp + } + } + shard.mu.Unlock() + } + return before.Truncate(time.Second) +} +func (t *RequestTracker) Clock() time.Time { return t.origin.Add(time.Since(t.origin)) } + +// RequestSamples preserves the normal payload while identifying its lifecycle pin. +type RequestSamples struct { + SampleContainer + Ticket *RequestTicket +} + +func (s RequestSamples) RequestTicket() *RequestTicket { return s.Ticket } + +// PushTrackedRequest keeps the lifecycle pin across both the engine queue and +// the output's mapping queue. A cancelled producer releases a never-enqueued pin. +func PushTrackedRequest(ctx context.Context, ch chan<- SampleContainer, ticket *RequestTicket, sample SampleContainer) bool { + if ticket != nil { + sample = RequestSamples{SampleContainer: sample, Ticket: ticket} + } + if PushIfNotDone(ctx, ch, sample) { + return true + } + ticket.Release() + return false +} +func RequestTime(ticket *RequestTicket, fallback time.Time) time.Time { + if ticket != nil { + return ticket.Time + } + return fallback +} diff --git a/metrics/request_tracker_test.go b/metrics/request_tracker_test.go new file mode 100644 index 000000000..55fff1897 --- /dev/null +++ b/metrics/request_tracker_test.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRequestTrackerPinsBeforeTimestampAssignment(t *testing.T) { + ch := make(chan SampleContainer, 1) + tracker, unbind := BindRequestTracker(ch) + defer unbind() + ticket := BeginRequest(ch) + require.NotNil(t, ticket) + require.False(t, tracker.Before().After(ticket.Time)) + ticket.Release() + ticket.Release() + for i := range tracker.shards { + require.Empty(t, tracker.shards[i].active) + } + unbind() + require.Nil(t, BeginRequest(ch)) +} + +func TestRequestTrackerConcurrentProducers(t *testing.T) { + ch := make(chan SampleContainer) + tracker, unbind := BindRequestTracker(ch) + defer unbind() + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + for range 1000 { + ticket := BeginRequest(ch) + require.False(t, tracker.Before().After(ticket.Time)) + ticket.Release() + } + }) + } + wg.Wait() + for i := range tracker.shards { + require.Empty(t, tracker.shards[i].active) + } +} diff --git a/output/manager.go b/output/manager.go index 0a65d67c1..6c5b5397c 100644 --- a/output/manager.go +++ b/output/manager.go @@ -1,6 +1,7 @@ package output import ( + "fmt" "sync" "time" @@ -40,7 +41,25 @@ func NewManager(outputs []Output, logger logrus.FieldLogger, testStopCallback fu // all of its buffered metrics have been sent to all outputs. The second // callback will call the Stop() or StopWithTestError() method of every output. func (om *Manager) Start(samplesChan chan metrics.SampleContainer) (wait func(), finish func(error), err error) { + var unbind func() + for _, out := range om.outputs { + if tracked, ok := out.(interface{ SetRequestTracker(*metrics.RequestTracker) }); ok { + if mode, ok := out.(interface{ UsesRequestTracking() bool }); ok && !mode.UsesRequestTracking() { + continue + } + if unbind != nil { + unbind() + return nil, nil, fmt.Errorf("only one request-tracked output is supported") + } + tracker, release := metrics.BindRequestTracker(samplesChan) + unbind = release + tracked.SetRequestTracker(tracker) + } + } if err := om.startOutputs(); err != nil { + if unbind != nil { + unbind() + } return nil, nil, err } @@ -78,6 +97,9 @@ func (om *Manager) Start(samplesChan chan metrics.SampleContainer) (wait func(), finish = func(testErr error) { wait() // just in case, though it shouldn't be needed if API is used correctly om.stopOutputs(testErr, len(om.outputs)) + if unbind != nil { + unbind() + } } return wait, finish, nil }