diff --git a/FORK.md b/FORK.md
index f0108a00e..5d3980111 100644
--- a/FORK.md
+++ b/FORK.md
@@ -23,6 +23,8 @@ code to the k6 executable.
## Fork-specific changes
- Adds the `timescaledb-breakingit` output for writing load-test data through a PG-proxy.
+- Adds the opt-in `breaktest-db-breakingit` direct output alongside TimescaleDB; see
+ [configuration and current limits](internal/output/breakingit/BREAKTEST_DB.md).
- Adds optional detailed request and response data for failed HTTP requests.
- Adds the optional `httpRequestUrl` option to keep a named request's real URL in sample metadata.
- Extends browser network and error metrics, including optional error screenshots.
diff --git a/ai-tests/breaktest-db/smoke.py b/ai-tests/breaktest-db/smoke.py
new file mode 100644
index 000000000..33db2f3bc
--- /dev/null
+++ b/ai-tests/breaktest-db/smoke.py
@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+"""Isolated k6 -> Rust protocol test with the pg-proxy binary COPY writer as an oracle.
+
+Requires Docker, Python 3, a built k6 fork and a built BreakTest DB service image.
+Creates a temporary service/container and directory; never connects to customer DBs.
+"""
+import argparse
+import email.parser
+import gzip
+import http.server
+import io
+import json
+import os
+from pathlib import Path
+import secrets
+import struct
+import socket
+import html
+import subprocess
+import tempfile
+import threading
+import time
+import urllib.request
+import uuid
+
+
+
+def copy_rows(table, payload):
+ schemas = {
+ "requests_raw": "tssssbiisiii",
+ "transactions": "tsssbiii",
+ "requests_error": "tssssssiissssssu",
+ "virtual_users": "tsssiii",
+ }
+ schema = schemas[table]
+ stream = io.BytesIO(payload)
+ if stream.read(19) != b"PGCOPY\n\xff\r\n\x00" + bytes(8):
+ raise ValueError("Invalid binary COPY header")
+ while True:
+ count, = struct.unpack(">h", stream.read(2))
+ if count == -1:
+ if stream.read(1):
+ raise ValueError("Trailing bytes after COPY data")
+ return
+ if count != len(schema):
+ raise ValueError(f"Unexpected {table} row width {count}")
+ row = []
+ for kind in schema:
+ size, = struct.unpack(">i", stream.read(4))
+ if size == -1:
+ row.append(None)
+ continue
+ if size < 0:
+ raise ValueError("Invalid COPY field length")
+ field = stream.read(size)
+ if len(field) != size:
+ raise ValueError("Truncated COPY field")
+ if kind == "s":
+ value = field.decode()
+ elif kind == "u":
+ value = str(uuid.UUID(bytes=field))
+ else:
+ value, = struct.unpack({"i": ">i", "t": ">q", "b": ">?"}[kind], field)
+ row.append(value)
+ yield row
+
+
+def run(args):
+ name = "k6-breaktest-db-smoke-" + uuid.uuid4().hex[:12]
+ admin = secrets.token_hex(32)
+ producers = [secrets.token_hex(32) for _ in range(args.lanes)]
+ producer = ",".join(producers)
+ oracle = {}
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, *_):
+ pass
+
+ def do_GET(self):
+ if self.path == '/redirect':
+ self.send_response(302);self.send_header('Location','/ok');self.end_headers();return
+ if self.path == '/reset':
+ self.connection.shutdown(socket.SHUT_RDWR);self.connection.close();return
+ body=b'
fixture' if self.path=='/page' else b'fixture response'
+ self.send_response(503 if self.path in ('/failure','/slowfailure') else 200)
+ self.send_header('Content-Type','text/html')
+ self.send_header('Content-Length',str(len(body)))
+ self.send_header('Cache-Control','max-age=3600' if self.path=='/cached' else 'no-store')
+ self.end_headers()
+ if self.path=='/slowfailure':
+ self.wfile.write(body[:3]);self.wfile.flush();time.sleep(1.2);body=body[3:]
+ self.wfile.write(body)
+
+ def do_POST(self):
+ if self.path != "/ingest/batch":
+ self.send_error(404)
+ return
+ data = self.rfile.read(int(self.headers["Content-Length"]))
+ if self.headers.get("Content-Encoding") == "gzip":
+ data = gzip.decompress(data)
+ msg = email.parser.BytesParser().parsebytes(
+ b"MIME-Version: 1.0\r\nContent-Type: "
+ + self.headers["Content-Type"].encode() + b"\r\n\r\n" + data)
+ for part in msg.get_payload():
+ table = part["X-Table"].removeprefix("public.")
+ if part["X-Format"] != "binary":
+ raise ValueError(f"Expected binary COPY for {table}")
+ rows = copy_rows(table, part.get_payload(decode=True))
+ oracle.setdefault(table, []).extend(rows)
+ self.send_response(200)
+ self.end_headers()
+ self.wfile.write(b"OK")
+
+ target = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=target.serve_forever, daemon=True)
+ thread.start()
+ with tempfile.TemporaryDirectory(prefix=name) as temp:
+ root = Path(temp)
+ data_dir = root / "data"
+ data_dir.mkdir()
+ env = {**os.environ, "BREAKTEST_DB_ADMIN_TOKEN": admin}
+ with socket.socket() as port_socket:
+ port_socket.bind(('127.0.0.1',0))
+ fixed_port=port_socket.getsockname()[1]
+ try:
+ subprocess.run(["docker", "run", "-d", "--name", name,
+ "--label", "breaktest.k6-writer-smoke=true", "--memory", "512m",
+ "-e", "BREAKTEST_DB_ADMIN_TOKEN", "-p", f"127.0.0.1:{fixed_port}:8099",
+ "-v", f"{data_dir}:/data", args.image,
+ "serve", "/data", "0.0.0.0:8099"], env=env, check=True, capture_output=True)
+ port = subprocess.check_output(["docker", "port", name, "8099/tcp"], text=True).strip().rsplit(":", 1)[1]
+ service = f"http://127.0.0.1:{port}"
+
+ def post(path, payload):
+ request = urllib.request.Request(service + path, data=json.dumps(payload).encode(),
+ headers={"Authorization": "Bearer " + admin, "Content-Type": "application/json"})
+ with urllib.request.urlopen(request, timeout=30) as response:
+ return json.load(response)
+
+ for _ in range(100):
+ try:
+ post("/health", {})
+ break
+ except OSError:
+ time.sleep(.05)
+ origin = int(time.time() - 946684800 - 10) * 1_000_000
+ post("/runs/create", {"run_id": "k6-smoke", "source_tokens": producers,
+ "origin_us": origin, "customer_id": "isolated-k6-smoke"})
+ script = root / "smoke.js"
+ script.write_text('''import http from 'k6/http';
+import { group, check, sleep } from 'k6';
+import { Trend } from 'k6/metrics';
+const customLatency = new Trend('custom_latency');
+export const options = { vus: 2, iterations: Number(__ENV.FIXTURE_ITERATIONS) };
+export default function () {
+ group('Checkout', function () {
+ http.get(__ENV.TARGET + '/ok');
+ const r = http.get(__ENV.TARGET + '/failure');
+ check(r, { 'expected failure': x => x.status === 200 });
+ });
+ if (__ENV.FIXTURE_CUSTOM === 'true') customLatency.add(37);
+ sleep(0.15);
+}
+''')
+ if args.browser:
+ script.write_text("""import { browser } from 'k6/browser';
+import { group, check, sleep } from 'k6';
+export const options = { scenarios: { browser: {executor:'per-vu-iterations',vus:1,iterations:Number(__ENV.FIXTURE_ITERATIONS),options:{browser:{type:'chromium'}}} } };
+export default async function(){
+ const page=await browser.newPage();
+ try { await group('Checkout',async function(){
+ await page.goto(__ENV.TARGET+'/page');
+ const status=await page.evaluate(async()=>{
+ await fetch('/redirect');await fetch('/cached');await fetch('/cached');
+ try {await fetch('/reset')}catch(e){}
+ return (await fetch('/slowfailure').then(async r=>{await r.text();return r.status}));
+ });
+ check(status,{'expected failure': s=>s===200});
+ }); sleep(.3); } finally {await page.close();}
+}
+""")
+ target_url = f"http://127.0.0.1:{target.server_port}"
+ writer_env = {**os.environ, "BREAKTEST_DB_TOKEN": producer, "runId": "k6-smoke",
+ "nodeName": "k6-probe", "location": "Local", "PG_PROXY_TOKEN": "isolated-oracle",
+ "TARGET": target_url, "FIXTURE_ITERATIONS": str(args.iterations), "FIXTURE_CUSTOM": str(args.custom_metrics).lower(), "K6_NO_USAGE_REPORT": "true", "K6_HTTP_ERROR_DATA": "true"}
+ for key in ("syntheticMonitoring", "K6_BROWSER_SCREENSHOT_ON_ERROR", "CUSTOM_URL_PATTERNS", "BREAKINGIT_TRANSACTION_SOURCE"):
+ writer_env.pop(key, None)
+ command = [str(Path(args.k6).resolve()), "run", "--quiet",
+ "--out", "breaktest-db-breakingit=" + service,
+ "--out", "timescaledb-breakingit=" + target_url, str(script)]
+ if args.container_image:
+ root.chmod(0o755); script.chmod(0o644)
+ for key in ('TARGET',): writer_env[key]=writer_env[key].replace('127.0.0.1','host.docker.internal')
+ command=['docker','run','--rm','--shm-size','512m','--entrypoint','/usr/bin/k6',
+ '-v',str(Path(args.k6).resolve())+':/usr/bin/k6:ro','-v',str(script)+':/fixture.js:ro',
+ '-e','K6_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium','-e','K6_BROWSER_ARGS=no-sandbox',
+ '-e','K6_BROWSER_HEADLESS=true',
+ *[item for key in ('BREAKTEST_DB_TOKEN','runId','nodeName','location','PG_PROXY_TOKEN','TARGET','K6_NO_USAGE_REPORT','K6_HTTP_ERROR_DATA','FIXTURE_ITERATIONS','FIXTURE_CUSTOM') for item in ('-e',key)],
+ args.container_image,'run','--quiet','--out','breaktest-db-breakingit='+service.replace('127.0.0.1','host.docker.internal'),
+ '--out','timescaledb-breakingit='+target_url.replace('127.0.0.1','host.docker.internal'),'/fixture.js']
+ observations=[]
+ with (root/'k6.log').open('w+') as log:
+ process=subprocess.Popen(command,env=writer_env,text=True,stdout=log,stderr=log)
+ began=time.monotonic(); restarted=False
+ try:
+ while process.poll() is None:
+ if time.monotonic()-began>90:raise TimeoutError('k6 smoke deadline')
+ try:
+ observation=post('/runs/k6-smoke/snapshot',{})
+ if not observation['sealed']: observations.append(observation)
+ except OSError: pass
+ if args.restart_live and not restarted and time.monotonic()-began>=2:
+ subprocess.run(['docker','kill','--signal=KILL',name],check=True,capture_output=True)
+ subprocess.run(['docker','start',name],check=True,capture_output=True)
+ restarted=True
+ time.sleep(.1)
+ log.seek(0)
+ text=log.read()
+ assert process.returncode==0 and 'BREAKTEST_DB_FAILED' not in text,text
+ finally:
+ if process.poll() is None: process.kill(); process.wait()
+ live=[s for s in observations if all(not x['finished'] for x in s['sources']) and min(x['watermark'] for x in s['sources'])>origin and any(r['complete'] for r in s['rows'])]
+ assert live, 'No live watermark-completed aggregates observed'
+ end = int(time.time() - 946684800 + 10) * 1_000_000
+ final = post("/runs/k6-smoke/finalize", {})
+ assert final["sealed"], final
+ if args.restart_live:
+ assert restarted and final['recovered'],final
+ assert all(g.get('max_second') is not None for g in final['gaps']),final
+ assert final['run_complete'] == (not final['gaps'] and not final['unknown_coverage']),final
+ else: assert final['run_complete'] and not final['gaps'],final
+ expected_counts = {"requests_raw": args.iterations*(3 if args.custom_metrics else 2), "transactions": args.iterations, "requests_error": args.iterations*2}
+ counts = {}
+ for table in ("requests_raw", "transactions", "requests_error", "virtual_users"):
+ actual = post("/runs/k6-smoke/query", {"table": table, "start_us": origin, "end_us": end})
+ counts[table] = len(actual["rows"])
+ if not args.restart_live: assert counts[table] == len(oracle[table]), (table, counts[table], len(oracle[table]))
+ else: assert counts[table] <= len(oracle[table]), (table, counts)
+ if table in expected_counts and not args.restart_live and not args.browser:
+ assert counts[table] == expected_counts[table], (table, counts)
+ else:
+ assert counts[table] > 0
+ # Match all fields including timestamp, except independently-generated UUIDs.
+ def normalized(row):
+ values = []
+ for i, value in enumerate(row):
+ if table == "requests_error" and i == 15:
+ continue
+ if value is None or value == "":
+ value = None
+ elif isinstance(value, bool):
+ value = "true" if value else "false"
+ else:
+ value = str(value)
+ values.append(value)
+ return json.dumps(values, ensure_ascii=False)
+ left = sorted(normalized(row) for row in actual["rows"])
+ right = sorted(normalized(row) for row in oracle[table])
+ if args.restart_live:
+ from collections import Counter
+ assert not (Counter(left)-Counter(right)), (table,'duplicate or corrupted recovered rows')
+ if len(left)= builtinOutput(len(_builtinOutputIndex)-1) {
@@ -36,9 +36,10 @@ func _builtinOutputNoOp() {
_ = x[builtinOutputOpentelemetry-(9)]
_ = x[builtinOutputSummary-(10)]
_ = x[builtinOutputTimescaledbBreakingit-(11)]
+ _ = x[builtinOutputBreakTestDBBreakingit-(12)]
}
-var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit}
+var _builtinOutputValues = []builtinOutput{builtinOutputCloud, builtinOutputCSV, builtinOutputDatadog, builtinOutputExperimentalPrometheusRW, builtinOutputInfluxdb, builtinOutputJSON, builtinOutputKafka, builtinOutputStatsd, builtinOutputExperimentalOpentelemetry, builtinOutputOpentelemetry, builtinOutputSummary, builtinOutputTimescaledbBreakingit, builtinOutputBreakTestDBBreakingit}
var _builtinOutputNameToValueMap = map[string]builtinOutput{
_builtinOutputName[0:5]: builtinOutputCloud,
@@ -65,6 +66,8 @@ var _builtinOutputNameToValueMap = map[string]builtinOutput{
_builtinOutputLowerName[103:110]: builtinOutputSummary,
_builtinOutputName[110:132]: builtinOutputTimescaledbBreakingit,
_builtinOutputLowerName[110:132]: builtinOutputTimescaledbBreakingit,
+ _builtinOutputName[132:155]: builtinOutputBreakTestDBBreakingit,
+ _builtinOutputLowerName[132:155]: builtinOutputBreakTestDBBreakingit,
}
var _builtinOutputNames = []string{
@@ -80,6 +83,7 @@ var _builtinOutputNames = []string{
_builtinOutputName[90:103],
_builtinOutputName[103:110],
_builtinOutputName[110:132],
+ _builtinOutputName[132:155],
}
// builtinOutputString retrieves an enum value from the enum constants string name.
diff --git a/internal/cmd/outputs.go b/internal/cmd/outputs.go
index a641fc2ac..c95a11882 100644
--- a/internal/cmd/outputs.go
+++ b/internal/cmd/outputs.go
@@ -35,6 +35,7 @@ const (
builtinOutputOpentelemetry
builtinOutputSummary
builtinOutputTimescaledbBreakingit
+ builtinOutputBreakTestDBBreakingit
)
func getPossibleIDList(constrs map[string]output.Constructor) string {
diff --git a/internal/cmd/outputs_full.go b/internal/cmd/outputs_full.go
index ae701b283..45e1fd671 100644
--- a/internal/cmd/outputs_full.go
+++ b/internal/cmd/outputs_full.go
@@ -48,6 +48,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) {
},
"web-dashboard": dashboard.New,
builtinOutputTimescaledbBreakingit.String(): breakingit.New,
+ builtinOutputBreakTestDBBreakingit.String(): breakingit.NewBreakTestDB,
builtinOutputExperimentalOpentelemetry.String(): func(params output.Params) (output.Output, error) {
params.Logger.Warnf("OpenTelemetry output has been graduated as a stable output."+
"You can now use just %q instead of %q. The experimental version will be removed in future versions.",
diff --git a/internal/cmd/outputs_lean.go b/internal/cmd/outputs_lean.go
index c58dc328a..4527636e7 100644
--- a/internal/cmd/outputs_lean.go
+++ b/internal/cmd/outputs_lean.go
@@ -56,6 +56,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) {
return nil, errors.New("the web-dashboard output is not available in this lean build of k6")
},
builtinOutputTimescaledbBreakingit.String(): breakingit.New,
+ builtinOutputBreakTestDBBreakingit.String(): breakingit.NewBreakTestDB,
}
exts := ext.Get(ext.OutputExtension)
diff --git a/internal/cmd/outputs_test.go b/internal/cmd/outputs_test.go
index 0501f2b64..572dc72f1 100644
--- a/internal/cmd/outputs_test.go
+++ b/internal/cmd/outputs_test.go
@@ -12,7 +12,15 @@ func TestBuiltinOutputString(t *testing.T) {
"cloud", "csv", "datadog", "experimental-prometheus-rw",
"influxdb", "json", "kafka", "statsd",
"experimental-opentelemetry", "opentelemetry",
- "summary", "timescaledb-breakingit",
+ "summary", "timescaledb-breakingit", "breaktest-db-breakingit",
}
assert.Equal(t, exp, builtinOutputStrings())
}
+
+func TestBreakTestDBAndTimescaleConstructors(t *testing.T) {
+ t.Parallel()
+ constructors, err := getAllOutputConstructors()
+ assert.NoError(t, err)
+ assert.Contains(t, constructors, "timescaledb-breakingit")
+ assert.Contains(t, constructors, "breaktest-db-breakingit")
+}
diff --git a/internal/js/modules/k6/browser/common/network_manager.go b/internal/js/modules/k6/browser/common/network_manager.go
index 7dbc58ac3..77cdc3d09 100644
--- a/internal/js/modules/k6/browser/common/network_manager.go
+++ b/internal/js/modules/k6/browser/common/network_manager.go
@@ -286,7 +286,13 @@ func (m *NetworkManager) emitRequestMetrics(req *Request) {
})
}
-func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //nolint:funlen
+func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request, completion browserCompletion) { //nolint:funlen
+ transferred := false
+ defer func() {
+ if !transferred {
+ completion.ticket.Release()
+ }
+ }()
state := m.vu.State()
if state == nil {
return // CDP events can arrive after iteration teardown.
@@ -368,67 +374,70 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //n
return
}
- pushIfNotDone(m.vu.Context(), m.logger, state.Samples, k6metrics.ConnectedSamples{
+ bundle := k6metrics.ConnectedSamples{
Samples: []k6metrics.Sample{
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqDuration, Tags: tags},
Value: timings.durationMs,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqBlocked, Tags: tags},
Value: timings.blocked,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqConnecting, Tags: tags},
Value: timings.connecting,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqTLSHandshaking, Tags: tags},
Value: timings.tlsHandshaking,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqSending, Tags: tags},
Value: timings.sending,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqWaiting, Tags: tags},
Value: timings.waiting,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqReceiving, Tags: tags},
Value: timings.receiving,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserDataReceived, Tags: tags},
Value: float64(bodySize),
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
},
},
- })
+ }
- if resp != nil {
+ {
+ if resp == nil {
+ failed = 1
+ }
sample := k6metrics.Sample{
TimeSeries: k6metrics.TimeSeries{Metric: m.customMetrics.BrowserHTTPReqFailed, Tags: tags},
Value: failed,
- Time: wallTime,
+ Time: completion.at,
Metadata: tagsAndMeta.Metadata,
}
- if failed == 1 {
+ if failed == 1 && resp != nil {
reqHeaders, reqBody := req.formatErrorRequestData()
sample.HTTPErrorReqHeaders = reqHeaders
sample.HTTPErrorReqBody = reqBody
@@ -438,10 +447,9 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { //n
sample.HTTPErrorResHeaders = resHeaders
sample.HTTPErrorResBody = resBody
}
- pushIfNotDone(m.vu.Context(), m.logger, state.Samples, k6metrics.ConnectedSamples{
- Samples: []k6metrics.Sample{sample},
- })
+ bundle.Samples = append(bundle.Samples, sample)
}
+ transferred = k6metrics.PushTrackedRequest(m.vu.Context(), state.Samples, completion.ticket, bundle)
}
type browserResponseTimings struct {
@@ -542,7 +550,13 @@ func (m *NetworkManager) calcReceiving(req *Request, t *network.ResourceTiming,
// ERR_CONNECTION_REFUSED). This ensures outputs like breakingit receive metrics for
// failed navigations/requests. The failure reason (e.g. net::ERR_NAME_NOT_RESOLVED) is
// added as the "error" tag so CSV and other outputs can show it.
-func (m *NetworkManager) emitRequestFailedMetrics(req *Request) {
+func (m *NetworkManager) emitRequestFailedMetrics(req *Request, completion browserCompletion) {
+ transferred := false
+ defer func() {
+ if !transferred {
+ completion.ticket.Release()
+ }
+ }()
state := m.vu.State()
if state == nil {
return
@@ -596,7 +610,7 @@ func (m *NetworkManager) emitRequestFailedMetrics(req *Request) {
} {
samples = append(samples, k6metrics.Sample{
TimeSeries: k6metrics.TimeSeries{Metric: metric, Tags: tags},
- Time: wallTime, Metadata: tagsAndMeta.Metadata,
+ Time: completion.at, Metadata: tagsAndMeta.Metadata,
})
}
samples[0].Value = durationMs
@@ -604,7 +618,7 @@ func (m *NetworkManager) emitRequestFailedMetrics(req *Request) {
failed.Value = 1
failed.HTTPErrorReqHeaders = reqHeaders
failed.HTTPErrorReqBody = reqBody
- k6metrics.PushIfNotDone(m.vu.Context(), state.Samples, k6metrics.ConnectedSamples{Samples: samples})
+ transferred = k6metrics.PushTrackedRequest(m.vu.Context(), state.Samples, completion.ticket, k6metrics.ConnectedSamples{Samples: samples})
}
// handleURLTag will check if the url tag needs to be grouped by testing
@@ -626,6 +640,7 @@ func handleURLTag(mi eventInterceptor, url string, method string, tags *k6metric
func (m *NetworkManager) handleRequestRedirect(
req *Request, redirectResponse *network.Response, hasExtraInfo bool, timestamp *cdp.MonotonicTime,
) {
+ completion := m.beginBrowserCompletion()
resp := NewHTTPResponse(m.ctx, req, redirectResponse, timestamp)
m.extraInfoTracker.processResponse(req.requestID, resp, hasExtraInfo)
req.responseMu.Lock()
@@ -638,7 +653,7 @@ func (m *NetworkManager) handleRequestRedirect(
// them (the redirect's ExtraInfo may still be in flight here).
m.wg.Go(func() {
resp.WaitForRawHeaders()
- m.emitResponseMetrics(resp, req)
+ m.emitResponseMetrics(resp, req, completion)
})
m.deleteRequestByID(req.requestID)
@@ -740,7 +755,7 @@ func (m *NetworkManager) onLoadingFailed(event *network.EventLoadingFailed) {
// Emit browser metrics for failed requests so outputs (e.g. breakingit) can store them.
if !isInternalURL(req.url) {
- m.emitRequestFailedMetrics(req)
+ m.emitRequestFailedMetrics(req, m.beginBrowserCompletion())
}
m.eventInterceptor.onRequestFailed(req)
@@ -757,6 +772,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished)
return
}
+ completion := m.beginBrowserCompletion()
req.responseEndWall = event.Timestamp.Time()
req.responseEndTiming = float64(event.Timestamp.Time().Sub(req.timestamp).Milliseconds())
req.encodedDataLength = int64(event.EncodedDataLength)
@@ -768,6 +784,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished)
// Skip data and blob URLs when emitting metrics, since they're internal to the browser.
if isInternalURL(req.url) {
+ completion.ticket.Release()
return
}
// Emit the response metrics in a separate goroutine, once the raw headers
@@ -784,7 +801,7 @@ func (m *NetworkManager) onLoadingFinished(event *network.EventLoadingFinished)
if resp != nil {
resp.WaitForRawHeaders()
}
- m.emitResponseMetrics(resp, req)
+ m.emitResponseMetrics(resp, req, completion)
})
}
diff --git a/internal/js/modules/k6/browser/common/network_manager_test.go b/internal/js/modules/k6/browser/common/network_manager_test.go
index d1338a4a0..7fe402094 100644
--- a/internal/js/modules/k6/browser/common/network_manager_test.go
+++ b/internal/js/modules/k6/browser/common/network_manager_test.go
@@ -311,7 +311,7 @@ func TestNetworkManagerEmitRequestResponseMetricsTimingSkew(t *testing.T) {
&network.Response{Timing: &network.ResourceTiming{}},
(*cdp.MonotonicTime)(&tt.res.ts),
)
- nm.emitResponseMetrics(res, req)
+ nm.emitResponseMetrics(res, req, browserCompletion{at: tt.wantRes.wt})
n = vu.AssertSamples(func(s k6metrics.Sample) {
assert.Equalf(t, tt.wantRes.wt, s.Time, "timing skew in %s", s.Metric.Name)
})
@@ -363,7 +363,7 @@ func TestNetworkManagerMetricsRetainRequestTagsAndMetadata(t *testing.T) {
URL: req.URL(),
Timing: &network.ResourceTiming{},
}, (*cdp.MonotonicTime)(&now))
- nm.emitResponseMetrics(resp, req)
+ nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion())
n := vu.AssertSamples(func(sample k6metrics.Sample) {
group, ok := sample.Tags.Get(k6metrics.TagGroup.String())
@@ -379,7 +379,7 @@ func TestNetworkManagerMetricsRetainRequestTagsAndMetadata(t *testing.T) {
// The fork also emits samples when navigation fails before any response.
// Those must use the request's context, not the VU's restored context.
req.setErrorText("net::ERR_CONNECTION_REFUSED")
- nm.emitRequestFailedMetrics(req)
+ nm.emitRequestFailedMetrics(req, nm.beginBrowserCompletion())
n = vu.AssertSamples(func(sample k6metrics.Sample) {
group, _ := sample.Tags.Get(k6metrics.TagGroup.String())
assert.Equal(t, "::request", group)
@@ -621,7 +621,7 @@ func TestNetworkManagerReceivingMetric(t *testing.T) {
},
(*cdp.MonotonicTime)(&reqTimestamp),
)
- nm.emitResponseMetrics(resp, req)
+ nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion())
var receivingVal float64
vu.AssertSamples(func(s k6metrics.Sample) {
@@ -687,7 +687,7 @@ func TestNetworkManagerReceivingMetricNoHeadersEndWall(t *testing.T) {
},
(*cdp.MonotonicTime)(&reqTimestamp),
)
- nm.emitResponseMetrics(resp, req)
+ nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion())
var receivingVal float64
vu.AssertSamples(func(s k6metrics.Sample) {
@@ -751,7 +751,7 @@ func TestNetworkManagerReceivingMetricCachedResponse(t *testing.T) {
},
(*cdp.MonotonicTime)(&reqTimestamp),
)
- nm.emitResponseMetrics(resp, req)
+ nm.emitResponseMetrics(resp, req, nm.beginBrowserCompletion())
var receivingVal float64
receivingFound := false
diff --git a/internal/js/modules/k6/browser/common/request_completion.go b/internal/js/modules/k6/browser/common/request_completion.go
new file mode 100644
index 000000000..aaf7570ab
--- /dev/null
+++ b/internal/js/modules/k6/browser/common/request_completion.go
@@ -0,0 +1,24 @@
+package common
+
+import (
+ "time"
+
+ k6metrics "go.k6.io/k6/v2/metrics"
+)
+
+// Browser completion is observed at terminal CDP-event dispatch, before async
+// header/body retrieval. Chromium phase durations remain in their CDP time base;
+// the result's timestamp is the generator-observed sampler completion, for ALL
+// outputs. Retimestamping at output delivery would hide backlog and is forbidden.
+type browserCompletion struct {
+ at time.Time
+ ticket *k6metrics.RequestTicket
+}
+
+func (m *NetworkManager) beginBrowserCompletion() browserCompletion {
+ var ticket *k6metrics.RequestTicket
+ if state := m.vu.State(); state != nil {
+ ticket = k6metrics.BeginRequest(state.Samples)
+ }
+ return browserCompletion{at: k6metrics.RequestTime(ticket, time.Now()), ticket: ticket}
+}
diff --git a/internal/js/modules/k6/browser/common/request_completion_test.go b/internal/js/modules/k6/browser/common/request_completion_test.go
new file mode 100644
index 000000000..1f275dd3f
--- /dev/null
+++ b/internal/js/modules/k6/browser/common/request_completion_test.go
@@ -0,0 +1,57 @@
+package common
+
+import (
+ "net/url"
+ "testing"
+ "time"
+
+ "github.com/chromedp/cdproto/network"
+ "github.com/stretchr/testify/require"
+ "go.k6.io/k6/v2/internal/js/modules/k6/browser/k6ext"
+ "go.k6.io/k6/v2/internal/js/modules/k6/browser/k6ext/k6test"
+ "go.k6.io/k6/v2/internal/js/modules/k6/browser/log"
+ k6metrics "go.k6.io/k6/v2/metrics"
+)
+
+func TestBrowserCompletionPinsHeaderDelayAndOutputQueue(t *testing.T) {
+ samples := make(chan k6metrics.SampleContainer, 10)
+ tracker, unbind := k6metrics.BindRequestTracker(samples)
+ defer unbind()
+ vu := k6test.NewVU(t, k6test.WithSamples(samples))
+ vu.ActivateVU()
+ nm := &NetworkManager{vu: vu, customMetrics: k6ext.RegisterCustomMetrics(k6metrics.NewRegistry()), eventInterceptor: &EventInterceptorMock{}, logger: log.NewNullLogger()}
+ u, _ := url.Parse("https://example.test/error")
+ req := &Request{url: u, wallTime: time.Now().Add(-time.Second), tagsAndMeta: vu.State().Tags.GetCurrentValues()}
+ completion := nm.beginBrowserCompletion()
+ // A captured terminal event waits through delayed headers/body extraction.
+ require.NotNil(t, completion.ticket)
+ time.Sleep(1100 * time.Millisecond)
+ require.False(t, tracker.Before().After(completion.at))
+ resp := &Response{url: u.String(), status: 503, remoteAddress: &RemoteAddress{}, wallTime: completion.at.Add(-500 * time.Millisecond), timing: &network.ResourceTiming{}, body: []byte("error detail"), request: req}
+ nm.emitResponseMetrics(resp, req, completion)
+ require.False(t, tracker.Before().After(completion.at)) // buffered output still pins it
+ frame := (<-samples).(k6metrics.RequestSamples)
+ require.Len(t, frame.GetSamples(), 9)
+ for _, sample := range frame.GetSamples() {
+ require.Equal(t, completion.at, sample.Time)
+ }
+ require.Equal(t, float64(1), frame.GetSamples()[8].Value)
+ frame.Ticket.Release()
+ require.True(t, tracker.Before().After(completion.at))
+}
+
+func TestBrowserCacheSkipReleasesCompletionPin(t *testing.T) {
+ samples := make(chan k6metrics.SampleContainer, 10)
+ tracker, unbind := k6metrics.BindRequestTracker(samples)
+ defer unbind()
+ vu := k6test.NewVU(t, k6test.WithSamples(samples))
+ vu.ActivateVU()
+ nm := &NetworkManager{vu: vu, customMetrics: k6ext.RegisterCustomMetrics(k6metrics.NewRegistry()), eventInterceptor: &EventInterceptorMock{}, logger: log.NewNullLogger()}
+ u, _ := url.Parse("https://example.test/cached")
+ req := &Request{url: u, wallTime: time.Now(), tagsAndMeta: vu.State().Tags.GetCurrentValues(), fromMemoryCache: true}
+ completion := nm.beginBrowserCompletion()
+ nm.emitResponseMetrics(&Response{url: u.String(), remoteAddress: &RemoteAddress{}, wallTime: time.Now(), request: req, body: []byte("cached")}, req, completion)
+ require.Empty(t, samples)
+ time.Sleep(1100 * time.Millisecond)
+ require.True(t, tracker.Before().After(completion.at))
+}
diff --git a/internal/js/modules/k6/metrics/metrics.go b/internal/js/modules/k6/metrics/metrics.go
index 5e0bc3bdf..bae63e2c0 100644
--- a/internal/js/modules/k6/metrics/metrics.go
+++ b/internal/js/modules/k6/metrics/metrics.go
@@ -115,16 +115,17 @@ func (m Metric) add(v sobek.Value, addTags sobek.Value) (bool, error) {
return false, fmt.Errorf("cannot add tags for the '%s' custom metric: %w", m.metric.Name, err)
}
+ ticket := metrics.BeginRequest(state.Samples)
sample := metrics.Sample{
TimeSeries: metrics.TimeSeries{
Metric: m.metric,
Tags: ctm.Tags,
},
- Time: time.Now(),
+ Time: metrics.RequestTime(ticket, time.Now()),
Metadata: ctm.Metadata,
Value: vfloat,
}
- metrics.PushIfNotDone(m.vu.Context(), state.Samples, sample)
+ metrics.PushTrackedRequest(m.vu.Context(), state.Samples, ticket, sample)
return true, nil
}
diff --git a/internal/js/modules/k6/websockets/websockets.go b/internal/js/modules/k6/websockets/websockets.go
index 90529fc9f..4938a72c1 100644
--- a/internal/js/modules/k6/websockets/websockets.go
+++ b/internal/js/modules/k6/websockets/websockets.go
@@ -306,7 +306,8 @@ func (w *webSocket) establishConnection(ctx context.Context, params *wsParams) {
w.started = time.Now()
handshakeDone := make(chan struct{})
go w.closePendingConnectionOnContextDone(ctx, handshakeDone)
- start := time.Now()
+ ticket := metrics.BeginRequest(w.vu.State().Samples)
+ start := metrics.RequestTime(ticket, time.Now())
conn, httpResponse, connErr := w.dial(ctx, params)
w.connectionMu.Lock()
w.pendingConnection = nil
@@ -345,7 +346,7 @@ func (w *webSocket) establishConnection(ctx context.Context, params *wsParams) {
}
//nolint:contextcheck // Handshake cancellation must not suppress connection metrics.
- w.emitConnectionMetrics(w.vu.Context(), start, connectionDuration)
+ w.emitConnectionMetrics(w.vu.Context(), start, connectionDuration, ticket)
if connErr != nil {
// Pass the error to the user script before exiting immediately
w.tq.Queue(func() error {
@@ -437,10 +438,14 @@ func (w *webSocket) closePendingConnection() {
}
// emitConnectionMetrics emits the metrics for a websocket connection.
-func (w *webSocket) emitConnectionMetrics(ctx context.Context, start time.Time, duration float64) {
+func (w *webSocket) emitConnectionMetrics(ctx context.Context, start time.Time, duration float64, tickets ...*metrics.RequestTicket) {
+ var ticket *metrics.RequestTicket
+ if len(tickets) > 0 {
+ ticket = tickets[0]
+ }
state := w.vu.State()
- metrics.PushIfNotDone(ctx, state.Samples, metrics.ConnectedSamples{
+ metrics.PushTrackedRequest(ctx, state.Samples, ticket, metrics.ConnectedSamples{
Samples: []metrics.Sample{
{
TimeSeries: metrics.TimeSeries{Metric: state.BuiltinMetrics.WSSessions, Tags: w.tagsAndMeta.Tags},
@@ -827,7 +832,6 @@ func (w *webSocket) ping() {
}
func (w *webSocket) trackPong(pingID string) {
- pongTimestamp := time.Now()
pingTimestamp, ok := w.sendPings.timestamps[pingID]
if !ok {
@@ -838,7 +842,9 @@ func (w *webSocket) trackPong(pingID string) {
return
}
- metrics.PushIfNotDone(w.vu.Context(), w.vu.State().Samples, metrics.Sample{
+ ticket := metrics.BeginRequest(w.vu.State().Samples)
+ pongTimestamp := metrics.RequestTime(ticket, time.Now())
+ metrics.PushTrackedRequest(w.vu.Context(), w.vu.State().Samples, ticket, metrics.Sample{
TimeSeries: metrics.TimeSeries{
Metric: w.builtinMetrics.WSPing,
Tags: w.tagsAndMeta.Tags,
diff --git a/internal/js/modules/k6/ws/ws.go b/internal/js/modules/k6/ws/ws.go
index 5389fc788..dfe186a44 100644
--- a/internal/js/modules/k6/ws/ws.go
+++ b/internal/js/modules/k6/ws/ws.go
@@ -268,7 +268,8 @@ func (mi *WS) dial(
wsd.Jar = args.cookieJar
}
- connStart := time.Now()
+ ticket := metrics.BeginRequest(state.Samples)
+ connStart := metrics.RequestTime(ticket, time.Now())
conn, httpResponse, dialErr := wsd.DialContext(ctx, url, args.headers)
connEnd := time.Now()
@@ -303,7 +304,7 @@ func (mi *WS) dial(
builtinMetrics: state.BuiltinMetrics,
}
- connEndHook := socket.pushSessionMetrics(connStart, connEnd)
+ connEndHook := socket.pushSessionMetrics(connStart, connEnd, ticket)
return &socket, httpResponse, connEndHook, dialErr
}
@@ -392,7 +393,6 @@ func (s *Socket) Ping() {
}
func (s *Socket) trackPong(pingID string) {
- pongTimestamp := time.Now()
if _, ok := s.pingSendTimestamps[pingID]; !ok {
// We received a pong for a ping we didn't send; ignore
@@ -400,8 +400,10 @@ func (s *Socket) trackPong(pingID string) {
return
}
pingTimestamp := s.pingSendTimestamps[pingID]
+ ticket := metrics.BeginRequest(s.samplesOutput)
+ pongTimestamp := metrics.RequestTime(ticket, time.Now())
- metrics.PushIfNotDone(s.ctx, s.samplesOutput, metrics.Sample{
+ metrics.PushTrackedRequest(s.ctx, s.samplesOutput, ticket, metrics.Sample{
TimeSeries: metrics.TimeSeries{
Metric: s.builtinMetrics.WSPing,
Tags: s.tagsAndMeta.Tags,
@@ -515,10 +517,14 @@ func (s *Socket) closeConnection(code int) error {
return err
}
-func (s *Socket) pushSessionMetrics(connStart, connEnd time.Time) func() {
+func (s *Socket) pushSessionMetrics(connStart, connEnd time.Time, tickets ...*metrics.RequestTicket) func() {
+ var ticket *metrics.RequestTicket
+ if len(tickets) > 0 {
+ ticket = tickets[0]
+ }
connDuration := metrics.D(connEnd.Sub(connStart))
- metrics.PushIfNotDone(s.ctx, s.samplesOutput, metrics.ConnectedSamples{
+ metrics.PushTrackedRequest(s.ctx, s.samplesOutput, ticket, metrics.ConnectedSamples{
Samples: []metrics.Sample{
{
TimeSeries: metrics.TimeSeries{
diff --git a/internal/output/breakingit/BREAKTEST_DB.md b/internal/output/breakingit/BREAKTEST_DB.md
new file mode 100644
index 000000000..ecc2f47bd
--- /dev/null
+++ b/internal/output/breakingit/BREAKTEST_DB.md
@@ -0,0 +1,136 @@
+# Direct BreakTest DB output
+
+The fork has two independent built-in outputs, in both normal and `lean` builds:
+
+- `timescaledb-breakingit`: the pg-proxy binary COPY writer.
+- `breaktest-db-breakingit`: direct zstd-compressed PostgreSQL binary COPY batches
+ to the BreakTest BreakTest DB service. It does not contact PostgreSQL or PG-proxy.
+
+The implementation lives in this fork, alongside the active built-in TimescaleDB
+writer. It does not use the older xk6 extension in the BreakTest repository.
+
+## Invocation and application contract
+
+The application must first provision a BreakTest DB run and one to four producer tokens per
+load generator. Pass the run ID and that generator's token when invoking k6:
+
+```sh
+export runId=''
+export BREAKTEST_DB_TOKEN=','
+export location='Amsterdam'
+export nodeName='generator-1'
+k6 run --out breaktest-db-breakingit=https://storage.example.test script.js
+```
+
+The URL is the service base URL (an optional routing prefix is retained). The
+writer appends `/runs//hello`, `/ingest/batch`, `/watermark` and `/finish` to it. TLS
+certificates are verified, connections reused, and redirects disabled so tokens
+cannot be redirected to another origin. Never pass the service admin token.
+
+`BREAKINGIT_TRANSACTION_SOURCE` and `CUSTOM_URL_PATTERNS` retain their existing
+mapping behavior. Native groups, legacy `pages`, requests, WebSocket/custom trend
+request rows, failed checks, browser errors and VUs use the existing mapping code.
+Empty fields remain NULL in the binary encoding, and negative/unavailable
+connect times become NULL. Error UUIDs are generated independently per output.
+
+The application selects the output from the frozen test backend, provisions all
+lane tokens, and finalizes only after every lane finishes. BreakTest's integration
+requires generator capability `k6-breaktest-db-v1`, verified against the installed binary
+checksum; this prevents routing multi-lane jobs to the older single-source writer.
+Generator machine statistics remain the responsibility of the existing agent.
+
+## Delivery and limits
+
+- Flush every 200 ms or at a 2 MiB encoded-row budget. Four queued batches per lane bound
+ outgoing buffering; a full queue applies backpressure rather than dropping rows.
+ The building batch, one waiting enqueue, sender/compression buffers, and k6's own
+ sample containers consume additional memory; 8 MiB per lane is not a total RSS limit.
+- One to four ordered compression/sender workers per generator, with independent
+ sequences and shared keep-alive connections. Batches are distributed round-robin.
+ Watermark-only messages reach idle lanes too. No high-load capacity claim yet.
+- Incomplete HTTP joins are limited to 4,096 groups / an estimated 2 MiB payload
+ budget. Oversized rows or exhausted join budgets stop the run explicitly.
+- HTTP/browser joins include timestamp and tag-set identity. All required request
+ metrics must arrive before a row is emitted; delayed browser failure payloads
+ are not treated as success. Ambiguous duplicate metrics or incomplete groups at
+ shutdown fail the output instead of publishing an apparently complete run.
+- Retry transient failures and throttling with the exact same compressed bytes,
+ boundary, and sequence, for at most 30 seconds per operation. Validate ACKs.
+ Rejection, invalid ACK, or exhausted retries stops k6 and forbids `/finish`.
+- Drain accepted sample containers and all queued requests before `/finish`.
+ The producer does not seal the run itself. A final barrier proves delivery of
+ this producer's data; it is not a durable-on-disk/fsync acknowledgement.
+- A fresh producer source is required at startup. Tracked native and browser runs recover after a
+ server incarnation change using the anchored source clock and checkpoint sequence:
+ skip a held batch proven checkpointed; otherwise renumber and replay only that
+ unacknowledged batch. Previously acknowledged, lost rows are not replayed; the
+ service exposes its relaxed-durability coverage gaps. Same-incarnation conflicts
+ remain fatal.
+
+## Watermarks and unsupported data
+
+Native and browser k6 runs send live request watermarks, using the same sequence-backed,
+all-source barrier as the Java writer. The output manager installs a per-run
+tracker before producers start. HTTP completion assigns its timestamp and pin
+atomically; WebSocket connecting/ping and custom metrics are similarly tracked.
+Pins survive the engine sample channel and pending metric joins until rows are
+serialized. A sharded tracker computes the oldest unresolved timestamp. It uses
+an anchored monotonic clock, so a wall-clock step cannot invalidate a sent barrier.
+
+After detaching all preceding rows, the writer queues the same whole-second
+barrier on every lane. A lane sends it only after acknowledging its earlier
+batches. The service closes the common prefix into immediate 1s aggregates; open
+seconds remain queryable through the pending/raw path. Event-time completeness
+still does not imply fsync durability. Transactions and monitoring are separate
+streams, just as in the Java request-watermark contract.
+
+Browser requests acquire a completion pin when the generator dispatches the
+terminal CDP event (including redirects and loading failures), before deferred
+header/body retrieval. They emit one complete timing/failure bundle. The pin
+survives deferred work and the output queue; skipped cached requests release it.
+Mixed browser/native scenarios use the same tracker and lane barriers.
+
+**Timestamp contract:** browser sampler timestamps now mean generator-observed
+terminal CDP completion, for both BreakTest DB and TimescaleDB outputs. Previously,
+successful rows used the response-header timestamp. Chromium phase/duration
+measurements remain in their original time base. CDP transport delay precedes
+this observation; this is not a claim that Chromium's original event clock is
+watermarked. Nothing is retimestamped at output delivery to hide writer backlog.
+Uninstrumented request metric producers fail closed.
+
+Synthetic monitoring and browser screenshots are explicitly unsupported and
+rejected. Ordinary browser request timings, failure response data, native groups
+and browser error messages are supported. An unexpected screenshot-bearing error
+also fails the output, rather than storing a dangling screenshot reference.
+
+## Validation
+
+```sh
+go test -race ./metrics ./output ./lib/netext/httpext ./internal/output/breakingit
+go test -tags lean ./internal/cmd -run 'TestBuiltinOutputString|TestBreakTestDBAndTimescaleConstructors'
+go test ./internal/cmd -run 'TestBuiltinOutputString|TestBreakTestDBAndTimescaleConstructors'
+go build -tags lean -o /tmp/k6-breaktest-db-writer .
+python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-breaktest-db-writer \
+ --image breaktest-db:local --lanes 4
+python3 ai-tests/breaktest-db/smoke.py --k6 /tmp/k6-breaktest-db-writer --lanes 4 --restart-live
+```
+
+The smoke test creates its own disposable Docker service and temporary directory,
+then runs both writers on the **same** metric stream. The TimescaleDB writer goes
+to an isolated binary COPY capture server; it does not write an actual TimescaleDB database.
+The test compares every stored column against the independently decoded proxy rows, except independently
+generated error UUIDs. It checks request, transaction, error and VU counts, complete
+sealing and reads after storage-service restart. It never connects to existing
+customer databases. This establishes protocol/mapping compatibility, not a
+TimescaleDB performance benchmark or browser end-to-end acceptance.
+
+The live smoke test also requires unfinished sources with advancing watermarks and
+complete aggregate buckets before finalization. Crash mode kills only its own
+throwaway service, verifies retained rows against the oracle without duplicates,
+and requires explicit incomplete coverage whenever rows were lost.
+
+Browser validation also runs real Chromium with redirects, iframe requests,
+connection failure, caching and a delayed 503 response, comparing both writers
+on the same stream. Use `--browser --container-image IMAGE` with a Linux binary
+and an image containing Chromium. This mapping oracle is distinct from the
+actual-database high-load benchmark in the BreakTest integration repository.
diff --git a/internal/output/breakingit/README.md b/internal/output/breakingit/README.md
index 99605eb87..19bd20587 100644
--- a/internal/output/breakingit/README.md
+++ b/internal/output/breakingit/README.md
@@ -134,3 +134,9 @@ so producers can fill the next batch during retries. Exhausted retries discard
only the failed batch; newly collected rows remain queued for the next send.
Synthetic run IDs use UUID binary fields, with the same deterministic fallback
as JMeter for legacy identifiers that are not UUIDs.
+
+## BreakTest DB alternative
+
+The fork also includes `breaktest-db-breakingit` for direct binary delivery to the
+BreakTest DB service. See [configuration, delivery semantics and current
+limits](BREAKTEST_DB.md). The existing TimescaleDB output remains available.
diff --git a/internal/output/breakingit/batch_test.go b/internal/output/breakingit/batch_test.go
index df8360ed9..4f686bdda 100644
--- a/internal/output/breakingit/batch_test.go
+++ b/internal/output/breakingit/batch_test.go
@@ -64,9 +64,5 @@ func TestBatchRetryPreservesConcurrentSamples(t *testing.T) {
t.Fatal("missing batch")
}
}
- require.Eventually(t, func() bool {
- logger.mu.Lock()
- defer logger.mu.Unlock()
- return !logger.flushing && logger.bufTrans.Len() == 0
- }, 5*time.Second, 10*time.Millisecond)
+ require.Empty(t, logger.bufTrans.Bytes())
}
diff --git a/internal/output/breakingit/breakingit.go b/internal/output/breakingit/breakingit.go
index 22f597aa4..97f3f4520 100644
--- a/internal/output/breakingit/breakingit.go
+++ b/internal/output/breakingit/breakingit.go
@@ -62,6 +62,9 @@ const (
// Logger writes k6 metric samples to pg-proxy using COPY via multipart batch
type Logger struct {
+ rowSink func(string, []string) error // direct BreakTest DB delivery, when configured
+ flushMu sync.Mutex // serialize sends, including the final drain
+ flushErr error // first exhausted delivery, guarded by flushMu
out io.Writer
proxyURL string
database string
@@ -78,7 +81,6 @@ type Logger struct {
firstData chan struct{}
firstQueued bool
cleanupDone chan struct{}
- flushing bool // flag to prevent recursive flush calls
// pending screenshots to upsert via /upsert endpoint (low-volume)
pendingScreenshots [][]any
@@ -191,11 +193,19 @@ func (l *Logger) Stop() error {
<-l.cleanupDone
<-l.batchDone
// Finish incomplete groups before the final batch.
- for _, g := range l.pendingHTTPGroups {
+ l.mu.Lock()
+ groups := l.pendingHTTPGroups
+ l.pendingHTTPGroups = make(map[int64]*MetricGroup)
+ l.mu.Unlock()
+ for _, g := range groups {
l.processHTTPGroup(g)
}
- l.flush()
- return nil
+ l.flushMu.Lock()
+ defer l.flushMu.Unlock()
+ for l.flushOnce() {
+ // Drain rows accepted during the preceding send.
+ }
+ return l.flushErr
}
func (l *Logger) batchLoop() {
@@ -250,17 +260,19 @@ func (l *Logger) cleanupProcessor() {
}
func (l *Logger) flush() {
+ l.flushMu.Lock()
+ defer l.flushMu.Unlock()
+ l.flushOnce()
+}
+
+// flushOnce sends one detached batch. The caller holds flushMu.
+func (l *Logger) flushOnce() bool {
l.mu.Lock()
- // Prevent recursive flush calls
- if l.flushing {
- l.mu.Unlock()
- return
- }
hasBatchData := l.bufHTTP.Len() > 0 || l.bufVUs.Len() > 0 || l.bufTrans.Len() > 0 || l.bufErrors.Len() > 0
hasScreenshots := len(l.pendingScreenshots) > 0
if !hasBatchData && !hasScreenshots {
l.mu.Unlock()
- return
+ return false
}
// Transfer ownership so producers can fill the next batch during network I/O.
@@ -273,7 +285,6 @@ func (l *Logger) flush() {
// Snapshot pending screenshots
screenshotsSnapshot := l.pendingScreenshots
l.pendingScreenshots = nil
- l.flushing = true
l.mu.Unlock()
// Send screenshots via upsert endpoint (separate from batch COPY path)
@@ -284,18 +295,13 @@ func (l *Logger) flush() {
}
// Skip batch if no COPY data
- if !hasBatchData {
- l.mu.Lock()
- l.flushing = false
- l.mu.Unlock()
- return
+ if hasBatchData {
+ body, boundary := l.batchBody(bufHTTPSnapshot, bufVUsSnapshot, bufTransSnapshot, bufErrorsSnapshot)
+ if err := l.sendBatch(body, boundary); err != nil && l.flushErr == nil {
+ l.flushErr = err
+ }
}
-
- body, boundary := l.batchBody(bufHTTPSnapshot, bufVUsSnapshot, bufTransSnapshot, bufErrorsSnapshot)
- l.sendBatch(body, boundary)
- l.mu.Lock()
- l.flushing = false
- l.mu.Unlock()
+ return true
}
// upsertScreenshots sends screenshots to the pg-proxy upsert endpoint.
@@ -480,7 +486,7 @@ func (l *Logger) processHTTPGroup(g *MetricGroup) {
}
}
l.mu.Lock()
- l.writeBinaryRow(l.bufHTTP, row)
+ l.writeRow(l.bufHTTP, row)
// If not successful, also emit into requests_error(_sm)
if success == "false" {
l.writeHTTPError(g, transactionName, samplerName, responseCode, responseTime, connectTime, isBrowser)
@@ -561,7 +567,7 @@ func (l *Logger) processWebSocketMetric(s metrics.Sample, ts int64) {
}
}
l.mu.Lock()
- l.writeBinaryRow(l.bufHTTP, row)
+ l.writeRow(l.bufHTTP, row)
l.mu.Unlock()
}
@@ -643,7 +649,7 @@ func (l *Logger) processCustomTrendMetric(s metrics.Sample, ts int64) {
}
}
l.mu.Lock()
- l.writeBinaryRow(l.bufHTTP, row)
+ l.writeRow(l.bufHTTP, row)
l.mu.Unlock()
}
@@ -716,7 +722,7 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) {
})
_, _ = fmt.Fprintf(l.out, "[screenshot] queued for upsert, pendingScreenshots=%d\n", len(l.pendingScreenshots))
}
- l.writeBinaryRow(l.bufErrors, row)
+ l.writeRow(l.bufErrors, row)
l.mu.Unlock()
} else {
row := []any{
@@ -743,7 +749,7 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) {
screenshotHash, screenshot, "image/png", len(screenshot),
})
}
- l.writeBinaryRow(l.bufErrors, row)
+ l.writeRow(l.bufErrors, row)
l.mu.Unlock()
}
}
@@ -987,7 +993,7 @@ func (l *Logger) processPageSample(s metrics.Sample, ts int64) {
}
}
l.mu.Lock()
- l.writeBinaryRow(l.bufTrans, row)
+ l.writeRow(l.bufTrans, row)
l.mu.Unlock()
}
@@ -1033,7 +1039,7 @@ func (l *Logger) processGroupSample(s metrics.Sample, ts int64) {
}
}
l.mu.Lock()
- l.writeBinaryRow(l.bufTrans, row)
+ l.writeRow(l.bufTrans, row)
l.mu.Unlock()
}
@@ -1048,7 +1054,7 @@ func (l *Logger) processVUsSample(s metrics.Sample, ts int64) {
"", // finished_threads (unknown)
}
l.mu.Lock()
- l.writeBinaryRow(l.bufVUs, row)
+ l.writeRow(l.bufVUs, row)
l.mu.Unlock()
}
@@ -1102,7 +1108,7 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) {
fmt.Sprintf("%v", l.envTags["run_id"]), // run_id last
}
l.mu.Lock()
- l.writeBinaryRow(l.bufErrors, row)
+ l.writeRow(l.bufErrors, row)
l.mu.Unlock()
} else {
row := []any{
@@ -1124,7 +1130,7 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) {
id, // id (UUIDv7 generated from measurement timestamp)
}
l.mu.Lock()
- l.writeBinaryRow(l.bufErrors, row)
+ l.writeRow(l.bufErrors, row)
l.mu.Unlock()
}
}
@@ -1191,23 +1197,22 @@ func (l *Logger) batchBody(
return gzBody.Bytes(), mw.Boundary()
}
-func (l *Logger) sendBatch(body []byte, boundary string) bool {
+func (l *Logger) sendBatch(body []byte, boundary string) error {
start := time.Now()
delay := time.Duration(initialBackoffMs) * time.Millisecond
- for attempts := 1; time.Since(start) < maxRetryTimeMs*time.Millisecond; attempts++ {
+ for attempts := 1; ; attempts++ {
err := l.postBatch(body, boundary)
if err == nil {
- return true
+ return nil
}
elapsed := time.Since(start)
_, _ = fmt.Fprintf(l.out, "pg-proxy ingest attempt %d failed (%v elapsed): %v\n", attempts, elapsed, err)
if elapsed+delay >= maxRetryTimeMs*time.Millisecond {
- break
+ return fmt.Errorf("pg-proxy delivery failed after %d attempts: %w", attempts, err)
}
time.Sleep(delay)
delay = min(delay*2, maxBackoffMs*time.Millisecond)
}
- return false
}
func (l *Logger) postBatch(body []byte, boundary string) error {
@@ -1303,7 +1308,7 @@ func (l *Logger) writeHTTPError(g *MetricGroup,
id, // id (UUIDv7)
fmt.Sprintf("%v", l.envTags["run_id"]),
}
- l.writeBinaryRow(l.bufErrors, erow)
+ l.writeRow(l.bufErrors, erow)
} else {
erow := []any{
time.Unix(0, g.Timestamp),
@@ -1323,7 +1328,7 @@ func (l *Logger) writeHTTPError(g *MetricGroup,
g.ErrorResBody, // response_data
id, // id (UUIDv7)
}
- l.writeBinaryRow(l.bufErrors, erow)
+ l.writeRow(l.bufErrors, erow)
}
}
@@ -1423,3 +1428,32 @@ func requestTimings(values map[string]float64) (connectTime, latency, responseTi
int64(connect + values["http_req_sending"] + values["http_req_waiting"]),
int64(connect + values["http_req_duration"])
}
+
+// writeRow routes the shared metric mapping to the selected storage writer with mu held.
+func (l *Logger) writeRow(buf *bytes.Buffer, row []any) {
+ if l.rowSink == nil {
+ l.writeBinaryRow(buf, row)
+ return
+ }
+ table := "requests_raw"
+ switch buf {
+ case l.bufTrans:
+ table = "transactions"
+ case l.bufVUs:
+ table = "virtual_users"
+ case l.bufErrors:
+ table = "requests_error"
+ }
+ fields := make([]string, len(row))
+ for i, value := range row {
+ switch v := value.(type) {
+ case time.Time:
+ fields[i] = v.UTC().Format(time.RFC3339Nano)
+ case string:
+ fields[i] = v
+ default:
+ fields[i] = fmt.Sprint(v)
+ }
+ }
+ _ = l.rowSink(table, fields) // The direct writer records failures and stops the run.
+}
diff --git a/internal/output/breakingit/breaktest_db.go b/internal/output/breakingit/breaktest_db.go
new file mode 100644
index 000000000..c27aace7e
--- /dev/null
+++ b/internal/output/breakingit/breaktest_db.go
@@ -0,0 +1,491 @@
+package breakingit
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "net/textproto"
+ "net/url"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/klauspost/compress/zstd"
+ "go.k6.io/k6/v2/metrics"
+ "go.k6.io/k6/v2/output"
+)
+
+// BreakTestDBOutput reuses BreakTest's metric mappings, with bounded direct binary delivery.
+// Only the sender owns incarnation/sequence. An ACK is acceptance, not an fsync guarantee.
+type dbGroupKey struct {
+ time int64
+ tags *metrics.TagSet
+}
+
+type BreakTestDBOutput struct {
+ groups map[dbGroupKey]*MetricGroup
+ groupTickets map[dbGroupKey]*metrics.RequestTicket
+ tracker *metrics.RequestTracker
+ lanes []*BreakTestDBOutput
+ owner *BreakTestDBOutput
+ nextLane int
+ watermark int64
+ queuedWatermark int64
+ tokens []string
+
+ mapper *Logger
+ client *http.Client
+ base, token, runID string
+ incarnation string
+ sequence uint64
+ retryTimeout time.Duration
+ encoder *zstd.Encoder
+ queue chan dbBatch
+ done, tickStop, tickDone, failed chan struct{}
+ batchMu sync.Mutex
+ parts map[string]*bytes.Buffer
+ batchBytes int
+ addMu sync.Mutex // Serializes sample mapping and pending-group completion.
+ failureMu sync.Mutex
+ failure error
+ stopCallback func(error)
+ started bool
+ stopOnce sync.Once
+ stopErr error
+}
+type dbBatch struct {
+ body []byte
+ contentType string
+ beforeUS int64
+}
+
+// NewBreakTestDB is registered separately from timescaledb-breakingit in full and lean builds.
+func NewBreakTestDB(params output.Params) (output.Output, error) {
+ env := params.Environment
+ u, err := url.Parse(params.ConfigArgument)
+ if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
+ return nil, errors.New("BreakTest DB requires an HTTP(S) base URL without credentials, query or fragment")
+ }
+ runID, token := env["runId"], env["BREAKTEST_DB_TOKEN"]
+ if !regexp.MustCompile(`^[A-Za-z0-9_-]+$`).MatchString(runID) || token == "" {
+ return nil, errors.New("BreakTest DB requires runId and a run-scoped BREAKTEST_DB_TOKEN")
+ }
+ if strings.EqualFold(env["syntheticMonitoring"], "true") {
+ return nil, errors.New("BreakTest DB k6 output does not support synthetic monitoring")
+ }
+ if strings.EqualFold(env["K6_BROWSER_SCREENSHOT_ON_ERROR"], "true") {
+ return nil, errors.New("BreakTest DB k6 output does not yet support browser screenshot storage")
+ }
+ tokens := strings.Split(token, ",")
+ seen := map[string]bool{}
+ if len(tokens) > 4 {
+ return nil, errors.New("BreakTest DB supports 1-4 ingestion lanes")
+ }
+ for _, t := range tokens {
+ if t == "" || seen[t] {
+ return nil, errors.New("invalid BreakTest DB lane tokens")
+ }
+ seen[t] = true
+ }
+ source, err := parseTransactionSource(env["BREAKINGIT_TRANSACTION_SOURCE"])
+ if err != nil {
+ return nil, err
+ }
+ patterns := map[string]string{}
+ for _, pair := range strings.Split(env["CUSTOM_URL_PATTERNS"], ",") {
+ if pair == "" {
+ continue
+ }
+ parts := strings.Split(pair, "==")
+ if len(parts) != 2 {
+ return nil, errors.New("invalid CUSTOM_URL_PATTERNS entry")
+ }
+ patterns[parts[0]] = parts[1]
+ }
+ tags := map[string]string{"run_id": runID, "location": env["location"], "node_name": env["nodeName"], "scenario_name": env["scenarioName"]}
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxConnsPerHost = len(tokens)
+ c := &BreakTestDBOutput{
+ base: strings.TrimRight(u.String(), "/") + "/runs/" + runID, token: tokens[0], tokens: tokens, runID: runID,
+ client: &http.Client{Transport: transport, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }},
+ retryTimeout: 30 * time.Second, queue: make(chan dbBatch, 4), done: make(chan struct{}),
+ tickStop: make(chan struct{}), tickDone: make(chan struct{}), failed: make(chan struct{}), parts: map[string]*bytes.Buffer{},
+ }
+ c.groups = map[dbGroupKey]*MetricGroup{}
+ c.groupTickets = map[dbGroupKey]*metrics.RequestTicket{}
+ c.mapper = &Logger{out: params.StdOut, envTags: tags, customPatterns: patterns, transactionSource: source,
+ bufHTTP: &bytes.Buffer{}, bufVUs: &bytes.Buffer{}, bufTrans: &bytes.Buffer{}, bufErrors: &bytes.Buffer{},
+ pendingHTTPGroups: map[int64]*MetricGroup{}, rowSink: c.addRow}
+ return c, nil
+}
+
+func (c *BreakTestDBOutput) SetRequestTracker(t *metrics.RequestTracker) { c.tracker = t }
+
+func (c *BreakTestDBOutput) Description() string { return "breaktest-db-breakingit" }
+func (c *BreakTestDBOutput) SetTestRunStopCallback(callback func(error)) {
+ c.failureMu.Lock()
+ c.stopCallback = callback
+ c.failureMu.Unlock()
+}
+func (c *BreakTestDBOutput) err() error {
+ if c.owner != nil {
+ return c.owner.err()
+ }
+ c.failureMu.Lock()
+ defer c.failureMu.Unlock()
+ return c.failure
+}
+func (c *BreakTestDBOutput) fail(err error) {
+ if c.owner != nil {
+ c.owner.fail(err)
+ return
+ }
+ if err == nil {
+ return
+ }
+ c.failureMu.Lock()
+ if c.failure != nil {
+ c.failureMu.Unlock()
+ return
+ }
+ c.failure = fmt.Errorf("BREAKTEST_DB_FAILED: final publication forbidden: %w", err)
+ failure, callback := c.failure, c.stopCallback
+ close(c.failed)
+ c.failureMu.Unlock()
+ // Never invoke the engine callback under a mapping/buffer lock.
+ if callback != nil {
+ go callback(failure)
+ }
+}
+func (c *BreakTestDBOutput) Start() error {
+ c.lanes = []*BreakTestDBOutput{c}
+ for _, token := range c.tokens[1:] {
+ c.lanes = append(c.lanes, &BreakTestDBOutput{owner: c, tracker: c.tracker, base: c.base, runID: c.runID, token: token,
+ client: c.client, retryTimeout: c.retryTimeout, queue: make(chan dbBatch, 4), done: make(chan struct{}), failed: c.failed})
+ }
+ for _, lane := range c.lanes {
+ if err := lane.hello(); err != nil {
+ for _, opened := range c.lanes {
+ if opened.encoder != nil {
+ opened.encoder.Close()
+ }
+ }
+ return err
+ }
+ encoder, err := newDBEncoder()
+ if err != nil {
+ return err
+ }
+ lane.encoder = encoder
+ }
+ // One keep-alive connection and compression worker per ordered source.
+
+ c.started = true
+ for _, lane := range c.lanes {
+ go lane.sender()
+ }
+ go func() {
+ defer close(c.tickDone)
+ tick := time.NewTicker(200 * time.Millisecond)
+ defer tick.Stop()
+ for {
+ select {
+ case <-c.tickStop:
+ return
+ case <-c.failed:
+ return
+ case <-tick.C:
+ c.addMu.Lock()
+ c.completeGroups(false)
+ c.flush()
+ c.publishWatermark()
+ c.addMu.Unlock()
+ }
+ }
+ }()
+ return nil
+}
+
+func (c *BreakTestDBOutput) AddMetricSamples(samples []metrics.SampleContainer) {
+ c.addMu.Lock()
+ defer c.addMu.Unlock()
+ if c.err() != nil {
+ return
+ }
+ for _, sc := range samples {
+ httpContainer := false
+ var ticket *metrics.RequestTicket
+ if tracked, ok := sc.(interface{ RequestTicket() *metrics.RequestTicket }); ok {
+ ticket = tracked.RequestTicket()
+ }
+ for _, sample := range sc.GetSamples() {
+ if sample.Metric.Name == "browser_errors" && sample.Tags.Map()["screenshot"] != "" {
+ c.fail(errors.New("browser screenshots require a BreakTest DB payload integration"))
+ return
+ }
+ }
+ for _, sample := range sc.GetSamples() {
+ key := httpMetricCanonicalKey(sample.Metric.Name)
+ if key == "" {
+ requestMetric := strings.HasPrefix(sample.Metric.Name, "custom_") || sample.Metric.Name == "ws_connecting" || sample.Metric.Name == "ws_ping"
+ if c.tracker != nil && requestMetric && ticket == nil {
+ c.fail(errors.New("request metric lacks producer completion tracking"))
+ return
+ }
+ c.mapper.AddMetricSamples([]metrics.SampleContainer{sample})
+ continue
+ }
+ httpContainer = true
+ if c.tracker != nil && ticket == nil {
+ c.fail(errors.New("HTTP metrics lack producer completion tracking; live watermark would be unsafe"))
+ return
+ }
+ id := dbGroupKey{sample.Time.UnixNano(), sample.Tags}
+ if ticket != nil {
+ c.groupTickets[id] = ticket
+ }
+ g := c.groups[id]
+ if g == nil {
+ g = &MetricGroup{Timestamp: id.time, Tags: map[string]string{}, Values: map[string]float64{}}
+ copyTagsForHTTP(g.Tags, sample.Tags.Map(), c.mapper.envTags, c.mapper.customPatterns)
+ c.groups[id] = g
+ }
+ if _, duplicate := g.Values[key]; duplicate {
+ c.fail(errors.New("ambiguous HTTP metric identity: duplicate metric before request completion"))
+ return
+ }
+ g.Values[key] = sample.Value
+ if key == "http_req_failed" {
+ g.ErrorReqHeaders = sample.HTTPErrorReqHeaders
+ g.ErrorReqBody = sample.HTTPErrorReqBody
+ g.ErrorResHeaders = sample.HTTPErrorResHeaders
+ g.ErrorResBody = sample.HTTPErrorResBody
+ }
+ }
+ c.completeGroups(false)
+ if !httpContainer {
+ ticket.Release()
+ }
+
+ if c.err() != nil {
+ return
+ }
+ // Bound incomplete metric joins as well as outgoing payloads. Complete HTTP
+ // containers usually leave this map empty; browser fragments may remain.
+ n := 0
+ for _, g := range c.groups {
+ n += 256 + len(g.ErrorReqHeaders) + len(g.ErrorReqBody) + len(g.ErrorResHeaders) + len(g.ErrorResBody)
+ for k, v := range g.Tags {
+ n += len(k) + len(v) + 32
+ }
+ n += len(g.Values) * 64
+ }
+ if len(c.groups) > 4096 || n > dbBatchBytes {
+ c.fail(errors.New("incomplete metric join budget exceeded"))
+ return
+ }
+ }
+}
+
+// Complete only after every request metric has arrived. Browser failure payloads
+// may arrive much later than duration metrics; a timer is not proof of success.
+func (c *BreakTestDBOutput) completeGroups(final bool) {
+ for key, g := range c.groups {
+ complete := true
+ for _, name := range []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving", "http_req_failed"} {
+ if _, ok := g.Values[name]; !ok {
+ complete = false
+ break
+ }
+ }
+ if complete {
+ delete(c.groups, key)
+ c.mapper.processHTTPGroup(g)
+ c.groupTickets[key].Release()
+ delete(c.groupTickets, key)
+ } else if final {
+ c.fail(errors.New("incomplete HTTP metric group at final drain"))
+ return
+ }
+ }
+}
+func (c *BreakTestDBOutput) addRow(table string, row []string) error {
+ if err := c.err(); err != nil {
+ return err
+ }
+ data, err := encodeDBRow(table, row)
+ if err != nil {
+ c.fail(err)
+ return err
+ }
+ c.batchMu.Lock()
+ defer c.batchMu.Unlock()
+ if c.batchBytes+len(data) > dbBatchBytes {
+ if err = c.enqueueLocked(); err != nil {
+ return err
+ }
+ }
+ if c.parts[table] == nil {
+ c.parts[table] = &bytes.Buffer{}
+ }
+ _, _ = c.parts[table].Write(data)
+ c.batchBytes += len(data)
+ return nil
+}
+func (c *BreakTestDBOutput) flush() {
+ c.batchMu.Lock()
+ defer c.batchMu.Unlock()
+ if err := c.enqueueLocked(); err != nil {
+ c.fail(err)
+ }
+}
+func (c *BreakTestDBOutput) enqueueLocked() error {
+ if c.batchBytes == 0 {
+ return nil
+ }
+ var body bytes.Buffer
+ writer := multipart.NewWriter(&body)
+ for _, table := range []string{"requests_raw", "transactions", "requests_error", "virtual_users"} {
+ data := c.parts[table]
+ if data == nil {
+ continue
+ }
+ header := textproto.MIMEHeader{}
+ header.Set("X-Table", "public."+table)
+ header.Set("X-Format", "binary")
+ part, err := writer.CreatePart(header)
+ if err != nil {
+ return err
+ }
+ _, _ = part.Write([]byte("PGCOPY\n\xff\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00"))
+ _, _ = part.Write(data.Bytes())
+ _, _ = part.Write([]byte{255, 255})
+ }
+ if err := writer.Close(); err != nil {
+ return err
+ }
+ batch := dbBatch{body: body.Bytes(), contentType: "multipart/mixed; boundary=" + writer.Boundary()}
+ lane := c
+ if len(c.lanes) > 0 {
+ lane = c.lanes[c.nextLane]
+ c.nextLane = (c.nextLane + 1) % len(c.lanes)
+ }
+ select {
+ case <-c.failed:
+ return c.err()
+ case lane.queue <- batch:
+ c.parts = map[string]*bytes.Buffer{}
+ c.batchBytes = 0
+ return nil
+ }
+}
+func (c *BreakTestDBOutput) sender() {
+ defer close(c.done)
+ for batch := range c.queue {
+ if c.err() != nil {
+ continue
+ }
+ if batch.beforeUS != 0 {
+ c.fail(c.sendControl("/watermark", batch.beforeUS))
+ continue
+ }
+ compressed := c.encoder.EncodeAll(batch.body, nil)
+ c.fail(c.sendBatch(compressed, batch.contentType))
+ }
+}
+
+func (c *BreakTestDBOutput) Stop() error {
+ c.stopOnce.Do(func() {
+ if !c.started {
+ return
+ }
+ close(c.tickStop)
+ <-c.tickDone
+ c.addMu.Lock()
+ if c.err() == nil {
+ c.completeGroups(true)
+ c.flush()
+ }
+ c.addMu.Unlock()
+ for _, lane := range c.lanes {
+ close(lane.queue)
+ }
+ for _, lane := range c.lanes {
+ <-lane.done
+ }
+ for _, lane := range c.lanes {
+ if c.err() != nil {
+ break
+ }
+ c.fail(lane.sendControl("/finish", 0))
+ }
+ for _, lane := range c.lanes {
+ lane.encoder.Close()
+ }
+ c.client.CloseIdleConnections()
+ c.stopErr = c.err()
+ })
+ return c.stopErr
+}
+
+// Retries keep the same sequence, multipart boundary and compressed bytes.
+// Incarnation recovery is handled by the lane, after this request returns 409.
+func (c *BreakTestDBOutput) request(path string, body []byte, contentType string, seq uint64, result any) error {
+ ctx, cancel := context.WithTimeout(context.Background(), c.retryTimeout)
+ defer cancel()
+ for {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+path, bytes.NewReader(body))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ req.Header.Set("Content-Type", contentType)
+ req.Header.Set("X-Run-Id", c.runID)
+ if path != "/hello" {
+ req.Header.Set("X-Incarnation", c.incarnation)
+ req.Header.Set("X-Sequence", fmt.Sprint(seq))
+ }
+ if path == "/ingest/batch" {
+ req.Header.Set("Content-Encoding", "zstd")
+ }
+ response, err := c.client.Do(req)
+ if err == nil {
+ data, readErr := io.ReadAll(io.LimitReader(response.Body, 64*1024+1))
+ _ = response.Body.Close()
+ if len(data) > 64*1024 {
+ return errors.New("oversized BreakTest DB acknowledgement")
+ }
+ if response.StatusCode == http.StatusOK && readErr == nil {
+ if err = json.Unmarshal(data, result); err != nil {
+ return errors.New("invalid BreakTest DB acknowledgement JSON")
+ }
+ return nil
+ }
+ if readErr == nil && response.StatusCode != 429 && response.StatusCode < 500 {
+ return dbRejection(response.StatusCode)
+ }
+ }
+ timer := time.NewTimer(100 * time.Millisecond)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return fmt.Errorf("BreakTest DB %s retry deadline: %w", path, ctx.Err())
+ case <-c.failed:
+ timer.Stop()
+ return c.err()
+ case <-timer.C:
+ }
+ }
+}
+
+// The storage ingress caps zstd decoder windows at 2^18 bytes, independently
+// of the larger decoded batch budget. Enforce this even on full 2 MiB batches.
+func newDBEncoder() (*zstd.Encoder, error) {
+ return zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), zstd.WithWindowSize(1<<18))
+}
diff --git a/internal/output/breakingit/breaktest_db_binary.go b/internal/output/breakingit/breaktest_db_binary.go
new file mode 100644
index 000000000..64bfe1545
--- /dev/null
+++ b/internal/output/breakingit/breaktest_db_binary.go
@@ -0,0 +1,92 @@
+package breakingit
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "strconv"
+ "time"
+
+ "github.com/gofrs/uuid/v5"
+)
+
+const pgEpochSeconds int64 = 946684800
+const dbBatchBytes = 2 * 1024 * 1024
+
+// PostgreSQL binary COPY types in the existing non-SM BreakTest table order.
+// t = timestamp, s = text, i = int32, b = boolean, u = UUID.
+var customSchemas = map[string]string{
+ "requests_raw": "tssssbiisiii",
+ "transactions": "tsssbiii",
+ "requests_error": "tssssssiissssssu",
+ "virtual_users": "tsssiii",
+}
+
+func encodeDBRow(table string, row []string) ([]byte, error) {
+ schema := customSchemas[table]
+ if schema == "" || len(row) != len(schema) {
+ return nil, fmt.Errorf("invalid %s row width: %d", table, len(row))
+ }
+ // Reject before allocating a second copy of a potentially enormous error response.
+ size := 2 + 4*len(row)
+ for _, s := range row {
+ size += len(s)
+ }
+ if size > dbBatchBytes-1024 {
+ return nil, fmt.Errorf("%s row exceeds BreakTest DB batch budget", table)
+ }
+ b := bytes.NewBuffer(make([]byte, 0, size))
+ _ = binary.Write(b, binary.BigEndian, int16(len(row)))
+ for i, kind := range []byte(schema) {
+ value := row[i]
+ // Match the existing CSV writer's empty-field NULL semantics.
+ if value == "" {
+ _ = binary.Write(b, binary.BigEndian, int32(-1))
+ continue
+ }
+ var data []byte
+ switch kind {
+ case 's':
+ data = []byte(value)
+ case 't':
+ t, err := time.Parse(time.RFC3339Nano, value)
+ if err != nil {
+ return nil, fmt.Errorf("%s timestamp: %w", table, err)
+ }
+ n := (t.Unix()-pgEpochSeconds)*1_000_000 + int64(t.Nanosecond()/1000)
+ if n < 0 {
+ return nil, fmt.Errorf("timestamp before BreakTest DB epoch")
+ }
+ data = binary.BigEndian.AppendUint64(nil, uint64(n))
+ case 'i':
+ n, err := strconv.ParseInt(value, 10, 32)
+ if err != nil {
+ return nil, fmt.Errorf("%s integer column %d: %w", table, i, err)
+ }
+ // Unavailable connect times must never become negative averages.
+ if n < 0 && ((table == "requests_raw" && i == 9) || (table == "requests_error" && i == 8)) {
+ _ = binary.Write(b, binary.BigEndian, int32(-1))
+ continue
+ }
+ data = binary.BigEndian.AppendUint32(nil, uint32(n))
+ case 'b':
+ v, err := strconv.ParseBool(value)
+ if err != nil {
+ return nil, err
+ }
+ data = []byte{0}
+ if v {
+ data[0] = 1
+ }
+ case 'u':
+ id, err := uuid.FromString(value)
+ if err != nil {
+ return nil, err
+ }
+ data = id[:]
+ }
+ _ = binary.Write(b, binary.BigEndian, int32(len(data)))
+ _, _ = b.Write(data)
+ }
+ return b.Bytes(), nil
+}
diff --git a/internal/output/breakingit/breaktest_db_recovery.go b/internal/output/breakingit/breaktest_db_recovery.go
new file mode 100644
index 000000000..dd78d6c16
--- /dev/null
+++ b/internal/output/breakingit/breaktest_db_recovery.go
@@ -0,0 +1,106 @@
+package breakingit
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+)
+
+type dbRejection int
+
+func (s dbRejection) Error() string {
+ return fmt.Sprintf("BreakTest DB rejected request (HTTP %d)", int(s))
+}
+
+type sourceHello struct {
+ Incarnation string `json:"incarnation"`
+ Accepted *uint64 `json:"accepted_sequence"`
+ Checkpoint *uint64 `json:"checkpoint_sequence"`
+ Sealed bool `json:"sealed"`
+}
+
+func (c *BreakTestDBOutput) register() (sourceHello, error) {
+ body := []byte(`{}`)
+ if c.tracker != nil {
+ body, _ = json.Marshal(map[string]any{"source_clock_us": c.tracker.Clock().UnixMicro() - pgEpochSeconds*1_000_000, "clock_contract": "sample-end-monotonic-v1"})
+ }
+ var hello sourceHello
+ err := c.request("/hello", body, "application/json", 0, &hello)
+ return hello, err
+}
+
+// Acknowledged, uncheckpointed rows may be lost under relaxed durability. The
+// server exposes that loss as coverage gaps. Replay only the held unacknowledged
+// batch, never already acknowledged history. Each ordered lane recovers itself.
+func (c *BreakTestDBOutput) recoverSource(err error) (uint64, error) {
+ var status dbRejection
+ if c.tracker == nil || !errors.As(err, &status) || status != 409 {
+ return 0, err
+ }
+ hello, e := c.register()
+ if e != nil {
+ return 0, e
+ }
+ if hello.Sealed || hello.Incarnation == "" || hello.Incarnation == c.incarnation || hello.Accepted == nil || hello.Checkpoint == nil || *hello.Accepted != *hello.Checkpoint {
+ return 0, err
+ }
+ c.incarnation = hello.Incarnation
+ c.sequence = *hello.Accepted
+ return *hello.Checkpoint, nil
+}
+
+func (c *BreakTestDBOutput) sendBatch(body []byte, content string) error {
+ for attempt := 0; attempt < 4; attempt++ {
+ next := c.sequence + 1
+ var ack struct {
+ Accepted *uint64 `json:"accepted"`
+ }
+ err := c.request("/ingest/batch", body, content, next, &ack)
+ if err == nil {
+ if ack.Accepted == nil || *ack.Accepted != next {
+ return errors.New("invalid BreakTest DB batch acknowledgement")
+ }
+ c.sequence = next
+ return nil
+ }
+ checkpoint, e := c.recoverSource(err)
+ if e != nil {
+ return e
+ }
+ if checkpoint > next {
+ return errors.New("checkpoint exceeds outstanding batch sequence")
+ }
+ if next <= checkpoint {
+ return nil
+ } // Lost ACK, but checkpoint proves this batch persisted.
+ }
+ return errors.New("BreakTest DB restarted repeatedly during batch delivery")
+}
+
+func (c *BreakTestDBOutput) sendControl(path string, before int64) error {
+ body := []byte(nil)
+ if path == "/watermark" {
+ body, _ = json.Marshal(map[string]int64{"before_us": before})
+ }
+ for attempt := 0; attempt < 4; attempt++ {
+ var ack struct {
+ Accepted *uint64 `json:"accepted_sequence"`
+ Watermark int64 `json:"watermark_us"`
+ Finished bool `json:"finished"`
+ }
+ previous := c.sequence
+ err := c.request(path, body, "application/json", c.sequence, &ack)
+ if err == nil {
+ if ack.Accepted == nil || *ack.Accepted != c.sequence || (path == "/watermark" && ack.Watermark != before) || (path == "/finish" && !ack.Finished) {
+ return errors.New("invalid BreakTest DB control acknowledgement")
+ }
+ return nil
+ }
+ if checkpoint, e := c.recoverSource(err); e != nil {
+ return e
+ } else if checkpoint > previous {
+ return errors.New("checkpoint exceeds acknowledged sequence")
+ }
+ }
+ return errors.New("BreakTest DB restarted repeatedly during control delivery")
+}
diff --git a/internal/output/breakingit/breaktest_db_test.go b/internal/output/breakingit/breaktest_db_test.go
new file mode 100644
index 000000000..76ff9694c
--- /dev/null
+++ b/internal/output/breakingit/breaktest_db_test.go
@@ -0,0 +1,472 @@
+package breakingit
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "io"
+ "mime"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/stretchr/testify/require"
+ "go.k6.io/k6/v2/metrics"
+ "go.k6.io/k6/v2/output"
+)
+
+func newBreakTestDBTest(t *testing.T, endpoint string) *BreakTestDBOutput {
+ t.Helper()
+ out, err := NewBreakTestDB(output.Params{ConfigArgument: endpoint, StdOut: io.Discard, Environment: map[string]string{
+ "runId": "run-123", "BREAKTEST_DB_TOKEN": "producer-token", "location": "Amsterdam", "nodeName": "node-1",
+ }})
+ require.NoError(t, err)
+ return out.(*BreakTestDBOutput)
+}
+func decodeDBBatch(t *testing.T, data []byte, content string) map[string][][][]byte {
+ t.Helper()
+ decoder, err := zstd.NewReader(nil)
+ require.NoError(t, err)
+ defer decoder.Close()
+ plain, err := decoder.DecodeAll(data, nil)
+ require.NoError(t, err)
+ _, params, err := mime.ParseMediaType(content)
+ require.NoError(t, err)
+ reader := multipart.NewReader(bytes.NewReader(plain), params["boundary"])
+ result := map[string][][][]byte{}
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ require.Equal(t, "binary", part.Header.Get("X-Format"))
+ data, err := io.ReadAll(part)
+ require.NoError(t, err)
+ require.Equal(t, []byte("PGCOPY\n\xff\r\n\x00"), data[:11])
+ require.Equal(t, make([]byte, 8), data[11:19])
+ r := bytes.NewReader(data[19:])
+ table := strings.TrimPrefix(part.Header.Get("X-Table"), "public.")
+ for {
+ var n int16
+ require.NoError(t, binary.Read(r, binary.BigEndian, &n))
+ if n == -1 {
+ break
+ }
+ row := make([][]byte, n)
+ for i := range row {
+ var length int32
+ require.NoError(t, binary.Read(r, binary.BigEndian, &length))
+ if length == -1 {
+ continue
+ }
+ require.GreaterOrEqual(t, length, int32(0))
+ row[i] = make([]byte, length)
+ _, err = io.ReadFull(r, row[i])
+ require.NoError(t, err)
+ }
+ result[table] = append(result[table], row)
+ }
+ require.Zero(t, r.Len())
+ }
+ return result
+}
+func TestBreakTestDBBinaryMatchesProxyMapping(t *testing.T) {
+ t.Parallel()
+ rows := []struct {
+ table string
+ row []string
+ }{
+ {"requests_raw", []string{"2026-10-30T12:00:00.123456Z", "run-123", "NL", "Checkout", "/pay", "false", "", "123", "503", "-1", "25", "100"}},
+ {"transactions", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "120"}},
+ {"requests_error", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "Checkout", "/pay", "503", "100", "", "https://example.test/", "", "bad\nreply", "POST,\"a\"", "Header: value", "雪", "019a3524-3890-7000-8000-000000000001"}},
+ {"virtual_users", []string{"2026-10-30T12:00:00Z", "run-123", "NL", "node", "10", "", ""}},
+ }
+ c := newBreakTestDBTest(t, "http://localhost")
+ for _, r := range rows {
+ require.NoError(t, c.addRow(r.table, r.row))
+ }
+ c.flush()
+ batch := <-c.queue
+ encoder, err := zstd.NewWriter(nil)
+ require.NoError(t, err)
+ defer encoder.Close()
+ decoded := decodeDBBatch(t, encoder.EncodeAll(batch.body, nil), batch.contentType)
+ require.Len(t, decoded, 4)
+ require.Nil(t, decoded["requests_raw"][0][9])
+ require.Nil(t, decoded["requests_raw"][0][6])
+ require.Equal(t, uint32(123), binary.BigEndian.Uint32(decoded["requests_raw"][0][7]))
+ require.Equal(t, []byte("雪"), decoded["requests_error"][0][14])
+ require.Len(t, decoded["requests_error"][0][15], 16)
+ require.Equal(t, []byte{1}, decoded["transactions"][0][4])
+ require.Equal(t, int64(846676800123456), int64(binary.BigEndian.Uint64(decoded["requests_raw"][0][0])))
+ // The transaction row must be byte-identical on both binary COPY paths.
+ proxy := newTransactionTestLogger(false)
+ sample := transactionTestSample("group_duration", map[string]string{"group": "::Checkout"}, true)
+ proxy.AddMetricSamples([]metrics.SampleContainer{sample})
+ c.AddMetricSamples([]metrics.SampleContainer{sample})
+ require.NotEmpty(t, proxy.bufTrans.Bytes())
+ require.Equal(t, proxy.bufTrans.Bytes(), c.parts["transactions"].Bytes())
+}
+
+func TestBreakTestDBRetriesExactBytesAndFinalDrain(t *testing.T) {
+ t.Parallel()
+ var first []byte
+ var content string
+ var calls, finishes int
+ var mu sync.Mutex
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, "Bearer producer-token", r.Header.Get("Authorization"))
+ switch r.URL.Path {
+ case "/runs/run-123/hello":
+ fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0,"sealed":false}`)
+ case "/runs/run-123/ingest/batch":
+ require.Equal(t, "one", r.Header.Get("X-Incarnation"))
+ require.Equal(t, "1", r.Header.Get("X-Sequence"))
+ require.Equal(t, "zstd", r.Header.Get("Content-Encoding"))
+ data, err := io.ReadAll(r.Body)
+ require.NoError(t, err)
+ calls++
+ if calls == 1 {
+ first = data
+ content = r.Header.Get("Content-Type")
+ w.WriteHeader(503)
+ return
+ }
+ require.Equal(t, first, data)
+ require.Equal(t, content, r.Header.Get("Content-Type"))
+ fmt.Fprint(w, `{"accepted":1,"duplicate":true}`)
+ case "/runs/run-123/finish":
+ require.Equal(t, 2, calls)
+ require.Equal(t, "1", r.Header.Get("X-Sequence"))
+ finishes++
+ fmt.Fprint(w, `{"accepted_sequence":1,"finished":true}`)
+ default:
+ t.Errorf("unexpected route %s", r.URL.Path)
+ w.WriteHeader(404)
+ }
+ }))
+ defer server.Close()
+ c := newBreakTestDBTest(t, server.URL)
+ require.NoError(t, c.Start())
+ c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)})
+ require.NoError(t, c.Stop())
+ require.NoError(t, c.Stop())
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, 1, finishes)
+ decoded := decodeDBBatch(t, first, content)
+ require.Len(t, decoded["transactions"], 1)
+}
+
+func TestBreakTestDBFailureForbidsFinish(t *testing.T) {
+ for _, status := range []int{400, 401, 403, 409, 413, 429, 503, 200} {
+ t.Run(strconv.Itoa(status), func(t *testing.T) {
+ var finished bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/hello") {
+ fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0}`)
+ return
+ }
+ if strings.HasSuffix(r.URL.Path, "/finish") {
+ finished = true
+ }
+ w.WriteHeader(status)
+ fmt.Fprint(w, `{"accepted":999}`)
+ }))
+ defer server.Close()
+ c := newBreakTestDBTest(t, server.URL)
+ c.retryTimeout = 50 * time.Millisecond
+ stopped := make(chan error, 1)
+ c.SetTestRunStopCallback(func(err error) { stopped <- err })
+ require.NoError(t, c.Start())
+ c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("pages", nil, false)})
+ require.ErrorContains(t, c.Stop(), "final publication forbidden")
+ select {
+ case err := <-stopped:
+ require.Error(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("run not stopped")
+ }
+ require.False(t, finished)
+ })
+ }
+}
+
+func TestBreakTestDBConfigurationAndBounds(t *testing.T) {
+ t.Parallel()
+ for _, entry := range []struct{ k, v string }{{"syntheticMonitoring", "true"}, {"K6_BROWSER_SCREENSHOT_ON_ERROR", "true"}, {"runId", "../wrong"}, {"BREAKTEST_DB_TOKEN", ""}, {"BREAKINGIT_TRANSACTION_SOURCE", "wrong"}} {
+ env := map[string]string{"runId": "test", "BREAKTEST_DB_TOKEN": "token"}
+ env[entry.k] = entry.v
+ _, err := NewBreakTestDB(output.Params{ConfigArgument: "http://localhost", Environment: env})
+ require.Error(t, err)
+ }
+ _, err := NewBreakTestDB(output.Params{ConfigArgument: "http://user:password@localhost"})
+ require.Error(t, err)
+ c := newBreakTestDBTest(t, "http://localhost")
+ row := []string{"2026-10-30T00:00:00Z", "run-123", "NL", strings.Repeat("x", dbBatchBytes), "true", "", "", "1"}
+ require.Error(t, c.addRow("transactions", row))
+ require.Error(t, c.err())
+ require.Empty(t, c.queue)
+}
+
+func TestBreakTestDBHTTPJoinsAndLateFailures(t *testing.T) {
+ t.Parallel()
+ c := newBreakTestDBTest(t, "http://localhost")
+ var rows [][]string
+ c.mapper.rowSink = func(table string, row []string) error {
+ if table == "requests_raw" {
+ rows = append(rows, row)
+ }
+ return nil
+ }
+ reg := metrics.NewRegistry()
+ ts := time.Now()
+ names := []string{"browser_http_req_duration", "browser_http_req_blocked", "browser_http_req_connecting", "browser_http_req_tls_handshaking", "browser_http_req_sending", "browser_http_req_waiting", "browser_http_req_receiving", "browser_data_received"}
+ tagsA := reg.RootTagSet().With("name", "/a").With("resource_type", "fetch")
+ tagsB := reg.RootTagSet().With("name", "/b").With("resource_type", "fetch")
+ for _, tags := range []*metrics.TagSet{tagsA, tagsB} {
+ samples := metrics.Samples{}
+ for _, name := range names {
+ samples = append(samples, metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric(name, metrics.Trend), Tags: tags}, Time: ts, Value: 12})
+ }
+ c.AddMetricSamples([]metrics.SampleContainer{samples})
+ }
+ require.Empty(t, rows)
+ c.completeGroups(false)
+ require.Empty(t, rows) // A flush timer cannot infer request success.
+ for _, tags := range []*metrics.TagSet{tagsA, tagsB} {
+ sample := metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric("browser_http_req_failed", metrics.Rate), Tags: tags}, Time: ts, Value: 1, HTTPErrorResBody: "failure"}
+ c.AddMetricSamples([]metrics.SampleContainer{sample})
+ }
+ require.Len(t, rows, 2)
+ require.Empty(t, c.groups)
+ for _, row := range rows {
+ require.Equal(t, "false", row[5])
+ require.Equal(t, "12", row[7])
+ }
+}
+
+func TestBreakTestDBIncompleteDrainFails(t *testing.T) {
+ t.Parallel()
+ c := newBreakTestDBTest(t, "http://localhost")
+ c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("http_req_duration", nil, false)})
+ c.completeGroups(true)
+ require.ErrorContains(t, c.err(), "incomplete HTTP metric group")
+}
+
+func TestBreakTestDBQueueBackpressure(t *testing.T) {
+ t.Parallel()
+ c := newBreakTestDBTest(t, "http://localhost")
+ row := []string{"2026-10-30T12:00:00Z", "run-123", "NL", "Checkout", "true", "", "", "1"}
+ for range cap(c.queue) {
+ require.NoError(t, c.addRow("transactions", row))
+ c.flush()
+ }
+ blocked := make(chan struct{})
+ go func() { defer close(blocked); _ = c.addRow("transactions", row); c.flush() }()
+ select {
+ case <-blocked:
+ t.Fatal("full queue did not apply backpressure")
+ case <-time.After(20 * time.Millisecond):
+ }
+ c.fail(fmt.Errorf("sender unavailable"))
+ select {
+ case <-blocked:
+ case <-time.After(time.Second):
+ t.Fatal("failure did not unblock producer")
+ }
+ require.Len(t, c.queue, cap(c.queue))
+}
+
+func TestBreakTestDBStartupRejectsNonFreshAndMissingAck(t *testing.T) {
+ t.Parallel()
+ for _, body := range []string{`{}`, `{"incarnation":"a"}`, `{"incarnation":"a","accepted_sequence":1}`, `{"incarnation":"a","accepted_sequence":0,"sealed":true}`} {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) }))
+ c := newBreakTestDBTest(t, server.URL)
+ require.Error(t, c.Start())
+ require.False(t, c.started)
+ server.Close()
+ }
+}
+
+func TestBreakTestDBHTTPDuplicateIdentityFails(t *testing.T) {
+ t.Parallel()
+ c := newBreakTestDBTest(t, "http://localhost")
+ sample := transactionTestSample("http_req_duration", nil, false)
+ c.AddMetricSamples([]metrics.SampleContainer{sample, sample})
+ require.ErrorContains(t, c.err(), "ambiguous HTTP metric identity")
+}
+
+func TestBreakTestDBWatermarkWaitsForEveryLaneAndPendingProducer(t *testing.T) {
+ var mu sync.Mutex
+ sequences := map[string]uint64{}
+ watermarks := map[string]int64{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ token := r.Header.Get("Authorization")
+ seq, _ := strconv.ParseUint(r.Header.Get("X-Sequence"), 10, 64)
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/hello"):
+ fmt.Fprint(w, `{"incarnation":"one","accepted_sequence":0}`)
+ case strings.HasSuffix(r.URL.Path, "/ingest/batch"):
+ require.Equal(t, sequences[token]+1, seq)
+ sequences[token] = seq
+ fmt.Fprintf(w, `{"accepted":%d}`, seq)
+ case strings.HasSuffix(r.URL.Path, "/watermark"):
+ var body struct {
+ Before int64 `json:"before_us"`
+ }
+ require.NoError(t, json.NewDecoder(r.Body).Decode(&body))
+ require.Equal(t, sequences[token], seq)
+ watermarks[token] = body.Before
+ fmt.Fprintf(w, `{"accepted_sequence":%d,"watermark_us":%d}`, seq, body.Before)
+ case strings.HasSuffix(r.URL.Path, "/finish"):
+ require.Equal(t, sequences[token], seq)
+ fmt.Fprintf(w, `{"accepted_sequence":%d,"finished":true}`, seq)
+ default:
+ w.WriteHeader(404)
+ }
+ }))
+ defer server.Close()
+ c := newBreakTestDBTest(t, server.URL)
+ c.tokens = []string{"producer-token", "second"}
+ ch := make(chan metrics.SampleContainer, 1)
+ tracker, release := metrics.BindRequestTracker(ch)
+ defer release()
+ c.SetRequestTracker(tracker)
+ require.NoError(t, c.Start())
+ ticket := metrics.BeginRequest(ch) // Request not delivered to the output yet.
+ pin := ticket.Time.Truncate(time.Second).UnixMicro() - pgEpochSeconds*1_000_000
+ for range 4 {
+ c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("group_duration", nil, false)})
+ c.flush()
+ }
+ require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(watermarks) == 2 }, 2*time.Second, 10*time.Millisecond)
+ mu.Lock()
+ for _, watermark := range watermarks {
+ require.LessOrEqual(t, watermark, pin)
+ }
+ mu.Unlock()
+ // Cancellation before enqueue may release its pin; no request row exists.
+ ticket.Release()
+ require.Eventually(t, func() bool {
+ mu.Lock()
+ defer mu.Unlock()
+ return watermarks["Bearer producer-token"] > pin && watermarks["Bearer second"] > pin
+ }, 2*time.Second, 10*time.Millisecond)
+ require.NoError(t, c.Stop())
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, uint64(2), sequences["Bearer producer-token"])
+ require.Equal(t, uint64(2), sequences["Bearer second"])
+}
+
+func TestBreakTestDBRecoveryReplaysOnlyUnacknowledgedBatch(t *testing.T) {
+ for _, checkpoint := range []uint64{0, 2} {
+ t.Run(strconv.Itoa(int(checkpoint)), func(t *testing.T) {
+ calls := 0
+ var first []byte
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/hello") {
+ fmt.Fprintf(w, `{"incarnation":"new","accepted_sequence":%d,"checkpoint_sequence":%d}`, checkpoint, checkpoint)
+ return
+ }
+ calls++
+ body, _ := io.ReadAll(r.Body)
+ if calls == 1 {
+ first = body
+ require.Equal(t, "old", r.Header.Get("X-Incarnation"))
+ require.Equal(t, "2", r.Header.Get("X-Sequence"))
+ w.WriteHeader(409)
+ return
+ }
+ require.Equal(t, first, body)
+ require.Equal(t, "new", r.Header.Get("X-Incarnation"))
+ require.Equal(t, "1", r.Header.Get("X-Sequence"))
+ fmt.Fprint(w, `{"accepted":1}`)
+ }))
+ defer server.Close()
+ c := newBreakTestDBTest(t, server.URL)
+ ch := make(chan metrics.SampleContainer)
+ tracker, release := metrics.BindRequestTracker(ch)
+ defer release()
+ c.tracker = tracker
+ c.sequence = 1
+ c.incarnation = "old"
+ require.NoError(t, c.sendBatch([]byte("identical-compressed-body"), "multipart/mixed"))
+ if checkpoint == 0 {
+ require.Equal(t, 2, calls)
+ require.Equal(t, uint64(1), c.sequence)
+ } else {
+ require.Equal(t, 1, calls)
+ require.Equal(t, uint64(2), c.sequence)
+ }
+ })
+ }
+}
+
+func TestTrackedHTTPJoinKeepsPinUntilFailureMetric(t *testing.T) {
+ c := newBreakTestDBTest(t, "http://localhost")
+ channel := make(chan metrics.SampleContainer)
+ tracker, release := metrics.BindRequestTracker(channel)
+ defer release()
+ c.tracker = tracker
+ ticket := metrics.BeginRequest(channel)
+ reg := metrics.NewRegistry()
+ tags := reg.RootTagSet().With("name", "/tracked")
+ names := []string{"http_req_duration", "http_req_blocked", "http_req_connecting", "http_req_tls_handshaking", "http_req_sending", "http_req_waiting", "http_req_receiving"}
+ samples := metrics.Samples{}
+ for _, name := range names {
+ samples = append(samples, metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric(name, metrics.Trend), Tags: tags}, Time: ticket.Time, Value: 12})
+ }
+ c.AddMetricSamples([]metrics.SampleContainer{metrics.RequestSamples{SampleContainer: samples, Ticket: ticket}})
+ require.Len(t, c.groups, 1)
+ require.Len(t, c.groupTickets, 1)
+ require.False(t, tracker.Before().After(ticket.Time))
+ failed := metrics.Sample{TimeSeries: metrics.TimeSeries{Metric: reg.MustNewMetric("http_req_failed", metrics.Rate), Tags: tags}, Time: ticket.Time, Value: 1, HTTPErrorResBody: "delayed failure"}
+ c.AddMetricSamples([]metrics.SampleContainer{metrics.RequestSamples{SampleContainer: failed, Ticket: ticket}})
+ require.Empty(t, c.groups)
+ require.Empty(t, c.groupTickets)
+ require.NoError(t, c.err())
+ require.NotNil(t, c.parts["requests_raw"])
+ require.NotNil(t, c.parts["requests_error"])
+}
+
+func TestUntrackedRequestCannotCrossNativeWatermark(t *testing.T) {
+ c := newBreakTestDBTest(t, "http://localhost")
+ channel := make(chan metrics.SampleContainer)
+ tracker, release := metrics.BindRequestTracker(channel)
+ defer release()
+ c.tracker = tracker
+ c.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("custom_missing_hook", nil, false)})
+ require.ErrorContains(t, c.err(), "lacks producer completion tracking")
+ require.Empty(t, c.parts)
+}
+
+func TestBreakTestDBFullBatchRespectsStorageDecoderWindow(t *testing.T) {
+ encoder, err := newDBEncoder()
+ require.NoError(t, err)
+ defer encoder.Close()
+ plain := bytes.Repeat([]byte("large-request-metric-row\n"), 100000)
+ encoded := encoder.EncodeAll(plain, nil)
+ var header zstd.Header
+ require.NoError(t, header.Decode(encoded))
+ require.LessOrEqual(t, header.WindowSize, uint64(1<<18))
+ decoder, err := zstd.NewReader(nil, zstd.WithDecoderMaxWindow(1<<18))
+ require.NoError(t, err)
+ defer decoder.Close()
+ decoded, err := decoder.DecodeAll(encoded, nil)
+ require.NoError(t, err)
+ require.Equal(t, plain, decoded)
+}
diff --git a/internal/output/breakingit/breaktest_db_watermark.go b/internal/output/breakingit/breaktest_db_watermark.go
new file mode 100644
index 000000000..453fe81bd
--- /dev/null
+++ b/internal/output/breakingit/breaktest_db_watermark.go
@@ -0,0 +1,38 @@
+package breakingit
+
+import (
+ "errors"
+)
+
+func (c *BreakTestDBOutput) hello() error {
+ hello, err := c.register()
+ if err != nil {
+ return err
+ }
+ if hello.Sealed || hello.Incarnation == "" || hello.Accepted == nil || *hello.Accepted != 0 {
+ return errors.New("BreakTest DB requires a fresh unfinished producer source")
+ }
+ c.incarnation = hello.Incarnation
+ return nil
+}
+
+// Called under addMu, after detaching every serialized row. Tickets in the engine
+// queue and incomplete joins still pin Before(). Each FIFO carries its own barrier:
+// the database takes the minimum across all lanes and generators.
+func (c *BreakTestDBOutput) publishWatermark() {
+ if c.tracker == nil || c.err() != nil {
+ return
+ }
+ before := c.tracker.Before().UnixMicro() - pgEpochSeconds*1_000_000
+ if before <= c.queuedWatermark {
+ return
+ }
+ for _, lane := range c.lanes {
+ select {
+ case <-c.failed:
+ return
+ case lane.queue <- dbBatch{beforeUS: before}:
+ }
+ }
+ c.queuedWatermark = before
+}
diff --git a/internal/output/breakingit/cadence_test.go b/internal/output/breakingit/cadence_test.go
index 076e17e65..85fa598f6 100644
--- a/internal/output/breakingit/cadence_test.go
+++ b/internal/output/breakingit/cadence_test.go
@@ -25,7 +25,7 @@ func TestFirstBatchImmediateThenOneSecond(t *testing.T) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
- exporter, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test", "isSynthetic": fmt.Sprint(synthetic)}})
+ exporter, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test", "syntheticMonitoring": fmt.Sprint(synthetic)}})
require.NoError(t, err)
stopped := false
defer func() {
diff --git a/internal/output/breakingit/drain_test.go b/internal/output/breakingit/drain_test.go
new file mode 100644
index 000000000..62786fb7b
--- /dev/null
+++ b/internal/output/breakingit/drain_test.go
@@ -0,0 +1,161 @@
+package breakingit
+
+import (
+ "compress/gzip"
+ "errors"
+ "io"
+ "mime"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "go.k6.io/k6/v2/metrics"
+ "go.k6.io/k6/v2/output"
+)
+
+// The second request is the follow-up tail of an already running flush.
+// Stop must wait for it and for the third row appended while it is in flight.
+func TestTimescaleStopJoinsFollowupFlush(t *testing.T) {
+ first, second := make(chan struct{}), make(chan struct{})
+ releaseFirst, releaseSecond := make(chan struct{}), make(chan struct{})
+ var onceFirst, onceSecond sync.Once
+ var mu sync.Mutex
+ var received []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gz, err := gzip.NewReader(r.Body)
+ if err != nil {
+ t.Error(err)
+ w.WriteHeader(400)
+ return
+ }
+ defer gz.Close()
+ _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+ if err != nil {
+ t.Error(err)
+ w.WriteHeader(400)
+ return
+ }
+ reader := multipart.NewReader(gz, params["boundary"])
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ assert.Equal(t, "binary", part.Header.Get("X-Format"))
+ assert.Equal(t, "public.transactions", part.Header.Get("X-Table"))
+ rows, err := readBinaryRows(part, "tsssbiii", true)
+ if !assert.NoError(t, err) || !assert.Len(t, rows, 1) {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ mu.Lock()
+ received = append(received, rows[0][3])
+ n := len(received)
+ mu.Unlock()
+ if n == 1 {
+ close(first)
+ <-releaseFirst
+ }
+ if n == 2 {
+ close(second)
+ <-releaseSecond
+ }
+ }
+ w.WriteHeader(200)
+ }))
+ defer server.Close()
+ defer onceFirst.Do(func() { close(releaseFirst) })
+ defer onceSecond.Do(func() { close(releaseSecond) })
+ out, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test-token"}})
+ require.NoError(t, err)
+ l := out.(*Logger)
+ appendRow := func(name string) {
+ sample := transactionTestSample("group_duration", map[string]string{"group": "::" + name}, false)
+ l.AddMetricSamples([]metrics.SampleContainer{sample})
+ }
+ await := func(ch <-chan struct{}) {
+ select {
+ case <-ch:
+ case <-time.After(5 * time.Second):
+ t.Fatal("flush did not reach handler")
+ }
+ }
+ appendRow("first")
+ await(first)
+ appendRow("second")
+ onceFirst.Do(func() { close(releaseFirst) })
+ await(second)
+ appendRow("third")
+ stopped := make(chan error, 1)
+ go func() { stopped <- l.Stop() }()
+ select {
+ case err := <-stopped:
+ onceSecond.Do(func() { close(releaseSecond) })
+ t.Fatalf("Stop returned before the in-flight tail was acknowledged: %v", err)
+ case <-time.After(100 * time.Millisecond):
+ }
+ onceSecond.Do(func() { close(releaseSecond) })
+ select {
+ case err := <-stopped:
+ require.NoError(t, err)
+ case <-time.After(5 * time.Second):
+ t.Fatal("Stop failed to drain")
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, []string{"first", "second", "third"}, received)
+}
+
+func TestTimescaleStopDrainsUnexpiredMetricGroup(t *testing.T) {
+ var mu sync.Mutex
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ mu.Lock()
+ requests++
+ mu.Unlock()
+ w.WriteHeader(200)
+ }))
+ defer server.Close()
+ out, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test-token"}})
+ require.NoError(t, err)
+ l := out.(*Logger)
+ l.mu.Lock()
+ now := time.Now()
+ l.pendingHTTPGroups[now.UnixNano()] = &MetricGroup{Timestamp: now.UnixNano(), LastUpdate: now, Tags: map[string]string{"name": "tail-request"}, Values: map[string]float64{"http_req_duration": 12}}
+ l.mu.Unlock()
+ require.NoError(t, l.Stop())
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, 1, requests)
+ require.Empty(t, l.pendingHTTPGroups)
+}
+
+type unavailableProxy struct{}
+
+func (unavailableProxy) RoundTrip(*http.Request) (*http.Response, error) {
+ return nil, errors.New("proxy offline")
+}
+
+func TestTimescaleStopReportsExhaustedDelivery(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ out, err := New(output.Params{ConfigArgument: "http://proxy.test", StdOut: io.Discard,
+ Environment: map[string]string{"PG_PROXY_TOKEN": "test-token"}})
+ require.NoError(t, err)
+ logger := out.(*Logger)
+ logger.httpClient.Transport = unavailableProxy{}
+ sample := transactionTestSample("group_duration", map[string]string{"group": "::tail"}, false)
+ logger.AddMetricSamples([]metrics.SampleContainer{sample})
+ require.ErrorContains(t, logger.Stop(), "proxy offline")
+ })
+}
diff --git a/lib/netext/httpext/tracer.go b/lib/netext/httpext/tracer.go
index 673770d8a..1b8cc58f2 100644
--- a/lib/netext/httpext/tracer.go
+++ b/lib/netext/httpext/tracer.go
@@ -14,7 +14,8 @@ import (
// A Trail represents detailed information about an HTTP request.
// You'd typically get one from a Tracer.
type Trail struct {
- EndTime time.Time
+ requestTicket *metrics.RequestTicket
+ EndTime time.Time
// Total connect time (Connecting + TLSHandshaking)
ConnDuration time.Duration
@@ -121,6 +122,8 @@ func (tr *Trail) SaveSamples(builtinMetrics *metrics.BuiltinMetrics, ctm *metric
}...)
}
+func (tr *Trail) RequestTicket() *metrics.RequestTicket { return tr.requestTicket }
+
// GetSamples implements the metrics.SampleContainer interface.
func (tr *Trail) GetSamples() []metrics.Sample {
return tr.Samples
diff --git a/lib/netext/httpext/transport.go b/lib/netext/httpext/transport.go
index 8e293525f..e15e55931 100644
--- a/lib/netext/httpext/transport.go
+++ b/lib/netext/httpext/transport.go
@@ -84,7 +84,12 @@ func newTransport(
// Helper method to finish the tracer trail, assemble the tag values and emits
// the metric samples for the supplied unfinished request.
func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRequest {
+ ticket := metrics.BeginRequest(t.state.Samples)
trail := unfReq.tracer.Done()
+ if ticket != nil {
+ trail.EndTime = ticket.Time
+ }
+ trail.requestTicket = ticket
result := &finishedRequest{
unfinishedRequest: unfReq,
@@ -166,7 +171,9 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe
t.addFailedSample(trail, &tagsAndMeta, failed)
}
- metrics.PushIfNotDone(t.ctx, t.state.Samples, trail)
+ if !metrics.PushIfNotDone(t.ctx, t.state.Samples, trail) {
+ ticket.Release()
+ }
return result
}
diff --git a/metrics/request_tracker.go b/metrics/request_tracker.go
new file mode 100644
index 000000000..8ddc29927
--- /dev/null
+++ b/metrics/request_tracker.go
@@ -0,0 +1,110 @@
+package metrics
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+// RequestTracker pins request-end timestamps until the output has serialized the
+// entire request. Registration happens before assigning the timestamp, not when
+// the metrics eventually arrive at the output. Only opted-in outputs pay this cost.
+type requestShard struct {
+ mu sync.Mutex
+ active map[*RequestTicket]time.Time
+}
+type RequestTracker struct {
+ origin time.Time
+ next atomic.Uint64
+ shards [64]requestShard
+}
+
+type RequestTicket struct {
+ tracker *RequestTracker
+ shard uint64
+ Time time.Time
+}
+
+var requestTrackers sync.Map // chan<- SampleContainer -> *RequestTracker
+
+func BindRequestTracker(ch chan<- SampleContainer) (*RequestTracker, func()) {
+ t := &RequestTracker{origin: time.Now()}
+ for i := range t.shards {
+ t.shards[i].active = make(map[*RequestTicket]time.Time)
+ }
+ requestTrackers.Store(ch, t)
+ return t, func() { requestTrackers.CompareAndDelete(ch, t) }
+}
+
+// BeginRequest assigns an end timestamp using an anchored monotonic clock.
+// The pin and the timestamp are atomic with respect to Before().
+func BeginRequest(ch chan<- SampleContainer) *RequestTicket {
+ v, ok := requestTrackers.Load(ch)
+ if !ok {
+ return nil
+ }
+ return v.(*RequestTracker).Begin()
+}
+func (t *RequestTracker) Begin() *RequestTicket {
+ index := t.next.Add(1) % uint64(len(t.shards))
+ shard := &t.shards[index]
+ shard.mu.Lock()
+ defer shard.mu.Unlock()
+ stamp := t.Clock()
+ ticket := &RequestTicket{tracker: t, shard: index, Time: stamp}
+ shard.active[ticket] = stamp
+ return ticket
+}
+func (t *RequestTicket) Release() {
+ if t == nil {
+ return
+ }
+ shard := &t.tracker.shards[t.shard]
+ shard.mu.Lock()
+ delete(shard.active, t)
+ shard.mu.Unlock()
+}
+func (t *RequestTracker) Before() time.Time {
+ // New pins created during the scan have timestamps >= this captured clock.
+ before := t.Clock()
+ for i := range t.shards {
+ shard := &t.shards[i]
+ shard.mu.Lock()
+ for _, stamp := range shard.active {
+ if stamp.Before(before) {
+ before = stamp
+ }
+ }
+ shard.mu.Unlock()
+ }
+ return before.Truncate(time.Second)
+}
+func (t *RequestTracker) Clock() time.Time { return t.origin.Add(time.Since(t.origin)) }
+
+// RequestSamples preserves the normal payload while identifying its lifecycle pin.
+type RequestSamples struct {
+ SampleContainer
+ Ticket *RequestTicket
+}
+
+func (s RequestSamples) RequestTicket() *RequestTicket { return s.Ticket }
+
+// PushTrackedRequest keeps the lifecycle pin across both the engine queue and
+// the output's mapping queue. A cancelled producer releases a never-enqueued pin.
+func PushTrackedRequest(ctx context.Context, ch chan<- SampleContainer, ticket *RequestTicket, sample SampleContainer) bool {
+ if ticket != nil {
+ sample = RequestSamples{SampleContainer: sample, Ticket: ticket}
+ }
+ if PushIfNotDone(ctx, ch, sample) {
+ return true
+ }
+ ticket.Release()
+ return false
+}
+func RequestTime(ticket *RequestTicket, fallback time.Time) time.Time {
+ if ticket != nil {
+ return ticket.Time
+ }
+ return fallback
+}
diff --git a/metrics/request_tracker_test.go b/metrics/request_tracker_test.go
new file mode 100644
index 000000000..55fff1897
--- /dev/null
+++ b/metrics/request_tracker_test.go
@@ -0,0 +1,44 @@
+package metrics
+
+import (
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestRequestTrackerPinsBeforeTimestampAssignment(t *testing.T) {
+ ch := make(chan SampleContainer, 1)
+ tracker, unbind := BindRequestTracker(ch)
+ defer unbind()
+ ticket := BeginRequest(ch)
+ require.NotNil(t, ticket)
+ require.False(t, tracker.Before().After(ticket.Time))
+ ticket.Release()
+ ticket.Release()
+ for i := range tracker.shards {
+ require.Empty(t, tracker.shards[i].active)
+ }
+ unbind()
+ require.Nil(t, BeginRequest(ch))
+}
+
+func TestRequestTrackerConcurrentProducers(t *testing.T) {
+ ch := make(chan SampleContainer)
+ tracker, unbind := BindRequestTracker(ch)
+ defer unbind()
+ var wg sync.WaitGroup
+ for range 16 {
+ wg.Go(func() {
+ for range 1000 {
+ ticket := BeginRequest(ch)
+ require.False(t, tracker.Before().After(ticket.Time))
+ ticket.Release()
+ }
+ })
+ }
+ wg.Wait()
+ for i := range tracker.shards {
+ require.Empty(t, tracker.shards[i].active)
+ }
+}
diff --git a/output/manager.go b/output/manager.go
index 0a65d67c1..6c5b5397c 100644
--- a/output/manager.go
+++ b/output/manager.go
@@ -1,6 +1,7 @@
package output
import (
+ "fmt"
"sync"
"time"
@@ -40,7 +41,25 @@ func NewManager(outputs []Output, logger logrus.FieldLogger, testStopCallback fu
// all of its buffered metrics have been sent to all outputs. The second
// callback will call the Stop() or StopWithTestError() method of every output.
func (om *Manager) Start(samplesChan chan metrics.SampleContainer) (wait func(), finish func(error), err error) {
+ var unbind func()
+ for _, out := range om.outputs {
+ if tracked, ok := out.(interface{ SetRequestTracker(*metrics.RequestTracker) }); ok {
+ if mode, ok := out.(interface{ UsesRequestTracking() bool }); ok && !mode.UsesRequestTracking() {
+ continue
+ }
+ if unbind != nil {
+ unbind()
+ return nil, nil, fmt.Errorf("only one request-tracked output is supported")
+ }
+ tracker, release := metrics.BindRequestTracker(samplesChan)
+ unbind = release
+ tracked.SetRequestTracker(tracker)
+ }
+ }
if err := om.startOutputs(); err != nil {
+ if unbind != nil {
+ unbind()
+ }
return nil, nil, err
}
@@ -78,6 +97,9 @@ func (om *Manager) Start(samplesChan chan metrics.SampleContainer) (wait func(),
finish = func(testErr error) {
wait() // just in case, though it shouldn't be needed if API is used correctly
om.stopOutputs(testErr, len(om.outputs))
+ if unbind != nil {
+ unbind()
+ }
}
return wait, finish, nil
}