From 0e0a41f8cb26c6edcb601392b38a6509272bdfd6 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Tue, 13 Oct 2026 15:07:20 +0200 Subject: [PATCH 1/2] Send Breaking-IT metrics using binary COPY and paced batches --- internal/output/breakingit/README.md | 21 + internal/output/breakingit/batch_test.go | 4 +- internal/output/breakingit/binary.go | 133 +++++++ .../breakingit/binary_benchmark_test.go | 33 ++ internal/output/breakingit/binary_test.go | 193 +++++++++ internal/output/breakingit/breakingit.go | 373 ++++++++---------- internal/output/breakingit/cadence_test.go | 81 ++++ internal/output/breakingit/http_rows_test.go | 5 +- .../output/breakingit/transactions_test.go | 17 +- 9 files changed, 637 insertions(+), 223 deletions(-) create mode 100644 internal/output/breakingit/binary.go create mode 100644 internal/output/breakingit/binary_benchmark_test.go create mode 100644 internal/output/breakingit/binary_test.go create mode 100644 internal/output/breakingit/cadence_test.go diff --git a/internal/output/breakingit/README.md b/internal/output/breakingit/README.md index 4e7ef0e44..99605eb87 100644 --- a/internal/output/breakingit/README.md +++ b/internal/output/breakingit/README.md @@ -113,3 +113,24 @@ and context, while keeping any business checks and throwing on failures. PG-proxy connection. The default remains `true` for compatibility with existing private proxy deployments. This setting applies only to output delivery, not to HTTP requests generated by the test script. + +### Binary delivery and batching + +The Breaking-IT output sends PostgreSQL binary COPY parts (`X-Format: binary`) +through `/ingest/batch`, matching the JMeter pg-proxy writer. Requests, +transactions, errors, and virtual users retain their existing table and column +order, including the synthetic monitoring tables. The proxy must support binary +COPY. Screenshots continue to use the separate JSON upsert endpoint. + +The first complete row triggers an immediate batch so BreakTest can detect that +the test is running. Subsequent batches wait 1,000 ms after the previous send +finishes, for both load tests and synthetic monitoring. Slow sends and retries +never overlap. Shutdown drains incomplete HTTP groups and sends the remaining +rows immediately. The 100 ms HTTP-group cleanup timer only assembles rows; it +does not control network delivery. + +Multipart bodies use fast gzip compression. Sending detaches the current buffers, +so producers can fill the next batch during retries. Exhausted retries discard +only the failed batch; newly collected rows remain queued for the next send. +Synthetic run IDs use UUID binary fields, with the same deterministic fallback +as JMeter for legacy identifiers that are not UUIDs. diff --git a/internal/output/breakingit/batch_test.go b/internal/output/breakingit/batch_test.go index 198f9b042..df8360ed9 100644 --- a/internal/output/breakingit/batch_test.go +++ b/internal/output/breakingit/batch_test.go @@ -54,10 +54,12 @@ func TestBatchRetryPreservesConcurrentSamples(t *testing.T) { defer server.Close() logger.proxyURL, logger.httpClient = server.URL, server.Client() logger.flush() + require.Len(t, requests, 2) + logger.flush() for _, want := range []string{"original\n", "original\n", "arrived during retry\n"} { select { case got := <-requests: - assert.Equal(t, want, got) + assert.Equal(t, binaryCopyHeader+want+"\xff\xff", got) case <-time.After(5 * time.Second): t.Fatal("missing batch") } diff --git a/internal/output/breakingit/binary.go b/internal/output/breakingit/binary.go new file mode 100644 index 000000000..3b2ee12d7 --- /dev/null +++ b/internal/output/breakingit/binary.go @@ -0,0 +1,133 @@ +package breakingit + +import ( + "bytes" + "crypto/md5" //nolint:gosec // Java UUID.nameUUIDFromBytes compatibility, not cryptography. + "encoding/binary" + "fmt" + "math" + "strconv" + "time" + + "github.com/gofrs/uuid/v5" +) + +const ( + binaryCopyHeader = "PGCOPY\n\xff\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00" + postgresEpochMicros int64 = 946684800000000 +) + +// Column types follow the pg-proxy/JMeter COPY contract: timestamp, text, bool, int4, UUID. +func (l *Logger) binarySchema(buffer *bytes.Buffer) string { + switch buffer { + case l.bufHTTP: + if l.isSynthetic { + return "tssssssbiisiiiu" + } + return "tssssbiisiii" + case l.bufTrans: + if l.isSynthetic { + return "tsssssbiiiu" + } + return "tsssbiii" + case l.bufErrors: + if l.isSynthetic { + return "tsssssssiissssssuu" + } + return "tssssssiissssssu" + default: + return "tsssiii" + } +} + +// syntheticRunUUID matches Java's nameUUIDFromBytes fallback for legacy run identifiers. +func syntheticRunUUID(value string) uuid.UUID { + if id, err := uuid.FromString(value); err == nil { + return id + } + if value == "" { + value = "unknown" + } + id := uuid.UUID(md5.Sum([]byte(value))) //nolint:gosec // Protocol compatibility. + id[6] = (id[6] & 0x0f) | 0x30 + id[8] = (id[8] & 0x3f) | 0x80 + return id +} + +// writeBinaryRow is called with mu held. Buffers contain rows only; framing is added per batch. +func (l *Logger) writeBinaryRow(buffer *bytes.Buffer, row []any) { + schema := l.binarySchema(buffer) + // Reserve enough room for common rows, and write directly into unused buffer capacity. + size := 2 + len(row)*12 + for _, value := range row { + if text, ok := value.(string); ok { + size += len(text) + } + } + buffer.Grow(size) + data := binary.BigEndian.AppendUint16(buffer.AvailableBuffer(), uint16(len(row))) //nolint:gosec + for i, value := range row { + var err error + data, err = appendBinaryField(data, value, schema[i]) + if err != nil { + _, _ = fmt.Fprintf(l.out, "breakingit: cannot encode COPY row: %v\n", err) + return + } + } + + _, _ = buffer.Write(data) + if !l.firstQueued && l.firstData != nil { + l.firstQueued = true + l.firstData <- struct{}{} + } +} + +func appendBinaryField(data []byte, value any, kind byte) ([]byte, error) { + switch v := value.(type) { + case time.Time: + data = binary.BigEndian.AppendUint32(data, 8) + return binary.BigEndian.AppendUint64(data, uint64(v.UnixMicro()-postgresEpochMicros)), nil //nolint:gosec + case int64: + if v < math.MinInt32 || v > math.MaxInt32 { + return nil, fmt.Errorf("int4 out of range: %d", v) + } + data = binary.BigEndian.AppendUint32(data, 4) + return binary.BigEndian.AppendUint32(data, uint32(v)), nil //nolint:gosec // Signed int4 bits. + case string: + return appendBinaryString(data, v, kind) + default: + return nil, fmt.Errorf("unsupported field type %T", value) + } +} + +func appendBinaryString(data []byte, value string, kind byte) ([]byte, error) { + // Preserve CSV's NULL semantics for empty fields, except JMeter's run UUID fallback. + if value == "" && kind != 'u' { + return binary.BigEndian.AppendUint32(data, ^uint32(0)), nil + } + switch kind { + case 'b': + v, err := strconv.ParseBool(value) + if err != nil { + return nil, err + } + data = binary.BigEndian.AppendUint32(data, 1) + if v { + return append(data, 1), nil + } + return append(data, 0), nil + case 'i': + n, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return nil, err + } + return appendBinaryField(data, n, kind) + case 'u': + id := syntheticRunUUID(value) + data = binary.BigEndian.AppendUint32(data, 16) + return append(data, id[:]...), nil + default: + data = binary.BigEndian.AppendUint32(data, uint32(len(value))) //nolint:gosec // COPY field length. + return append(data, value...), nil + } +} diff --git a/internal/output/breakingit/binary_benchmark_test.go b/internal/output/breakingit/binary_benchmark_test.go new file mode 100644 index 000000000..0f9e63f2a --- /dev/null +++ b/internal/output/breakingit/binary_benchmark_test.go @@ -0,0 +1,33 @@ +package breakingit + +import ( + "bytes" + "encoding/csv" + "strconv" + "testing" + "time" +) + +// Compare row serialization with the previous per-row CSV writer, excluding network and gzip. +func BenchmarkTransactionEncoding(b *testing.B) { + at := time.Date(2026, 10, 13, 12, 0, 0, 123456000, time.UTC) + b.Run("CSV", func(b *testing.B) { + var buffer bytes.Buffer + b.ReportAllocs() + for b.Loop() { + buffer.Reset() + row := []string{at.UTC().Format(time.RFC3339Nano), "run-123", "Amsterdam", "Checkout", "true", "", "", strconv.FormatInt(123, 10)} + writer := csv.NewWriter(&buffer) + _ = writer.Write(row) + writer.Flush() + } + }) + b.Run("Binary", func(b *testing.B) { + logger := newTransactionTestLogger(false) + b.ReportAllocs() + for b.Loop() { + logger.bufTrans.Reset() + logger.writeBinaryRow(logger.bufTrans, []any{at, "run-123", "Amsterdam", "Checkout", "true", "", "", int64(123)}) + } + }) +} diff --git a/internal/output/breakingit/binary_test.go b/internal/output/breakingit/binary_test.go new file mode 100644 index 000000000..d0525fb2c --- /dev/null +++ b/internal/output/breakingit/binary_test.go @@ -0,0 +1,193 @@ +package breakingit + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "errors" + "fmt" + "io" + "mime/multipart" + "strconv" + "testing" + "time" + + "go.k6.io/k6/v2/metrics" + + "github.com/gofrs/uuid/v5" + "github.com/stretchr/testify/require" +) + +// Decode independently of the writer, including strict field widths and stream framing. +func readBinaryRows(reader io.Reader, schema string, framed bool) ([][]string, error) { + if framed { + header := make([]byte, 19) + if _, err := io.ReadFull(reader, header); err != nil { + return nil, err + } + if !bytes.Equal(header, []byte("PGCOPY\n\xff\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00")) { + return nil, fmt.Errorf("invalid header") + } + } + var rows [][]string + for { + var count int16 + err := binary.Read(reader, binary.BigEndian, &count) + if errors.Is(err, io.EOF) && !framed { + return rows, nil + } + if err != nil { + return nil, err + } + if count == -1 && framed { + rest, err := io.ReadAll(reader) + if len(rest) != 0 { + return nil, fmt.Errorf("trailing bytes") + } + return rows, err + } + if int(count) != len(schema) { + return nil, fmt.Errorf("column count %d for %s", count, schema) + } + row := make([]string, count) + for i, kind := range schema { + var size int32 + if err := binary.Read(reader, binary.BigEndian, &size); err != nil { + return nil, err + } + if size == -1 { + continue + } + if size < 0 || size > 10000000 { + return nil, fmt.Errorf("invalid length") + } + field := make([]byte, size) + if _, err := io.ReadFull(reader, field); err != nil { + return nil, err + } + widths := map[rune]int{'t': 8, 'i': 4, 'b': 1, 'u': 16} + if want, ok := widths[kind]; ok && len(field) != want { + return nil, fmt.Errorf("field %d width %d", i, size) + } + switch kind { + case 't': + row[i] = time.UnixMicro(int64(binary.BigEndian.Uint64(field)) + 946684800000000).UTC().Format(time.RFC3339Nano) + case 'i': + row[i] = strconv.FormatInt(int64(int32(binary.BigEndian.Uint32(field))), 10) + case 'b': + if field[0] > 1 { + return nil, fmt.Errorf("invalid bool") + } + row[i] = strconv.FormatBool(field[0] == 1) + case 'u': + id, err := uuid.FromBytes(field) + if err != nil { + return nil, err + } + row[i] = id.String() + default: + row[i] = string(field) + } + } + rows = append(rows, row) + } +} + +func TestBinaryEpochAndJavaUUID(t *testing.T) { + t.Parallel() + l := newTransactionTestLogger(false) + l.writeBinaryRow(l.bufVUs, []any{time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), "a,\"b\"\n日本", "", "node", "-1", "", ""}) + rows, err := readBinaryRows(l.bufVUs, "tsssiii", false) + require.NoError(t, err) + require.Equal(t, [][]string{{"2000-01-01T00:00:00Z", "a,\"b\"\n日本", "", "node", "-1", "", ""}}, rows) + require.Equal(t, "68ca2f05-25ba-39de-9dcd-ba9ac7e36f46", syntheticRunUUID("run-123").String()) +} + +func TestBinaryFieldBytes(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + value any + kind byte + want []byte + }{ + {time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), 't', []byte{0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0}}, + {time.Date(1999, 12, 31, 23, 59, 59, 999999000, time.UTC), 't', []byte{0, 0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255}}, + {int64(-2), 'i', []byte{0, 0, 0, 4, 255, 255, 255, 254}}, + {"true", 'b', []byte{0, 0, 0, 1, 1}}, + {"false", 'b', []byte{0, 0, 0, 1, 0}}, + {"", 'i', []byte{255, 255, 255, 255}}, + {"é", 's', []byte{0, 0, 0, 2, 195, 169}}, + {"00112233-4455-6677-8899-aabbccddeeff", 'u', []byte{0, 0, 0, 16, 0, 17, 34, 51, 68, 85, 102, 119, 136, 153, 170, 187, 204, 221, 238, 255}}, + } { + got, err := appendBinaryField(nil, tc.value, tc.kind) + require.NoError(t, err) + require.Equal(t, tc.want, got) + } +} + +func TestInvalidBinaryRowDoesNotCorruptBuffer(t *testing.T) { + t.Parallel() + logger := newTransactionTestLogger(false) + valid := []any{time.Now(), "run", "location", "node", int64(1), "", ""} + logger.writeBinaryRow(logger.bufVUs, valid) + before := bytes.Clone(logger.bufVUs.Bytes()) + invalid := append([]any(nil), valid...) + invalid[4] = int64(1) << 32 + logger.writeBinaryRow(logger.bufVUs, invalid) + require.Equal(t, before, logger.bufVUs.Bytes()) + logger.writeBinaryRow(logger.bufVUs, valid) + rows, err := readBinaryRows(logger.bufVUs, "tsssiii", false) + require.NoError(t, err) + require.Len(t, rows, 2) +} + +func TestAllBinaryBatchTables(t *testing.T) { + t.Parallel() + for _, synthetic := range []bool{false, true} { + t.Run(fmt.Sprint(synthetic), func(t *testing.T) { + t.Parallel() + logger := newTransactionTestLogger(synthetic) + for _, metric := range []string{"group_duration", "ws_ping", "custom_duration", "browser_errors", "checks", "vus"} { + sample := transactionTestSample(metric, map[string]string{"group": "::checkout"}, false) + if metric == "checks" { + sample.Value = 0 + } + logger.AddMetricSamples([]metrics.SampleContainer{sample}) + } + body, boundary := logger.batchBody(logger.bufHTTP.Bytes(), logger.bufVUs.Bytes(), logger.bufTrans.Bytes(), logger.bufErrors.Bytes()) + gz, err := gzip.NewReader(bytes.NewReader(body)) + require.NoError(t, err) + defer func() { require.NoError(t, gz.Close()) }() + reader := multipart.NewReader(gz, boundary) + schemas := map[string]string{"public.requests_raw": "tssssbiisiii", "public.requests_raw_sm": "tssssssbiisiiiu", "public.transactions": "tsssbiii", "public.transactions_sm": "tsssssbiiiu", "public.requests_error": "tssssssiissssssu", "public.requests_error_sm": "tsssssssiissssssuu", "public.virtual_users": "tsssiii"} + counts := map[string]int{} + for { + part, err := reader.NextPart() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + table := part.Header.Get("X-Table") + require.Equal(t, "binary", part.Header.Get("X-Format")) + schema, ok := schemas[table] + require.True(t, ok) + rows, err := readBinaryRows(part, schema, true) + require.NoError(t, err) + counts[table] = len(rows) + } + suffix := "" + if synthetic { + suffix = "_sm" + } + require.Equal(t, 2, counts["public.requests_raw"+suffix]) + require.Equal(t, 2, counts["public.requests_error"+suffix]) + require.Equal(t, 1, counts["public.transactions"+suffix]) + if synthetic { + require.Len(t, counts, 3) + } else { + require.Len(t, counts, 4) + require.Equal(t, 1, counts["public.virtual_users"]) + } + }) + } +} diff --git a/internal/output/breakingit/breakingit.go b/internal/output/breakingit/breakingit.go index 013243621..22f597aa4 100644 --- a/internal/output/breakingit/breakingit.go +++ b/internal/output/breakingit/breakingit.go @@ -8,7 +8,6 @@ import ( "crypto/rand" "crypto/sha256" "crypto/tls" - "encoding/csv" "encoding/hex" "encoding/json" "fmt" @@ -70,13 +69,16 @@ type Logger struct { httpClient *http.Client // batch buffers - mu sync.Mutex - bufHTTP *bytes.Buffer - bufVUs *bytes.Buffer - bufTrans *bytes.Buffer - bufErrors *bytes.Buffer - batchTimer *time.Timer - flushing bool // flag to prevent recursive flush calls + mu sync.Mutex + bufHTTP *bytes.Buffer + bufVUs *bytes.Buffer + bufTrans *bytes.Buffer + bufErrors *bytes.Buffer + batchTimer *time.Timer + firstData chan struct{} + firstQueued bool + cleanupDone chan struct{} + flushing bool // flag to prevent recursive flush calls // pending screenshots to upsert via /upsert endpoint (low-volume) pendingScreenshots [][]any @@ -169,6 +171,8 @@ func New(params output.Params) (output.Output, error) { isSynthetic: isSynthetic, transactionSource: transactionSource, } + l.firstData = make(chan struct{}, 1) + l.cleanupDone = make(chan struct{}) l.batchTimer = time.NewTimer(time.Second) go l.batchLoop() go l.cleanupProcessor() @@ -183,33 +187,36 @@ func (l *Logger) Start() error { return nil } // Stop flushes remaining samples and stops background processing. func (l *Logger) Stop() error { - _, _ = fmt.Fprintf(l.out, "[screenshot] Stop() called, waiting for batchLoop to finish\n") - l.batchTimer.Stop() close(l.cleanupStop) - // Wait for any in-progress batchLoop flush to complete + <-l.cleanupDone <-l.batchDone - _, _ = fmt.Fprintf(l.out, "[screenshot] batchLoop done, running final flush (pendingScreenshots=%d)\n", - len(l.pendingScreenshots)) - // Final flush to send any remaining data (including screenshots) + // Finish incomplete groups before the final batch. + for _, g := range l.pendingHTTPGroups { + l.processHTTPGroup(g) + } l.flush() - _, _ = fmt.Fprintf(l.out, "[screenshot] Stop() complete\n") return nil } func (l *Logger) batchLoop() { defer close(l.batchDone) + defer l.batchTimer.Stop() + select { + case <-l.firstData: + case <-l.cleanupStop: + return + } + if !l.batchTimer.Stop() { + select { + case <-l.batchTimer.C: + default: + } + } for { + l.flush() + l.batchTimer.Reset(time.Second) select { - case _, ok := <-l.batchTimer.C: - if !ok { - return - } - l.flush() - if l.isSynthetic { - l.batchTimer.Reset(5 * time.Second) - } else { - l.batchTimer.Reset(time.Second) - } + case <-l.batchTimer.C: case <-l.cleanupStop: return } @@ -217,6 +224,7 @@ func (l *Logger) batchLoop() { } func (l *Logger) cleanupProcessor() { + defer close(l.cleanupDone) t := time.NewTicker(100 * time.Millisecond) defer t.Stop() for { @@ -255,19 +263,13 @@ func (l *Logger) flush() { return } - // Snapshot buffer data and lengths for retry attempts - bufHTTPSnapshot := make([]byte, l.bufHTTP.Len()) - copy(bufHTTPSnapshot, l.bufHTTP.Bytes()) - bufHTTPLen := l.bufHTTP.Len() - bufVUsSnapshot := make([]byte, l.bufVUs.Len()) - copy(bufVUsSnapshot, l.bufVUs.Bytes()) - bufVUsLen := l.bufVUs.Len() - bufTransSnapshot := make([]byte, l.bufTrans.Len()) - copy(bufTransSnapshot, l.bufTrans.Bytes()) - bufTransLen := l.bufTrans.Len() - bufErrorsSnapshot := make([]byte, l.bufErrors.Len()) - copy(bufErrorsSnapshot, l.bufErrors.Bytes()) - bufErrorsLen := l.bufErrors.Len() + // Transfer ownership so producers can fill the next batch during network I/O. + bufHTTPSnapshot := l.bufHTTP.Bytes() + bufVUsSnapshot := l.bufVUs.Bytes() + bufTransSnapshot := l.bufTrans.Bytes() + bufErrorsSnapshot := l.bufErrors.Bytes() + l.bufHTTP, l.bufVUs = &bytes.Buffer{}, &bytes.Buffer{} + l.bufTrans, l.bufErrors = &bytes.Buffer{}, &bytes.Buffer{} // Snapshot pending screenshots screenshotsSnapshot := l.pendingScreenshots l.pendingScreenshots = nil @@ -290,38 +292,10 @@ func (l *Logger) flush() { } body, boundary := l.batchBody(bufHTTPSnapshot, bufVUsSnapshot, bufTransSnapshot, bufErrorsSnapshot) - success := l.sendBatch(body, boundary) - - // Reset buffers and check for new data that accumulated during retries + l.sendBatch(body, boundary) l.mu.Lock() - hasNewData := false - for _, item := range []struct { - buffer *bytes.Buffer - sent int - }{ - {l.bufHTTP, bufHTTPLen}, - {l.bufVUs, bufVUsLen}, - {l.bufTrans, bufTransLen}, - {l.bufErrors, bufErrorsLen}, - } { - if success { - item.buffer.Next(item.sent) - hasNewData = hasNewData || item.buffer.Len() > 0 - } else { - // Preserve the existing bounded-memory policy after exhausted retries. - item.buffer.Reset() - } - } - l.flushing = false l.mu.Unlock() - - // If new data accumulated during retries and we succeeded, flush immediately to send it - // This ensures no gap in data delivery - if success && hasNewData { - // Use a goroutine to avoid blocking and prevent potential deadlock - go l.flush() - } } // upsertScreenshots sends screenshots to the pg-proxy upsert endpoint. @@ -464,16 +438,16 @@ func (l *Logger) processHTTPGroup(g *MetricGroup) { // response_size from browser_data_received when present (browser only); k6 HTTP has no per-request // size in this group - responseSize := "" + var responseSize any = "" if v, ok := g.Values["data_received"]; ok && v >= 0 { - responseSize = fmtInt(v) + responseSize = int64(v) } - var row []string + var row []any if l.isSynthetic { // SM table has run_id last (existing table order) - row = []string{ - time.Unix(0, g.Timestamp).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, g.Timestamp), g.Tags["scenario_name"], g.Tags["location"], g.Tags["node_name"], @@ -490,8 +464,8 @@ func (l *Logger) processHTTPGroup(g *MetricGroup) { fmt.Sprintf("%v", l.envTags["run_id"]), } } else { - row = []string{ - time.Unix(0, g.Timestamp).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, g.Timestamp), g.Tags["run_id"], g.Tags["location"], transactionName, @@ -506,9 +480,7 @@ func (l *Logger) processHTTPGroup(g *MetricGroup) { } } l.mu.Lock() - w := csv.NewWriter(l.bufHTTP) - _ = w.Write(row) - w.Flush() + l.writeBinaryRow(l.bufHTTP, row) // If not successful, also emit into requests_error(_sm) if success == "false" { l.writeHTTPError(g, transactionName, samplerName, responseCode, responseTime, connectTime, isBrowser) @@ -548,13 +520,13 @@ func (l *Logger) processWebSocketMetric(s metrics.Sample, ts int64) { } // response_time is the metric value - responseTime := fmtInt(s.Value) + responseTime := int64(s.Value) - var row []string + var row []any if l.isSynthetic { // SM requests_raw_sm (existing table: run_id last) - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), tags["scenario_name"], tags["location"], tags["node_name"], @@ -573,8 +545,8 @@ func (l *Logger) processWebSocketMetric(s metrics.Sample, ts int64) { } else { // Non-SM: time, run_id, location, transaction_name, sampler_name, success, request_size, // response_size, response_code, response_connect_time, response_latency, response_time - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), tags["run_id"], tags["location"], transactionName, @@ -589,9 +561,7 @@ func (l *Logger) processWebSocketMetric(s metrics.Sample, ts int64) { } } l.mu.Lock() - w := csv.NewWriter(l.bufHTTP) - _ = w.Write(row) - w.Flush() + l.writeBinaryRow(l.bufHTTP, row) l.mu.Unlock() } @@ -632,13 +602,13 @@ func (l *Logger) processCustomTrendMetric(s metrics.Sample, ts int64) { success := "true" // response_time is the metric value - responseTime := fmtInt(s.Value) + responseTime := int64(s.Value) - var row []string + var row []any if l.isSynthetic { // SM requests_raw_sm (existing table: run_id last) - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), tags["scenario_name"], tags["location"], tags["node_name"], @@ -657,8 +627,8 @@ func (l *Logger) processCustomTrendMetric(s metrics.Sample, ts int64) { } else { // Non-SM: time, run_id, location, transaction_name, sampler_name, success, request_size, // response_size, response_code, response_connect_time, response_latency, response_time - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), tags["run_id"], tags["location"], transactionName, @@ -673,9 +643,7 @@ func (l *Logger) processCustomTrendMetric(s metrics.Sample, ts int64) { } } l.mu.Lock() - w := csv.NewWriter(l.bufHTTP) - _ = w.Write(row) - w.Flush() + l.writeBinaryRow(l.bufHTTP, row) l.mu.Unlock() } @@ -721,11 +689,11 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { // captured in the same flush snapshot (prevents the error row referencing // a hash that was never sent to sm_screenshots). if l.isSynthetic { - row := []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), // time - fmt.Sprintf("%v", l.envTags["scenario_name"]), // scenario_name - fmt.Sprintf("%v", l.envTags["location"]), // location - fmt.Sprintf("%v", l.envTags["node_name"]), // node_name + row := []any{ + time.Unix(0, ts), // time + fmt.Sprintf("%v", l.envTags["scenario_name"]), // scenario_name + fmt.Sprintf("%v", l.envTags["location"]), // location + fmt.Sprintf("%v", l.envTags["node_name"]), // node_name m["scenario"], // thread_group_name transactionName, // transaction_name samplerName, // sampler_name @@ -748,28 +716,26 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { }) _, _ = fmt.Fprintf(l.out, "[screenshot] queued for upsert, pendingScreenshots=%d\n", len(l.pendingScreenshots)) } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeBinaryRow(l.bufErrors, row) l.mu.Unlock() } else { - row := []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), // time - fmt.Sprintf("%v", l.envTags["run_id"]), // run_id - fmt.Sprintf("%v", l.envTags["location"]), // location - fmt.Sprintf("%v", l.envTags["node_name"]), // node_name - transactionName, // transaction_name - samplerName, // sampler_name - "", // response_code - "", // response_time - "", // connection_time - "", // url - "", // assertions - errMsg, // response_message - "", // request_headers - "", // response_headers - screenshotHash, // response_data (screenshot hash reference) - id, // id (UUIDv7) + row := []any{ + time.Unix(0, ts), // time + fmt.Sprintf("%v", l.envTags["run_id"]), // run_id + fmt.Sprintf("%v", l.envTags["location"]), // location + fmt.Sprintf("%v", l.envTags["node_name"]), // node_name + transactionName, // transaction_name + samplerName, // sampler_name + "", // response_code + "", // response_time + "", // connection_time + "", // url + "", // assertions + errMsg, // response_message + "", // request_headers + "", // response_headers + screenshotHash, // response_data (screenshot hash reference) + id, // id (UUIDv7) } l.mu.Lock() if screenshotHash != "" { @@ -777,15 +743,11 @@ func (l *Logger) processBrowserError(s metrics.Sample, ts int64) { screenshotHash, screenshot, "image/png", len(screenshot), }) } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeBinaryRow(l.bufErrors, row) l.mu.Unlock() } } -func fmtInt(v float64) string { return fmt.Sprintf("%d", int64(v)) } - // generateUUIDv7 generates a UUIDv7 using the provided timestamp (nanoseconds since Unix epoch) func generateUUIDv7(timestampNanos int64) string { // Convert nanoseconds to time.Time for the UUID generator @@ -997,10 +959,10 @@ func (l *Logger) processPageSample(s metrics.Sample, ts int64) { // SM transactions_sm (run_id last): time, scenario_name, location, node_name, thread_group_name, // transaction_name, success, request_size, response_size, response_time, run_id isSM := l.isSynthetic - var row []string + var row []any if isSM { - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), fmt.Sprintf("%v", l.envTags["scenario_name"]), fmt.Sprintf("%v", l.envTags["location"]), fmt.Sprintf("%v", l.envTags["node_name"]), @@ -1009,25 +971,23 @@ func (l *Logger) processPageSample(s metrics.Sample, ts int64) { "true", "", "", - fmtInt(s.Value), + int64(s.Value), fmt.Sprintf("%v", l.envTags["run_id"]), } } else { - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), fmt.Sprintf("%v", l.envTags["run_id"]), fmt.Sprintf("%v", l.envTags["location"]), transactionName, "true", "", "", - fmtInt(s.Value), + int64(s.Value), } } l.mu.Lock() - wt := csv.NewWriter(l.bufTrans) - _ = wt.Write(row) - wt.Flush() + l.writeBinaryRow(l.bufTrans, row) l.mu.Unlock() } @@ -1044,11 +1004,11 @@ func (l *Logger) processGroupSample(s metrics.Sample, ts int64) { nameRaw = "/" } isSM := l.isSynthetic - var row []string + var row []any if isSM { // SM transactions_sm (run_id last) - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), fmt.Sprintf("%v", l.envTags["scenario_name"]), fmt.Sprintf("%v", l.envTags["location"]), fmt.Sprintf("%v", l.envTags["node_name"]), @@ -1057,42 +1017,38 @@ func (l *Logger) processGroupSample(s metrics.Sample, ts int64) { success, "", "", - fmtInt(s.Value), + int64(s.Value), fmt.Sprintf("%v", l.envTags["run_id"]), } } else { - row = []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row = []any{ + time.Unix(0, ts), fmt.Sprintf("%v", l.envTags["run_id"]), fmt.Sprintf("%v", l.envTags["location"]), nameRaw, success, "", "", - fmtInt(s.Value), + int64(s.Value), } } l.mu.Lock() - wt := csv.NewWriter(l.bufTrans) - _ = wt.Write(row) - wt.Flush() + l.writeBinaryRow(l.bufTrans, row) l.mu.Unlock() } func (l *Logger) processVUsSample(s metrics.Sample, ts int64) { - row := []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), + row := []any{ + time.Unix(0, ts), fmt.Sprintf("%v", l.envTags["run_id"]), fmt.Sprintf("%v", l.envTags["location"]), fmt.Sprintf("%v", l.envTags["node_name"]), - fmt.Sprintf("%d", int64(s.Value)), // active_threads - "", // started_threads (unknown) - "", // finished_threads (unknown) + int64(s.Value), // active_threads + "", // started_threads (unknown) + "", // finished_threads (unknown) } l.mu.Lock() - wv := csv.NewWriter(l.bufVUs) - _ = wv.Write(row) - wv.Flush() + l.writeBinaryRow(l.bufVUs, row) l.mu.Unlock() } @@ -1125,11 +1081,11 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) { // Build row per schema (includes id field) if l.isSynthetic { // SM requests_error_sm (existing table: run_id last): time, scenario_name, ..., id, run_id - row := []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), // time - fmt.Sprintf("%v", l.envTags["scenario_name"]), // scenario_name - fmt.Sprintf("%v", l.envTags["location"]), // location - fmt.Sprintf("%v", l.envTags["node_name"]), // node_name + row := []any{ + time.Unix(0, ts), // time + fmt.Sprintf("%v", l.envTags["scenario_name"]), // scenario_name + fmt.Sprintf("%v", l.envTags["location"]), // location + fmt.Sprintf("%v", l.envTags["node_name"]), // node_name m["scenario"], // thread_group_name transactionName, // transaction_name samplerName, // sampler_name @@ -1146,33 +1102,29 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) { fmt.Sprintf("%v", l.envTags["run_id"]), // run_id last } l.mu.Lock() - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeBinaryRow(l.bufErrors, row) l.mu.Unlock() } else { - row := []string{ - time.Unix(0, ts).UTC().Format(time.RFC3339Nano), // time - fmt.Sprintf("%v", l.envTags["run_id"]), // run_id - fmt.Sprintf("%v", l.envTags["location"]), // location - fmt.Sprintf("%v", l.envTags["node_name"]), // node_name - transactionName, // transaction_name - samplerName, // sampler_name - responseCode, // response_code - responseTime, // response_time - "", // connection_time - url, // url - "", // assertions - checkMsg, // response_message - requestData, // request_headers (used for request data) - "", // response_headers - responseData, // response_data - id, // id (UUIDv7 generated from measurement timestamp) + row := []any{ + time.Unix(0, ts), // time + fmt.Sprintf("%v", l.envTags["run_id"]), // run_id + fmt.Sprintf("%v", l.envTags["location"]), // location + fmt.Sprintf("%v", l.envTags["node_name"]), // node_name + transactionName, // transaction_name + samplerName, // sampler_name + responseCode, // response_code + responseTime, // response_time + "", // connection_time + url, // url + "", // assertions + checkMsg, // response_message + requestData, // request_headers (used for request data) + "", // response_headers + responseData, // response_data + id, // id (UUIDv7 generated from measurement timestamp) } l.mu.Lock() - we := csv.NewWriter(l.bufErrors) - _ = we.Write(row) - we.Flush() + l.writeBinaryRow(l.bufErrors, row) l.mu.Unlock() } } @@ -1180,8 +1132,9 @@ func (l *Logger) processCheckSample(s metrics.Sample, ts int64) { func (l *Logger) batchBody( bufHTTPSnapshot, bufVUsSnapshot, bufTransSnapshot, bufErrorsSnapshot []byte, ) ([]byte, string) { - var body bytes.Buffer - mw := multipart.NewWriter(&body) + var gzBody bytes.Buffer + gz, _ := gzip.NewWriterLevel(&gzBody, gzip.BestSpeed) + mw := multipart.NewWriter(gz) if len(bufHTTPSnapshot) > 0 { h := make(textproto.MIMEHeader) @@ -1190,16 +1143,20 @@ func (l *Logger) batchBody( table = "public.requests_raw_sm" } h.Set("X-Table", table) - h.Set("X-Format", "csv") + h.Set("X-Format", "binary") pw, _ := mw.CreatePart(h) - _, _ = io.Copy(pw, bytes.NewReader(bufHTTPSnapshot)) + _, _ = pw.Write([]byte(binaryCopyHeader)) + _, _ = pw.Write(bufHTTPSnapshot) + _, _ = pw.Write([]byte{0xff, 0xff}) } if len(bufVUsSnapshot) > 0 && !l.isSynthetic { h := make(textproto.MIMEHeader) h.Set("X-Table", "public.virtual_users") - h.Set("X-Format", "csv") + h.Set("X-Format", "binary") pw, _ := mw.CreatePart(h) - _, _ = io.Copy(pw, bytes.NewReader(bufVUsSnapshot)) + _, _ = pw.Write([]byte(binaryCopyHeader)) + _, _ = pw.Write(bufVUsSnapshot) + _, _ = pw.Write([]byte{0xff, 0xff}) } if len(bufTransSnapshot) > 0 { h := make(textproto.MIMEHeader) @@ -1208,9 +1165,11 @@ func (l *Logger) batchBody( table = "public.transactions_sm" } h.Set("X-Table", table) - h.Set("X-Format", "csv") + h.Set("X-Format", "binary") pw, _ := mw.CreatePart(h) - _, _ = io.Copy(pw, bytes.NewReader(bufTransSnapshot)) + _, _ = pw.Write([]byte(binaryCopyHeader)) + _, _ = pw.Write(bufTransSnapshot) + _, _ = pw.Write([]byte{0xff, 0xff}) } if len(bufErrorsSnapshot) > 0 { h := make(textproto.MIMEHeader) @@ -1219,16 +1178,14 @@ func (l *Logger) batchBody( table = "public.requests_error_sm" } h.Set("X-Table", table) - h.Set("X-Format", "csv") + h.Set("X-Format", "binary") pw, _ := mw.CreatePart(h) - _, _ = io.Copy(pw, bytes.NewReader(bufErrorsSnapshot)) + _, _ = pw.Write([]byte(binaryCopyHeader)) + _, _ = pw.Write(bufErrorsSnapshot) + _, _ = pw.Write([]byte{0xff, 0xff}) } _ = mw.Close() - // gzip body - var gzBody bytes.Buffer - gz := gzip.NewWriter(&gzBody) - _, _ = gz.Write(body.Bytes()) _ = gz.Close() return gzBody.Bytes(), mw.Boundary() @@ -1276,7 +1233,7 @@ func (l *Logger) postBatch(body []byte, boundary string) error { // writeHTTPError requires the buffer lock to be held by the caller. func (l *Logger) writeHTTPError(g *MetricGroup, - transactionName, samplerName, responseCode, responseTime, connectTime string, isBrowser bool, + transactionName, samplerName, responseCode string, responseTime, connectTime int64, isBrowser bool, ) { // Build response_message from tags errText := fmt.Sprintf("%v", g.Tags["error"]) // may be empty @@ -1326,8 +1283,8 @@ func (l *Logger) writeHTTPError(g *MetricGroup, id := generateUUIDv7(g.Timestamp) if l.isSynthetic { // SM requests_error_sm (existing table: run_id last): time, scenario_name, ..., id, run_id - erow := []string{ - time.Unix(0, g.Timestamp).UTC().Format(time.RFC3339Nano), + erow := []any{ + time.Unix(0, g.Timestamp), g.Tags["scenario_name"], g.Tags["location"], g.Tags["node_name"], @@ -1346,12 +1303,10 @@ func (l *Logger) writeHTTPError(g *MetricGroup, id, // id (UUIDv7) fmt.Sprintf("%v", l.envTags["run_id"]), } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(erow) - we.Flush() + l.writeBinaryRow(l.bufErrors, erow) } else { - erow := []string{ - time.Unix(0, g.Timestamp).UTC().Format(time.RFC3339Nano), + erow := []any{ + time.Unix(0, g.Timestamp), g.Tags["run_id"], g.Tags["location"], g.Tags["node_name"], @@ -1368,9 +1323,7 @@ func (l *Logger) writeHTTPError(g *MetricGroup, g.ErrorResBody, // response_data id, // id (UUIDv7) } - we := csv.NewWriter(l.bufErrors) - _ = we.Write(erow) - we.Flush() + l.writeBinaryRow(l.bufErrors, erow) } } @@ -1464,9 +1417,9 @@ func parseCustomPatterns(out io.Writer, patterns string) map[string]string { } // requestTimings converts k6 timings to the JMeter-style fields expected by BreakTest. -func requestTimings(values map[string]float64) (connectTime, latency, responseTime string) { +func requestTimings(values map[string]float64) (connectTime, latency, responseTime int64) { connect := values["http_req_connecting"] + values["http_req_tls_handshaking"] - return fmtInt(connect), - fmtInt(connect + values["http_req_sending"] + values["http_req_waiting"]), - fmtInt(connect + values["http_req_duration"]) + return int64(connect), + int64(connect + values["http_req_sending"] + values["http_req_waiting"]), + int64(connect + values["http_req_duration"]) } diff --git a/internal/output/breakingit/cadence_test.go b/internal/output/breakingit/cadence_test.go new file mode 100644 index 000000000..076e17e65 --- /dev/null +++ b/internal/output/breakingit/cadence_test.go @@ -0,0 +1,81 @@ +package breakingit + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.k6.io/k6/v2/metrics" + "go.k6.io/k6/v2/output" +) + +func TestFirstBatchImmediateThenOneSecond(t *testing.T) { + t.Parallel() + for _, synthetic := range []bool{false, true} { + t.Run(fmt.Sprint(synthetic), func(t *testing.T) { + t.Parallel() + sent := make(chan time.Time, 10) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + sent <- time.Now() + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + exporter, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test", "isSynthetic": fmt.Sprint(synthetic)}}) + require.NoError(t, err) + stopped := false + defer func() { + if !stopped { + require.NoError(t, exporter.Stop()) + } + }() + // Starting the output must not send an empty heartbeat. + select { + case <-sent: + t.Fatal("empty batch") + case <-time.After(50 * time.Millisecond): + } + sample := transactionTestSample("group_duration", map[string]string{"group": "::hello"}, false) + exporter.AddMetricSamples([]metrics.SampleContainer{sample}) + var first time.Time + select { + case first = <-sent: + case <-time.After(500 * time.Millisecond): + t.Fatal("first data was delayed") + } + exporter.AddMetricSamples([]metrics.SampleContainer{sample}) + select { + case at := <-sent: + require.GreaterOrEqual(t, at.Sub(first), 900*time.Millisecond) + case <-time.After(2 * time.Second): + t.Fatal("second batch was delayed") + } + // Shutdown must flush rows without waiting for another tick. + exporter.AddMetricSamples([]metrics.SampleContainer{sample}) + require.NoError(t, exporter.Stop()) + stopped = true + require.Len(t, sent, 1) + }) + } +} + +func TestStopDrainsIncompleteHTTPGroups(t *testing.T) { + t.Parallel() + bodies := make(chan []byte, 4) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + bodies <- body + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + exporter, err := New(output.Params{ConfigArgument: server.URL, StdOut: io.Discard, Environment: map[string]string{"PG_PROXY_TOKEN": "test"}}) + require.NoError(t, err) + exporter.AddMetricSamples([]metrics.SampleContainer{transactionTestSample("http_req_duration", nil, false)}) + require.NoError(t, exporter.Stop()) + require.Len(t, bodies, 1) +} diff --git a/internal/output/breakingit/http_rows_test.go b/internal/output/breakingit/http_rows_test.go index 6e46bb805..662dca12d 100644 --- a/internal/output/breakingit/http_rows_test.go +++ b/internal/output/breakingit/http_rows_test.go @@ -1,7 +1,6 @@ package breakingit import ( - "encoding/csv" "fmt" "testing" "time" @@ -32,7 +31,7 @@ func TestFailedHTTPRows(t *testing.T) { ErrorReqHeaders: "POST /pay", ErrorReqBody: "payload", ErrorResHeaders: "HTTP/1.1 500", ErrorResBody: "failure", }) - requests, err := csv.NewReader(logger.bufHTTP).ReadAll() + requests, err := readBinaryRows(logger.bufHTTP, map[bool]string{false: "tssssbiisiii", true: "tssssssbiisiiiu"}[logger.isSynthetic], false) require.NoError(t, err) require.Len(t, requests, 1) offset := 3 @@ -41,7 +40,7 @@ func TestFailedHTTPRows(t *testing.T) { } assert.Equal(t, []string{"Checkout", "/pay", "false", "", "", "500", "5", "14", "25"}, requests[0][offset:offset+9]) - errors, err := csv.NewReader(logger.bufErrors).ReadAll() + errors, err := readBinaryRows(logger.bufErrors, map[bool]string{false: "tssssssiissssssu", true: "tsssssssiissssssuu"}[logger.isSynthetic], false) require.NoError(t, err) require.Len(t, errors, 1) offset = 4 diff --git a/internal/output/breakingit/transactions_test.go b/internal/output/breakingit/transactions_test.go index 8c8009ab3..9871491c0 100644 --- a/internal/output/breakingit/transactions_test.go +++ b/internal/output/breakingit/transactions_test.go @@ -3,7 +3,6 @@ package breakingit import ( "bytes" "compress/gzip" - "encoding/csv" "fmt" "io" "mime" @@ -58,7 +57,7 @@ func TestTransactionRows(t *testing.T) { {"native success", "group_duration", "01_Home", map[string]string{"group": "::01_Home"}, false}, {"native failure", "group_duration", "02_Pay", map[string]string{"group": "::02_Pay"}, true}, {"nested", "group_duration", "Checkout::Payment", map[string]string{"group": "::Checkout::Payment"}, true}, - {"CSV escaping", "group_duration", "A,\"B\"\nC", map[string]string{"group": "::A,\"B\"\nC"}, false}, + {"special characters", "group_duration", "A,\"B\"\nC", map[string]string{"group": "::A,\"B\"\nC"}, false}, {"group fallback", "group_duration", "fallback", map[string]string{"name": "fallback"}, false}, {"group default", "group_duration", "/", nil, false}, } @@ -70,7 +69,7 @@ func TestTransactionRows(t *testing.T) { sample := transactionTestSample(test.metric, test.tags, test.failed) sample.Tags = sample.Tags.With("scenario", "browser_checkout") l.AddMetricSamples([]metrics.SampleContainer{sample}) - rows, err := csv.NewReader(l.bufTrans).ReadAll() + rows, err := readBinaryRows(l.bufTrans, map[bool]string{false: "tsssbiii", true: "tsssssbiiiu"}[l.isSynthetic], false) require.NoError(t, err) expected := []string{ "2026-09-24T12:00:00.123Z", "run-123", "Amsterdam", test.expectedName, @@ -80,7 +79,7 @@ func TestTransactionRows(t *testing.T) { expected = []string{ "2026-09-24T12:00:00.123Z", "checkout", "Amsterdam", "node-1", "browser_checkout", test.expectedName, fmt.Sprint(!test.failed), - "", "", "123", "run-123", + "", "", "123", syntheticRunUUID("run-123").String(), } } assert.Equal(t, [][]string{expected}, rows) @@ -110,7 +109,7 @@ func TestTransactionSource(t *testing.T) { transactionTestSample("pages", map[string]string{"page": "legacy"}, false), transactionTestSample("group_duration", map[string]string{"group": "::native"}, false), }}) - rows, err := csv.NewReader(l.bufTrans).ReadAll() + rows, err := readBinaryRows(l.bufTrans, map[bool]string{false: "tsssbiii", true: "tsssssbiiiu"}[l.isSynthetic], false) require.NoError(t, err) var names []string for _, row := range rows { @@ -121,7 +120,7 @@ func TestTransactionSource(t *testing.T) { check := transactionTestSample("checks", map[string]string{"group": "::native", "check": "bad"}, false) check.Value = 0 l.AddMetricSamples([]metrics.SampleContainer{check}) - errors, err := csv.NewReader(l.bufErrors).ReadAll() + errors, err := readBinaryRows(l.bufErrors, map[bool]string{false: "tssssssiissssssu", true: "tsssssssiissssssuu"}[l.isSynthetic], false) require.NoError(t, err) require.Len(t, errors, 1) assert.Equal(t, "native", errors[0][4]) @@ -163,7 +162,7 @@ func TestBrowserRequestKeepsExplicitGroup(t *testing.T) { l.processHTTPGroup(&MetricGroup{ Tags: map[string]string{"group": test.group, "path": test.path, "resource_type": "Document"}, }) - rows, err := csv.NewReader(l.bufHTTP).ReadAll() + rows, err := readBinaryRows(l.bufHTTP, map[bool]string{false: "tssssbiisiii", true: "tssssssbiisiiiu"}[l.isSynthetic], false) require.NoError(t, err) require.Len(t, rows, 1) index := 3 @@ -203,8 +202,8 @@ func TestTransactionsSentToProxy(t *testing.T) { w.WriteHeader(http.StatusOK) return } - assert.Equal(t, "csv", part.Header.Get("X-Format")) - rows, err := csv.NewReader(part).ReadAll() + assert.Equal(t, "binary", part.Header.Get("X-Format")) + rows, err := readBinaryRows(part, map[bool]string{false: "tsssbiii", true: "tsssssbiiiu"}[synthetic], true) assert.NoError(t, err) batches <- batch{part.Header.Get("X-Table"), rows} _, err = reader.NextPart() From d2370bdcfc63ee5a846ad42c04e06ca3f2cec640 Mon Sep 17 00:00:00 2001 From: jgaalen Date: Tue, 13 Oct 2026 15:15:33 +0200 Subject: [PATCH 2/2] Add UTC build timestamps to Breaking-IT binary filenames --- FORK.md | 3 +++ build-linux-binaries.sh | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/FORK.md b/FORK.md index 0b6b01551..da23701f2 100644 --- a/FORK.md +++ b/FORK.md @@ -60,6 +60,9 @@ git pull --ff-only origin master The build script reads the source in the current checkout and writes Linux and macOS binaries for amd64 and arm64 into `dist/`. It does not install or deploy them. Its displayed `VERSION` value is informational; use the binary's `version` command and SHA-256 checksum to identify an artifact. +Filenames include the build-start timestamp in UTC after `-breakingit`, for example +`k6-linux-amd64-breakingit-20261013T143000Z`. All four binaries from one invocation share the +same timestamp. For the complete change history, compare the `master` branch with the upstream [grafana/k6](https://github.com/grafana/k6) repository. diff --git a/build-linux-binaries.sh b/build-linux-binaries.sh index 1d642da50..3b9c6081b 100755 --- a/build-linux-binaries.sh +++ b/build-linux-binaries.sh @@ -1,14 +1,16 @@ #!/bin/bash # Build k6 Linux and macOS binaries (amd64 and arm64) from the current code. # Usage: ./build-linux-binaries.sh [output-dir] -# Default output dir is "dist". Binaries: k6-linux-*-breakingit, k6-darwin-*-breakingit +# Default output dir is "dist". Binaries end in -breakingit-YYYYMMDDTHHMMSSZ (UTC). set -euo pipefail OUT_DIR="${1:-dist}" VERSION="${VERSION:-$(git describe --tags --always --dirty)}" +BUILD_TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" echo "--- Building k6 binaries: ${VERSION}" +echo "--- Build timestamp (UTC): ${BUILD_TIMESTAMP}" echo "--- Output directory: ${OUT_DIR}" mkdir -p "$OUT_DIR" @@ -17,7 +19,7 @@ LDFLAGS='-s -w' build_linux() { local arch="$1" - local name="k6-linux-${arch}-breakingit" + local name="k6-linux-${arch}-breakingit-${BUILD_TIMESTAMP}" echo "Building ${name}..." GOOS=linux GOARCH="${arch}" CGO_ENABLED=0 go build -tags lean -trimpath -ldflags "${LDFLAGS}" -o "${OUT_DIR}/${name}" . echo " -> ${OUT_DIR}/${name}" @@ -25,7 +27,7 @@ build_linux() { build_darwin() { local arch="$1" - local name="k6-darwin-${arch}-breakingit" + local name="k6-darwin-${arch}-breakingit-${BUILD_TIMESTAMP}" echo "Building ${name}..." GOOS=darwin GOARCH="${arch}" CGO_ENABLED=0 go build -tags lean -trimpath -ldflags "${LDFLAGS}" -o "${OUT_DIR}/${name}" . echo " -> ${OUT_DIR}/${name}"