diff --git a/package-lock.json b/package-lock.json index a01161dc..b3dfd66b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "node-gyp-build": "^4.8.4", - "pprof-format": "^2.3.2", + "pprof-format": "^2.4.0", "source-map": "^0.8.0" }, "devDependencies": { @@ -5086,9 +5086,9 @@ } }, "node_modules/pprof-format": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/pprof-format/-/pprof-format-2.3.2.tgz", - "integrity": "sha512-fcMM+sgKMOveSurz7i0BMHfy1hHNnI9+VWjd4kdclqQjSTuiHWaP69DGwvkw3zcLkM/F76LhVIKvPyrxmp6iiw==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/pprof-format/-/pprof-format-2.4.0.tgz", + "integrity": "sha512-7SDtSvsD996RFtlODgEmCR+GAfC3Dx6zlnef7duB1QNAkOStz+VAV60Ie/OQrkEijmBmcOZlKK0KQ/Oupd1frw==", "license": "MIT" }, "node_modules/prelude-ls": { diff --git a/package.json b/package.json index e1e7d466..d253d01e 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "license": "Apache-2.0", "dependencies": { "node-gyp-build": "^4.8.4", - "pprof-format": "^2.3.2", + "pprof-format": "^2.4.0", "source-map": "^0.8.0" }, "devDependencies": { diff --git a/ts/src/heap-profiler.ts b/ts/src/heap-profiler.ts index a9af3924..ccaa9418 100644 --- a/ts/src/heap-profiler.ts +++ b/ts/src/heap-profiler.ts @@ -29,7 +29,14 @@ import { DEFAULT_COLUMN_NUMBERS, serializeHeapProfile, } from './profile-serializer'; -import {NormalizedProfile, normalizeHeapProfile} from './profile-normalizer'; +import { + HeapProfileNormalizerOptions, + NormalizedProfile, + normalizeHeapProfile, + ProfileVisitor, + visitHeapProfile, + VisitResult, +} from './profile-normalizer'; import {SourceMapper} from './sourcemapper/sourcemapper'; import { AllocationProfileNode, @@ -130,6 +137,32 @@ export function normalizedProfile( }); } +/** + * Collects a profile and passes it to the visitor, normalized, instead of + * serializing it as pprof, so the caller chooses the output (e.g. a visitor of + * an OtelProfilesBuilder). Throws if heap profiler is not enabled. + */ +export function visitProfile( + visitor: ProfileVisitor, + options: Pick< + HeapProfileNormalizerOptions, + 'ignoreSamplesPath' | 'sourceMapper' | 'generateLabels' + > = {}, +): VisitResult { + return collectProfile(root => + visitHeapProfile( + addExternalMemoryNode(root, startedWithAllocations), + visitor, + { + ...options, + startTimeNanos: BigInt(Date.now()) * 1_000_000n, + intervalBytes: heapIntervalBytes, + allocations: startedWithAllocations, + }, + ), + ); +} + function collectProfile( processProfile: ( root: AllocationProfileNode | AllocationProfileNodeWithStats, diff --git a/ts/src/index.ts b/ts/src/index.ts index 95415114..69a40910 100644 --- a/ts/src/index.ts +++ b/ts/src/index.ts @@ -39,11 +39,29 @@ export { SampleLabel, SampleLabelValue, SampleType, + teeVisitor, TimeProfileNormalizerOptions, visitHeapProfile, visitTimeProfile, VisitResult, } from './profile-normalizer'; +export { + ColumnNumbers, + PprofProfileBuilder, + PprofSerializerOptions, +} from './profile-serializer'; +export { + FRAME_MISSING_SOURCE_MAP, + LinkExtractor, + OtelProfileOptions, + OtelProfilesBuilder, + OtelSerializerOptions, + PROFILE_HAS_MISSING_SOURCE_MAPS, + ProfileAttribute, + SampleLink, + serializeHeapProfileOtel, + serializeTimeProfileOtel, +} from './otel-serializer'; export {HeapLimitExtensionSize} from './heap-profiler'; @@ -57,6 +75,7 @@ export const time = { start: timeProfiler.start, stop: timeProfiler.stop, stopNormalized: timeProfiler.stopNormalized, + stopAndVisit: timeProfiler.stopAndVisit, profileV2: timeProfiler.profileV2, getContext: timeProfiler.getContext, setContext: timeProfiler.setContext, @@ -75,6 +94,7 @@ export const heap = { profile: heapProfiler.profile, profileV2: heapProfiler.profileV2, normalizedProfile: heapProfiler.normalizedProfile, + visitProfile: heapProfiler.visitProfile, convertProfile: heapProfiler.convertProfile, v8Profile: heapProfiler.v8Profile, monitorOutOfMemory: heapProfiler.monitorOutOfMemory, diff --git a/ts/src/otel-serializer.ts b/ts/src/otel-serializer.ts new file mode 100644 index 00000000..59a6d0ca --- /dev/null +++ b/ts/src/otel-serializer.ts @@ -0,0 +1,595 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Serializes normalized profiles as OpenTelemetry profiles, in an + * ExportProfilesServiceRequest of pprof-format/otel. + * + * The request has one profile per sample type, since an OTel profile has a + * single sample type, and all of them share one dictionary. A sample count + * is the exception when every sampling event is a single timestamped + * observation (time profiles normalized with samplePerContext): it is then + * folded into the profile of the period type, whose observations include + * every sampling event, even zero-valued ones such as idle samples' wall + * time. Otherwise the count keeps a profile of its own, so no count is lost. + * + * Samples with the same stack, attributes and link are merged into one, as + * the OTel format requires: timestamped samples accumulate a value and a + * timestamp per observation, and samples without a timestamp are summed into + * one value. Other zero values are left out of the profile of their sample + * type: e.g. zero CPU time, or the wall time of the synthetic non-JS threads + * sample. + * + * OtelProfilesBuilder collects several profiles, e.g. time and heap, into one + * request; serializeTimeProfileOtel() and serializeHeapProfileOtel() are + * shorthands for a request with a single profile. + * + * Frames whose script declares a source map that can't be found locally, and + * so are candidates for server-side unminification, are marked with a + * location attribute, and their profiles with a profile attribute. (pprof + * carries this as a profile comment, which OTel reserves for human-readable + * text.) + */ + +import { + AnyValueInput, + ExportProfilesServiceRequest, + FunctionInput, + InstrumentationScopeInput, + KeyValueAndUnitInput, + KeyValueInput, + LinkInput, + LocationInput, + ProfileInput, + SampleInput, + StackInput, + StringTable, +} from 'pprof-format/otel'; +import { + Frame, + frameFunctionName, + HeapProfileNormalizerOptions, + ProfileMetadata, + ProfileVisitor, + NormalizedSample, + SampleType, + SampleLabel, + TimeProfileNormalizerOptions, + visitHeapProfile, + visitTimeProfile, +} from './profile-normalizer'; +import { + AllocationProfileNode, + AllocationProfileNodeWithStats, + TimeProfile, +} from './v8-types'; + +/** + * Trace and span IDs that link a sample to a span. The builder copies them, + * and treats a link whose trace ID or span ID is all zeros, which is invalid, + * as no link. + */ +export interface SampleLink { + /** 16 bytes. */ + traceId: Uint8Array; + /** 8 bytes. */ + spanId: Uint8Array; +} + +/** + * Turns some of a sample's labels into a link to a span. Returns the link, if + * any, and the labels to keep as the sample's attributes. + */ +export type LinkExtractor = (labels: SampleLabel[]) => { + link?: SampleLink; + labels: SampleLabel[]; +}; + +export interface OtelSerializerOptions { + /** Attributes of the resource that produced the profile. */ + resourceAttributes?: KeyValueInput[]; + /** The instrumentation scope. Defaults to `@datadog/pprof`. */ + scope?: InstrumentationScopeInput; + /** Without one, all labels become attributes. */ + extractLink?: LinkExtractor; +} + +/** + * An attribute of a profile. The value can be structured, e.g. an + * `arrayValue` or a `kvlistValue`. + */ +export interface ProfileAttribute { + key: string; + value: AnyValueInput; +} + +/** Options for one profile added to an OtelProfilesBuilder. */ +export interface OtelProfileOptions { + /** + * Attributes of each OTel profile that the visit adds. They are interned in + * the request's attribute table, so repeating them on several profiles costs + * an index each. + */ + attributes?: ProfileAttribute[]; + /** + * The profile in another format, set as the original payload of each OTel + * profile that the visit adds. The payload is not copied. + */ + originalPayload?: {format: string; payload: Uint8Array}; +} + +const DEFAULT_SCOPE: InstrumentationScopeInput = {name: '@datadog/pprof'}; + +const SAMPLE_COUNT: SampleType = {type: 'sample', unit: 'count'}; + +/** The OTel profiles of one normalized profile, by its sample type index. */ +interface ProfileLayout { + profiles: ProfileInput[]; + /** Per profile: samples by identity, for merging. */ + samplesByIdentity: Array>; + /** + * Index of a sample count type that can be folded into the period type's + * profile, or -1. + */ + countIndex: number; + /** Index of the period type. */ + periodIndex: number; + /** + * Whether the count can still be folded: every sampling event so far was a + * single timestamped observation. + */ + foldable: boolean; +} + +// Boolean attributes marking frames with a missing source map, and profiles +// that have such frames. There is no semantic convention for this, so the keys +// are Datadog's own. +// TODO: confirm the keys with the Datadog profiling backend. +export const FRAME_MISSING_SOURCE_MAP = 'datadog.frame.missing_source_map'; +export const PROFILE_HAS_MISSING_SOURCE_MAPS = + 'datadog.profile.has_missing_source_maps'; + +/** + * Builds one ExportProfilesServiceRequest from any number of normalized + * profiles, which share the request's dictionary: e.g. the time and heap + * profiles of one collection cycle. Each profile is added through a visitor + * from visitor(), so that whatever drives the visit (the normalizer, or a + * profiler emitting samples itself) needs no knowledge of the OTel format. + * + * Functions, locations, stacks, attributes and links are deduplicated across + * all profiles, and each dictionary table starts with its zero value. A + * sample whose stack has no frames refers to the zero-value stack. + * + * All profiles of the request get the same time range, the union of their + * own: one collection cycle's profiles then cover the same period, even a heap + * profile, which is a snapshot at a single instant. + * + * Visitors run synchronously, and build() can be called after any number of + * completed visits. A visit that throws leaves partial data in the builder, + * so discard the builder after a failed visit. + */ +export class OtelProfilesBuilder { + private readonly strings = new StringTable(); + private readonly functionTable: FunctionInput[] = [{}]; + private readonly locationTable: LocationInput[] = [{}]; + private readonly stackTable: StackInput[] = [{}]; + private readonly attributeTable: KeyValueAndUnitInput[] = [{}]; + // An all-zero link, with IDs of the lengths codecs expect. + private readonly linkTable: LinkInput[] = [ + {traceId: new Uint8Array(16), spanId: new Uint8Array(8)}, + ]; + private readonly functionIndices = new Map(); + private readonly locationIndices = new Map(); + private readonly stackIndices = new Map(); + private readonly attributeIndices = new Map(); + private readonly linkIndices = new Map(); + private readonly profiles: ProfileInput[] = []; + + constructor(private readonly options: OtelSerializerOptions = {}) {} + + /** + * Returns a visitor that adds one normalized profile to the request, as one + * OTel profile per sample type. Use a new visitor for each profile. + */ + visitor(options: OtelProfileOptions = {}): ProfileVisitor { + let layout: ProfileLayout = { + profiles: [], + samplesByIdentity: [], + countIndex: -1, + periodIndex: -1, + foldable: false, + }; + return { + metadata: metadata => { + layout = this.layout(metadata); + const attributeIndices = (options.attributes ?? []).map(attribute => + this.profileAttributeIndex(attribute), + ); + for (const profile of layout.profiles) { + profile.attributeIndices = [...attributeIndices]; + if (options.originalPayload) { + profile.originalPayloadFormat = options.originalPayload.format; + profile.originalPayload = options.originalPayload.payload; + } + } + this.profiles.push(...layout.profiles); + }, + frame: frame => this.locationIndex(frame), + stack: frameIds => this.stackIndex(frameIds), + sample: sample => this.addSample(sample, layout), + end: ({hasMissingMapFiles}) => { + let profiles = layout.profiles; + if (layout.countIndex >= 0 && layout.foldable) { + const count = profiles[layout.countIndex]; + this.profiles.splice(this.profiles.indexOf(count), 1); + profiles = profiles.filter(profile => profile !== count); + } + if (hasMissingMapFiles) { + const attribute = this.trueAttribute(PROFILE_HAS_MISSING_SOURCE_MAPS); + for (const profile of profiles) { + profile.attributeIndices!.push(attribute); + } + } + }, + }; + } + + /** + * Creates the OTel profiles for a normalized profile's sample types. + * + * A sample count type can be folded into the profile of the period type + * (wall time, for time profiles) instead of keeping a profile of its own: + * that profile gets an observation for every sampling event, even one whose + * value is zero, so the count is its number of observations. That keeps + * idle samples, which have zero wall time, as timestamped evidence that + * sampling happened, without repeating every sample in a count profile. + * The fold is only lossless if every sampling event is a single timestamped + * observation, so the count profile is built anyway, and dropped at the end + * of the visit only if that turned out to be the case. + */ + private layout(metadata: ProfileMetadata): ProfileLayout { + const {sampleTypes, periodType} = metadata; + const sameType = (a: SampleType, b: SampleType) => + a.type === b.type && a.unit === b.unit; + let countIndex = sampleTypes.findIndex(t => sameType(t, SAMPLE_COUNT)); + const periodIndex = sampleTypes.findIndex(t => sameType(t, periodType)); + if (periodIndex < 0 || periodIndex === countIndex) countIndex = -1; + + const otelPeriodType = this.valueType(periodType); + const profiles = sampleTypes.map(sampleType => ({ + sampleType: this.valueType(sampleType), + periodType: otelPeriodType, + period: metadata.period, + // Sample timestamps should lie within the profile's time range, so it + // starts at the start of collection when that is known. + timeUnixNano: metadata.startTimeNanos ?? metadata.timeNanos, + durationNano: metadata.durationNanos ?? 0, + samples: [], + })); + return { + profiles, + samplesByIdentity: profiles.map(() => new Map()), + countIndex, + periodIndex, + foldable: countIndex >= 0, + }; + } + + /** Builds the request from the profiles added so far. */ + build(): ExportProfilesServiceRequest { + return new ExportProfilesServiceRequest({ + resourceProfiles: [ + { + resource: {attributes: this.options.resourceAttributes ?? []}, + scopeProfiles: [ + { + scope: this.options.scope ?? DEFAULT_SCOPE, + profiles: withCommonTimeRange(this.profiles), + }, + ], + }, + ], + dictionary: { + functionTable: this.functionTable, + locationTable: this.locationTable, + linkTable: this.linkTable, + stringTable: this.strings, + attributeTable: this.attributeTable, + stackTable: this.stackTable, + }, + }); + } + + private addSample(sample: NormalizedSample, layout: ProfileLayout) { + const {profiles, samplesByIdentity, countIndex, periodIndex} = layout; + // With a foldable count, the period type's profile observes every + // sampling event, even one with a zero value, such as an idle sample's + // wall time. + const count = countIndex >= 0 ? sample.values[countIndex] : 0; + const isSamplingEvent = count > 0; + // Folding is lossless only if the period type's observations are exactly + // the sampling events, one each: every sampling event is a single + // timestamped observation, and no other sample adds an observation. + if ( + isSamplingEvent + ? sample.timestamp === undefined || count !== 1 + : countIndex >= 0 && sample.values[periodIndex] !== 0 + ) { + layout.foldable = false; + } + let labels = sample.labels; + let linkIndex = 0; + if (this.options.extractLink) { + const extracted = this.options.extractLink(labels); + labels = extracted.labels; + if (extracted.link) { + linkIndex = this.linkIndex(extracted.link); + } + } + // Attributes form a set, so order them for the sample's identity. + const attributeIndices = labels + .map(label => this.attributeIndex(label)) + .sort((a, b) => a - b); + const timestamped = sample.timestamp !== undefined; + const identity = `${sample.stackId}|${attributeIndices.join(',')}|${linkIndex}|${timestamped}`; + + sample.values.forEach((value, i) => { + const profile = profiles[i]; + if (value === 0 && !(i === periodIndex && isSamplingEvent)) return; + const samples = samplesByIdentity[i]; + let otelSample = samples.get(identity); + if (otelSample === undefined) { + otelSample = { + stackIndex: sample.stackId, + attributeIndices, + linkIndex, + values: [], + timestampsUnixNano: [], + }; + samples.set(identity, otelSample); + profile.samples!.push(otelSample); + } + if (timestamped) { + otelSample.values!.push(value); + otelSample.timestampsUnixNano!.push(sample.timestamp!); + } else if (otelSample.values!.length === 0) { + otelSample.values!.push(value); + } else { + otelSample.values![0] = Number(otelSample.values![0]) + value; + } + }); + } + + private valueType({type, unit}: {type: string; unit: string}) { + return { + typeStrindex: this.strings.dedup(type), + unitStrindex: this.strings.dedup(unit), + }; + } + + // Functions are identified by what they encode as: their display name and + // resolved file. (After source mapping, one script can map to several + // files, so a script ID doesn't identify a file.) + private functionIndex(frame: Frame): number { + const name = frameFunctionName(frame); + const key = `${frame.file.length}:${frame.file}:${name}`; + let index = this.functionIndices.get(key); + if (index === undefined) { + index = + this.functionTable.push({ + nameStrindex: this.strings.dedup(name), + filenameStrindex: this.strings.dedup(frame.file), + }) - 1; + this.functionIndices.set(key, index); + } + return index; + } + + private locationIndex(frame: Frame): number { + const functionIndex = this.functionIndex(frame); + const line = frame.line ?? 0; + const column = frame.column ?? 0; + const key = `${functionIndex}:${line}:${column}:${frame.missingMapFile}`; + let index = this.locationIndices.get(key); + if (index === undefined) { + index = + this.locationTable.push({ + lines: [{functionIndex, line, column}], + attributeIndices: frame.missingMapFile + ? [this.trueAttribute(FRAME_MISSING_SOURCE_MAP)] + : [], + }) - 1; + this.locationIndices.set(key, index); + } + return index; + } + + private stackIndex(locationIndices: number[]): number { + // A sample without frames, e.g. an event, has the zero-value stack + if (locationIndices.length === 0) return 0; + const key = locationIndices.join(','); + let index = this.stackIndices.get(key); + if (index === undefined) { + index = this.stackTable.push({locationIndices}) - 1; + this.stackIndices.set(key, index); + } + return index; + } + + private attributeIndex([key, value]: SampleLabel): number { + // JSON-encoding the tuple keeps the key unambiguous whatever the strings + // contain. + const internKey = JSON.stringify([key, typeof value, String(value)]); + return this.internAttribute(internKey, () => ({ + keyStrindex: this.strings.dedup(key), + value: anyValue(value), + })); + } + + private profileAttributeIndex({key, value}: ProfileAttribute): number { + // A different shape from the label keys above, so the two never collide + return this.internAttribute( + JSON.stringify(['profile', key, canonicalValue(value)]), + () => ({keyStrindex: this.strings.dedup(key), value}), + ); + } + + private trueAttribute(key: string): number { + return this.internAttribute( + JSON.stringify([key, 'boolean', 'true']), + () => ({ + keyStrindex: this.strings.dedup(key), + value: {boolValue: true}, + }), + ); + } + + private internAttribute( + key: string, + create: () => KeyValueAndUnitInput, + ): number { + let index = this.attributeIndices.get(key); + if (index === undefined) { + index = this.attributeTable.push(create()) - 1; + this.attributeIndices.set(key, index); + } + return index; + } + + private linkIndex(link: SampleLink): number { + if (link.traceId.length !== 16 || link.spanId.length !== 8) { + throw new TypeError( + `A link needs a 16-byte trace ID and an 8-byte span ID, not ${link.traceId.length} and ${link.spanId.length} bytes`, + ); + } + // A trace or span ID of all zeros is invalid, so either one means no + // link: index 0 + if (isAllZero(link.traceId) || isAllZero(link.spanId)) return 0; + const key = `${hex(link.traceId)}:${hex(link.spanId)}`; + let index = this.linkIndices.get(key); + if (index === undefined) { + // Copy the IDs, in case the caller reuses its arrays + index = + this.linkTable.push({ + traceId: Uint8Array.from(link.traceId), + spanId: Uint8Array.from(link.spanId), + }) - 1; + this.linkIndices.set(key, index); + } + return index; + } +} + +/** + * Converts a label value to an attribute value. Integers become int64 + * attributes: like pprof's int64 labels, bigints of 2^63 and above (such as + * large unsigned 64-bit span IDs) wrap to negative values, and bits beyond 64 + * are lost. Labels whose exact unsigned value matters should be strings, or + * links. + */ +function anyValue(value: SampleLabel[1]): AnyValueInput { + if (typeof value === 'string') return {stringValue: value}; + if (typeof value === 'number' && !Number.isInteger(value)) { + return {doubleValue: value}; + } + return {intValue: value}; +} + +/** + * Returns the profiles, each with the union of their time ranges. The + * profiles are copied, so that each keeps its own range for later builds. + */ +function withCommonTimeRange(profiles: ProfileInput[]): ProfileInput[] { + if (profiles.length < 2) return profiles; + let start: bigint | undefined; + let end: bigint | undefined; + for (const profile of profiles) { + const profileStart = BigInt(profile.timeUnixNano ?? 0); + const profileEnd = profileStart + BigInt(profile.durationNano ?? 0); + if (start === undefined || profileStart < start) start = profileStart; + if (end === undefined || profileEnd > end) end = profileEnd; + } + const timeUnixNano = start!; + const durationNano = end! - start!; + return profiles.map(profile => ({...profile, timeUnixNano, durationNano})); +} + +/** + * A JSON-serializable form of an attribute value that is equal for equal + * values, and different for values of different types, e.g. int 1 and double + * 1. + */ +function canonicalValue(value: AnyValueInput): unknown { + if (value.stringValue !== undefined) return ['s', value.stringValue]; + if (value.boolValue !== undefined) return ['b', value.boolValue]; + if (value.intValue !== undefined) return ['i', String(value.intValue)]; + if (value.doubleValue !== undefined) return ['d', String(value.doubleValue)]; + if (value.bytesValue !== undefined) return ['y', hex(value.bytesValue)]; + if (value.arrayValue !== undefined) { + return ['a', (value.arrayValue.values ?? []).map(canonicalValue)]; + } + if (value.kvlistValue !== undefined) { + return [ + 'k', + (value.kvlistValue.values ?? []).map(({key, value}) => [ + key, + value === undefined ? null : canonicalValue(value), + ]), + ]; + } + if (value.stringValueStrindex !== undefined) { + return ['si', value.stringValueStrindex]; + } + return ['empty']; +} + +function isAllZero(bytes: Uint8Array): boolean { + return bytes.every(byte => byte === 0); +} + +function hex(bytes: Uint8Array): string { + return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length).toString( + 'hex', + ); +} + +/** + * Serializes a V8 time profile as OpenTelemetry profiles: wall time, CPU + * time if the profile has it, and a sample count unless it can be folded into + * the wall time profile. Pass `samplePerContext` to give every sample with a + * context a timestamp, which lets the count be folded. + */ +export function serializeTimeProfileOtel( + prof: TimeProfile, + options: TimeProfileNormalizerOptions & OtelSerializerOptions, +): ExportProfilesServiceRequest { + const builder = new OtelProfilesBuilder(options); + visitTimeProfile(prof, builder.visitor(), options); + return builder.build(); +} + +/** + * Serializes a V8 heap profile as OpenTelemetry profiles: in-use objects and + * space and, in allocation mode, allocated objects and space. + */ +export function serializeHeapProfileOtel( + prof: AllocationProfileNode | AllocationProfileNodeWithStats, + options: HeapProfileNormalizerOptions & OtelSerializerOptions, +): ExportProfilesServiceRequest { + const builder = new OtelProfilesBuilder(options); + visitHeapProfile(prof, builder.visitor(), options); + return builder.build(); +} diff --git a/ts/src/profile-normalizer.ts b/ts/src/profile-normalizer.ts index 5e2a026c..2cb10892 100644 --- a/ts/src/profile-normalizer.ts +++ b/ts/src/profile-normalizer.ts @@ -143,6 +143,46 @@ export interface VisitResult { hasMissingMapFiles: boolean; } +/** + * Returns a visitor that delivers one visit to several visitors, e.g. to build + * a profile in two formats from a single collection. Each visitor gets the + * frame and stack IDs it returned itself. The visitors share each sample's + * labels, so they must not modify them. + */ +export function teeVisitor(...visitors: ProfileVisitor[]): ProfileVisitor { + // For each visitor, its frame and stack IDs, by the tee's IDs + const frameIds: number[][] = visitors.map(() => []); + const stackIds: number[][] = visitors.map(() => []); + let frameCount = 0; + let stackCount = 0; + return { + metadata: metadata => { + for (const visitor of visitors) visitor.metadata(metadata); + }, + frame: frame => { + for (let i = 0; i < visitors.length; i++) { + frameIds[i].push(visitors[i].frame(frame)); + } + return frameCount++; + }, + stack: ids => { + for (let i = 0; i < visitors.length; i++) { + const ownFrameIds = frameIds[i]; + stackIds[i].push(visitors[i].stack(ids.map(id => ownFrameIds[id]))); + } + return stackCount++; + }, + sample: sample => { + for (let i = 0; i < visitors.length; i++) { + visitors[i].sample({...sample, stackId: stackIds[i][sample.stackId]}); + } + }, + end: result => { + for (const visitor of visitors) visitor.end?.(result); + }, + }; +} + /** A normalized profile collected into arrays. */ export interface NormalizedProfile extends ProfileMetadata, VisitResult { frames: Frame[]; diff --git a/ts/src/profile-serializer.ts b/ts/src/profile-serializer.ts index aecac7a2..264039ab 100644 --- a/ts/src/profile-serializer.ts +++ b/ts/src/profile-serializer.ts @@ -102,19 +102,87 @@ function packLineAndColumn( return (BigInt(column) << 32n) | BigInt(line); } +export interface PprofSerializerOptions { + /** + * Keys of labels with few distinct values, whose Label objects are shared + * between samples to save memory. + */ + lowCardinalityLabels?: string[]; + /** How frame columns are encoded. */ + columnNumbers?: ColumnNumbers; + /** + * Name of a numeric label that receives each sample's timestamp, in + * nanoseconds since the epoch, e.g. `end_timestamp_ns`. pprof samples have no + * timestamps, so without it they are not encoded. + */ + timestampLabel?: string; +} + +/** + * Builds a pprof profile from one normalized profile, delivered to the + * visitor from visitor(): by the normalizer, or by a profiler emitting + * samples itself. pprof samples have no timestamps, so sample timestamps are + * encoded as a label if the options name one, and not at all otherwise. + * + * The visit runs synchronously; call build() after it ends. + */ +export class PprofProfileBuilder { + private readonly profileVisitor: ProfileVisitor; + private readonly buildProfile: () => Profile; + private visited = false; + + constructor(options: PprofSerializerOptions = {}) { + [this.profileVisitor, this.buildProfile] = createPprofVisitor( + options.lowCardinalityLabels ?? [], + options.columnNumbers ?? DEFAULT_COLUMN_NUMBERS, + options.timestampLabel, + ); + } + + /** Returns the visitor for the profile. A builder takes a single visit. */ + visitor(): ProfileVisitor { + if (this.visited) { + throw new Error('A PprofProfileBuilder builds a single profile.'); + } + this.visited = true; + return this.profileVisitor; + } + + /** Builds the profile from the visit. */ + build(): Profile { + return this.buildProfile(); + } +} + /** * Encodes a normalized profile as a profile proto. * * @param visit - function that delivers the normalized profile to a visitor. - * @param lowCardinalityLabels - keys of labels with few distinct values, whose - * Label objects are shared between samples to save memory. */ function serialize( visit: (visitor: ProfileVisitor) => VisitResult, lowCardinalityLabels: string[] = [], columnNumbers: ColumnNumbers = DEFAULT_COLUMN_NUMBERS, ): Profile { + const builder = new PprofProfileBuilder({ + lowCardinalityLabels, + columnNumbers, + }); + visit(builder.visitor()); + return builder.build(); +} + +/** Returns a visitor that collects a pprof profile, and a function to build it. */ +function createPprofVisitor( + lowCardinalityLabels: string[], + columnNumbers: ColumnNumbers, + timestampLabel?: string, +): [ProfileVisitor, () => Profile] { const stringTable = new StringTable(); + const timestampKey = + timestampLabel === undefined + ? undefined + : stringTable.dedup(timestampLabel); const samples: Sample[] = []; const locations: Location[] = []; const stacks: number[][] = []; @@ -127,8 +195,9 @@ function serialize( labelCaches[stringTable.dedup(l)] = new Map(); } let profile: ProfileInput = {}; + let hasMissingMapFiles = false; - const {hasMissingMapFiles} = visit({ + const visitor: ProfileVisitor = { metadata: metadata => { const valueType = (t: SampleType) => new ValueType({ @@ -153,26 +222,39 @@ function serialize( }, stack: frameIds => stacks.push(frameIds) - 1, sample: sample => { + const labels = dedupLabels(buildLabels(sample.labels)); + if (timestampKey !== undefined && sample.timestamp !== undefined) { + // A timestamp is unique to its sample, so its Label isn't shared + labels.push(new Label({key: timestampKey, num: sample.timestamp})); + } samples.push( new Sample({ locationId: stacks[sample.stackId], value: sample.values, - label: dedupLabels(buildLabels(sample.labels)), + label: labels, }), ); }, - }); - - profile.sample = samples; - profile.location = locations; - profile.function = functions; - profile.stringTable = stringTable; + end: result => { + hasMissingMapFiles = result.hasMissingMapFiles; + }, + }; - if (hasMissingMapFiles) { - profile.comment = [stringTable.dedup('dd:has-missing-map-files')]; - } + const build = () => { + const input: ProfileInput = { + ...profile, + sample: samples, + location: locations, + function: functions, + stringTable, + }; + if (hasMissingMapFiles) { + input.comment = [stringTable.dedup('dd:has-missing-map-files')]; + } + return new Profile(input); + }; - return new Profile(profile); + return [visitor, build]; function getLine(frame: Frame): Line { // Both 'pack' and 'emit' carry the column only for frames whose source map diff --git a/ts/src/time-profiler.ts b/ts/src/time-profiler.ts index 40cf3ff2..9d20ac8c 100644 --- a/ts/src/time-profiler.ts +++ b/ts/src/time-profiler.ts @@ -27,7 +27,10 @@ import { import { NormalizedProfile, normalizeTimeProfile, + ProfileVisitor, TimeProfileNormalizerOptions, + visitTimeProfile, + VisitResult, } from './profile-normalizer'; import {SourceMapper} from './sourcemapper/sourcemapper'; import { @@ -302,6 +305,29 @@ export function stopV2( ); } +/** + * Like stop(), but passes the normalized profile to the visitor instead of + * serializing it as pprof, so the caller chooses the output (e.g. a visitor + * of an OtelProfilesBuilder). + */ +export function stopAndVisit( + restart: boolean, + visitor: ProfileVisitor, + options: Pick< + TimeProfileNormalizerOptions, + 'generateLabels' | 'samplePerContext' + > = {}, +): VisitResult { + return stopAndProcess(restart, profile => + visitTimeProfile(profile, visitor, { + ...options, + intervalMicros: gIntervalMicros, + sourceMapper: gSourceMapper, + recomputeSamplingInterval: true, + }), + ); +} + export function getState() { if (!gProfiler) { throw new Error('Wall profiler is not started'); diff --git a/ts/test/test-heap-profiler.ts b/ts/test/test-heap-profiler.ts index 743302f9..8f330f9d 100644 --- a/ts/test/test-heap-profiler.ts +++ b/ts/test/test-heap-profiler.ts @@ -152,6 +152,41 @@ describe('HeapProfiler', () => { assert.ok(normalized.frames.some(f => f.name === '(external)')); }); + it('should pass the profile to a visitor with visitProfile', async () => { + profileStub = sinon + .stub(v8HeapProfiler, 'getAllocationProfile') + .returns(copy(v8HeapProfile)); + memoryUsageStub = sinon.stub(process, 'memoryUsage').returns({ + external: 1024, + rss: 2048, + heapTotal: 4096, + heapUsed: 2048, + arrayBuffers: 512, + }); + const intervalBytes = 1024 * 512; + heapProfiler.start(intervalBytes, 32); + const values: number[][] = []; + let period = 0; + let frames = 0; + const result = heapProfiler.visitProfile({ + metadata: metadata => { + period = metadata.period; + }, + frame: () => frames++, + stack: () => 0, + sample: sample => { + values.push(sample.values); + }, + }); + assert.deepStrictEqual(result, {hasMissingMapFiles: false}); + assert.strictEqual(period, intervalBytes); + assert.ok(frames > 0); + assert.deepStrictEqual( + values, + heapProfileWithExternal.sample.map(s => s.value), + ); + }); + it('should return a profile equal to the expected profile when including all samples', async () => { profileStub = sinon .stub(v8HeapProfiler, 'getAllocationProfile') diff --git a/ts/test/test-otel-serializer.ts b/ts/test/test-otel-serializer.ts new file mode 100644 index 00000000..d6e03037 --- /dev/null +++ b/ts/test/test-otel-serializer.ts @@ -0,0 +1,1224 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as sinon from 'sinon'; +import * as tmp from 'tmp'; +import {Profile} from 'pprof-format'; +import { + ExportProfilesServiceRequest, + KeyValueAndUnit, + ProfilesDictionary, +} from 'pprof-format/otel'; + +import { + FRAME_MISSING_SOURCE_MAP, + OtelProfilesBuilder, + PROFILE_HAS_MISSING_SOURCE_MAPS, + serializeHeapProfileOtel, + serializeTimeProfileOtel, +} from '../src/otel-serializer'; +import { + PprofProfileBuilder, + serializeHeapProfile, + serializeTimeProfile, +} from '../src/profile-serializer'; +import { + teeVisitor, + visitHeapProfile, + visitTimeProfile, +} from '../src/profile-normalizer'; +import {SourceMapper} from '../src/sourcemapper/sourcemapper'; +import { + AllocationProfileNode, + AllocationProfileNodeWithStats, + TimeProfile, + TimeProfileNode, +} from '../src/v8-types'; +import {v8HeapProfile, v8TimeProfile} from './profiles-for-tests'; + +const assert = require('assert'); + +/** Encodes and decodes the request, so that tests check the wire format. */ +function roundTrip( + request: ExportProfilesServiceRequest, +): ExportProfilesServiceRequest { + return ExportProfilesServiceRequest.decode(request.encode()); +} + +function attributeValue(attribute: KeyValueAndUnit): string { + const value = attribute.value!; + if (value.stringValue !== undefined) return value.stringValue; + if (value.intValue !== undefined) return String(value.intValue); + return String(value.doubleValue); +} + +/** + * Totals of each OTel profile's values, keyed by sample type, then by stack + * (function names, leaf first) and sorted attributes. Zero totals, such as + * idle wall time, are left out. + */ +function otelTotals(request: ExportProfilesServiceRequest) { + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + const totals = new Map>(); + const [profiles] = request.resourceProfiles[0].scopeProfiles.map( + s => s.profiles, + ); + for (const profile of profiles) { + const type = strings[profile.sampleType!.typeStrindex]; + const byKey = new Map(); + for (const sample of profile.samples) { + const stack = dictionary.stackTable[ + sample.stackIndex + ].locationIndices.map(location => { + const line = dictionary.locationTable[location].lines[0]; + return strings[ + dictionary.functionTable[line.functionIndex].nameStrindex + ]; + }); + const attributes = sample.attributeIndices + .map(i => dictionary.attributeTable[i]) + .map(a => `${strings[a.keyStrindex]}=${attributeValue(a)}`) + .sort(); + const key = JSON.stringify([stack, attributes]); + const total = sample.values.reduce((t, v) => t + Number(v), 0); + if (total !== 0) byKey.set(key, (byKey.get(key) ?? 0) + total); + } + totals.set(type, byKey); + } + return totals; +} + +/** The profiles of the request's only scope. */ +function profilesOf(request: ExportProfilesServiceRequest) { + return request.resourceProfiles[0].scopeProfiles[0].profiles; +} + +/** The sample types of the request's profiles, in order. */ +function profileTypes(request: ExportProfilesServiceRequest): string[] { + const strings = request.dictionary!.stringTable.strings; + return profilesOf(request).map(p => strings[p.sampleType!.typeStrindex]); +} + +function profileOfType(request: ExportProfilesServiceRequest, type: string) { + const index = profileTypes(request).indexOf(type); + assert.ok(index >= 0, `has a ${type} profile`); + return profilesOf(request)[index]; +} + +/** The name of the leaf function of a stack. */ +function leafName( + request: ExportProfilesServiceRequest, + sample: {stackIndex: number}, +): string { + const dictionary = request.dictionary!; + const location = dictionary.stackTable[sample.stackIndex].locationIndices[0]; + const fn = dictionary.locationTable[location].lines[0].functionIndex; + return dictionary.stringTable.strings[ + dictionary.functionTable[fn].nameStrindex + ]; +} + +/** The totals of a pprof profile, as otelTotals(), leaving out zero values. */ +function pprofTotals(profile: Profile) { + const strings = profile.stringTable.strings; + const totals = new Map>(); + profile.sampleType.forEach((sampleType, i) => { + const byKey = new Map(); + for (const sample of profile.sample) { + const value = Number(sample.value[i]); + if (value === 0) continue; + const stack = sample.locationId.map(id => { + const location = profile.location[Number(id) - 1]; + const fn = profile.function[Number(location.line[0].functionId) - 1]; + return strings[Number(fn.name)]; + }); + const attributes = sample.label + .map( + l => + `${strings[Number(l.key)]}=${l.str ? strings[Number(l.str)] : String(l.num)}`, + ) + .sort(); + const key = JSON.stringify([stack, attributes]); + byKey.set(key, (byKey.get(key) ?? 0) + value); + } + totals.set(strings[Number(sampleType.type)], byKey); + }); + return totals; +} + +/** + * Checks that an OTel request has the same totals as a pprof profile. If the + * OTel request folded the sample count into the wall profile, the count must + * equal the number of wall observations. + */ +function assertSameTotals(otel: ExportProfilesServiceRequest, pprof: Profile) { + const expected = pprofTotals(pprof); + if (expected.has('sample') && !profileTypes(otel).includes('sample')) { + expected.delete('sample'); + const sampleCount = pprof.sample.reduce( + (total, sample) => total + Number(sample.value[0]), + 0, + ); + const observations = profileOfType(otel, 'wall').samples.reduce( + (total, sample) => total + sample.values.length, + 0, + ); + assert.strictEqual(observations, sampleCount, 'wall observations'); + } + assert.deepStrictEqual(otelTotals(otel), expected); +} + +function makeLeaf( + name: string, + props: Partial = {}, +): TimeProfileNode { + return { + name, + scriptName: 'script.js', + scriptId: 1, + lineNumber: 1, + columnNumber: 1, + hitCount: 1, + children: [], + ...props, + }; +} + +function makeTimeProfile(children: TimeProfileNode[]): TimeProfile { + return { + startTime: 0, + endTime: 10 * 1000 * 1000, + hasCpuTime: true, + topDownRoot: { + name: '(root)', + scriptName: 'root', + scriptId: 0, + lineNumber: 0, + columnNumber: 0, + hitCount: 0, + children, + }, + }; +} + +/** + * A profile with labelled and unlabelled contexts, idle, GC and CPU time. As + * in profiles from the native profiler with contexts enabled, each node has + * exactly one context per hit. + */ +function richTimeProfile(): TimeProfile { + const prof = makeTimeProfile([ + makeLeaf('handler', { + hitCount: 2, + children: [ + makeLeaf('query', { + scriptId: 2, + hitCount: 3, + contexts: [ + { + context: {endpoint: 'GET /a', span: 1n}, + timestamp: 1n, + cpuTime: 10, + }, + { + context: {endpoint: 'GET /a', span: 1n}, + timestamp: 2n, + cpuTime: 20, + }, + { + context: {endpoint: 'GET /b', span: 2n}, + timestamp: 3n, + cpuTime: 30, + }, + ], + }), + ], + contexts: [ + {context: {}, timestamp: 4n, cpuTime: 40}, + {context: {endpoint: 'GET /a', retries: 3}, timestamp: 5n, cpuTime: 50}, + ], + }), + makeLeaf('(idle)', { + scriptId: 3, + hitCount: 2, + contexts: [ + {context: {endpoint: 'GET /a'}, timestamp: 6n, cpuTime: 0}, + {context: {}, timestamp: 7n, cpuTime: 5}, + ], + }), + makeLeaf('(garbage collector)', { + scriptId: 4, + hitCount: 2, + contexts: [ + {context: {}, timestamp: 8n, cpuTime: 0}, + {context: {}, timestamp: 9n, cpuTime: 60}, + ], + }), + ]); + prof.nonJSThreadsCpuTime = 500; + return prof; +} + +describe('otel-serializer', () => { + let dateStub: sinon.SinonStub<[], number>; + + before(() => { + dateStub = sinon.stub(Date, 'now').returns(0); + }); + after(() => { + dateStub.restore(); + }); + + describe('serializeTimeProfileOtel', () => { + for (const [name, prof, samplePerContext, folded] of [ + // No contexts: every sample is summed, so the count is kept + ['the fixture', v8TimeProfile, false, false], + // Unlabelled contexts are summed without samplePerContext + ['a profile with contexts', richTimeProfile(), false, false], + [ + 'a profile with contexts, with a sample per context', + richTimeProfile(), + true, + true, + ], + ] as Array<[string, TimeProfile, boolean, boolean]>) { + it(`has the same totals as the pprof serializer for ${name}`, () => { + const otel = roundTrip( + serializeTimeProfileOtel(prof, { + intervalMicros: 1000, + samplePerContext, + }), + ); + const pprof = Profile.decode(serializeTimeProfile(prof, 1000).encode()); + assertSameTotals(otel, pprof); + assert.strictEqual( + !profileTypes(otel).includes('sample'), + folded, + 'sample count folded', + ); + }); + } + + it('folds the count into wall time when every sample is an observation', () => { + const request = roundTrip( + serializeTimeProfileOtel(richTimeProfile(), { + intervalMicros: 1000, + samplePerContext: true, + }), + ); + const strings = request.dictionary!.stringTable.strings; + const profiles = profilesOf(request); + assert.deepStrictEqual( + profiles.map(p => [ + strings[p.sampleType!.typeStrindex], + strings[p.sampleType!.unitStrindex], + ]), + [ + ['wall', 'nanoseconds'], + ['cpu', 'nanoseconds'], + ], + ); + for (const profile of profiles) { + assert.strictEqual(strings[profile.periodType!.typeStrindex], 'wall'); + assert.strictEqual(Number(profile.period), 1000 * 1000); + assert.strictEqual( + Number(profile.durationNano), + 10 * 1000 * 1000 * 1000, + ); + } + }); + + it('keeps the count when a sampling event is summed', () => { + // Seven unlabelled idle hits are summed into one sample with zero wall + // time, from which the count couldn't be recovered + const prof = makeTimeProfile([ + makeLeaf('(idle)', { + hitCount: 7, + contexts: Array.from({length: 7}, (_, i) => ({ + context: {}, + timestamp: BigInt(i + 1), + cpuTime: i === 0 ? 1 : 0, + })), + }), + ]); + const request = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000}), + ); + assert.deepStrictEqual(profileTypes(request), ['sample', 'wall', 'cpu']); + assert.deepStrictEqual( + profileOfType(request, 'sample').samples.map(s => s.values.map(Number)), + [[7]], + ); + }); + + it('starts the profiles at the start of collection', () => { + // 1.7e18 nanoseconds is beyond the precision of a double + dateStub.returns(1700000000123); + try { + const prof = { + ...makeTimeProfile([ + makeLeaf('foo', { + contexts: [{context: {}, timestamp: 1700000000005_000n}], + }), + ]), + startTime: 5_000_000, + endTime: 15_000_000, + epochOffset: 1700000000000_000 - 5_000_000, + }; + const request = roundTrip( + serializeTimeProfileOtel(prof, { + intervalMicros: 1000, + samplePerContext: true, + }), + ); + const wall = profileOfType(request, 'wall'); + assert.strictEqual(wall.timeUnixNano, 1700000000000_000_000n); + const [timestamp] = wall.samples[0].timestampsUnixNano; + const start = BigInt(wall.timeUnixNano); + assert.ok(timestamp >= start); + assert.ok(timestamp < start + BigInt(wall.durationNano)); + } finally { + dateStub.returns(0); + } + }); + + it('starts each dictionary table with its zero value', () => { + const {dictionary} = roundTrip( + serializeTimeProfileOtel(richTimeProfile(), {intervalMicros: 1000}), + ); + const d = dictionary as ProfilesDictionary; + assert.strictEqual(d.stringTable.strings[0], ''); + assert.strictEqual(d.functionTable[0].nameStrindex, 0); + assert.deepStrictEqual(d.locationTable[0].lines, []); + assert.deepStrictEqual(d.stackTable[0].locationIndices, []); + assert.strictEqual(d.attributeTable[0].value, undefined); + assert.deepStrictEqual(d.linkTable[0].traceId, new Uint8Array(16)); + assert.deepStrictEqual(d.linkTable[0].spanId, new Uint8Array(8)); + assert.strictEqual(d.mappingTable.length, 1); + }); + + it('merges timestamped samples with the same stack and attributes', () => { + const request = roundTrip( + serializeTimeProfileOtel(richTimeProfile(), { + intervalMicros: 1000, + samplePerContext: true, + }), + ); + const dictionary = request.dictionary!; + const query = profileOfType(request, 'wall').samples.find( + sample => + leafName(request, sample) === 'query' && + sample.attributeIndices.some( + i => attributeValue(dictionary.attributeTable[i]) === 'GET /a', + ), + )!; + assert.deepStrictEqual(query.values.map(Number), [1e6, 1e6]); + assert.deepStrictEqual(query.timestampsUnixNano, [1000, 2000]); + }); + + it('sums samples without a timestamp into one value', () => { + const prof = makeTimeProfile([makeLeaf('foo', {hitCount: 5})]); + const request = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000}), + ); + assert.deepStrictEqual( + profileOfType(request, 'sample').samples.map(s => s.values.map(Number)), + [[5]], + ); + const wall = profileOfType(request, 'wall'); + assert.deepStrictEqual( + wall.samples.map(s => s.values.map(Number)), + [[5e6]], + ); + assert.deepStrictEqual(wall.samples[0].timestampsUnixNano, []); + }); + + it('keeps idle samples in the wall profile with zero wall time', () => { + const request = roundTrip( + serializeTimeProfileOtel(richTimeProfile(), { + intervalMicros: 1000, + samplePerContext: true, + }), + ); + const wall = profileOfType(request, 'wall'); + const cpu = profileOfType(request, 'cpu'); + // Idle samples are timestamped evidence of sampling, without wall time + const idle = wall.samples.filter(s => leafName(request, s) === '(idle)'); + assert.deepStrictEqual( + idle.flatMap(s => s.values.map(Number)), + [0, 0], + ); + assert.deepStrictEqual( + idle.flatMap(s => s.timestampsUnixNano.map(Number)).sort(), + [6000, 7000], + ); + // Their CPU time is kept; zero CPU time is left out + assert.deepStrictEqual( + cpu.samples + .filter(s => leafName(request, s) === '(idle)') + .flatMap(s => s.values.map(Number)), + [5], + ); + // Non-JS threads aren't sampling events: only CPU time, no wall time + const nonJS = (s: {stackIndex: number}) => + leafName(request, s) === 'Non JS threads activity'; + assert.ok(!wall.samples.some(nonJS)); + assert.ok(cpu.samples.some(nonJS)); + }); + + it('turns labels into links with extractLink', () => { + const traceId = Uint8Array.from({length: 16}, (_, i) => i + 1); + const request = roundTrip( + serializeTimeProfileOtel(richTimeProfile(), { + intervalMicros: 1000, + extractLink: labels => { + const span = labels.find(([key]) => key === 'span'); + return { + link: span && { + traceId, + spanId: Buffer.from( + span[1].toString(16).padStart(16, '0'), + 'hex', + ), + }, + labels: labels.filter(([key]) => key !== 'span'), + }; + }, + }), + ); + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + assert.strictEqual(dictionary.linkTable.length, 3); + assert.deepStrictEqual(dictionary.linkTable[1].traceId, traceId); + assert.ok( + dictionary.attributeTable.every(a => strings[a.keyStrindex] !== 'span'), + ); + const spanIds = dictionary.linkTable + .slice(1) + .map(link => Number(Buffer.from(link.spanId).readBigUInt64BE())); + assert.deepStrictEqual(spanIds.sort(), [1, 2]); + const linked = profileOfType(request, 'wall').samples.filter( + s => s.linkIndex !== 0, + ); + assert.ok(linked.length > 0); + for (const sample of linked) { + assert.ok(sample.linkIndex < dictionary.linkTable.length); + } + }); + + it('validates and copies link IDs', () => { + const prof = makeTimeProfile([ + makeLeaf('foo', {contexts: [{context: {span: 1}, timestamp: 1n}]}), + ]); + const serialize = (link: {traceId: Uint8Array; spanId: Uint8Array}) => + serializeTimeProfileOtel(prof, { + intervalMicros: 1000, + extractLink: labels => ({link, labels}), + }); + assert.throws( + () => + serialize({traceId: new Uint8Array(8), spanId: new Uint8Array(8)}), + TypeError, + ); + // A trace or span ID of all zeros is invalid, so either means no link + for (const [traceByte, spanByte] of [ + [0, 0], + [0, 1], + [1, 0], + ]) { + const request = roundTrip( + serialize({ + traceId: new Uint8Array(16).fill(traceByte), + spanId: new Uint8Array(8).fill(spanByte), + }), + ); + assert.strictEqual( + request.dictionary!.linkTable.length, + 1, + `trace ${traceByte}, span ${spanByte}`, + ); + assert.strictEqual( + profileOfType(request, 'wall').samples[0].linkIndex, + 0, + ); + } + // A caller reusing its arrays doesn't change links already added + const spanId = new Uint8Array(8).fill(1); + const request = serialize({traceId: new Uint8Array(16).fill(1), spanId}); + spanId.fill(2); + assert.deepStrictEqual( + roundTrip(request).dictionary!.linkTable[1].spanId, + new Uint8Array(8).fill(1), + ); + }); + + it('keeps attributes apart whatever their keys and values contain', () => { + const prof = makeTimeProfile([ + makeLeaf('foo', { + hitCount: 2, + contexts: [ + {context: {a: 'b\0string\0c'}, timestamp: 1n}, + {context: {'a\0string\0b': 'c'}, timestamp: 2n}, + ], + }), + ]); + const request = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000}), + ); + assert.strictEqual(request.dictionary!.attributeTable.length, 3); + assert.strictEqual(profileOfType(request, 'wall').samples.length, 2); + }); + + it('sets the resource attributes and scope', () => { + const request = roundTrip( + serializeTimeProfileOtel(v8TimeProfile, { + intervalMicros: 1000, + resourceAttributes: [ + {key: 'service.name', value: {stringValue: 'svc'}}, + ], + scope: {name: 'my-profiler', version: '1.2.3'}, + }), + ); + const [resourceProfiles] = request.resourceProfiles; + assert.strictEqual( + resourceProfiles.resource!.attributes[0].value!.stringValue, + 'svc', + ); + assert.strictEqual( + resourceProfiles.scopeProfiles[0].scope!.name, + 'my-profiler', + ); + assert.strictEqual( + resourceProfiles.scopeProfiles[0].scope!.version, + '1.2.3', + ); + const defaultScope = roundTrip( + serializeTimeProfileOtel(v8TimeProfile, {intervalMicros: 1000}), + ).resourceProfiles[0].scopeProfiles[0].scope!; + assert.strictEqual(defaultScope.name, '@datadog/pprof'); + }); + + it('encodes number labels as int or double attributes', () => { + // pprof can only hold integer labels; OTel attributes keep fractions + const prof = makeTimeProfile([ + makeLeaf('foo', { + contexts: [ + {context: {ratio: 0.5, count: 3, big: 2n ** 60n}, timestamp: 1n}, + ], + }), + ]); + const {dictionary} = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000}), + ); + const strings = dictionary!.stringTable.strings; + const byKey = new Map( + dictionary!.attributeTable + .slice(1) + .map(a => [strings[a.keyStrindex], a.value!]), + ); + assert.strictEqual(byKey.get('ratio')!.doubleValue, 0.5); + assert.strictEqual(Number(byKey.get('count')!.intValue), 3); + assert.strictEqual(BigInt(byKey.get('big')!.intValue!), 2n ** 60n); + }); + + it('carries the columns of frames', () => { + const prof = makeTimeProfile([ + makeLeaf('foo', {lineNumber: 3, columnNumber: 7}), + ]); + const {dictionary} = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000}), + ); + const line = dictionary!.locationTable[1].lines[0]; + assert.strictEqual(Number(line.line), 3); + assert.strictEqual(Number(line.column), 7); + }); + + it('keeps functions from different original files apart', () => { + // One generated script whose two positions map to different original + // files, with the same function name and position + const sourceMapper = { + mappingInfo: (loc: {line: number}) => ({ + file: loc.line === 1 ? '/app/a.ts' : '/app/b.ts', + line: 1, + column: 1, + name: 'render', + }), + } as unknown as SourceMapper; + const prof = makeTimeProfile([ + makeLeaf('x', {scriptId: 42, lineNumber: 1, columnNumber: 5}), + makeLeaf('y', {scriptId: 42, lineNumber: 2, columnNumber: 5}), + ]); + const request = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000, sourceMapper}), + ); + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + assert.deepStrictEqual( + dictionary.functionTable + .slice(1) + .map(f => strings[f.filenameStrindex]) + .sort(), + ['/app/a.ts', '/app/b.ts'], + ); + assert.strictEqual(dictionary.locationTable.length, 3); + assert.strictEqual(dictionary.stackTable.length, 3); + }); + + it('shares functions that encode the same, across scripts', () => { + const prof = makeTimeProfile([ + makeLeaf('foo', {scriptId: 1, lineNumber: 5}), + makeLeaf('foo', {scriptId: 2, lineNumber: 7}), + ]); + const {dictionary} = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000}), + ); + assert.strictEqual(dictionary!.functionTable.length, 2); + assert.strictEqual(dictionary!.locationTable.length, 3); + }); + + it('marks frames with missing source maps and their profiles', async () => { + const fs = await import('fs'); + const path = await import('path'); + const dir = tmp.dirSync().name; + const script = path.join(dir, 'missing-map.js'); + fs.writeFileSync(script, '//# sourceMappingURL=nonexistent.js.map\n'); + const sourceMapper = await SourceMapper.create([dir]); + const prof = makeTimeProfile([ + makeLeaf('minified', {scriptName: script}), + makeLeaf('plain', {scriptId: 2}), + ]); + const request = roundTrip( + serializeTimeProfileOtel(prof, {intervalMicros: 1000, sourceMapper}), + ); + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + const attributes = (indices: number[]) => + indices.map(i => { + const attribute = dictionary.attributeTable[i]; + return [strings[attribute.keyStrindex], attribute.value!.boolValue]; + }); + + const marked = dictionary.locationTable + .slice(1) + .map(location => [ + strings[ + dictionary.functionTable[location.lines[0].functionIndex] + .nameStrindex + ], + attributes(location.attributeIndices), + ]); + assert.deepStrictEqual(marked.sort(), [ + ['minified', [[FRAME_MISSING_SOURCE_MAP, true]]], + ['plain', []], + ]); + for (const profile of profilesOf(request)) { + assert.deepStrictEqual(attributes(profile.attributeIndices), [ + [PROFILE_HAS_MISSING_SOURCE_MAPS, true], + ]); + } + }); + + it('marks nothing when all source maps are found', () => { + const request = roundTrip( + serializeTimeProfileOtel(richTimeProfile(), {intervalMicros: 1000}), + ); + assert.ok( + request.dictionary!.locationTable.every( + l => l.attributeIndices.length === 0, + ), + ); + for (const profile of profilesOf(request)) { + assert.deepStrictEqual(profile.attributeIndices, []); + } + }); + }); + + describe('serializeHeapProfileOtel', () => { + it('has the same totals as the pprof serializer', () => { + const otel = roundTrip( + serializeHeapProfileOtel(v8HeapProfile, { + startTimeNanos: 0n, + intervalBytes: 512, + }), + ); + const pprof = Profile.decode( + serializeHeapProfile(v8HeapProfile, 0, 512).encode(), + ); + assertSameTotals(otel, pprof); + assert.deepStrictEqual(profileTypes(otel), [ + 'inuse_objects', + 'inuse_space', + ]); + }); + + it('has allocation profiles in allocation mode', () => { + const withStats = ( + node: AllocationProfileNode, + ): AllocationProfileNodeWithStats => ({ + ...node, + allocations: node.allocations.map(a => ({ + inuseObjects: a.count, + inuseSpaceBytes: a.sizeBytes * a.count, + allocObjects: 2 * a.count, + allocSpaceBytes: 2 * a.sizeBytes * a.count, + })), + children: node.children.map(withStats), + }); + const prof = withStats(v8HeapProfile); + const otel = roundTrip( + serializeHeapProfileOtel(prof, { + startTimeNanos: 0n, + intervalBytes: 512, + allocations: true, + }), + ); + const pprof = Profile.decode( + serializeHeapProfile( + prof, + 0, + 512, + undefined, + undefined, + undefined, + true, + ).encode(), + ); + assertSameTotals(otel, pprof); + assert.strictEqual(profilesOf(otel).length, 4); + }); + }); + + describe('OtelProfilesBuilder', () => { + const heapNode = ( + name: string, + props: Partial = {}, + ): AllocationProfileNode => ({ + name, + scriptName: 'script.js', + scriptId: 1, + lineNumber: 1, + columnNumber: 1, + allocations: [{sizeBytes: 64, count: 2}], + children: [], + ...props, + }); + const heapRoot = (children: AllocationProfileNode[]) => + heapNode('(root)', { + scriptName: '', + scriptId: 0, + allocations: [], + children, + }); + const timeOptions = {intervalMicros: 1000, samplePerContext: true}; + + it('puts several profiles in one request, with the same totals', () => { + const builder = new OtelProfilesBuilder(); + visitTimeProfile(richTimeProfile(), builder.visitor(), timeOptions); + visitHeapProfile(v8HeapProfile, builder.visitor(), { + startTimeNanos: 0n, + intervalBytes: 512, + }); + const combined = roundTrip(builder.build()); + assert.deepStrictEqual(profileTypes(combined), [ + 'wall', + 'cpu', + 'inuse_objects', + 'inuse_space', + ]); + const separate = new Map([ + ...otelTotals( + roundTrip(serializeTimeProfileOtel(richTimeProfile(), timeOptions)), + ), + ...otelTotals( + roundTrip( + serializeHeapProfileOtel(v8HeapProfile, { + startTimeNanos: 0n, + intervalBytes: 512, + }), + ), + ), + ]); + assert.deepStrictEqual(otelTotals(combined), separate); + }); + + it('gives all profiles the union of their time ranges', () => { + // A time profile of 10 seconds from 1e18, and a heap profile taken 2 + // seconds after it ended + const startNanos = 1_000_000_000_000_000_000n; + dateStub.returns(Number((startNanos + 12_000_000_000n) / 1_000_000n)); + try { + const builder = new OtelProfilesBuilder(); + visitTimeProfile( + { + ...makeTimeProfile([ + makeLeaf('foo', { + contexts: [{context: {}, timestamp: 1_000_000_000_005_000n}], + }), + ]), + startTime: 0, + endTime: 10_000_000, + epochOffset: 1_000_000_000_000_000, + }, + builder.visitor(), + timeOptions, + ); + visitHeapProfile(v8HeapProfile, builder.visitor(), { + startTimeNanos: startNanos + 12_000_000_000n, + intervalBytes: 512, + }); + const request = roundTrip(builder.build()); + const ranges = profilesOf(request).map(p => [ + BigInt(p.timeUnixNano), + BigInt(p.durationNano), + ]); + assert.strictEqual(ranges.length, 4); + for (const range of ranges) { + assert.deepStrictEqual(range, [startNanos, 12_000_000_000n]); + } + // A later build starts again from each profile's own range + visitHeapProfile(v8HeapProfile, builder.visitor(), { + startTimeNanos: startNanos + 20_000_000_000n, + intervalBytes: 512, + }); + for (const profile of profilesOf(roundTrip(builder.build()))) { + assert.strictEqual(BigInt(profile.timeUnixNano), startNanos); + assert.strictEqual(BigInt(profile.durationNano), 20_000_000_000n); + } + } finally { + dateStub.returns(0); + } + }); + + it('shares dictionary entries between profiles', () => { + const builder = new OtelProfilesBuilder(); + visitTimeProfile( + makeTimeProfile([ + makeLeaf('shared', {children: [makeLeaf('timeOnly', {scriptId: 2})]}), + ]), + builder.visitor(), + timeOptions, + ); + visitHeapProfile( + heapRoot([ + heapNode('shared', {children: [heapNode('heapOnly', {scriptId: 3})]}), + ]), + builder.visitor(), + {startTimeNanos: 0n, intervalBytes: 512}, + ); + const request = roundTrip(builder.build()); + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + const names = dictionary.functionTable + .slice(1) + .map(f => strings[f.nameStrindex]); + assert.deepStrictEqual(names.sort(), ['heapOnly', 'shared', 'timeOnly']); + // No duplicate locations or stacks + const keys = (items: unknown[]) => + items.map(item => JSON.stringify(item)); + const locations = keys(dictionary.locationTable.map(l => l.lines)); + assert.strictEqual(new Set(locations).size, locations.length); + const stacks = keys(dictionary.stackTable.map(st => st.locationIndices)); + assert.strictEqual(new Set(stacks).size, stacks.length); + // The frame both profiles share is one location, with one stack + const sharedStack = (samples: Array<{stackIndex: number}>) => + samples.find(s => leafName(request, s) === 'shared')!.stackIndex; + assert.strictEqual( + sharedStack(profileOfType(request, 'sample').samples), + sharedStack(profileOfType(request, 'inuse_space').samples), + ); + }); + + it("doesn't fold a count that observations wouldn't match", () => { + // A zero-count sample with a period value would add an observation that + // isn't a sampling event, so the count can't be its number of + // observations + const builder = new OtelProfilesBuilder(); + const visitor = builder.visitor(); + const cpu = {type: 'cpu', unit: 'nanoseconds'}; + visitor.metadata({ + sampleTypes: [{type: 'sample', unit: 'count'}, cpu], + periodType: cpu, + period: 1, + timeNanos: 0n, + }); + const stackId = visitor.stack([ + visitor.frame({name: 'f', file: 'a.js', missingMapFile: false}), + ]); + visitor.sample({stackId, values: [1, 100], labels: [], timestamp: 1n}); + visitor.sample({stackId, values: [0, 50], labels: [], timestamp: 2n}); + visitor.end!({hasMissingMapFiles: false}); + const request = roundTrip(builder.build()); + assert.deepStrictEqual(profileTypes(request), ['sample', 'cpu']); + assert.deepStrictEqual( + profileOfType(request, 'sample').samples.map(s => s.values.map(Number)), + [[1]], + ); + }); + + const cpu = {type: 'cpu', unit: 'nanoseconds'}; + /** Adds a profile with one CPU sample through the visitor. */ + function visitCpuSample( + visitor: ReturnType, + hasMissingMapFiles = false, + ) { + visitor.metadata({ + sampleTypes: [cpu], + periodType: cpu, + period: 1, + timeNanos: 0n, + }); + const stackId = visitor.stack([ + visitor.frame({name: 'f', file: 'a.js', missingMapFile: false}), + ]); + visitor.sample({stackId, values: [10], labels: []}); + visitor.end!({hasMissingMapFiles}); + } + + it("sets a visit's attributes and original payload on its profiles", () => { + const builder = new OtelProfilesBuilder(); + const payload = new TextEncoder().encode('{"start":"now"}'); + visitTimeProfile( + richTimeProfile(), + builder.visitor({ + attributes: [ + {key: 'datadog.profile.snapshot', value: {stringValue: 'periodic'}}, + { + key: 'datadog.profile.endpoint_counts', + value: { + kvlistValue: { + values: [ + {key: 'GET /a', value: {intValue: 3}}, + {key: 'GET /b', value: {intValue: 1}}, + ], + }, + }, + }, + ], + originalPayload: {format: 'datadog-event-json', payload}, + }), + timeOptions, + ); + visitCpuSample(builder.visitor()); + const request = roundTrip(builder.build()); + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + const [wall, cpuOfTime, cpuAlone] = profilesOf(request); + for (const profile of [wall, cpuOfTime]) { + const attributes = profile.attributeIndices.map( + i => dictionary.attributeTable[i], + ); + assert.deepStrictEqual( + attributes.map(a => strings[a.keyStrindex]), + ['datadog.profile.snapshot', 'datadog.profile.endpoint_counts'], + ); + assert.strictEqual(attributes[0].value!.stringValue, 'periodic'); + assert.deepStrictEqual( + attributes[1].value!.kvlistValue!.values.map(kv => [ + kv.key, + Number(kv.value!.intValue), + ]), + [ + ['GET /a', 3], + ['GET /b', 1], + ], + ); + assert.strictEqual(profile.originalPayloadFormat, 'datadog-event-json'); + assert.deepStrictEqual( + Buffer.from(profile.originalPayload), + Buffer.from(payload), + ); + } + // The two profiles of the visit share the attribute table entries + assert.deepStrictEqual(wall.attributeIndices, cpuOfTime.attributeIndices); + // Another visit's profiles get neither + assert.deepStrictEqual(cpuAlone.attributeIndices, []); + assert.strictEqual(cpuAlone.originalPayloadFormat, ''); + assert.strictEqual(cpuAlone.originalPayload.length, 0); + }); + + it('interns profile attributes by value', () => { + const builder = new OtelProfilesBuilder(); + const list = (...ints: number[]) => ({ + arrayValue: {values: ints.map(intValue => ({intValue}))}, + }); + visitCpuSample( + builder.visitor({ + attributes: [ + {key: 'a', value: list(1, 2)}, + {key: 'n', value: {intValue: 1}}, + ], + }), + ); + visitCpuSample( + builder.visitor({ + attributes: [ + {key: 'a', value: list(1, 2)}, + {key: 'a', value: list(2, 1)}, + {key: 'n', value: {doubleValue: 1}}, + ], + }), + ); + const [first, second] = profilesOf(roundTrip(builder.build())); + // Equal values share an entry; order and type tell values apart + assert.strictEqual(first.attributeIndices[0], second.attributeIndices[0]); + assert.strictEqual( + new Set([...first.attributeIndices, ...second.attributeIndices]).size, + 4, + ); + }); + + it('adds the missing source map attribute to the visit attributes', () => { + const builder = new OtelProfilesBuilder(); + visitCpuSample( + builder.visitor({ + attributes: [{key: 'k', value: {stringValue: 'v'}}], + }), + true, + ); + const request = roundTrip(builder.build()); + const dictionary = request.dictionary!; + const strings = dictionary.stringTable.strings; + assert.deepStrictEqual( + profilesOf(request)[0].attributeIndices.map( + i => strings[dictionary.attributeTable[i].keyStrindex], + ), + ['k', PROFILE_HAS_MISSING_SOURCE_MAPS], + ); + }); + + it('gives samples without frames the zero-value stack', () => { + const builder = new OtelProfilesBuilder(); + const visitor = builder.visitor(); + const timeline = {type: 'timeline', unit: 'nanoseconds'}; + visitor.metadata({ + sampleTypes: [timeline], + periodType: timeline, + period: 1, + timeNanos: 0n, + }); + const stackId = visitor.stack([]); + assert.strictEqual(stackId, 0); + visitor.sample({stackId, values: [5], labels: [], timestamp: 1n}); + visitor.end!({hasMissingMapFiles: false}); + const request = roundTrip(builder.build()); + assert.strictEqual(request.dictionary!.stackTable.length, 1); + assert.deepStrictEqual( + profilesOf(request)[0].samples.map(s => [s.stackIndex, s.values]), + [[0, [5]]], + ); + }); + + it('marks only the profiles that have missing source maps', async () => { + const fs = await import('fs'); + const path = await import('path'); + const dir = tmp.dirSync().name; + const script = path.join(dir, 'missing-map.js'); + fs.writeFileSync(script, '//# sourceMappingURL=nonexistent.js.map\n'); + const sourceMapper = await SourceMapper.create([dir]); + const builder = new OtelProfilesBuilder(); + visitTimeProfile( + makeTimeProfile([ + makeLeaf('minified', { + scriptName: script, + contexts: [{context: {}, timestamp: 1n, cpuTime: 1}], + }), + ]), + builder.visitor(), + {...timeOptions, sourceMapper}, + ); + visitHeapProfile(v8HeapProfile, builder.visitor(), { + startTimeNanos: 0n, + intervalBytes: 512, + }); + const request = roundTrip(builder.build()); + const strings = request.dictionary!.stringTable.strings; + const flagged = profilesOf(request).map(p => [ + strings[p.sampleType!.typeStrindex], + p.attributeIndices.length, + ]); + assert.deepStrictEqual(flagged, [ + ['wall', 1], + ['cpu', 1], + ['inuse_objects', 0], + ['inuse_space', 0], + ]); + }); + }); + + describe('package exports', () => { + it('exports the OTel builder and serializers', async () => { + const pprof = await import('../src'); + assert.strictEqual(pprof.OtelProfilesBuilder, OtelProfilesBuilder); + assert.strictEqual( + pprof.serializeTimeProfileOtel, + serializeTimeProfileOtel, + ); + assert.strictEqual( + pprof.serializeHeapProfileOtel, + serializeHeapProfileOtel, + ); + }); + }); + + describe('teeVisitor', () => { + // Time profiles are timestamped when normalized; the stubbed Date.now() + // makes separate visits comparable + it('builds pprof and OTel from one visit, as separate visits would', () => { + const options = {intervalMicros: 1000, samplePerContext: true}; + // Visit the OTel builder first, so that the pprof builder sees + // different frame and stack IDs than it returns itself + const otel = new OtelProfilesBuilder(); + visitHeapProfile(v8HeapProfile, otel.visitor(), { + startTimeNanos: 0n, + intervalBytes: 512, + }); + const pprof = new PprofProfileBuilder(); + visitTimeProfile( + richTimeProfile(), + teeVisitor(otel.visitor(), pprof.visitor()), + options, + ); + + const separateOtel = new OtelProfilesBuilder(); + visitHeapProfile(v8HeapProfile, separateOtel.visitor(), { + startTimeNanos: 0n, + intervalBytes: 512, + }); + visitTimeProfile(richTimeProfile(), separateOtel.visitor(), options); + const separatePprof = new PprofProfileBuilder(); + visitTimeProfile(richTimeProfile(), separatePprof.visitor(), options); + + assert.deepStrictEqual( + Buffer.from(otel.build().encode()), + Buffer.from(separateOtel.build().encode()), + ); + assert.deepStrictEqual( + Buffer.from(pprof.build().encode()), + Buffer.from(separatePprof.build().encode()), + ); + }); + + it('is exported from the package', async () => { + const pprof = await import('../src'); + assert.strictEqual(pprof.teeVisitor, teeVisitor); + }); + }); + + after(() => { + tmp.setGracefulCleanup(); + }); +}); diff --git a/ts/test/test-profile-serializer.ts b/ts/test/test-profile-serializer.ts index 7beed1f7..b4c97d2e 100644 --- a/ts/test/test-profile-serializer.ts +++ b/ts/test/test-profile-serializer.ts @@ -18,9 +18,11 @@ import * as tmp from 'tmp'; import { NON_JS_THREADS_FUNCTION_NAME, + PprofProfileBuilder, serializeHeapProfile, serializeTimeProfile, } from '../src/profile-serializer'; +import {visitTimeProfile} from '../src/profile-normalizer'; import {SourceMapper} from '../src/sourcemapper/sourcemapper'; import {Label, Profile} from 'pprof-format'; import { @@ -862,4 +864,120 @@ describe('profile-serializer', () => { ); }); }); + + describe('PprofProfileBuilder', () => { + it('builds the same profile as serializeTimeProfile', () => { + const builder = new PprofProfileBuilder({columnNumbers: 'emit'}); + visitTimeProfile(v8TimeProfile, builder.visitor(), { + intervalMicros: 1000, + }); + const expected = serializeTimeProfile( + v8TimeProfile, + 1000, + undefined, + false, + undefined, + [], + 'emit', + ); + assert.deepStrictEqual( + Buffer.from(builder.build().encode()), + Buffer.from(expected.encode()), + ); + }); + + it('takes samples from a visitor that emits them itself', () => { + const builder = new PprofProfileBuilder(); + const visitor = builder.visitor(); + const timeline = {type: 'timeline', unit: 'nanoseconds'}; + visitor.metadata({ + sampleTypes: [timeline], + periodType: timeline, + period: 1, + timeNanos: 5n, + }); + const stackId = visitor.stack([]); + visitor.sample({ + stackId, + values: [7], + labels: [ + ['event', 'gc'], + ['end_timestamp_ns', 9n], + ], + timestamp: 9n, + }); + visitor.end!({hasMissingMapFiles: true}); + const profile = Profile.decode(builder.build().encode()); + const strings = profile.stringTable.strings; + assert.strictEqual(profile.sample.length, 1); + const [sample] = profile.sample; + assert.deepStrictEqual(sample.locationId, []); + assert.deepStrictEqual(sample.value.map(Number), [7]); + assert.deepStrictEqual( + sample.label.map(l => [ + strings[Number(l.key)], + l.str ? strings[Number(l.str)] : Number(l.num), + ]), + [ + ['event', 'gc'], + ['end_timestamp_ns', 9], + ], + ); + assert.deepStrictEqual( + profile.comment.map(c => strings[Number(c)]), + ['dd:has-missing-map-files'], + ); + }); + + it('encodes sample timestamps as the label the options name', () => { + const builder = new PprofProfileBuilder({ + timestampLabel: 'end_timestamp_ns', + }); + const visitor = builder.visitor(); + const wall = {type: 'wall', unit: 'nanoseconds'}; + visitor.metadata({ + sampleTypes: [wall], + periodType: wall, + period: 1, + timeNanos: 0n, + }); + const stackId = visitor.stack([]); + // Beyond Number.MAX_SAFE_INTEGER, like real epoch nanoseconds + const timestamp = 1_700_000_000_123_456_789n; + visitor.sample({ + stackId, + values: [10], + labels: [['thread name', 'Main Event Loop']], + timestamp, + }); + visitor.sample({stackId, values: [20], labels: []}); + visitor.end!({hasMissingMapFiles: false}); + const profile = Profile.decode(builder.build().encode()); + const strings = profile.stringTable.strings; + const labels = profile.sample.map(s => + s.label.map(l => [ + strings[Number(l.key)], + l.str ? strings[Number(l.str)] : BigInt(l.num), + ]), + ); + assert.deepStrictEqual(labels, [ + [ + ['thread name', 'Main Event Loop'], + ['end_timestamp_ns', timestamp], + ], + [], + ]); + }); + + it('takes a single visit', () => { + const builder = new PprofProfileBuilder(); + builder.visitor(); + assert.throws(() => builder.visitor(), /single profile/); + }); + + it('is exported from the package', async () => { + const pprof = await import('../src'); + assert.strictEqual(pprof.PprofProfileBuilder, PprofProfileBuilder); + }); + }); }); diff --git a/ts/test/test-time-profiler.ts b/ts/test/test-time-profiler.ts index 80d7f012..60b8d62d 100644 --- a/ts/test/test-time-profiler.ts +++ b/ts/test/test-time-profiler.ts @@ -20,6 +20,7 @@ import {time, getNativeThreadId} from '../src'; import {profileV2, stopV2} from '../src/time-profiler'; import * as v8TimeProfiler from '../src/time-profiler-bindings'; import * as profileSerializer from '../src/profile-serializer'; +import {OtelProfilesBuilder} from '../src/otel-serializer'; import {SourceMapper} from '../src/sourcemapper/sourcemapper'; import {timeProfile, v8TimeProfile} from './profiles-for-tests'; import {hrtime} from 'process'; @@ -579,6 +580,52 @@ describe('Time Profiler', () => { ); }); + it('should stay usable when a stopAndVisit visitor throws', () => { + const throwing = { + metadata: () => { + throw new Error('boom'); + }, + frame: () => 0, + stack: () => 0, + sample: () => {}, + }; + time.start({intervalMicros: PROFILE_OPTIONS.intervalMicros}); + assert.throws(() => time.stopAndVisit(false, throwing), /boom/); + assert.ok(!time.isStarted(), 'stopped despite the error'); + time.start({intervalMicros: PROFILE_OPTIONS.intervalMicros}); + assert.throws(() => time.stopAndVisit(true, throwing), /boom/); + assert.ok(time.isStarted(), 'still running after a restart'); + time.stopAndVisit(false, new OtelProfilesBuilder().visitor()); + assert.ok(!time.isStarted()); + }); + + it('should pass the profile to a visitor with stopAndVisit', () => { + time.start({intervalMicros: PROFILE_OPTIONS.intervalMicros}); + timeProfilerStub.stop.resetHistory(); + const builder = new OtelProfilesBuilder(); + const result = time.stopAndVisit(true, builder.visitor()); + sinon.assert.calledOnce(timeProfilerStub.stop); + assert.ok(time.isStarted(), 'still running after a restart'); + assert.deepStrictEqual(result, {hasMissingMapFiles: false}); + time.stopAndVisit(false, builder.visitor()); + assert.ok(!time.isStarted()); + + const request = builder.build(); + const profiles = request.resourceProfiles[0].scopeProfiles[0].profiles; + // Two visits of a profile without contexts (so its samples are summed, + // and the count keeps a profile) or CPU time: sample count and wall + assert.strictEqual(profiles.length, 4); + const expectedCount = timeProfile.sample.reduce( + (total, sample) => total + Number(sample.value[0]), + 0, + ); + const count = profiles[0].samples.reduce( + (total, sample) => total + Number(sample.values[0]), + 0, + ); + assert.strictEqual(count, expectedCount); + }); + it('should serialize with the source mapper still set when stopping', () => { // Regression test: stop() used to tear down the profiler state (via // handleStopNoRestart, which clears gSourceMapper) *before* serializing,