From 2d1d6c6ace1238efe0e0661514d42576ca0f6ee7 Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Fri, 20 Nov 2026 11:35:49 +0000 Subject: [PATCH 1/6] test(profiling-ffi): add scoped failing allocator and contract tests Experimental test-only harness: a global allocator proxy that returns null at a chosen allocation ordinal, a controller that runs one worker process per ordinal and mode, and strict contract and toy-scenario tests. Jira: PROF-16808 --- libdd-profiling-ffi/Cargo.toml | 6 + .../tests/allocation_failure/allocator.rs | 326 +++++++++++++ .../tests/allocation_failure/contract.rs | 254 ++++++++++ .../tests/allocation_failure/controller.rs | 461 ++++++++++++++++++ .../tests/allocation_failure/main.rs | 118 +++++ .../tests/allocation_failure/scenarios.rs | 75 +++ 6 files changed, 1240 insertions(+) create mode 100644 libdd-profiling-ffi/tests/allocation_failure/allocator.rs create mode 100644 libdd-profiling-ffi/tests/allocation_failure/contract.rs create mode 100644 libdd-profiling-ffi/tests/allocation_failure/controller.rs create mode 100644 libdd-profiling-ffi/tests/allocation_failure/main.rs create mode 100644 libdd-profiling-ffi/tests/allocation_failure/scenarios.rs diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index fff19a2219..f53105bede 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -94,5 +94,11 @@ libdd-ffe-ffi = { path = "../libdd-ffe-ffi", default-features = false, optional libdd-shared-runtime-ffi = { path = "../libdd-shared-runtime-ffi", default-features = false, optional = true } libdd-otel-thread-ctx-ffi = { path = "../libdd-otel-thread-ctx-ffi", default-features = false, optional = true } +# Experimental allocation-failure harness: replaces the global allocator of its own binary. +[[test]] +name = "allocation_failure" +path = "tests/allocation_failure/main.rs" +harness = false + [package.metadata.cargo-machete] ignored = ["libdd-regex"] diff --git a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs new file mode 100644 index 0000000000..3292fdb7a4 --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs @@ -0,0 +1,326 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! `ScopedFailingAllocator`: a global allocator proxy that can return null for a chosen +//! allocation of a scope. +//! +//! Outside a scope, every call is forwarded unchanged. Inside a scope (per thread), every +//! allocation attempt (`alloc`, `alloc_zeroed`, `realloc`) gets an ordinal, is recorded in a +//! preallocated trace, and fails when the scope's policy says so. `dealloc` is never counted and +//! never fails. +//! +//! The allocator callbacks only use constant-initialized thread-locals and atomics: no +//! allocation, formatting, logging, stack walking or panic. + +use std::alloc::{GlobalAlloc, Layout}; +use std::cell::Cell; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + +/// Maximum number of events kept in the trace (and in the expected trace). +pub const TRACE_CAPACITY: usize = 1 << 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Policy { + /// Count and record, never fail. + Observe, + /// Fail only the allocation with this ordinal. + FailOnceAt(u64), + /// Fail every allocation from this ordinal on. + FailFrom(u64), +} + +impl Policy { + fn fails(self, ordinal: u64) -> bool { + match self { + Policy::Observe => false, + Policy::FailOnceAt(n) => ordinal == n, + Policy::FailFrom(n) => ordinal >= n, + } + } + + fn target(self) -> u64 { + match self { + Policy::Observe => u64::MAX, + Policy::FailOnceAt(n) | Policy::FailFrom(n) => n, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Domain { + Global = 0, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Operation { + Alloc = 0, + AllocZeroed = 1, + Realloc = 2, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Event { + pub ordinal: u64, + pub domain: Domain, + pub operation: Operation, + /// Requested size (the new size for `realloc`). + pub size: u64, + pub align: u64, + pub thread: u32, +} + +impl Event { + /// The fields that must match between runs of the same scenario (the thread ID may not). + pub fn same_request(&self, other: &Event) -> bool { + self.ordinal == other.ordinal + && self.domain == other.domain + && self.operation == other.operation + && self.size == other.size + && self.align == other.align + } +} + +struct Slot { + ordinal_thread: AtomicU64, + size: AtomicU64, + align_domain_op: AtomicU64, +} + +impl Slot { + const fn new() -> Self { + Slot { + ordinal_thread: AtomicU64::new(0), + size: AtomicU64::new(0), + align_domain_op: AtomicU64::new(0), + } + } + + fn store(&self, ev: &Event) { + self.ordinal_thread + .store((ev.ordinal << 32) | u64::from(ev.thread), Ordering::Relaxed); + self.size.store(ev.size, Ordering::Relaxed); + self.align_domain_op.store( + (ev.align << 16) | (u64::from(ev.domain as u8) << 8) | u64::from(ev.operation as u8), + Ordering::Relaxed, + ); + } + + fn load(&self) -> Event { + let ordinal_thread = self.ordinal_thread.load(Ordering::Relaxed); + let align_domain_op = self.align_domain_op.load(Ordering::Relaxed); + Event { + ordinal: ordinal_thread >> 32, + thread: (ordinal_thread & 0xffff_ffff) as u32, + size: self.size.load(Ordering::Relaxed), + align: align_domain_op >> 16, + domain: Domain::Global, + operation: match align_domain_op & 0xff { + 0 => Operation::Alloc, + 1 => Operation::AllocZeroed, + _ => Operation::Realloc, + }, + } + } +} + +static TRACE: [Slot; TRACE_CAPACITY] = [const { Slot::new() }; TRACE_CAPACITY]; +static TRACE_LEN: AtomicUsize = AtomicUsize::new(0); +static TRACE_OVERFLOW: AtomicBool = AtomicBool::new(false); + +/// The baseline trace a worker must reproduce before the failure is injected. +static EXPECTED: [Slot; TRACE_CAPACITY] = [const { Slot::new() }; TRACE_CAPACITY]; +static EXPECTED_LEN: AtomicUsize = AtomicUsize::new(0); +static CHECK_EXPECTED: AtomicBool = AtomicBool::new(false); +/// First ordinal whose event differs from the expected trace (`u64::MAX`: none). +static DIVERGED_AT: AtomicU64 = AtomicU64::new(u64::MAX); + +static INJECTED: AtomicU64 = AtomicU64::new(0); +static NEXT_THREAD_ID: AtomicU32 = AtomicU32::new(1); + +thread_local! { + static DEPTH: Cell = const { Cell::new(0) }; + static POLICY: Cell = const { Cell::new(Policy::Observe) }; + static ORDINAL: Cell = const { Cell::new(0) }; + static THREAD_ID: Cell = const { Cell::new(0) }; +} + +/// Debugger breakpoint target: called just before the allocator returns null. +#[unsafe(no_mangle)] +#[inline(never)] +pub fn alloc_chaos_injection_hook(ordinal: u64) { + std::hint::black_box(ordinal); +} + +fn thread_id() -> u32 { + THREAD_ID + .try_with(|id| { + if id.get() == 0 { + id.set(NEXT_THREAD_ID.fetch_add(1, Ordering::Relaxed)); + } + id.get() + }) + .unwrap_or(0) +} + +/// Counts and records one allocation attempt of the current thread. Returns `true` if it must +/// fail. Returns `false` outside a scope. +fn on_allocation(operation: Operation, size: usize, align: usize) -> bool { + let Ok(Some((policy, ordinal))) = DEPTH.try_with(|depth| { + if depth.get() == 0 { + return None; + } + let policy = POLICY.with(Cell::get); + let ordinal = ORDINAL.with(|o| { + let current = o.get(); + o.set(current + 1); + current + }); + Some((policy, ordinal)) + }) else { + return false; + }; + + let event = Event { + ordinal, + domain: Domain::Global, + operation, + size: size as u64, + align: align as u64, + thread: thread_id(), + }; + let index = TRACE_LEN.fetch_add(1, Ordering::Relaxed); + match TRACE.get(index) { + Some(slot) => slot.store(&event), + None => TRACE_OVERFLOW.store(true, Ordering::Relaxed), + } + + if CHECK_EXPECTED.load(Ordering::Relaxed) && ordinal <= policy.target() { + let matches = usize::try_from(ordinal) + .ok() + .filter(|&i| i < EXPECTED_LEN.load(Ordering::Relaxed)) + .is_some_and(|i| EXPECTED[i].load().same_request(&event)); + if !matches { + let _ = DIVERGED_AT.compare_exchange( + u64::MAX, + ordinal, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + } + + if policy.fails(ordinal) && DIVERGED_AT.load(Ordering::Relaxed) == u64::MAX { + INJECTED.fetch_add(1, Ordering::Relaxed); + alloc_chaos_injection_hook(ordinal); + return true; + } + false +} + +pub struct ScopedFailingAllocator(pub A); + +unsafe impl GlobalAlloc for ScopedFailingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if on_allocation(Operation::Alloc, layout.size(), layout.align()) { + return std::ptr::null_mut(); + } + unsafe { self.0.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + if on_allocation(Operation::AllocZeroed, layout.size(), layout.align()) { + return std::ptr::null_mut(); + } + unsafe { self.0.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // Returning null leaves the old block allocated and unchanged. + if on_allocation(Operation::Realloc, new_size, layout.align()) { + return std::ptr::null_mut(); + } + unsafe { self.0.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { self.0.dealloc(ptr, layout) } + } +} + +/// An active scope on the current thread. Dropping the outermost guard ends the scope. +pub struct Scope { + _not_send: std::marker::PhantomData<*const ()>, +} + +impl Scope { + /// Starts a scope on the current thread with `policy` and ordinal 0. A nested `enter` joins + /// the active scope: it keeps the outer policy and ordinal counter, and `policy` is ignored. + pub fn enter(policy: Policy) -> Scope { + DEPTH.with(|depth| { + if depth.get() == 0 { + POLICY.with(|p| p.set(policy)); + ORDINAL.with(|o| o.set(0)); + } + depth.set(depth.get() + 1); + }); + Scope { + _not_send: std::marker::PhantomData, + } + } +} + +impl Drop for Scope { + fn drop(&mut self) { + DEPTH.with(|depth| depth.set(depth.get() - 1)); + } +} + +/// Clears the trace, the expected trace and the counters. Call it outside any scope. +pub fn reset() { + TRACE_LEN.store(0, Ordering::Relaxed); + TRACE_OVERFLOW.store(false, Ordering::Relaxed); + EXPECTED_LEN.store(0, Ordering::Relaxed); + CHECK_EXPECTED.store(false, Ordering::Relaxed); + DIVERGED_AT.store(u64::MAX, Ordering::Relaxed); + INJECTED.store(0, Ordering::Relaxed); +} + +/// Loads the baseline trace: the allocator then checks every event up to the policy's target +/// ordinal against it, and does not inject a failure once an event differs. +pub fn set_expected(events: &[Event]) -> Result<(), String> { + if events.len() > TRACE_CAPACITY { + return Err(format!("expected trace too long: {}", events.len())); + } + for (slot, event) in EXPECTED.iter().zip(events) { + slot.store(event); + } + EXPECTED_LEN.store(events.len(), Ordering::Relaxed); + CHECK_EXPECTED.store(true, Ordering::Relaxed); + Ok(()) +} + +/// The recorded events. Call it outside any scope (it allocates). +pub fn recorded() -> Result, String> { + if TRACE_OVERFLOW.load(Ordering::Relaxed) { + return Err(format!("trace overflow: more than {TRACE_CAPACITY} events")); + } + let len = TRACE_LEN.load(Ordering::Relaxed); + Ok(TRACE[..len].iter().map(Slot::load).collect()) +} + +pub fn injected() -> u64 { + INJECTED.load(Ordering::Relaxed) +} + +pub fn diverged_at() -> Option { + match DIVERGED_AT.load(Ordering::Relaxed) { + u64::MAX => None, + ordinal => Some(ordinal), + } +} + +pub fn current_thread_id() -> u32 { + thread_id() +} diff --git a/libdd-profiling-ffi/tests/allocation_failure/contract.rs b/libdd-profiling-ffi/tests/allocation_failure/contract.rs new file mode 100644 index 0000000000..4cbb001e9c --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/contract.rs @@ -0,0 +1,254 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Contract tests of `ScopedFailingAllocator`, run in-process. +//! +//! Inside a scope, the tests only call the raw allocation functions and store results in locals; +//! assertions (which may allocate) run after the scope ends. + +use crate::allocator::{self, Domain, Event, Operation, Policy, Scope}; +use std::alloc::{Layout, alloc, alloc_zeroed, dealloc, realloc}; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; + +const LAYOUT: Layout = match Layout::from_size_align(48, 16) { + Ok(l) => l, + Err(_) => panic!("invalid layout"), +}; + +fn layout(size: usize) -> Layout { + Layout::from_size_align(size, LAYOUT.align()).unwrap() +} + +/// Frees `p` if it is not null. +unsafe fn free(p: *mut u8, layout: Layout) { + if !p.is_null() { + unsafe { dealloc(p, layout) } + } +} + +fn filled_with(p: *const u8, len: usize, byte: u8) -> bool { + unsafe { std::slice::from_raw_parts(p, len) } + .iter() + .all(|&b| b == byte) +} + +pub fn pass_through() { + allocator::reset(); + unsafe { + let a = alloc(LAYOUT); + assert!(!a.is_null()); + a.write_bytes(0xab, LAYOUT.size()); + + let z = alloc_zeroed(LAYOUT); + assert!(!z.is_null()); + assert!(filled_with(z, LAYOUT.size(), 0)); + + let r = realloc(a, LAYOUT, 4096); + assert!(!r.is_null()); + assert!(filled_with(r, LAYOUT.size(), 0xab)); + + dealloc(r, layout(4096)); + dealloc(z, LAYOUT); + } + assert_eq!(allocator::recorded().unwrap(), vec![]); + assert_eq!(allocator::injected(), 0); +} + +pub fn ordinal_counting() { + allocator::reset(); + let small = Layout::from_size_align(8, 8).unwrap(); + let (a, b, z, r); + { + let _scope = Scope::enter(Policy::Observe); + unsafe { + a = alloc(LAYOUT); + b = alloc(small); + dealloc(b, small); + z = alloc_zeroed(LAYOUT); + r = realloc(a, LAYOUT, 96); + } + } + unsafe { + free(r, layout(96)); + free(z, LAYOUT); + } + let thread = allocator::current_thread_id(); + let got: Vec<_> = allocator::recorded() + .unwrap() + .iter() + .map(|e| (e.ordinal, e.operation, e.size, e.align, e.thread)) + .collect(); + assert_eq!( + got, + vec![ + (0, Operation::Alloc, 48, 16, thread), + (1, Operation::Alloc, 8, 8, thread), + (2, Operation::AllocZeroed, 48, 16, thread), + (3, Operation::Realloc, 96, 16, thread), + ] + ); + assert!(!a.is_null() && !b.is_null() && !z.is_null() && !r.is_null()); + assert_eq!(allocator::injected(), 0); +} + +pub fn fail_once() { + allocator::reset(); + let mut p = [std::ptr::null_mut(); 4]; + { + let _scope = Scope::enter(Policy::FailOnceAt(1)); + for slot in &mut p { + *slot = unsafe { alloc(LAYOUT) }; + } + } + let nulls: Vec = p.iter().map(|x| x.is_null()).collect(); + for x in p { + unsafe { free(x, LAYOUT) }; + } + assert_eq!(nulls, vec![false, true, false, false]); + assert_eq!(allocator::injected(), 1); + assert_eq!(allocator::recorded().unwrap().len(), 4); +} + +pub fn fail_from() { + allocator::reset(); + let base = unsafe { alloc(LAYOUT) }; + assert!(!base.is_null()); + let (a, b, z, r); + { + let _scope = Scope::enter(Policy::FailFrom(1)); + unsafe { + a = alloc(LAYOUT); + b = alloc(LAYOUT); + z = alloc_zeroed(LAYOUT); + r = realloc(base, LAYOUT, 256); + } + } + let nulls = [a.is_null(), b.is_null(), z.is_null(), r.is_null()]; + unsafe { + free(a, LAYOUT); + free(b, LAYOUT); + free(z, LAYOUT); + if r.is_null() { + dealloc(base, LAYOUT); + } else { + dealloc(r, layout(256)); + } + } + assert_eq!(nulls, [false, true, true, true]); + assert_eq!(allocator::injected(), 3); +} + +pub fn realloc_preserves_block() { + allocator::reset(); + let p = unsafe { alloc(LAYOUT) }; + assert!(!p.is_null()); + unsafe { p.write_bytes(0x5a, LAYOUT.size()) }; + let r = { + let _scope = Scope::enter(Policy::FailOnceAt(0)); + unsafe { realloc(p, LAYOUT, 1 << 20) } + }; + assert!(r.is_null()); + assert!(filled_with(p, LAYOUT.size(), 0x5a)); + // The old block is still owned by the caller and can grow later. + let r = unsafe { realloc(p, LAYOUT, 4096) }; + assert!(!r.is_null()); + assert!(filled_with(r, LAYOUT.size(), 0x5a)); + unsafe { dealloc(r, layout(4096)) }; + assert_eq!(allocator::injected(), 1); +} + +pub fn nested_scopes() { + allocator::reset(); + let mut p = [std::ptr::null_mut(); 4]; + { + let _outer = Scope::enter(Policy::FailOnceAt(2)); + p[0] = unsafe { alloc(LAYOUT) }; + { + // Joins the outer scope: same counter, its own policy is ignored. + let _inner = Scope::enter(Policy::FailFrom(0)); + p[1] = unsafe { alloc(LAYOUT) }; + p[2] = unsafe { alloc(LAYOUT) }; + } + p[3] = unsafe { alloc(LAYOUT) }; + } + let after = unsafe { alloc(LAYOUT) }; + let nulls: Vec = p.iter().map(|x| x.is_null()).collect(); + for x in p { + unsafe { free(x, LAYOUT) }; + } + unsafe { free(after, LAYOUT) }; + assert_eq!(nulls, vec![false, false, true, false]); + assert!(!after.is_null()); + let ordinals: Vec = allocator::recorded() + .unwrap() + .iter() + .map(|e| e.ordinal) + .collect(); + assert_eq!(ordinals, vec![0, 1, 2, 3]); +} + +pub fn thread_isolation() { + static GO: AtomicBool = AtomicBool::new(false); + static DONE: AtomicBool = AtomicBool::new(false); + static OTHER: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + + allocator::reset(); + let other = std::thread::spawn(|| { + while !GO.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + OTHER.store(unsafe { alloc(LAYOUT) }, Ordering::Relaxed); + DONE.store(true, Ordering::Release); + }); + let mine = { + let _scope = Scope::enter(Policy::FailFrom(0)); + GO.store(true, Ordering::Release); + while !DONE.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + unsafe { alloc(LAYOUT) } + }; + other.join().unwrap(); + let theirs = OTHER.load(Ordering::Relaxed); + unsafe { free(theirs, LAYOUT) }; + assert!(mine.is_null(), "the scoped thread's allocation must fail"); + assert!(!theirs.is_null(), "another thread is not in the scope"); + let trace = allocator::recorded().unwrap(); + assert_eq!(trace.len(), 1, "{trace:?}"); + assert_eq!(trace[0].thread, allocator::current_thread_id()); +} + +pub fn prefix_divergence() { + allocator::reset(); + let expected = Event { + ordinal: 0, + domain: Domain::Global, + operation: Operation::Alloc, + size: 32, + align: 16, + thread: 0, + }; + allocator::set_expected(&[ + expected, + Event { + ordinal: 1, + ..expected + }, + ]) + .unwrap(); + let (a, b); + { + let _scope = Scope::enter(Policy::FailOnceAt(1)); + // 48 bytes instead of the expected 32: the run diverged, so nothing is injected. + a = unsafe { alloc(LAYOUT) }; + b = unsafe { alloc(layout(32)) }; + } + let nulls = [a.is_null(), b.is_null()]; + unsafe { + free(a, LAYOUT); + free(b, layout(32)); + } + assert_eq!(nulls, [false, false]); + assert_eq!(allocator::diverged_at(), Some(0)); + assert_eq!(allocator::injected(), 0); +} diff --git a/libdd-profiling-ffi/tests/allocation_failure/controller.rs b/libdd-profiling-ffi/tests/allocation_failure/controller.rs new file mode 100644 index 0000000000..c393ebe2f2 --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/controller.rs @@ -0,0 +1,461 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Controller/worker protocol. +//! +//! The controller runs the scenario in a baseline worker (no failure) to get its allocation +//! trace, checks it is stable, then runs one fresh worker process per ordinal and mode, and +//! classifies each run. +//! +//! A worker is this test binary started as ` __worker `. It reads +//! the baseline trace on stdin, runs the scenario inside a scope, then prints its trace and +//! result on stdout. + +use crate::allocator::{self, Domain, Event, Operation, Policy, Scope}; +use crate::scenarios::{self, ScenarioError}; +use std::fmt::Write as _; +use std::io::{Read, Write}; +use std::process::{Command, ExitStatus, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +pub const WORKER_ARG: &str = "__worker"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + Baseline, + FailOnce, + FailFrom, +} + +impl Mode { + fn as_str(self) -> &'static str { + match self { + Mode::Baseline => "baseline", + Mode::FailOnce => "fail-once", + Mode::FailFrom => "fail-from", + } + } + + fn parse(s: &str) -> Option { + [Mode::Baseline, Mode::FailOnce, Mode::FailFrom] + .into_iter() + .find(|m| m.as_str() == s) + } + + fn policy(self, ordinal: u64) -> Policy { + match self { + Mode::Baseline => Policy::Observe, + Mode::FailOnce => Policy::FailOnceAt(ordinal), + Mode::FailFrom => Policy::FailFrom(ordinal), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Outcome { + /// The API returned an allocation error. + OomError, + /// The API succeeded despite the failed allocation. + SuccessFallback, + Panic, + Abort, + Timeout, + /// The event prefix differs from the baseline, or the ordinal was not reached. + Nondeterministic, + /// Any other abnormal exit (signal, unexpected exit code). + Crash, +} + +impl Outcome { + pub fn as_str(self) -> &'static str { + match self { + Outcome::OomError => "oom-error", + Outcome::SuccessFallback => "success-fallback", + Outcome::Panic => "panic", + Outcome::Abort => "abort", + Outcome::Timeout => "timeout", + Outcome::Nondeterministic => "nondeterministic", + Outcome::Crash => "crash", + } + } +} + +// --------------------------------------------------------------------------------------------- +// Worker side +// --------------------------------------------------------------------------------------------- + +static PANICKED: AtomicBool = AtomicBool::new(false); + +fn format_event(e: &Event) -> String { + format!( + "event {} {:?} {:?} {} {} {}", + e.ordinal, e.domain, e.operation, e.size, e.align, e.thread + ) +} + +fn parse_event(line: &str) -> Option { + let mut it = line.strip_prefix("event ")?.split(' '); + let ordinal = it.next()?.parse().ok()?; + let domain = match it.next()? { + "Global" => Domain::Global, + _ => return None, + }; + let operation = match it.next()? { + "Alloc" => Operation::Alloc, + "AllocZeroed" => Operation::AllocZeroed, + "Realloc" => Operation::Realloc, + _ => return None, + }; + Some(Event { + ordinal, + domain, + operation, + size: it.next()?.parse().ok()?, + align: it.next()?.parse().ok()?, + thread: it.next()?.parse().ok()?, + }) +} + +/// Workers abort on purpose: do not write a core file for each of them. +fn disable_core_dumps() { + #[cfg(unix)] + { + let limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `limit` is a valid `rlimit` for the duration of the call. + unsafe { libc::setrlimit(libc::RLIMIT_CORE, &limit) }; + } +} + +/// Entry point of a worker process. Exits the process. +pub fn worker_main(args: &[String]) -> ! { + let (Some(name), Some(mode), Some(ordinal)) = ( + args.first(), + args.get(1).and_then(|m| Mode::parse(m)), + args.get(2).and_then(|n| n.parse::().ok()), + ) else { + eprintln!("usage: {WORKER_ARG} "); + std::process::exit(2); + }; + let Some(scenario) = scenarios::find(name) else { + eprintln!("unknown scenario {name}"); + std::process::exit(2); + }; + disable_core_dumps(); + + let mut input = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut input) { + eprintln!("cannot read the expected trace: {e}"); + std::process::exit(2); + } + let expected: Vec = input.lines().filter_map(parse_event).collect(); + + // The hook writes a fixed marker without allocating, so it still works when every + // allocation fails. + std::panic::set_hook(Box::new(|_| { + PANICKED.store(true, Ordering::Relaxed); + let _ = std::io::stderr().write_all(b"alloc-chaos: panic\n"); + })); + + allocator::reset(); + if mode != Mode::Baseline + && let Err(e) = allocator::set_expected(&expected) + { + eprintln!("{e}"); + std::process::exit(2); + } + + let mut fixture = (scenario.setup)(); + let result = { + let _scope = Scope::enter(mode.policy(ordinal)); + (scenario.run)(&mut fixture) + }; + drop(fixture); + + let trace = match allocator::recorded() { + Ok(trace) => trace, + Err(e) => { + eprintln!("{e}"); + std::process::exit(2); + } + }; + let mut out = String::new(); + for e in &trace { + let _ = writeln!(out, "{}", format_event(e)); + } + let result = match result { + Ok(()) => "ok", + Err(ScenarioError::Oom) => "oom-error", + }; + let _ = writeln!(out, "result {result}"); + let _ = writeln!(out, "injected {}", allocator::injected()); + if let Some(at) = allocator::diverged_at() { + let _ = writeln!(out, "diverged {at}"); + } + let _ = writeln!(out, "panicked {}", PANICKED.load(Ordering::Relaxed)); + let _ = std::io::stdout().write_all(out.as_bytes()); + std::process::exit(0); +} + +// --------------------------------------------------------------------------------------------- +// Controller side +// --------------------------------------------------------------------------------------------- + +#[derive(Debug)] +pub struct WorkerRun { + pub outcome: Outcome, + pub trace: Vec, + /// Exit status, signal, or the reason of the classification. + pub detail: String, + pub stderr: String, +} + +fn timeout() -> Duration { + let secs = std::env::var("ALLOC_CHAOS_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(60); + Duration::from_secs(secs) +} + +fn is_abort(status: &ExitStatus) -> bool { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + status.signal() == Some(libc::SIGABRT) + } + #[cfg(windows)] + { + // `abort()` from the CRT exits with 3; Rust's `process::abort` uses `__fastfail` + // (STATUS_STACK_BUFFER_OVERRUN). + matches!(status.code(), Some(3) | Some(-1073740791)) + } + #[cfg(not(any(unix, windows)))] + { + let _ = status; + false + } +} + +fn spawn_reader(mut r: R) -> std::thread::JoinHandle { + std::thread::spawn(move || { + let mut s = String::new(); + let _ = r.read_to_string(&mut s); + s + }) +} + +pub fn run_worker( + scenario: &str, + mode: Mode, + ordinal: u64, + baseline: &[Event], +) -> Result { + let exe = std::env::current_exe().map_err(|e| format!("current_exe: {e}"))?; + let mut child = Command::new(exe) + .args([WORKER_ARG, scenario, mode.as_str(), &ordinal.to_string()]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("cannot start the worker: {e}"))?; + + let mut input = String::new(); + for e in baseline { + let _ = writeln!(input, "{}", format_event(e)); + } + let mut stdin = child.stdin.take().ok_or("no worker stdin")?; + let stdout = spawn_reader(child.stdout.take().ok_or("no worker stdout")?); + let stderr = spawn_reader(child.stderr.take().ok_or("no worker stderr")?); + // The worker may die before reading its input: ignore a broken pipe. + let _ = stdin.write_all(input.as_bytes()); + drop(stdin); + + let deadline = Instant::now() + timeout(); + let status = loop { + match child.try_wait().map_err(|e| format!("wait: {e}"))? { + Some(status) => break Some(status), + None if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + None => std::thread::sleep(Duration::from_millis(5)), + } + }; + let stdout = stdout.join().unwrap_or_default(); + let stderr = stderr.join().unwrap_or_default(); + let trace: Vec = stdout.lines().filter_map(parse_event).collect(); + let field = |key: &str| { + stdout + .lines() + .find_map(|l| l.strip_prefix(key)?.strip_prefix(' ')) + .map(str::to_owned) + }; + + let (outcome, detail) = match status { + None => (Outcome::Timeout, format!("killed after {:?}", timeout())), + Some(status) if status.success() => { + let panicked = field("panicked").as_deref() == Some("true"); + let injected = field("injected") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + let result = field("result").unwrap_or_default(); + if let Some(at) = field("diverged") { + ( + Outcome::Nondeterministic, + format!("event {at} differs from the baseline"), + ) + } else if mode != Mode::Baseline && injected == 0 { + ( + Outcome::Nondeterministic, + format!("ordinal {ordinal} not reached"), + ) + } else if panicked { + (Outcome::Panic, format!("panic caught, result {result}")) + } else if result == "oom-error" { + (Outcome::OomError, String::new()) + } else if result == "ok" { + (Outcome::SuccessFallback, String::new()) + } else { + (Outcome::Crash, format!("no result line, {status}")) + } + } + Some(status) if stderr.contains("alloc-chaos: panic") => { + (Outcome::Panic, status.to_string()) + } + Some(status) if is_abort(&status) => (Outcome::Abort, status.to_string()), + Some(status) => (Outcome::Crash, status.to_string()), + }; + Ok(WorkerRun { + outcome, + trace, + detail, + stderr, + }) +} + +#[derive(Debug)] +pub struct Finding { + pub mode: Mode, + pub ordinal: u64, + pub event: Event, + pub run: WorkerRun, +} + +#[derive(Debug)] +pub struct Report { + pub scenario: &'static str, + pub baseline: Vec, + pub runs: Vec, +} + +impl Report { + pub fn outcomes(&self, mode: Mode) -> Vec { + self.runs + .iter() + .filter(|r| r.mode == mode) + .map(|r| r.run.outcome) + .collect() + } + + pub fn to_markdown(&self) -> String { + let mut s = String::new(); + let _ = writeln!(s, "## {}\n", self.scenario); + let _ = writeln!(s, "Baseline: {} events.\n", self.baseline.len()); + let _ = writeln!( + s, + "| mode | ordinal | operation | size | align | thread | outcome | detail |" + ); + let _ = writeln!(s, "|---|---|---|---|---|---|---|---|"); + for f in &self.runs { + let _ = writeln!( + s, + "| {} | {} | {:?} | {} | {} | {} | {} | {} |", + f.mode.as_str(), + f.ordinal, + f.event.operation, + f.event.size, + f.event.align, + f.event.thread, + f.run.outcome.as_str(), + f.run.detail + ); + } + s + } +} + +/// Runs the baseline twice (it must succeed and be stable), then one worker per ordinal and +/// mode. Returns `Err` only for harness errors. +pub fn enumerate(scenario: &'static str) -> Result { + let baseline = run_worker(scenario, Mode::Baseline, 0, &[])?; + if baseline.outcome != Outcome::SuccessFallback { + return Err(format!( + "{scenario}: baseline failed: {} {}\n{}", + baseline.outcome.as_str(), + baseline.detail, + baseline.stderr + )); + } + let again = run_worker(scenario, Mode::Baseline, 0, &[])?; + let stable = again.trace.len() == baseline.trace.len() + && again + .trace + .iter() + .zip(&baseline.trace) + .all(|(a, b)| a.same_request(b)); + if !stable { + return Err(format!( + "{scenario}: baseline trace is not stable ({} then {} events)", + baseline.trace.len(), + again.trace.len() + )); + } + if let Some(e) = baseline + .trace + .iter() + .find(|e| e.thread != baseline.trace[0].thread) + { + return Err(format!( + "{scenario}: baseline has events on another thread: {e:?}" + )); + } + + let mut runs = Vec::new(); + for event in &baseline.trace { + for mode in [Mode::FailOnce, Mode::FailFrom] { + let run = run_worker(scenario, mode, event.ordinal, &baseline.trace)?; + runs.push(Finding { + mode, + ordinal: event.ordinal, + event: *event, + run, + }); + } + } + let report = Report { + scenario, + baseline: baseline.trace, + runs, + }; + write_report(&report); + Ok(report) +} + +fn write_report(report: &Report) { + let markdown = report.to_markdown(); + println!("{markdown}"); + let dir = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join("allocation_failure"); + let path = dir.join(format!("{}.md", report.scenario)); + if let Err(e) = + std::fs::create_dir_all(&dir).and_then(|()| std::fs::write(&path, markdown.as_bytes())) + { + eprintln!("cannot write {}: {e}", path.display()); + } else { + println!("report: {}", path.display()); + } +} diff --git a/libdd-profiling-ffi/tests/allocation_failure/main.rs b/libdd-profiling-ffi/tests/allocation_failure/main.rs new file mode 100644 index 0000000000..40ac77c75c --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/main.rs @@ -0,0 +1,118 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Experimental allocation-failure harness: finds the paths where a failed allocation (the +//! allocator returns null) crashes instead of returning an out-of-memory error. +//! +//! This test runs as its own binary (`harness = false` in `Cargo.toml`) so that +//! `ScopedFailingAllocator` is the global allocator of this process only. `main` implements the +//! small protocol expected by `cargo nextest` (see +//! https://nexte.st/docs/design/custom-test-harnesses/): `--list --format terse` support and +//! per-test selection with `--exact`. Started with `__worker`, the binary is a worker process of +//! the controller (see `controller.rs`). +//! +//! Contract and toy-scenario tests are strict. Debugging an injected failure: run a worker under +//! a debugger with a breakpoint on `alloc_chaos_injection_hook`. + +mod allocator; +mod contract; +mod controller; +mod scenarios; + +use allocator::ScopedFailingAllocator; +use controller::{Mode, Outcome}; +use std::alloc::System; + +#[global_allocator] +static ALLOCATOR: ScopedFailingAllocator = ScopedFailingAllocator(System); + +/// Enumerates a toy scenario and checks every run has the `expected` outcome. +fn toy(name: &'static str, expected: Outcome) { + let report = controller::enumerate(name).unwrap_or_else(|e| panic!("harness error: {e}")); + assert!(!report.baseline.is_empty(), "{name}: no allocation"); + for mode in [Mode::FailOnce, Mode::FailFrom] { + let outcomes = report.outcomes(mode); + assert_eq!(outcomes.len(), report.baseline.len()); + assert!( + outcomes.iter().all(|&o| o == expected), + "{name}: expected {} in {mode:?}, got {outcomes:?}", + expected.as_str() + ); + } +} + +const TESTS: &[(&str, fn())] = &[ + ("contract::pass_through", contract::pass_through), + ("contract::ordinal_counting", contract::ordinal_counting), + ("contract::fail_once", contract::fail_once), + ("contract::fail_from", contract::fail_from), + ( + "contract::realloc_preserves_block", + contract::realloc_preserves_block, + ), + ("contract::nested_scopes", contract::nested_scopes), + ("contract::thread_isolation", contract::thread_isolation), + ("contract::prefix_divergence", contract::prefix_divergence), + ("toy::box_new", || { + toy(scenarios::TOY_BOX_NEW, Outcome::Abort) + }), + ("toy::vec_push", || { + toy(scenarios::TOY_VEC_PUSH, Outcome::Abort) + }), + ("toy::vec_try_reserve", || { + toy(scenarios::TOY_VEC_TRY_RESERVE, Outcome::OomError) + }), + ("toy::string_format", || { + toy(scenarios::TOY_STRING_FORMAT, Outcome::Abort) + }), +]; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + + if args.first().map(String::as_str) == Some(controller::WORKER_ARG) { + controller::worker_main(&args[1..]); + } + + if args.iter().any(|arg| arg == "--list") { + if !args.iter().any(|arg| arg == "--ignored") { + for (name, _) in TESTS { + println!("{name}: test"); + } + } + return; + } + + // Miri cannot start worker processes. + if cfg!(miri) { + println!("skipped under Miri: the harness starts worker processes"); + return; + } + + let exact = args.iter().any(|a| a == "--exact"); + let filters: Vec<&str> = args + .iter() + .filter(|a| !a.starts_with("--")) + .map(String::as_str) + .collect(); + let mut failed = 0; + for (name, test) in TESTS { + let selected = filters.is_empty() + || filters + .iter() + .any(|f| if exact { f == name } else { name.contains(f) }); + if !selected { + continue; + } + match std::panic::catch_unwind(test) { + Ok(()) => println!("test {name} ... ok"), + Err(_) => { + println!("test {name} ... FAILED"); + failed += 1; + } + } + } + if failed > 0 { + std::process::exit(101); + } +} diff --git a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs new file mode 100644 index 0000000000..5a7aa946a7 --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs @@ -0,0 +1,75 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Scenarios: a fixture built outside the scope, and one call run inside it. + +use std::any::Any; +use std::hint::black_box; + +#[derive(Debug)] +pub enum ScenarioError { + /// The call reported an allocation failure. + Oom, +} + +pub struct Scenario { + pub name: &'static str, + pub setup: fn() -> Box, + pub run: fn(&mut Box) -> Result<(), ScenarioError>, +} + +fn no_fixture() -> Box { + Box::new(()) +} + +/// Toy scenarios: plain Rust allocations, no libdatadog code. They check the harness itself. +pub const TOY_BOX_NEW: &str = "toy_box_new"; +pub const TOY_VEC_PUSH: &str = "toy_vec_push"; +pub const TOY_VEC_TRY_RESERVE: &str = "toy_vec_try_reserve"; +pub const TOY_STRING_FORMAT: &str = "toy_string_format"; + +pub const SCENARIOS: &[Scenario] = &[ + Scenario { + name: TOY_BOX_NEW, + setup: no_fixture, + run: |_| { + black_box(Box::new(black_box([7u8; 64]))); + Ok(()) + }, + }, + Scenario { + name: TOY_VEC_PUSH, + setup: no_fixture, + run: |_| { + let mut v = Vec::new(); + for i in 0..100u64 { + v.push(black_box(i)); + } + black_box(v); + Ok(()) + }, + }, + Scenario { + name: TOY_VEC_TRY_RESERVE, + setup: no_fixture, + run: |_| { + let mut v: Vec = Vec::new(); + v.try_reserve(black_box(1024)) + .map_err(|_| ScenarioError::Oom)?; + black_box(v); + Ok(()) + }, + }, + Scenario { + name: TOY_STRING_FORMAT, + setup: no_fixture, + run: |_| { + black_box(format!("{}-{}", black_box(42), black_box("forty-two"))); + Ok(()) + }, + }, +]; + +pub fn find(name: &str) -> Option<&'static Scenario> { + SCENARIOS.iter().find(|s| s.name == name) +} From 7ec63fffd05b77376d040407e7769be2d4128002 Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Fri, 20 Nov 2026 12:51:16 +0000 Subject: [PATCH 2/6] test(profiling-ffi): enumerate allocation failures of tag push Run ddog_Vec_Tag_push in one-shot and sticky modes for an empty vector, a vector at capacity, and a long key and value. Crash outcomes are recorded with the stack and the owned call site, and do not fail the test. Jira: PROF-16808 --- libdd-profiling-ffi/Cargo.toml | 4 + .../tests/allocation_failure/controller.rs | 247 +++++++++++++++++- .../tests/allocation_failure/main.rs | 59 ++++- .../tests/allocation_failure/scenarios.rs | 107 +++++++- 4 files changed, 410 insertions(+), 7 deletions(-) diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index f53105bede..2af5f75abd 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -94,6 +94,10 @@ libdd-ffe-ffi = { path = "../libdd-ffe-ffi", default-features = false, optional libdd-shared-runtime-ffi = { path = "../libdd-shared-runtime-ffi", default-features = false, optional = true } libdd-otel-thread-ctx-ffi = { path = "../libdd-otel-thread-ctx-ffi", default-features = false, optional = true } +[dev-dependencies] +libdd-common = { path = "../libdd-common" } +libdd-common-ffi = { path = "../libdd-common-ffi", features = ["std"] } + # Experimental allocation-failure harness: replaces the global allocator of its own binary. [[test]] name = "allocation_failure" diff --git a/libdd-profiling-ffi/tests/allocation_failure/controller.rs b/libdd-profiling-ffi/tests/allocation_failure/controller.rs index c393ebe2f2..dbd6cb7540 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/controller.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/controller.rs @@ -29,7 +29,7 @@ pub enum Mode { } impl Mode { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Mode::Baseline => "baseline", Mode::FailOnce => "fail-once", @@ -153,11 +153,13 @@ pub fn worker_main(args: &[String]) -> ! { } let expected: Vec = input.lines().filter_map(parse_event).collect(); - // The hook writes a fixed marker without allocating, so it still works when every - // allocation fails. - std::panic::set_hook(Box::new(|_| { + // The marker is written without allocating, so it still works when every allocation fails. + // The previous hook prints the `RUST_BACKTRACE` trace. + let previous_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { PANICKED.store(true, Ordering::Relaxed); let _ = std::io::stderr().write_all(b"alloc-chaos: panic\n"); + previous_hook(info); })); allocator::reset(); @@ -189,6 +191,10 @@ pub fn worker_main(args: &[String]) -> ! { let result = match result { Ok(()) => "ok", Err(ScenarioError::Oom) => "oom-error", + Err(ScenarioError::Rejected) => { + eprintln!("the call rejected its input"); + std::process::exit(2); + } }; let _ = writeln!(out, "result {result}"); let _ = writeln!(out, "injected {}", allocator::injected()); @@ -257,6 +263,7 @@ pub fn run_worker( let exe = std::env::current_exe().map_err(|e| format!("current_exe: {e}"))?; let mut child = Command::new(exe) .args([WORKER_ARG, scenario, mode.as_str(), &ordinal.to_string()]) + .env("RUST_BACKTRACE", "1") .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -344,6 +351,10 @@ pub struct Finding { pub ordinal: u64, pub event: Event, pub run: WorkerRun, + /// First stack frame in libdatadog code, from a debugger stopped on the injection hook. + pub call_site: Option, + /// Frame lines of that stack. + pub allocation_stack: Option, } #[derive(Debug)] @@ -385,6 +396,25 @@ impl Report { f.run.detail ); } + for f in &self.runs { + if f.call_site.is_none() && f.allocation_stack.is_none() { + continue; + } + let _ = writeln!( + s, + "\n### {} #{} {}\n\nCall site: {}\n", + f.mode.as_str(), + f.ordinal, + f.run.outcome.as_str(), + f.call_site.as_deref().unwrap_or("(none)") + ); + if let Some(stack) = &f.allocation_stack { + let _ = writeln!(s, "```\n{stack}\n```"); + } + if let Some(panic) = f.run.stderr.split("stack backtrace:").nth(1) { + let _ = writeln!(s, "\nPanic backtrace:\n```\nstack backtrace:{panic}\n```"); + } + } s } } @@ -392,6 +422,15 @@ impl Report { /// Runs the baseline twice (it must succeed and be stable), then one worker per ordinal and /// mode. Returns `Err` only for harness errors. pub fn enumerate(scenario: &'static str) -> Result { + enumerate_inner(scenario, false) +} + +/// Like [`enumerate`], then captures the allocation stack of every run that is not `oom-error`. +pub fn enumerate_with_stacks(scenario: &'static str) -> Result { + enumerate_inner(scenario, true) +} + +fn enumerate_inner(scenario: &'static str, stacks: bool) -> Result { let baseline = run_worker(scenario, Mode::Baseline, 0, &[])?; if baseline.outcome != Outcome::SuccessFallback { return Err(format!( @@ -434,9 +473,14 @@ pub fn enumerate(scenario: &'static str) -> Result { ordinal: event.ordinal, event: *event, run, + call_site: None, + allocation_stack: None, }); } } + if stacks { + capture_stacks(scenario, &baseline.trace, &mut runs); + } let report = Report { scenario, baseline: baseline.trace, @@ -459,3 +503,198 @@ fn write_report(report: &Report) { println!("report: {}", path.display()); } } + +fn capture_stacks(scenario: &str, baseline: &[Event], runs: &mut [Finding]) { + let Some(debugger) = debugger() else { + eprintln!("{scenario}: no gdb or lldb, allocation stacks were not captured"); + return; + }; + let dir = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join("allocation_failure"); + if std::fs::create_dir_all(&dir).is_err() { + return; + } + for finding in runs { + if matches!( + finding.run.outcome, + Outcome::OomError | Outcome::Nondeterministic | Outcome::Timeout + ) { + continue; + } + let input = dir.join(format!( + "stdin-{}-{}-{}-{}.txt", + scenario, + finding.mode.as_str(), + finding.ordinal, + std::process::id() + )); + let mut text = String::new(); + for event in baseline { + let _ = writeln!(text, "{}", format_event(event)); + } + if std::fs::write(&input, text).is_err() { + continue; + } + match debugger_backtrace(debugger, scenario, finding.mode, finding.ordinal, &input) { + Ok(raw) => { + let frames = frame_lines(&raw); + finding.call_site = frames.iter().find(|line| is_owned_frame(line)).cloned(); + finding.allocation_stack = Some(frames.join("\n")); + } + Err(error) => finding.allocation_stack = Some(error), + } + } +} + +fn debugger() -> Option<&'static str> { + if command_ok("gdb") { + Some("gdb") + } else if command_ok("lldb") { + Some("lldb") + } else { + None + } +} + +fn command_ok(program: &str) -> bool { + Command::new(program) + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) +} + +fn debugger_backtrace( + debugger: &str, + scenario: &str, + mode: Mode, + ordinal: u64, + input: &std::path::Path, +) -> Result { + let exe = std::env::current_exe().map_err(|e| format!("current_exe: {e}"))?; + let mut cmd = Command::new(debugger); + let ordinal = ordinal.to_string(); + match debugger { + "gdb" => { + // `run < file` replaces the inferior arguments, so the redirect goes in `set args`. + // The executable has to come before `-ex`, or gdb runs those commands too early. + cmd.arg("-nx") + .arg("-batch") + .arg(&exe) + .arg("-ex") + .arg("set pagination off") + .arg("-ex") + .arg("set confirm off") + .arg("-ex") + .arg("set debuginfod enabled off") + .arg("-ex") + .arg(format!( + "set args {WORKER_ARG} {scenario} {} {ordinal} < \"{}\"", + mode.as_str(), + input.display() + )) + .arg("-ex") + .arg("break alloc_chaos_injection_hook") + .arg("-ex") + .arg("run") + .arg("-ex") + .arg("bt 80"); + } + "lldb" => { + cmd.arg("-b") + .arg("-o") + .arg("settings set auto-confirm true") + .arg("-o") + .arg("breakpoint set --name alloc_chaos_injection_hook") + .arg("-o") + .arg(format!( + "process launch -i {} -- {} {} {} {}", + input.display(), + WORKER_ARG, + scenario, + mode.as_str(), + ordinal + )) + .arg("-o") + .arg("bt") + .arg("-o") + .arg("quit") + .arg(&exe); + } + other => return Err(format!("unknown debugger {other}")), + } + cmd.env("RUST_BACKTRACE", "1"); + let (status, stdout, stderr) = output_with_timeout(&mut cmd, Duration::from_secs(120))?; + if status.is_none() { + return Err("debugger timed out".to_string()); + } + let combined = format!("{stdout}\n{stderr}"); + if !combined.contains("alloc_chaos_injection_hook") { + return Err(format!("debugger did not stop on the hook\n{combined}")); + } + Ok(combined) +} + +fn output_with_timeout( + cmd: &mut Command, + limit: Duration, +) -> Result<(Option, String, String), String> { + let mut child = cmd + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("cannot start the debugger: {e}"))?; + let stdout = spawn_reader(child.stdout.take().ok_or("no debugger stdout")?); + let stderr = spawn_reader(child.stderr.take().ok_or("no debugger stderr")?); + let deadline = Instant::now() + limit; + let status = loop { + match child.try_wait().map_err(|e| format!("wait: {e}"))? { + Some(status) => break Some(status), + None if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + None => std::thread::sleep(Duration::from_millis(20)), + } + }; + Ok(( + status, + stdout.join().unwrap_or_default(), + stderr.join().unwrap_or_default(), + )) +} + +fn frame_lines(raw: &str) -> Vec { + raw.lines() + .filter_map(|line| { + let trimmed = line.trim(); + let marker = trimmed.trim_start_matches('*').trim(); + if marker.starts_with('#') || marker.starts_with("frame #") { + Some(trimmed.to_string()) + } else { + None + } + }) + .take(80) + .collect() +} + +/// A frame in libdatadog code. Runtime frames are skipped even when a generic argument names one +/// of our types, and harness frames are skipped too. +fn is_owned_frame(line: &str) -> bool { + if line.contains("allocation_failure::") || line.contains("/allocation_failure/") { + return false; + } + if let Some((_, file)) = line.rsplit_once(" at ") { + return file.contains("libdd-") || file.contains("libdd_"); + } + line.split(" in ").nth(1).is_some_and(|rest| { + rest.split('(') + .next() + .unwrap_or(rest) + .trim_start() + .starts_with("libdd_") + }) +} diff --git a/libdd-profiling-ffi/tests/allocation_failure/main.rs b/libdd-profiling-ffi/tests/allocation_failure/main.rs index 40ac77c75c..ba93640e92 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/main.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/main.rs @@ -11,8 +11,9 @@ //! per-test selection with `--exact`. Started with `__worker`, the binary is a worker process of //! the controller (see `controller.rs`). //! -//! Contract and toy-scenario tests are strict. Debugging an injected failure: run a worker under -//! a debugger with a breakpoint on `alloc_chaos_injection_hook`. +//! Contract and toy-scenario tests are strict. Discovery scenarios record each outcome, including +//! a crash, and fail only on a harness error. For those, a debugger stopped on +//! `alloc_chaos_injection_hook` records the stack and the owned call site. mod allocator; mod contract; @@ -26,6 +27,51 @@ use std::alloc::System; #[global_allocator] static ALLOCATOR: ScopedFailingAllocator = ScopedFailingAllocator(System); +/// Discovery scenario: every ordinal in both modes is recorded. A crash is a finding. +/// A nondeterministic trace is a harness error. +fn discover(name: &'static str) { + let report = + controller::enumerate_with_stacks(name).unwrap_or_else(|e| panic!("harness error: {e}")); + let nondeterministic: Vec<_> = report + .runs + .iter() + .filter(|finding| finding.run.outcome == Outcome::Nondeterministic) + .map(|finding| { + format!( + "{} #{} {}", + finding.mode.as_str(), + finding.ordinal, + finding.run.detail + ) + }) + .collect(); + assert!( + nondeterministic.is_empty(), + "{name}: nondeterministic trace: {nondeterministic:?}" + ); + let missing: Vec = report + .runs + .iter() + .filter(|finding| { + let needs_stack = !matches!( + finding.run.outcome, + Outcome::OomError | Outcome::Nondeterministic | Outcome::Timeout + ); + needs_stack + && finding.allocation_stack.is_some() + && !finding + .call_site + .as_deref() + .is_some_and(|site| site.contains("libdd")) + }) + .map(|finding| format!("{} #{}", finding.mode.as_str(), finding.ordinal)) + .collect(); + assert!( + missing.is_empty(), + "{name}: no owned call site for {missing:?}" + ); +} + /// Enumerates a toy scenario and checks every run has the `expected` outcome. fn toy(name: &'static str, expected: Outcome) { let report = controller::enumerate(name).unwrap_or_else(|e| panic!("harness error: {e}")); @@ -65,6 +111,15 @@ const TESTS: &[(&str, fn())] = &[ ("toy::string_format", || { toy(scenarios::TOY_STRING_FORMAT, Outcome::Abort) }), + ("scenario::tag_push_empty", || { + discover(scenarios::TAG_PUSH_EMPTY) + }), + ("scenario::tag_push_at_capacity", || { + discover(scenarios::TAG_PUSH_AT_CAPACITY) + }), + ("scenario::tag_push_long", || { + discover(scenarios::TAG_PUSH_LONG) + }), ]; fn main() { diff --git a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs index 5a7aa946a7..22ffdf6fa8 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs @@ -10,6 +10,8 @@ use std::hint::black_box; pub enum ScenarioError { /// The call reported an allocation failure. Oom, + /// The call rejected its input. A setup bug, not a finding. + Rejected, } pub struct Scenario { @@ -70,6 +72,109 @@ pub const SCENARIOS: &[Scenario] = &[ }, ]; +pub const TAG_PUSH_EMPTY: &str = "tag_push_empty"; +pub const TAG_PUSH_AT_CAPACITY: &str = "tag_push_at_capacity"; +pub const TAG_PUSH_LONG: &str = "tag_push_long"; + +/// Bytes in the key and in the value of [`TAG_PUSH_LONG`]. +const LONG_LEN: usize = 4096; + +struct TagPush { + tags: libdd_common_ffi::Vec, + key: String, + value: String, +} + +fn tag_push( + key: &str, + value: &str, + tags: libdd_common_ffi::Vec, +) -> TagPush { + TagPush { + tags, + key: key.to_string(), + value: value.to_string(), + } +} + +/// Pushes short tags until `len == capacity`, so the next push grows the buffer. +fn fill_to_capacity() -> libdd_common_ffi::Vec { + let mut tags = libdd_common_ffi::tags::ddog_Vec_Tag_new(); + for n in 0..64 { + let std_vec = tags.into_std(); + let full = !std_vec.is_empty() && std_vec.len() == std_vec.capacity(); + tags = std_vec.into(); + if full { + return tags; + } + let key = format!("k{n}"); + let value = format!("v{n}"); + // SAFETY: `tags` comes from this API, and both slices come from live Rust strings. + let result = unsafe { + libdd_common_ffi::tags::ddog_Vec_Tag_push( + &mut tags, + libdd_common_ffi::CharSlice::from(key.as_str()), + libdd_common_ffi::CharSlice::from(value.as_str()), + ) + }; + assert!( + matches!(result, libdd_common_ffi::tags::PushTagResult::Ok), + "setup push {n} was rejected" + ); + } + panic!("the tag vector never reached capacity"); +} + +fn run_tag_push(fixture: &mut Box) -> Result<(), ScenarioError> { + let fixture = fixture.downcast_mut::().unwrap(); + // SAFETY: `tags` comes from this API, and both slices come from live Rust strings. + let result = unsafe { + libdd_common_ffi::tags::ddog_Vec_Tag_push( + &mut fixture.tags, + libdd_common_ffi::CharSlice::from(fixture.key.as_str()), + libdd_common_ffi::CharSlice::from(fixture.value.as_str()), + ) + }; + if matches!(result, libdd_common_ffi::tags::PushTagResult::Ok) { + Ok(()) + } else { + Err(ScenarioError::Rejected) + } +} + pub fn find(name: &str) -> Option<&'static Scenario> { - SCENARIOS.iter().find(|s| s.name == name) + SCENARIOS + .iter() + .chain(TAG_SCENARIOS) + .find(|s| s.name == name) } + +const TAG_SCENARIOS: &[Scenario] = &[ + Scenario { + name: TAG_PUSH_EMPTY, + setup: || { + Box::new(tag_push( + "env", + "prod", + libdd_common_ffi::tags::ddog_Vec_Tag_new(), + )) + }, + run: run_tag_push, + }, + Scenario { + name: TAG_PUSH_AT_CAPACITY, + setup: || Box::new(tag_push("env", "prod", fill_to_capacity())), + run: run_tag_push, + }, + Scenario { + name: TAG_PUSH_LONG, + setup: || { + Box::new(TagPush { + tags: libdd_common_ffi::tags::ddog_Vec_Tag_new(), + key: "k".repeat(LONG_LEN), + value: "v".repeat(LONG_LEN), + }) + }, + run: run_tag_push, + }, +]; From 2abe6fea8cdcf73f9d26bb89b90a52ec14f93af6 Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Fri, 20 Nov 2026 13:27:57 +0000 Subject: [PATCH 3/6] test(profiling-ffi): cover arena allocation failure in the chaos harness The test-only feature shares one policy and event counter between logical arena requests and backing-chunk growth, so those failures can be enumerated without changing production types. Jira: PROF-16808 --- Cargo.lock | 1 + libdd-alloc/Cargo.toml | 4 + libdd-alloc/src/chaos.rs | 102 ++++++++++++ libdd-alloc/src/lib.rs | 4 + libdd-profiling-ffi/Cargo.toml | 2 + .../tests/allocation_failure/allocator.rs | 43 +++++- .../tests/allocation_failure/controller.rs | 14 +- .../tests/allocation_failure/main.rs | 93 ++++++++++- .../tests/allocation_failure/scenarios.rs | 146 ++++++++++++++++++ libdd-profiling/Cargo.toml | 3 + libdd-profiling/src/arena.rs | 31 ++++ .../src/collections/string_table/mod.rs | 17 +- libdd-profiling/src/lib.rs | 2 + .../src/profiles/collections/set.rs | 7 +- .../src/profiles/collections/slice_set.rs | 6 +- 15 files changed, 454 insertions(+), 21 deletions(-) create mode 100644 libdd-alloc/src/chaos.rs create mode 100644 libdd-profiling/src/arena.rs diff --git a/Cargo.lock b/Cargo.lock index 6536d7107f..a6803a1ad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3463,6 +3463,7 @@ dependencies = [ "function_name", "hyper", "libc", + "libdd-alloc", "libdd-common", "libdd-common-ffi", "libdd-crashtracker-ffi", diff --git a/libdd-alloc/Cargo.toml b/libdd-alloc/Cargo.toml index 9f095e2b93..9c069fd988 100644 --- a/libdd-alloc/Cargo.toml +++ b/libdd-alloc/Cargo.toml @@ -11,6 +11,10 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[features] +# Test-only allocation-failure proxy. Off for every production build. +allocation-chaos-test = [] + [dependencies] allocator-api2.workspace = true diff --git a/libdd-alloc/src/chaos.rs b/libdd-alloc/src/chaos.rs new file mode 100644 index 0000000000..45ec5bcf0d --- /dev/null +++ b/libdd-alloc/src/chaos.rs @@ -0,0 +1,102 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Test-only proxy in front of an [`Allocator`]. It asks the process-wide observer whether this +//! allocation must fail, then forwards the call. `deallocate` is never observed and never fails. +//! +//! The observer lives in the allocation-failure harness and shares its ordinal counter with the +//! global allocator proxy, so one trace can hold `Global`, `ArenaRequest` and `ArenaBacking` +//! events. With no observer registered, every call is forwarded. + +use crate::{AllocError, Allocator}; +use core::alloc::Layout; +use core::marker::PhantomData; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicPtr, Ordering}; + +pub const DOMAIN_ARENA_REQUEST: u8 = 1; +pub const DOMAIN_ARENA_BACKING: u8 = 2; +pub const OP_ALLOC: u8 = 0; +pub const OP_ALLOC_ZEROED: u8 = 1; + +/// `(domain, operation, size, align) -> fail`. Must not allocate. +pub type ObserveFn = fn(u8, u8, usize, usize) -> bool; + +static OBSERVE: AtomicPtr<()> = AtomicPtr::new(core::ptr::null_mut()); + +pub fn register_observer(observe: ObserveFn) { + OBSERVE.store(observe as *const () as *mut (), Ordering::Relaxed); +} + +#[inline(always)] +fn observe(domain: u8, op: u8, size: usize, align: usize) -> bool { + let ptr = OBSERVE.load(Ordering::Relaxed); + if ptr.is_null() { + return false; + } + // SAFETY: `ptr` was stored from an `ObserveFn` by `register_observer`. + let observe: ObserveFn = unsafe { core::mem::transmute(ptr) }; + observe(domain, op, size, align) +} + +/// Marker: count this allocator's calls as logical arena requests. +#[derive(Clone, Copy, Debug)] +pub struct ArenaRequest; + +/// Marker: count this allocator's calls as arena backing-chunk growth. +#[derive(Clone, Copy, Debug)] +pub struct ArenaBacking; + +trait ChaosDomain { + const ID: u8; +} + +impl ChaosDomain for ArenaRequest { + const ID: u8 = DOMAIN_ARENA_REQUEST; +} + +impl ChaosDomain for ArenaBacking { + const ID: u8 = DOMAIN_ARENA_BACKING; +} + +/// Forwards to `A`, failing allocations the observer selects. +#[derive(Clone, Copy)] +pub struct ChaosAllocator { + inner: A, + _domain: PhantomData, +} + +impl ChaosAllocator { + pub const fn new(inner: A) -> Self { + Self { + inner, + _domain: PhantomData, + } + } + + pub fn inner(&self) -> &A { + &self.inner + } +} + +unsafe impl Allocator for ChaosAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + if observe(D::ID, OP_ALLOC, layout.size(), layout.align()) { + return Err(AllocError); + } + self.inner.allocate(layout) + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + if observe(D::ID, OP_ALLOC_ZEROED, layout.size(), layout.align()) { + return Err(AllocError); + } + self.inner.allocate_zeroed(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // SAFETY: the caller guarantees `ptr` and `layout` match an allocation from this + // allocator, which forwarded it to `inner`. + unsafe { self.inner.deallocate(ptr, layout) } + } +} diff --git a/libdd-alloc/src/lib.rs b/libdd-alloc/src/lib.rs index 5ab491b7b7..78e30def3a 100644 --- a/libdd-alloc/src/lib.rs +++ b/libdd-alloc/src/lib.rs @@ -9,11 +9,15 @@ #![cfg_attr(not(test), deny(clippy::unimplemented))] mod chain; +#[cfg(feature = "allocation-chaos-test")] +mod chaos; mod linear; mod utils; mod virtual_alloc; pub use chain::*; +#[cfg(feature = "allocation-chaos-test")] +pub use chaos::{ArenaBacking, ArenaRequest, ChaosAllocator, ObserveFn, register_observer}; pub use linear::*; pub use virtual_alloc::*; diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index 2af5f75abd..00743d394a 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -95,8 +95,10 @@ libdd-shared-runtime-ffi = { path = "../libdd-shared-runtime-ffi", default-featu libdd-otel-thread-ctx-ffi = { path = "../libdd-otel-thread-ctx-ffi", default-features = false, optional = true } [dev-dependencies] +libdd-alloc = { path = "../libdd-alloc", features = ["allocation-chaos-test"] } libdd-common = { path = "../libdd-common" } libdd-common-ffi = { path = "../libdd-common-ffi", features = ["std"] } +libdd-profiling = { path = "../libdd-profiling", features = ["allocation-chaos-test"] } # Experimental allocation-failure harness: replaces the global allocator of its own binary. [[test]] diff --git a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs index 3292fdb7a4..59e4b0a151 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs @@ -50,6 +50,8 @@ impl Policy { #[repr(u8)] pub enum Domain { Global = 0, + ArenaRequest = 1, + ArenaBacking = 2, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -115,7 +117,11 @@ impl Slot { thread: (ordinal_thread & 0xffff_ffff) as u32, size: self.size.load(Ordering::Relaxed), align: align_domain_op >> 16, - domain: Domain::Global, + domain: match (align_domain_op >> 8) & 0xff { + 1 => Domain::ArenaRequest, + 2 => Domain::ArenaBacking, + _ => Domain::Global, + }, operation: match align_domain_op & 0xff { 0 => Operation::Alloc, 1 => Operation::AllocZeroed, @@ -164,9 +170,24 @@ fn thread_id() -> u32 { .unwrap_or(0) } +/// Observer for the arena chaos wrappers. `domain` and `op` are the `libdd_alloc` codes. +pub fn observe_allocation(domain: u8, op: u8, size: usize, align: usize) -> bool { + let domain = match domain { + 1 => Domain::ArenaRequest, + 2 => Domain::ArenaBacking, + _ => Domain::Global, + }; + let operation = match op { + 0 => Operation::Alloc, + 1 => Operation::AllocZeroed, + _ => Operation::Realloc, + }; + on_allocation(domain, operation, size, align) +} + /// Counts and records one allocation attempt of the current thread. Returns `true` if it must /// fail. Returns `false` outside a scope. -fn on_allocation(operation: Operation, size: usize, align: usize) -> bool { +fn on_allocation(domain: Domain, operation: Operation, size: usize, align: usize) -> bool { let Ok(Some((policy, ordinal))) = DEPTH.try_with(|depth| { if depth.get() == 0 { return None; @@ -184,7 +205,7 @@ fn on_allocation(operation: Operation, size: usize, align: usize) -> bool { let event = Event { ordinal, - domain: Domain::Global, + domain, operation, size: size as u64, align: align as u64, @@ -223,14 +244,24 @@ pub struct ScopedFailingAllocator(pub A); unsafe impl GlobalAlloc for ScopedFailingAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - if on_allocation(Operation::Alloc, layout.size(), layout.align()) { + if on_allocation( + Domain::Global, + Operation::Alloc, + layout.size(), + layout.align(), + ) { return std::ptr::null_mut(); } unsafe { self.0.alloc(layout) } } unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - if on_allocation(Operation::AllocZeroed, layout.size(), layout.align()) { + if on_allocation( + Domain::Global, + Operation::AllocZeroed, + layout.size(), + layout.align(), + ) { return std::ptr::null_mut(); } unsafe { self.0.alloc_zeroed(layout) } @@ -238,7 +269,7 @@ unsafe impl GlobalAlloc for ScopedFailingAllocator { unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { // Returning null leaves the old block allocated and unchanged. - if on_allocation(Operation::Realloc, new_size, layout.align()) { + if on_allocation(Domain::Global, Operation::Realloc, new_size, layout.align()) { return std::ptr::null_mut(); } unsafe { self.0.realloc(ptr, layout, new_size) } diff --git a/libdd-profiling-ffi/tests/allocation_failure/controller.rs b/libdd-profiling-ffi/tests/allocation_failure/controller.rs index dbd6cb7540..968823ca1a 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/controller.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/controller.rs @@ -99,6 +99,8 @@ fn parse_event(line: &str) -> Option { let ordinal = it.next()?.parse().ok()?; let domain = match it.next()? { "Global" => Domain::Global, + "ArenaRequest" => Domain::ArenaRequest, + "ArenaBacking" => Domain::ArenaBacking, _ => return None, }; let operation = match it.next()? { @@ -379,15 +381,16 @@ impl Report { let _ = writeln!(s, "Baseline: {} events.\n", self.baseline.len()); let _ = writeln!( s, - "| mode | ordinal | operation | size | align | thread | outcome | detail |" + "| mode | ordinal | domain | operation | size | align | thread | outcome | detail |" ); - let _ = writeln!(s, "|---|---|---|---|---|---|---|---|"); + let _ = writeln!(s, "|---|---|---|---|---|---|---|---|---|"); for f in &self.runs { let _ = writeln!( s, - "| {} | {} | {:?} | {} | {} | {} | {} | {} |", + "| {} | {} | {:?} | {:?} | {} | {} | {} | {} | {} |", f.mode.as_str(), f.ordinal, + f.event.domain, f.event.operation, f.event.size, f.event.align, @@ -688,6 +691,11 @@ fn is_owned_frame(line: &str) -> bool { return false; } if let Some((_, file)) = line.rsplit_once(" at ") { + // The chaos wrapper is test infrastructure, like this harness. The call site is the + // production line that asked for the memory. + if file.contains("chaos.rs") || file.contains("/allocation_failure/") { + return false; + } return file.contains("libdd-") || file.contains("libdd_"); } line.split(" in ").nth(1).is_some_and(|rest| { diff --git a/libdd-profiling-ffi/tests/allocation_failure/main.rs b/libdd-profiling-ffi/tests/allocation_failure/main.rs index ba93640e92..d4d18ac59f 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/main.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/main.rs @@ -20,8 +20,8 @@ mod contract; mod controller; mod scenarios; -use allocator::ScopedFailingAllocator; -use controller::{Mode, Outcome}; +use allocator::{Domain, ScopedFailingAllocator}; +use controller::{Mode, Outcome, Report}; use std::alloc::System; #[global_allocator] @@ -30,6 +30,29 @@ static ALLOCATOR: ScopedFailingAllocator = ScopedFailingAllocator(System /// Discovery scenario: every ordinal in both modes is recorded. A crash is a finding. /// A nondeterministic trace is a harness error. fn discover(name: &'static str) { + let _ = load_discovery(name); +} + +/// Same as [`discover`], then checks which allocation domains the baseline contains. +fn discover_arena(name: &'static str, required: &[Domain], absent: &[Domain]) -> Report { + let report = load_discovery(name); + let present: Vec = report.baseline.iter().map(|e| e.domain).collect(); + for domain in required { + assert!( + present.contains(domain), + "{name}: baseline {present:?} is missing {domain:?}" + ); + } + for domain in absent { + assert!( + !present.contains(domain), + "{name}: baseline {present:?} unexpectedly contains {domain:?}" + ); + } + report +} + +fn load_discovery(name: &'static str) -> Report { let report = controller::enumerate_with_stacks(name).unwrap_or_else(|e| panic!("harness error: {e}")); let nondeterministic: Vec<_> = report @@ -70,6 +93,7 @@ fn discover(name: &'static str) { missing.is_empty(), "{name}: no owned call site for {missing:?}" ); + report } /// Enumerates a toy scenario and checks every run has the `expected` outcome. @@ -120,9 +144,74 @@ const TESTS: &[(&str, fn())] = &[ ("scenario::tag_push_long", || { discover(scenarios::TAG_PUSH_LONG) }), + ("scenario::arena_first", || { + discover_arena( + scenarios::ARENA_FIRST, + &[Domain::ArenaRequest, Domain::ArenaBacking], + &[], + ); + }), + ("scenario::arena_spare", || { + discover_arena( + scenarios::ARENA_SPARE, + &[Domain::ArenaRequest], + &[Domain::ArenaBacking], + ); + }), + ("scenario::arena_exhausted", || { + discover_arena( + scenarios::ARENA_EXHAUSTED, + &[Domain::ArenaRequest], + &[Domain::ArenaBacking], + ); + }), + ("scenario::arena_grow", || { + discover_arena( + scenarios::ARENA_GROW, + &[Domain::ArenaRequest, Domain::ArenaBacking], + &[], + ); + }), + ("scenario::arena_oversized", || { + let report = discover_arena( + scenarios::ARENA_OVERSIZED, + &[Domain::ArenaRequest, Domain::ArenaBacking], + &[], + ); + let hint = scenarios::ARENA_CHUNK_HINT as u64; + assert!( + report + .baseline + .iter() + .any(|event| { event.domain == Domain::ArenaBacking && event.size > hint }), + "oversized item did not grow a backing chunk past the hint" + ); + }), + ("scenario::arena_string_table", || { + discover_arena( + scenarios::ARENA_STRING_TABLE, + &[Domain::ArenaRequest, Domain::ArenaBacking], + &[], + ); + }), + ("scenario::arena_set", || { + discover_arena( + scenarios::ARENA_SET, + &[Domain::ArenaRequest, Domain::ArenaBacking], + &[], + ); + }), + ("scenario::arena_slice_set", || { + discover_arena( + scenarios::ARENA_SLICE_SET, + &[Domain::ArenaRequest, Domain::ArenaBacking], + &[], + ); + }), ]; fn main() { + libdd_alloc::register_observer(allocator::observe_allocation); let args: Vec = std::env::args().skip(1).collect(); if args.first().map(String::as_str) == Some(controller::WORKER_ARG) { diff --git a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs index 22ffdf6fa8..8020df10af 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs @@ -146,6 +146,7 @@ pub fn find(name: &str) -> Option<&'static Scenario> { SCENARIOS .iter() .chain(TAG_SCENARIOS) + .chain(ARENA_SCENARIOS) .find(|s| s.name == name) } @@ -178,3 +179,148 @@ const TAG_SCENARIOS: &[Scenario] = &[ run: run_tag_push, }, ]; + +pub const ARENA_FIRST: &str = "arena_first"; +pub const ARENA_SPARE: &str = "arena_spare"; +pub const ARENA_EXHAUSTED: &str = "arena_exhausted"; +pub const ARENA_GROW: &str = "arena_grow"; +pub const ARENA_OVERSIZED: &str = "arena_oversized"; +pub const ARENA_STRING_TABLE: &str = "arena_string_table"; +pub const ARENA_SET: &str = "arena_set"; +pub const ARENA_SLICE_SET: &str = "arena_slice_set"; + +/// Chunk hint for the direct arena scenarios. Above the chain's minimum node size, so an item +/// of `ARENA_CHUNK_HINT + 1` cannot fit in the hinted chunk. +pub const ARENA_CHUNK_HINT: usize = 64 * 1024; + +type Backing = + libdd_alloc::ChaosAllocator; +type RequestArena = + libdd_alloc::ChaosAllocator, libdd_alloc::ArenaRequest>; + +fn request_arena() -> RequestArena { + let backing = libdd_alloc::ChaosAllocator::new(libdd_alloc::VirtualAllocator {}); + libdd_alloc::ChaosAllocator::new(libdd_alloc::ChainAllocator::new_in( + ARENA_CHUNK_HINT, + backing, + )) +} + +fn alloc_bytes(arena: &RequestArena, size: usize) -> Result<(), ScenarioError> { + let layout = std::alloc::Layout::from_size_align(size, 1).map_err(|_| ScenarioError::Oom)?; + libdd_alloc::Allocator::allocate(arena, layout) + .map(|_| ()) + .map_err(|_| ScenarioError::Oom) +} + +fn alloc_ok(arena: &RequestArena, size: usize) { + alloc_bytes(arena, size).expect("setup allocation"); +} + +/// One byte is already in the chunk, and `leave` bytes of spare capacity remain. +fn arena_leaving(leave: usize) -> RequestArena { + let arena = request_arena(); + alloc_ok(&arena, 1); + let remaining = arena.inner().remaining_capacity(); + assert!( + remaining > leave, + "chunk remaining {remaining} cannot leave {leave}" + ); + alloc_ok(&arena, remaining - leave); + assert_eq!(arena.inner().remaining_capacity(), leave); + arena +} + +fn run_arena(fixture: &mut Box, size: usize) -> Result<(), ScenarioError> { + alloc_bytes(fixture.downcast_mut::().unwrap(), size) +} + +fn run_arena_8(fixture: &mut Box) -> Result<(), ScenarioError> { + run_arena(fixture, 8) +} + +fn run_arena_32(fixture: &mut Box) -> Result<(), ScenarioError> { + run_arena(fixture, 32) +} + +fn run_arena_1(fixture: &mut Box) -> Result<(), ScenarioError> { + run_arena(fixture, 1) +} + +fn run_arena_oversized(fixture: &mut Box) -> Result<(), ScenarioError> { + run_arena(fixture, ARENA_CHUNK_HINT + 1) +} + +const ARENA_SCENARIOS: &[Scenario] = &[ + Scenario { + name: ARENA_FIRST, + setup: || Box::new(request_arena()), + run: run_arena_8, + }, + Scenario { + name: ARENA_SPARE, + setup: || Box::new(arena_leaving(1024)), + run: run_arena_8, + }, + Scenario { + name: ARENA_EXHAUSTED, + setup: || Box::new(arena_leaving(32)), + run: run_arena_32, + }, + Scenario { + name: ARENA_GROW, + setup: || Box::new(arena_leaving(0)), + run: run_arena_1, + }, + Scenario { + name: ARENA_OVERSIZED, + setup: || Box::new(request_arena()), + run: run_arena_oversized, + }, + Scenario { + name: ARENA_STRING_TABLE, + setup: || Box::new(libdd_profiling::collections::string_table::StringTable::new()), + run: |fixture| { + let table = fixture + .downcast_mut::() + .unwrap(); + table + .try_intern("endpoint") + .map(|_| ()) + .map_err(|_| ScenarioError::Oom) + }, + }, + Scenario { + name: ARENA_SET, + setup: || { + Box::new( + libdd_profiling::profiles::collections::Set::::try_new().expect("set setup"), + ) + }, + run: |fixture| { + let set = fixture + .downcast_mut::>() + .unwrap(); + set.try_insert(7) + .map(|_| ()) + .map_err(|_| ScenarioError::Oom) + }, + }, + Scenario { + name: ARENA_SLICE_SET, + setup: || { + Box::new( + libdd_profiling::profiles::collections::SliceSet::::try_with_capacity(14) + .expect("slice set setup"), + ) + }, + run: |fixture| { + let set = fixture + .downcast_mut::>() + .unwrap(); + set.try_insert(&[1, 2, 3]) + .map(|_| ()) + .map_err(|_| ScenarioError::Oom) + }, + }, +]; diff --git a/libdd-profiling/Cargo.toml b/libdd-profiling/Cargo.toml index 706f1b1617..4056c7a3f2 100644 --- a/libdd-profiling/Cargo.toml +++ b/libdd-profiling/Cargo.toml @@ -18,6 +18,9 @@ bench = false [features] default = [] +# Test-only. Wraps profiling arenas with the allocation-failure proxy from libdd-alloc. +# Production builds leave this off, so the arena type stays `ChainAllocator`. +allocation-chaos-test = ["libdd-alloc/allocation-chaos-test"] cxx = ["dep:cxx", "dep:cxx-build"] otel = ["dep:opentelemetry-proto"] diff --git a/libdd-profiling/src/arena.rs b/libdd-profiling/src/arena.rs new file mode 100644 index 0000000000..7eecc07f5b --- /dev/null +++ b/libdd-profiling/src/arena.rs @@ -0,0 +1,31 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Profiling arena used by the string table and the profile sets. +//! +//! Without `allocation-chaos-test` this is `ChainAllocator`. With that +//! test-only feature, logical requests and backing-chunk growth are wrapped so the +//! allocation-failure harness can fail either one. + +use libdd_alloc::{ChainAllocator, VirtualAllocator}; + +#[cfg(not(feature = "allocation-chaos-test"))] +pub(crate) type ProfilingArena = ChainAllocator; + +#[cfg(feature = "allocation-chaos-test")] +pub(crate) type ProfilingArena = libdd_alloc::ChaosAllocator< + ChainAllocator>, + libdd_alloc::ArenaRequest, +>; + +pub(crate) fn profiling_arena(chunk_size_hint: usize) -> ProfilingArena { + #[cfg(not(feature = "allocation-chaos-test"))] + { + ChainAllocator::new_in(chunk_size_hint, VirtualAllocator {}) + } + #[cfg(feature = "allocation-chaos-test")] + { + let backing = libdd_alloc::ChaosAllocator::new(VirtualAllocator {}); + libdd_alloc::ChaosAllocator::new(ChainAllocator::new_in(chunk_size_hint, backing)) + } +} diff --git a/libdd-profiling/src/collections/string_table/mod.rs b/libdd-profiling/src/collections/string_table/mod.rs index df3d617528..41ad2a5c14 100644 --- a/libdd-profiling/src/collections/string_table/mod.rs +++ b/libdd-profiling/src/collections/string_table/mod.rs @@ -7,9 +7,12 @@ pub mod wordpress_test_data; pub use error::*; +use crate::arena::{ProfilingArena, profiling_arena}; use crate::collections::identifiable::StringId; use crate::iter::{IntoLendingIterator, LendingIterator}; -use libdd_alloc::{AllocError, Allocator, ChainAllocator, VirtualAllocator}; +#[cfg(test)] +use libdd_alloc::VirtualAllocator; +use libdd_alloc::{AllocError, Allocator, ChainAllocator}; use std::alloc::Layout; /// A trait that indicates an allocator is arena allocator, meaning it doesn't @@ -51,6 +54,12 @@ pub trait ArenaAllocator: Allocator { impl ArenaAllocator for ChainAllocator {} +#[cfg(feature = "allocation-chaos-test")] +impl ArenaAllocator + for libdd_alloc::ChaosAllocator, libdd_alloc::ArenaRequest> +{ +} + type Hasher = core::hash::BuildHasherDefault; type HashSet = indexmap::IndexSet; @@ -58,7 +67,7 @@ type HashSet = indexmap::IndexSet; /// that the strings were inserted. pub struct StringTable { /// The bytes of each string stored in `strings` are allocated here. - bytes: ChainAllocator, + bytes: ProfilingArena, /// The ordered hash set of unique strings. The order becomes the StringId. /// The static lifetime is a lie, it is tied to the `bytes`, which is only @@ -86,7 +95,7 @@ impl StringTable { // string tables in memory at a given time. Talk to .NET profiling // engineers before making this any bigger. const SIZE_HINT: usize = 4 * 1024 * 1024; - let bytes = ChainAllocator::new_in(SIZE_HINT, VirtualAllocator {}); + let bytes = profiling_arena(SIZE_HINT); let mut strings = HashSet::with_hasher(Hasher::default()); // It varies by implementation, but frequently I've noticed that the @@ -178,7 +187,7 @@ pub struct StringTableIter { /// This is actually used, the compiler doesn't know that the static /// references in `iter` actually point in here. #[allow(unused)] - bytes: ChainAllocator, + bytes: ProfilingArena, /// The strings of the string table, in order of insertion. /// The static lifetimes are a lie, they are tied to the `bytes`. When diff --git a/libdd-profiling/src/lib.rs b/libdd-profiling/src/lib.rs index 17e26dcc79..456f2f7048 100644 --- a/libdd-profiling/src/lib.rs +++ b/libdd-profiling/src/lib.rs @@ -6,6 +6,8 @@ #![cfg_attr(not(test), deny(clippy::todo))] #![cfg_attr(not(test), deny(clippy::unimplemented))] +mod arena; + pub mod api; pub mod api2; pub mod collections; diff --git a/libdd-profiling/src/profiles/collections/set.rs b/libdd-profiling/src/profiles/collections/set.rs index 6504f0c0f0..2fce5ea700 100644 --- a/libdd-profiling/src/profiles/collections/set.rs +++ b/libdd-profiling/src/profiles/collections/set.rs @@ -3,10 +3,11 @@ use super::SetError; use super::SetHasher as Hasher; +use crate::arena::{ProfilingArena, profiling_arena}; use core::hint::unreachable_unchecked; use core::{fmt, mem, ptr}; use hashbrown::HashTable; -use libdd_alloc::{Allocator, ChainAllocator, VirtualAllocator}; +use libdd_alloc::Allocator; use std::ffi::c_void; use std::hash::{BuildHasher, Hash}; @@ -56,7 +57,7 @@ impl fmt::Debug for Set { } pub struct Set { - pub(crate) arena: ChainAllocator, + pub(crate) arena: ProfilingArena, pub(crate) table: HashTable>, } @@ -146,7 +147,7 @@ impl Drop for Set { impl Set { pub(crate) fn try_with_capacity(capacity: usize) -> Result { - let arena = ChainAllocator::new_in(Self::SIZE_HINT, VirtualAllocator {}); + let arena = profiling_arena(Self::SIZE_HINT); let mut table = HashTable::new(); // SAFETY: new empty table cannot require rehash, callback unreachable. diff --git a/libdd-profiling/src/profiles/collections/slice_set.rs b/libdd-profiling/src/profiles/collections/slice_set.rs index 28d8689942..f79e9d0a6e 100644 --- a/libdd-profiling/src/profiles/collections/slice_set.rs +++ b/libdd-profiling/src/profiles/collections/slice_set.rs @@ -2,17 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 use super::{SetError, ThinSlice}; +use crate::arena::{ProfilingArena, profiling_arena}; use core::hash::{BuildHasher, Hash}; use core::hint::unreachable_unchecked; use hashbrown::HashTable; -use libdd_alloc::{ChainAllocator, VirtualAllocator}; use super::SetHasher as Hasher; /// Holds unique slices and provides handles to fetch them later. pub struct SliceSet { /// The bytes of each slice stored in `slices` are allocated here. - pub(crate) arena: ChainAllocator, + pub(crate) arena: ProfilingArena, /// The unordered hash set of unique slices. /// The static lifetimes are a lie; they are tied to the `arena`, which is @@ -26,7 +26,7 @@ impl SliceSet { const SIZE_HINT: usize = 1024 * 1024; pub fn try_with_capacity(capacity: usize) -> Result { - let arena = ChainAllocator::new_in(Self::SIZE_HINT, VirtualAllocator {}); + let arena = profiling_arena(Self::SIZE_HINT); let mut slices = HashTable::new(); // SAFETY: we just made the empty hash table, so there's nothing that From 9d70a30c9a9e003e877f45c5e8f103bb7351926a Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Fri, 20 Nov 2026 14:41:51 +0000 Subject: [PATCH 4/6] test(profiling-ffi): keep arena chaos wrappers out of the public API Move the arena chaos wrappers from libdd-alloc into a crate-private module of libdd-profiling, compiled only under the allocation_chaos_test cfg (LIBDD_ALLOCATION_CHAOS_TEST=1). Drop the allocation-chaos-test Cargo features so published crates gain no public items or features. The harness reaches the observer through a cfg-gated extern static and builds its own arena wrappers. Jira: PROF-16808 --- libdd-alloc/Cargo.toml | 4 - libdd-alloc/src/chaos.rs | 102 --------------- libdd-alloc/src/lib.rs | 4 - libdd-profiling-ffi/Cargo.toml | 3 +- libdd-profiling-ffi/build.rs | 8 ++ .../tests/allocation_failure/allocator.rs | 8 +- .../tests/allocation_failure/arena.rs | 112 +++++++++++++++++ .../tests/allocation_failure/controller.rs | 17 ++- .../tests/allocation_failure/main.rs | 26 ++-- .../tests/allocation_failure/scenarios.rs | 11 +- libdd-profiling/Cargo.toml | 3 - libdd-profiling/build.rs | 10 ++ libdd-profiling/src/arena.rs | 117 ++++++++++++++++-- .../src/collections/string_table/mod.rs | 6 - 14 files changed, 274 insertions(+), 157 deletions(-) delete mode 100644 libdd-alloc/src/chaos.rs create mode 100644 libdd-profiling-ffi/tests/allocation_failure/arena.rs diff --git a/libdd-alloc/Cargo.toml b/libdd-alloc/Cargo.toml index 9c069fd988..9f095e2b93 100644 --- a/libdd-alloc/Cargo.toml +++ b/libdd-alloc/Cargo.toml @@ -11,10 +11,6 @@ edition.workspace = true rust-version.workspace = true license.workspace = true -[features] -# Test-only allocation-failure proxy. Off for every production build. -allocation-chaos-test = [] - [dependencies] allocator-api2.workspace = true diff --git a/libdd-alloc/src/chaos.rs b/libdd-alloc/src/chaos.rs deleted file mode 100644 index 45ec5bcf0d..0000000000 --- a/libdd-alloc/src/chaos.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -//! Test-only proxy in front of an [`Allocator`]. It asks the process-wide observer whether this -//! allocation must fail, then forwards the call. `deallocate` is never observed and never fails. -//! -//! The observer lives in the allocation-failure harness and shares its ordinal counter with the -//! global allocator proxy, so one trace can hold `Global`, `ArenaRequest` and `ArenaBacking` -//! events. With no observer registered, every call is forwarded. - -use crate::{AllocError, Allocator}; -use core::alloc::Layout; -use core::marker::PhantomData; -use core::ptr::NonNull; -use core::sync::atomic::{AtomicPtr, Ordering}; - -pub const DOMAIN_ARENA_REQUEST: u8 = 1; -pub const DOMAIN_ARENA_BACKING: u8 = 2; -pub const OP_ALLOC: u8 = 0; -pub const OP_ALLOC_ZEROED: u8 = 1; - -/// `(domain, operation, size, align) -> fail`. Must not allocate. -pub type ObserveFn = fn(u8, u8, usize, usize) -> bool; - -static OBSERVE: AtomicPtr<()> = AtomicPtr::new(core::ptr::null_mut()); - -pub fn register_observer(observe: ObserveFn) { - OBSERVE.store(observe as *const () as *mut (), Ordering::Relaxed); -} - -#[inline(always)] -fn observe(domain: u8, op: u8, size: usize, align: usize) -> bool { - let ptr = OBSERVE.load(Ordering::Relaxed); - if ptr.is_null() { - return false; - } - // SAFETY: `ptr` was stored from an `ObserveFn` by `register_observer`. - let observe: ObserveFn = unsafe { core::mem::transmute(ptr) }; - observe(domain, op, size, align) -} - -/// Marker: count this allocator's calls as logical arena requests. -#[derive(Clone, Copy, Debug)] -pub struct ArenaRequest; - -/// Marker: count this allocator's calls as arena backing-chunk growth. -#[derive(Clone, Copy, Debug)] -pub struct ArenaBacking; - -trait ChaosDomain { - const ID: u8; -} - -impl ChaosDomain for ArenaRequest { - const ID: u8 = DOMAIN_ARENA_REQUEST; -} - -impl ChaosDomain for ArenaBacking { - const ID: u8 = DOMAIN_ARENA_BACKING; -} - -/// Forwards to `A`, failing allocations the observer selects. -#[derive(Clone, Copy)] -pub struct ChaosAllocator { - inner: A, - _domain: PhantomData, -} - -impl ChaosAllocator { - pub const fn new(inner: A) -> Self { - Self { - inner, - _domain: PhantomData, - } - } - - pub fn inner(&self) -> &A { - &self.inner - } -} - -unsafe impl Allocator for ChaosAllocator { - fn allocate(&self, layout: Layout) -> Result, AllocError> { - if observe(D::ID, OP_ALLOC, layout.size(), layout.align()) { - return Err(AllocError); - } - self.inner.allocate(layout) - } - - fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { - if observe(D::ID, OP_ALLOC_ZEROED, layout.size(), layout.align()) { - return Err(AllocError); - } - self.inner.allocate_zeroed(layout) - } - - unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - // SAFETY: the caller guarantees `ptr` and `layout` match an allocation from this - // allocator, which forwarded it to `inner`. - unsafe { self.inner.deallocate(ptr, layout) } - } -} diff --git a/libdd-alloc/src/lib.rs b/libdd-alloc/src/lib.rs index 78e30def3a..5ab491b7b7 100644 --- a/libdd-alloc/src/lib.rs +++ b/libdd-alloc/src/lib.rs @@ -9,15 +9,11 @@ #![cfg_attr(not(test), deny(clippy::unimplemented))] mod chain; -#[cfg(feature = "allocation-chaos-test")] -mod chaos; mod linear; mod utils; mod virtual_alloc; pub use chain::*; -#[cfg(feature = "allocation-chaos-test")] -pub use chaos::{ArenaBacking, ArenaRequest, ChaosAllocator, ObserveFn, register_observer}; pub use linear::*; pub use virtual_alloc::*; diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index 00743d394a..caacdf1a09 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -95,10 +95,9 @@ libdd-shared-runtime-ffi = { path = "../libdd-shared-runtime-ffi", default-featu libdd-otel-thread-ctx-ffi = { path = "../libdd-otel-thread-ctx-ffi", default-features = false, optional = true } [dev-dependencies] -libdd-alloc = { path = "../libdd-alloc", features = ["allocation-chaos-test"] } +libdd-alloc = { path = "../libdd-alloc" } libdd-common = { path = "../libdd-common" } libdd-common-ffi = { path = "../libdd-common-ffi", features = ["std"] } -libdd-profiling = { path = "../libdd-profiling", features = ["allocation-chaos-test"] } # Experimental allocation-failure harness: replaces the global allocator of its own binary. [[test]] diff --git a/libdd-profiling-ffi/build.rs b/libdd-profiling-ffi/build.rs index 9ae5902028..e0b06f5dfb 100644 --- a/libdd-profiling-ffi/build.rs +++ b/libdd-profiling-ffi/build.rs @@ -5,6 +5,14 @@ extern crate build_common; use build_common::{embed_windows_version_info, generate_and_configure_header}; fn main() { + // Must match libdd-profiling/build.rs: the allocation-failure test reaches the arena + // observer that libdd-profiling defines only under this cfg. + println!("cargo:rerun-if-env-changed=LIBDD_ALLOCATION_CHAOS_TEST"); + println!("cargo:rustc-check-cfg=cfg(allocation_chaos_test)"); + if std::env::var_os("LIBDD_ALLOCATION_CHAOS_TEST").is_some_and(|v| v == "1") { + println!("cargo:rustc-cfg=allocation_chaos_test"); + } + let header_name = "profiling.h"; generate_and_configure_header(header_name); embed_windows_version_info("datadog_profiling_ffi.dll", "Datadog libdatadog FFI"); diff --git a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs index 59e4b0a151..04dd50873d 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs @@ -170,7 +170,8 @@ fn thread_id() -> u32 { .unwrap_or(0) } -/// Observer for the arena chaos wrappers. `domain` and `op` are the `libdd_alloc` codes. +/// Observer for the `libdd-profiling` arena proxy. `domain` and `op` use that proxy's codes. +#[cfg_attr(not(allocation_chaos_test), allow(dead_code))] pub fn observe_allocation(domain: u8, op: u8, size: usize, align: usize) -> bool { let domain = match domain { 1 => Domain::ArenaRequest, @@ -185,6 +186,11 @@ pub fn observe_allocation(domain: u8, op: u8, size: usize, align: usize) -> bool on_allocation(domain, operation, size, align) } +/// Same as [`observe_allocation`], for the harness's own arena wrappers. +pub fn observe(domain: Domain, operation: Operation, size: usize, align: usize) -> bool { + on_allocation(domain, operation, size, align) +} + /// Counts and records one allocation attempt of the current thread. Returns `true` if it must /// fail. Returns `false` outside a scope. fn on_allocation(domain: Domain, operation: Operation, size: usize, align: usize) -> bool { diff --git a/libdd-profiling-ffi/tests/allocation_failure/arena.rs b/libdd-profiling-ffi/tests/allocation_failure/arena.rs new file mode 100644 index 0000000000..e6bb9a019b --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/arena.rs @@ -0,0 +1,112 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Arena wiring of the harness. +//! +//! Built with `LIBDD_ALLOCATION_CHAOS_TEST=1`, `libdd-profiling` wraps its arenas in a +//! crate-private proxy that calls the observer stored in `ALLOC_CHAOS_OBSERVER`. Without it, +//! profiling arenas are not observed and the arena-domain checks are skipped. The direct arena +//! scenarios always use the same wrapping built here, around a plain `ChainAllocator`, so that +//! the chunk hint can be small. + +use crate::allocator::{self, Domain, Operation}; +use libdd_alloc::{AllocError, Allocator, ChainAllocator, VirtualAllocator}; +use std::alloc::Layout; +use std::marker::PhantomData; +use std::ptr::NonNull; + +/// Whether `libdd-profiling` arenas report to the harness in this build. +pub const PROFILING_ARENAS_OBSERVED: bool = cfg!(allocation_chaos_test); + +#[cfg(allocation_chaos_test)] +unsafe extern "Rust" { + static ALLOC_CHAOS_OBSERVER: std::sync::atomic::AtomicPtr<()>; +} + +/// Routes the `libdd-profiling` arena proxy to the allocator's counter. Call it first in `main`. +pub fn register_observer() { + #[cfg(allocation_chaos_test)] + { + let observe: fn(u8, u8, usize, usize) -> bool = allocator::observe_allocation; + // SAFETY: under this cfg, `libdd-profiling` defines the static as an `AtomicPtr<()>`. + unsafe { + ALLOC_CHAOS_OBSERVER.store( + observe as *const () as *mut (), + std::sync::atomic::Ordering::Relaxed, + ) + }; + } +} + +pub trait ArenaDomain { + const DOMAIN: Domain; +} + +#[derive(Clone, Copy)] +pub struct Request; + +#[derive(Clone, Copy)] +pub struct Backing; + +impl ArenaDomain for Request { + const DOMAIN: Domain = Domain::ArenaRequest; +} + +impl ArenaDomain for Backing { + const DOMAIN: Domain = Domain::ArenaBacking; +} + +#[derive(Clone, Copy)] +pub struct Chaos { + inner: A, + _domain: PhantomData, +} + +impl Chaos { + pub fn new(inner: A) -> Self { + Self { + inner, + _domain: PhantomData, + } + } + + pub fn inner(&self) -> &A { + &self.inner + } +} + +unsafe impl Allocator for Chaos { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + if allocator::observe(D::DOMAIN, Operation::Alloc, layout.size(), layout.align()) { + return Err(AllocError); + } + self.inner.allocate(layout) + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + if allocator::observe( + D::DOMAIN, + Operation::AllocZeroed, + layout.size(), + layout.align(), + ) { + return Err(AllocError); + } + self.inner.allocate_zeroed(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // SAFETY: forwarded unchanged to the allocator that made the block. + unsafe { self.inner.deallocate(ptr, layout) } + } +} + +/// Same shape as the `libdd-profiling` arena with the feature on. +pub type RequestArena = Chaos>, Request>; + +pub fn request_arena(chunk_size_hint: usize) -> RequestArena { + Chaos::new(ChainAllocator::new_in( + chunk_size_hint, + Chaos::new(VirtualAllocator {}), + )) +} diff --git a/libdd-profiling-ffi/tests/allocation_failure/controller.rs b/libdd-profiling-ffi/tests/allocation_failure/controller.rs index 968823ca1a..784252f875 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/controller.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/controller.rs @@ -548,7 +548,13 @@ fn capture_stacks(scenario: &str, baseline: &[Event], runs: &mut [Finding]) { } } +/// `ALLOC_CHAOS_DEBUGGER=gdb|lldb` forces one; otherwise gdb, then lldb. fn debugger() -> Option<&'static str> { + match std::env::var("ALLOC_CHAOS_DEBUGGER").as_deref() { + Ok("gdb") => return command_ok("gdb").then_some("gdb"), + Ok("lldb") => return command_ok("lldb").then_some("lldb"), + _ => {} + } if command_ok("gdb") { Some("gdb") } else if command_ok("lldb") { @@ -604,10 +610,17 @@ fn debugger_backtrace( .arg("bt 80"); } "lldb" => { + // The default frame format prints the file name only; owned frames are recognized + // by their full path. cmd.arg("-b") .arg("-o") .arg("settings set auto-confirm true") .arg("-o") + .arg( + "settings set frame-format \"frame #${frame.index}: ${function.name}\ + { at ${line.file.fullpath}:${line.number}}\\n\"", + ) + .arg("-o") .arg("breakpoint set --name alloc_chaos_injection_hook") .arg("-o") .arg(format!( @@ -619,7 +632,7 @@ fn debugger_backtrace( ordinal )) .arg("-o") - .arg("bt") + .arg("bt 80") .arg("-o") .arg("quit") .arg(&exe); @@ -693,7 +706,7 @@ fn is_owned_frame(line: &str) -> bool { if let Some((_, file)) = line.rsplit_once(" at ") { // The chaos wrapper is test infrastructure, like this harness. The call site is the // production line that asked for the memory. - if file.contains("chaos.rs") || file.contains("/allocation_failure/") { + if file.contains("libdd-profiling/src/arena.rs") || file.contains("/allocation_failure/") { return false; } return file.contains("libdd-") || file.contains("libdd_"); diff --git a/libdd-profiling-ffi/tests/allocation_failure/main.rs b/libdd-profiling-ffi/tests/allocation_failure/main.rs index d4d18ac59f..3ffc0a87cb 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/main.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/main.rs @@ -16,6 +16,7 @@ //! `alloc_chaos_injection_hook` records the stack and the owned call site. mod allocator; +mod arena; mod contract; mod controller; mod scenarios; @@ -52,6 +53,15 @@ fn discover_arena(name: &'static str, required: &[Domain], absent: &[Domain]) -> report } +/// Arena domains a `libdd-profiling` collection must show, when its arenas are observed. +fn profiling_arena_domains() -> &'static [Domain] { + if arena::PROFILING_ARENAS_OBSERVED { + &[Domain::ArenaRequest, Domain::ArenaBacking] + } else { + &[] + } +} + fn load_discovery(name: &'static str) -> Report { let report = controller::enumerate_with_stacks(name).unwrap_or_else(|e| panic!("harness error: {e}")); @@ -190,28 +200,20 @@ const TESTS: &[(&str, fn())] = &[ ("scenario::arena_string_table", || { discover_arena( scenarios::ARENA_STRING_TABLE, - &[Domain::ArenaRequest, Domain::ArenaBacking], + profiling_arena_domains(), &[], ); }), ("scenario::arena_set", || { - discover_arena( - scenarios::ARENA_SET, - &[Domain::ArenaRequest, Domain::ArenaBacking], - &[], - ); + discover_arena(scenarios::ARENA_SET, profiling_arena_domains(), &[]); }), ("scenario::arena_slice_set", || { - discover_arena( - scenarios::ARENA_SLICE_SET, - &[Domain::ArenaRequest, Domain::ArenaBacking], - &[], - ); + discover_arena(scenarios::ARENA_SLICE_SET, profiling_arena_domains(), &[]); }), ]; fn main() { - libdd_alloc::register_observer(allocator::observe_allocation); + arena::register_observer(); let args: Vec = std::env::args().skip(1).collect(); if args.first().map(String::as_str) == Some(controller::WORKER_ARG) { diff --git a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs index 8020df10af..f96a07c801 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs @@ -193,17 +193,10 @@ pub const ARENA_SLICE_SET: &str = "arena_slice_set"; /// of `ARENA_CHUNK_HINT + 1` cannot fit in the hinted chunk. pub const ARENA_CHUNK_HINT: usize = 64 * 1024; -type Backing = - libdd_alloc::ChaosAllocator; -type RequestArena = - libdd_alloc::ChaosAllocator, libdd_alloc::ArenaRequest>; +use crate::arena::RequestArena; fn request_arena() -> RequestArena { - let backing = libdd_alloc::ChaosAllocator::new(libdd_alloc::VirtualAllocator {}); - libdd_alloc::ChaosAllocator::new(libdd_alloc::ChainAllocator::new_in( - ARENA_CHUNK_HINT, - backing, - )) + crate::arena::request_arena(ARENA_CHUNK_HINT) } fn alloc_bytes(arena: &RequestArena, size: usize) -> Result<(), ScenarioError> { diff --git a/libdd-profiling/Cargo.toml b/libdd-profiling/Cargo.toml index 4056c7a3f2..706f1b1617 100644 --- a/libdd-profiling/Cargo.toml +++ b/libdd-profiling/Cargo.toml @@ -18,9 +18,6 @@ bench = false [features] default = [] -# Test-only. Wraps profiling arenas with the allocation-failure proxy from libdd-alloc. -# Production builds leave this off, so the arena type stays `ChainAllocator`. -allocation-chaos-test = ["libdd-alloc/allocation-chaos-test"] cxx = ["dep:cxx", "dep:cxx-build"] otel = ["dep:opentelemetry-proto"] diff --git a/libdd-profiling/build.rs b/libdd-profiling/build.rs index 73dd8d8d1f..1f96b2c012 100644 --- a/libdd-profiling/build.rs +++ b/libdd-profiling/build.rs @@ -2,6 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 fn main() { + // Test-only: `LIBDD_ALLOCATION_CHAOS_TEST=1` wraps the profiling arenas for the + // allocation-failure harness of libdd-profiling-ffi. A cfg rather than a Cargo feature, so + // feature unification can never turn it on in a shipped build. + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=LIBDD_ALLOCATION_CHAOS_TEST"); + println!("cargo:rustc-check-cfg=cfg(allocation_chaos_test)"); + if std::env::var_os("LIBDD_ALLOCATION_CHAOS_TEST").is_some_and(|v| v == "1") { + println!("cargo:rustc-cfg=allocation_chaos_test"); + } + // Build CXX bridge if feature is enabled #[cfg(feature = "cxx")] { diff --git a/libdd-profiling/src/arena.rs b/libdd-profiling/src/arena.rs index 7eecc07f5b..05467eb026 100644 --- a/libdd-profiling/src/arena.rs +++ b/libdd-profiling/src/arena.rs @@ -3,29 +3,122 @@ //! Profiling arena used by the string table and the profile sets. //! -//! Without `allocation-chaos-test` this is `ChainAllocator`. With that -//! test-only feature, logical requests and backing-chunk growth are wrapped so the -//! allocation-failure harness can fail either one. +//! This is `ChainAllocator`. Only under the test-only cfg +//! `allocation_chaos_test` (see `build.rs`) are logical requests and backing-chunk growth +//! wrapped, so the allocation-failure harness can fail either one. Nothing here is public API. use libdd_alloc::{ChainAllocator, VirtualAllocator}; -#[cfg(not(feature = "allocation-chaos-test"))] +#[cfg(not(allocation_chaos_test))] pub(crate) type ProfilingArena = ChainAllocator; -#[cfg(feature = "allocation-chaos-test")] -pub(crate) type ProfilingArena = libdd_alloc::ChaosAllocator< - ChainAllocator>, - libdd_alloc::ArenaRequest, +#[cfg(allocation_chaos_test)] +pub(crate) type ProfilingArena = chaos::ChaosAllocator< + ChainAllocator>, + chaos::ArenaRequest, >; pub(crate) fn profiling_arena(chunk_size_hint: usize) -> ProfilingArena { - #[cfg(not(feature = "allocation-chaos-test"))] + #[cfg(not(allocation_chaos_test))] { ChainAllocator::new_in(chunk_size_hint, VirtualAllocator {}) } - #[cfg(feature = "allocation-chaos-test")] + #[cfg(allocation_chaos_test)] { - let backing = libdd_alloc::ChaosAllocator::new(VirtualAllocator {}); - libdd_alloc::ChaosAllocator::new(ChainAllocator::new_in(chunk_size_hint, backing)) + let backing = chaos::ChaosAllocator::new(VirtualAllocator {}); + chaos::ChaosAllocator::new(ChainAllocator::new_in(chunk_size_hint, backing)) } } + +/// Test-only proxy in front of an arena allocator. It asks the harness observer whether this +/// allocation must fail, then forwards the call. `deallocate` is never observed and never fails. +#[cfg(allocation_chaos_test)] +pub(crate) mod chaos { + use crate::collections::string_table::ArenaAllocator; + use core::alloc::Layout; + use core::marker::PhantomData; + use core::ptr::NonNull; + use core::sync::atomic::{AtomicPtr, Ordering}; + use libdd_alloc::{AllocError, Allocator, ChainAllocator}; + + const DOMAIN_ARENA_REQUEST: u8 = 1; + const DOMAIN_ARENA_BACKING: u8 = 2; + const OP_ALLOC: u8 = 0; + const OP_ALLOC_ZEROED: u8 = 1; + + /// `fn(domain, operation, size, align) -> fail`, stored by the allocation-failure harness. + /// The harness reaches it by symbol name, so it is not part of the Rust API. Null means + /// every call is forwarded. + #[unsafe(no_mangle)] + static ALLOC_CHAOS_OBSERVER: AtomicPtr<()> = AtomicPtr::new(core::ptr::null_mut()); + + #[inline(always)] + fn observe(domain: u8, op: u8, size: usize, align: usize) -> bool { + let ptr = ALLOC_CHAOS_OBSERVER.load(Ordering::Relaxed); + if ptr.is_null() { + return false; + } + // SAFETY: the harness only stores a `fn(u8, u8, usize, usize) -> bool`. + let observe: fn(u8, u8, usize, usize) -> bool = unsafe { core::mem::transmute(ptr) }; + observe(domain, op, size, align) + } + + #[derive(Clone, Copy)] + pub(crate) struct ArenaRequest; + + #[derive(Clone, Copy)] + pub(crate) struct ArenaBacking; + + pub(crate) trait ChaosDomain { + const ID: u8; + } + + impl ChaosDomain for ArenaRequest { + const ID: u8 = DOMAIN_ARENA_REQUEST; + } + + impl ChaosDomain for ArenaBacking { + const ID: u8 = DOMAIN_ARENA_BACKING; + } + + #[derive(Clone, Copy)] + pub(crate) struct ChaosAllocator { + inner: A, + _domain: PhantomData, + } + + impl ChaosAllocator { + pub(crate) const fn new(inner: A) -> Self { + Self { + inner, + _domain: PhantomData, + } + } + } + + unsafe impl Allocator for ChaosAllocator { + #[inline(always)] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + if observe(D::ID, OP_ALLOC, layout.size(), layout.align()) { + return Err(AllocError); + } + self.inner.allocate(layout) + } + + #[inline(always)] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + if observe(D::ID, OP_ALLOC_ZEROED, layout.size(), layout.align()) { + return Err(AllocError); + } + self.inner.allocate_zeroed(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // SAFETY: the caller guarantees `ptr` and `layout` match an allocation from this + // allocator, which forwarded it to `inner`. + unsafe { self.inner.deallocate(ptr, layout) } + } + } + + impl ArenaAllocator for ChaosAllocator, ArenaRequest> {} +} diff --git a/libdd-profiling/src/collections/string_table/mod.rs b/libdd-profiling/src/collections/string_table/mod.rs index 41ad2a5c14..ebc8d3c382 100644 --- a/libdd-profiling/src/collections/string_table/mod.rs +++ b/libdd-profiling/src/collections/string_table/mod.rs @@ -54,12 +54,6 @@ pub trait ArenaAllocator: Allocator { impl ArenaAllocator for ChainAllocator {} -#[cfg(feature = "allocation-chaos-test")] -impl ArenaAllocator - for libdd_alloc::ChaosAllocator, libdd_alloc::ArenaRequest> -{ -} - type Hasher = core::hash::BuildHasherDefault; type HashSet = indexmap::IndexSet; From 7ad6df84dadbb027307e6273d59706b68c4e1dba Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Fri, 20 Nov 2026 15:09:29 +0000 Subject: [PATCH 5/6] test(profiling-ffi): enumerate allocation failures in the profile lifecycle Add the Tier 1 lifecycle scenarios: Profile_new, Profile_add (.NET shaped samples: first, interned, new strings, timestamped, growth), Profile_serialize (empty, populated, endpoints, upscaling, large), Exporter_new and Exporter_send_blocking (cold and warm runtime, local agent, tags, metadata, attachment). They call the C symbols through mirrored repr(C) types. Each scenario checks its post-failure contract once allocations work again: no handle from a failed constructor, the profile still usable after a failed add or serialize, the encoded profile taken by send and the exporter able to send again. Workers now write the stack of their first injected failure, and the controller symbolizes it; a debugger is only a fallback. Workers run in parallel. Allocations of other threads are counted, not enumerated. zstd allocates through the Rust allocator in test builds. Jira: PROF-16808 --- Cargo.lock | 2 + libdd-profiling-ffi/Cargo.toml | 3 + .../tests/allocation_failure/allocator.rs | 138 ++- .../tests/allocation_failure/controller.rs | 268 +++++- .../tests/allocation_failure/lifecycle.rs | 838 ++++++++++++++++++ .../tests/allocation_failure/main.rs | 113 ++- .../tests/allocation_failure/scenarios.rs | 51 +- 7 files changed, 1365 insertions(+), 48 deletions(-) create mode 100644 libdd-profiling-ffi/tests/allocation_failure/lifecycle.rs diff --git a/Cargo.lock b/Cargo.lock index a6803a1ad7..e75d7e32aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3459,6 +3459,7 @@ version = "1.0.0" dependencies = [ "allocator-api2", "anyhow", + "backtrace", "build_common", "function_name", "hyper", @@ -3482,6 +3483,7 @@ dependencies = [ "symbolizer-ffi", "thiserror 2.0.17", "tokio-util", + "zstd", ] [[package]] diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index caacdf1a09..d033ed84ee 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -95,9 +95,12 @@ libdd-shared-runtime-ffi = { path = "../libdd-shared-runtime-ffi", default-featu libdd-otel-thread-ctx-ffi = { path = "../libdd-otel-thread-ctx-ffi", default-features = false, optional = true } [dev-dependencies] +backtrace = "0.3.74" libdd-alloc = { path = "../libdd-alloc" } libdd-common = { path = "../libdd-common" } libdd-common-ffi = { path = "../libdd-common-ffi", features = ["std"] } +# zstd contexts allocate through the Rust global allocator, so the harness can fail them. +zstd = { workspace = true, features = ["with-rust-allocator"] } # Experimental allocation-failure harness: replaces the global allocator of its own binary. [[test]] diff --git a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs index 04dd50873d..186d716362 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/allocator.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/allocator.rs @@ -7,7 +7,8 @@ //! Outside a scope, every call is forwarded unchanged. Inside a scope (per thread), every //! allocation attempt (`alloc`, `alloc_zeroed`, `realloc`) gets an ordinal, is recorded in a //! preallocated trace, and fails when the scope's policy says so. `dealloc` is never counted and -//! never fails. +//! never fails. While a scope is active, allocations of the other threads are only counted per +//! thread: they are forwarded unchanged and never fail. //! //! The allocator callbacks only use constant-initialized thread-locals and atomics: no //! allocation, formatting, logging, stack walking or panic. @@ -145,6 +146,29 @@ static DIVERGED_AT: AtomicU64 = AtomicU64::new(u64::MAX); static INJECTED: AtomicU64 = AtomicU64::new(0); static NEXT_THREAD_ID: AtomicU32 = AtomicU32::new(1); +/// Number of threads currently inside a scope. +static ACTIVE_SCOPES: AtomicU32 = AtomicU32::new(0); + +/// Maximum number of other threads whose allocations are counted while a scope is active. +pub const OTHER_THREAD_CAPACITY: usize = 16; + +/// Allocations made by threads outside any scope while another thread is in one. They are +/// counted per thread and never fail: they are reported, not enumerated. +struct OtherThread { + thread: AtomicU32, + count: AtomicU64, + bytes: AtomicU64, +} + +static OTHER_THREADS: [OtherThread; OTHER_THREAD_CAPACITY] = [const { + OtherThread { + thread: AtomicU32::new(0), + count: AtomicU64::new(0), + bytes: AtomicU64::new(0), + } +}; OTHER_THREAD_CAPACITY]; +static OTHER_THREADS_OVERFLOW: AtomicU64 = AtomicU64::new(0); + thread_local! { static DEPTH: Cell = const { Cell::new(0) }; static POLICY: Cell = const { Cell::new(Policy::Observe) }; @@ -159,6 +183,59 @@ pub fn alloc_chaos_injection_hook(ordinal: u64) { std::hint::black_box(ordinal); } +static WRITE_STACK: AtomicBool = AtomicBool::new(false); + +/// Workers write the stack of their first injected failure to stderr. +pub fn write_stack_on_first_failure() { + WRITE_STACK.store(true, Ordering::Relaxed); +} + +/// Maximum number of return addresses written for the first injected failure. +const STACK_FRAMES: usize = 128; +pub const STACK_BASE_PREFIX: &str = "alloc-chaos-base "; +pub const STACK_FRAME_PREFIX: &str = "alloc-chaos-frame "; + +/// Writes `prefix`, `value` in hexadecimal and a newline to stderr, without allocating. +fn write_hex_line(prefix: &str, value: usize) { + use std::io::Write; + let mut line = [0u8; 64]; + let prefix = prefix.as_bytes(); + line[..prefix.len()].copy_from_slice(prefix); + let mut len = prefix.len(); + let digits = (usize::BITS as usize / 4).max(1); + let mut started = false; + for shift in (0..digits).rev() { + let nibble = (value >> (shift * 4)) & 0xf; + if nibble != 0 || started || shift == 0 { + started = true; + line[len] = b"0123456789abcdef"[nibble]; + len += 1; + } + } + line[len] = b'\n'; + let _ = std::io::stderr().write_all(&line[..=len]); +} + +/// Writes the return addresses of the current stack to stderr, with the address of +/// [`alloc_chaos_injection_hook`] so the controller can map them into its own address space. +/// Written right away: the process may abort just after. +fn write_stack() { + write_hex_line( + STACK_BASE_PREFIX, + alloc_chaos_injection_hook as *const () as usize, + ); + let mut frames = 0; + // SAFETY: the closure only formats into a stack buffer and writes to stderr. Concurrent + // unsynchronized traces are allowed on the supported platforms. + unsafe { + backtrace::trace_unsynchronized(|frame| { + write_hex_line(STACK_FRAME_PREFIX, frame.ip() as usize); + frames += 1; + frames < STACK_FRAMES + }); + } +} + fn thread_id() -> u32 { THREAD_ID .try_with(|id| { @@ -191,6 +268,31 @@ pub fn observe(domain: Domain, operation: Operation, size: usize, align: usize) on_allocation(domain, operation, size, align) } +fn count_other_thread(size: usize) { + let thread = thread_id(); + if thread == 0 { + // The thread-local is gone: the thread is exiting. + OTHER_THREADS_OVERFLOW.fetch_add(1, Ordering::Relaxed); + return; + } + for slot in &OTHER_THREADS { + let owner = + match slot + .thread + .compare_exchange(0, thread, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => thread, + Err(owner) => owner, + }; + if owner == thread { + slot.count.fetch_add(1, Ordering::Relaxed); + slot.bytes.fetch_add(size as u64, Ordering::Relaxed); + return; + } + } + OTHER_THREADS_OVERFLOW.fetch_add(1, Ordering::Relaxed); +} + /// Counts and records one allocation attempt of the current thread. Returns `true` if it must /// fail. Returns `false` outside a scope. fn on_allocation(domain: Domain, operation: Operation, size: usize, align: usize) -> bool { @@ -206,6 +308,9 @@ fn on_allocation(domain: Domain, operation: Operation, size: usize, align: usize }); Some((policy, ordinal)) }) else { + if ACTIVE_SCOPES.load(Ordering::Relaxed) > 0 { + count_other_thread(size); + } return false; }; @@ -239,7 +344,9 @@ fn on_allocation(domain: Domain, operation: Operation, size: usize, align: usize } if policy.fails(ordinal) && DIVERGED_AT.load(Ordering::Relaxed) == u64::MAX { - INJECTED.fetch_add(1, Ordering::Relaxed); + if INJECTED.fetch_add(1, Ordering::Relaxed) == 0 && WRITE_STACK.load(Ordering::Relaxed) { + write_stack(); + } alloc_chaos_injection_hook(ordinal); return true; } @@ -299,6 +406,7 @@ impl Scope { if depth.get() == 0 { POLICY.with(|p| p.set(policy)); ORDINAL.with(|o| o.set(0)); + ACTIVE_SCOPES.fetch_add(1, Ordering::Relaxed); } depth.set(depth.get() + 1); }); @@ -310,10 +418,34 @@ impl Scope { impl Drop for Scope { fn drop(&mut self) { - DEPTH.with(|depth| depth.set(depth.get() - 1)); + DEPTH.with(|depth| { + depth.set(depth.get() - 1); + if depth.get() == 0 { + ACTIVE_SCOPES.fetch_sub(1, Ordering::Relaxed); + } + }); } } +/// Allocations of other threads while a scope was active: `(thread, count, bytes)`, plus the +/// number of allocations from threads past [`OTHER_THREAD_CAPACITY`]. +pub fn other_threads() -> (Vec<(u32, u64, u64)>, u64) { + let threads = OTHER_THREADS + .iter() + .filter_map(|slot| { + let thread = slot.thread.load(Ordering::Relaxed); + (thread != 0).then(|| { + ( + thread, + slot.count.load(Ordering::Relaxed), + slot.bytes.load(Ordering::Relaxed), + ) + }) + }) + .collect(); + (threads, OTHER_THREADS_OVERFLOW.load(Ordering::Relaxed)) +} + /// Clears the trace, the expected trace and the counters. Call it outside any scope. pub fn reset() { TRACE_LEN.store(0, Ordering::Relaxed); diff --git a/libdd-profiling-ffi/tests/allocation_failure/controller.rs b/libdd-profiling-ffi/tests/allocation_failure/controller.rs index 784252f875..fed44945d9 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/controller.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/controller.rs @@ -58,6 +58,8 @@ pub enum Outcome { OomError, /// The API succeeded despite the failed allocation. SuccessFallback, + /// The API returned, but its handles are not in their documented state afterwards. + ContractViolation, Panic, Abort, Timeout, @@ -72,6 +74,7 @@ impl Outcome { match self { Outcome::OomError => "oom-error", Outcome::SuccessFallback => "success-fallback", + Outcome::ContractViolation => "contract-violation", Outcome::Panic => "panic", Outcome::Abort => "abort", Outcome::Timeout => "timeout", @@ -165,6 +168,7 @@ pub fn worker_main(args: &[String]) -> ! { })); allocator::reset(); + allocator::write_stack_on_first_failure(); if mode != Mode::Baseline && let Err(e) = allocator::set_expected(&expected) { @@ -177,6 +181,10 @@ pub fn worker_main(args: &[String]) -> ! { let _scope = Scope::enter(mode.policy(ordinal)); (scenario.run)(&mut fixture) }; + let (other_threads, other_overflow) = allocator::other_threads(); + let contract = scenario + .check + .map(|check| check(&mut fixture, result.is_err())); drop(fixture); let trace = match allocator::recorded() { @@ -199,6 +207,18 @@ pub fn worker_main(args: &[String]) -> ! { } }; let _ = writeln!(out, "result {result}"); + match contract { + None | Some(Ok(())) => {} + Some(Err(violation)) => { + let _ = writeln!(out, "contract {}", violation.replace('\n', " ")); + } + } + for (thread, count, bytes) in other_threads { + let _ = writeln!(out, "other-thread {thread} {count} {bytes}"); + } + if other_overflow > 0 { + let _ = writeln!(out, "other-thread-overflow {other_overflow}"); + } let _ = writeln!(out, "injected {}", allocator::injected()); if let Some(at) = allocator::diverged_at() { let _ = writeln!(out, "diverged {at}"); @@ -219,6 +239,10 @@ pub struct WorkerRun { /// Exit status, signal, or the reason of the classification. pub detail: String, pub stderr: String, + /// Allocations of other threads during the scope: `(thread, count, bytes)`. + pub other_threads: Vec<(u32, u64, u64)>, + /// Other-thread allocations that could not be attributed to a thread. + pub other_threads_unattributed: u64, } fn timeout() -> Duration { @@ -325,6 +349,11 @@ pub fn run_worker( ) } else if panicked { (Outcome::Panic, format!("panic caught, result {result}")) + } else if let Some(violation) = field("contract") { + ( + Outcome::ContractViolation, + format!("result {result}: {violation}"), + ) } else if result == "oom-error" { (Outcome::OomError, String::new()) } else if result == "ok" { @@ -339,11 +368,27 @@ pub fn run_worker( Some(status) if is_abort(&status) => (Outcome::Abort, status.to_string()), Some(status) => (Outcome::Crash, status.to_string()), }; + let other_threads = stdout + .lines() + .filter_map(|line| { + let mut it = line.strip_prefix("other-thread ")?.split(' '); + Some(( + it.next()?.parse().ok()?, + it.next()?.parse().ok()?, + it.next()?.parse().ok()?, + )) + }) + .collect(); + let other_threads_unattributed = field("other-thread-overflow") + .and_then(|s| s.parse().ok()) + .unwrap_or(0); Ok(WorkerRun { outcome, trace, detail, stderr, + other_threads, + other_threads_unattributed, }) } @@ -363,6 +408,12 @@ pub struct Finding { pub struct Report { pub scenario: &'static str, pub baseline: Vec, + /// Allocations of other threads during the baseline scope: `(thread, count, bytes)`. + /// They are reported, not enumerated. + pub other_threads: Vec<(u32, u64, u64)>, + pub other_threads_unattributed: u64, + /// Set for an I/O-bound scenario whose two baselines differ. + pub unstable_baseline: Option, pub runs: Vec, } @@ -379,6 +430,30 @@ impl Report { let mut s = String::new(); let _ = writeln!(s, "## {}\n", self.scenario); let _ = writeln!(s, "Baseline: {} events.\n", self.baseline.len()); + if let Some(reason) = &self.unstable_baseline { + let _ = writeln!( + s, + "I/O-bound scenario: {reason}. Runs past the common prefix can be \ + nondeterministic.\n" + ); + } + if !self.other_threads.is_empty() || self.other_threads_unattributed > 0 { + let _ = writeln!( + s, + "Other threads during the baseline (not enumerated, never failed):\n" + ); + for (thread, count, bytes) in &self.other_threads { + let _ = writeln!(s, "- thread {thread}: {count} allocations, {bytes} bytes"); + } + if self.other_threads_unattributed > 0 { + let _ = writeln!( + s, + "- unattributed: {} allocations", + self.other_threads_unattributed + ); + } + let _ = writeln!(s); + } let _ = writeln!( s, "| mode | ordinal | domain | operation | size | align | thread | outcome | detail |" @@ -425,15 +500,17 @@ impl Report { /// Runs the baseline twice (it must succeed and be stable), then one worker per ordinal and /// mode. Returns `Err` only for harness errors. pub fn enumerate(scenario: &'static str) -> Result { - enumerate_inner(scenario, false) + enumerate_inner(scenario, false, false) } /// Like [`enumerate`], then captures the allocation stack of every run that is not `oom-error`. -pub fn enumerate_with_stacks(scenario: &'static str) -> Result { - enumerate_inner(scenario, true) +/// With `io_bound`, the trace depends on I/O timing: an unstable baseline is reported, not a +/// harness error. +pub fn enumerate_with_stacks(scenario: &'static str, io_bound: bool) -> Result { + enumerate_inner(scenario, true, io_bound) } -fn enumerate_inner(scenario: &'static str, stacks: bool) -> Result { +fn enumerate_inner(scenario: &'static str, stacks: bool, io_bound: bool) -> Result { let baseline = run_worker(scenario, Mode::Baseline, 0, &[])?; if baseline.outcome != Outcome::SuccessFallback { return Err(format!( @@ -450,12 +527,17 @@ fn enumerate_inner(scenario: &'static str, stacks: bool) -> Result Result = baseline + .trace + .iter() + .flat_map(|event| [(*event, Mode::FailOnce), (*event, Mode::FailFrom)]) + .collect(); + let results = run_parallel(&jobs, |&(event, mode)| { + run_worker(scenario, mode, event.ordinal, &baseline.trace) + }); + let mut runs = Vec::with_capacity(jobs.len()); + for ((event, mode), run) in jobs.into_iter().zip(results) { + runs.push(Finding { + mode, + ordinal: event.ordinal, + event, + run: run?, + call_site: None, + allocation_stack: None, + }); } if stacks { capture_stacks(scenario, &baseline.trace, &mut runs); @@ -487,6 +574,9 @@ fn enumerate_inner(scenario: &'static str, stacks: bool) -> Result usize { + std::env::var("ALLOC_CHAOS_JOBS") + .ok() + .and_then(|s| s.parse().ok()) + .or_else(|| std::thread::available_parallelism().ok().map(usize::from)) + .unwrap_or(1) + .max(1) +} + +/// Runs `f` on every item on [`jobs`] threads. Results keep the order of `items`. +fn run_parallel(items: &[T], f: impl Fn(&T) -> R + Sync) -> Vec { + let next = std::sync::atomic::AtomicUsize::new(0); + let results: Vec>> = + items.iter().map(|_| std::sync::Mutex::new(None)).collect(); + std::thread::scope(|scope| { + for _ in 0..jobs().min(items.len()) { + scope.spawn(|| { + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(item) = items.get(index) else { + break; + }; + let result = f(item); + *results[index].lock().unwrap_or_else(|e| e.into_inner()) = Some(result); + } + }); + } + }); + results + .into_iter() + .map(|slot| { + slot.into_inner() + .unwrap_or_else(|e| e.into_inner()) + .expect("every item ran") + }) + .collect() +} + +/// Symbolizes the stack a worker wrote on its first injected failure. The addresses are mapped +/// into this process with the address of `alloc_chaos_injection_hook`: same executable. +fn worker_stack(stderr: &str) -> Option> { + let parse = |line: &str, prefix: &str| { + line.strip_prefix(prefix) + .and_then(|hex| usize::from_str_radix(hex.trim(), 16).ok()) }; - let dir = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join("allocation_failure"); - if std::fs::create_dir_all(&dir).is_err() { - return; + let base = stderr + .lines() + .find_map(|line| parse(line, allocator::STACK_BASE_PREFIX))?; + let here = allocator::alloc_chaos_injection_hook as *const () as usize; + let mut frames = Vec::new(); + for (i, ip) in stderr + .lines() + .filter_map(|line| parse(line, allocator::STACK_FRAME_PREFIX)) + .enumerate() + { + let addr = ip.wrapping_add(here).wrapping_sub(base); + // A return address points after the call: look up the call itself. + let lookup = if i == 0 { addr } else { addr.wrapping_sub(1) }; + let before = frames.len(); + backtrace::resolve(lookup as *mut std::ffi::c_void, |symbol| { + let name = symbol + .name() + .map(|name| format!("{name:#}")) + .unwrap_or_else(|| "??".to_string()); + let frame = match (symbol.filename(), symbol.lineno()) { + (Some(file), Some(line)) => { + format!("#{} {name} at {}:{line}", frames.len(), file.display()) + } + _ => format!("#{} {name}", frames.len()), + }; + frames.push(frame); + }); + if frames.len() == before { + frames.push(format!("#{} 0x{addr:x}", frames.len())); + } } - for finding in runs { + (!frames.is_empty()).then_some(frames) +} + +/// Maximum number of runs replayed under a debugger when their own stack has no owned frame. +const REPLAY_LIMIT: usize = 32; + +/// Takes each run's stack from the worker's own output. A run without an owned frame there, or +/// every run with `ALLOC_CHAOS_DEBUGGER=gdb|lldb`, is replayed under a debugger stopped on +/// `alloc_chaos_injection_hook`. +fn capture_stacks(scenario: &str, baseline: &[Event], runs: &mut [Finding]) { + let force_debugger = std::env::var_os("ALLOC_CHAOS_DEBUGGER").is_some(); + let mut replay = Vec::new(); + for (index, finding) in runs.iter_mut().enumerate() { if matches!( finding.run.outcome, Outcome::OomError | Outcome::Nondeterministic | Outcome::Timeout ) { continue; } + if !force_debugger { + match worker_stack(&finding.run.stderr) { + Some(frames) => { + finding.call_site = frames.iter().find(|line| is_owned_frame(line)).cloned(); + finding.allocation_stack = Some(frames.join("\n")); + } + None => { + finding.allocation_stack = Some("the worker wrote no stack".to_string()); + } + } + if finding.call_site.is_some() { + continue; + } + } + replay.push(index); + } + if replay.is_empty() { + return; + } + // A debugger run takes seconds. When symbolization fails for many runs, report the missing + // call sites rather than replaying for hours. + if !force_debugger && replay.len() > REPLAY_LIMIT { + eprintln!( + "{scenario}: {} runs have no owned frame, more than {REPLAY_LIMIT}: not replayed", + replay.len() + ); + return; + } + let Some(debugger) = debugger() else { + eprintln!("{scenario}: no gdb or lldb, allocation stacks were not replayed"); + return; + }; + let dir = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join("allocation_failure"); + if std::fs::create_dir_all(&dir).is_err() { + return; + } + for index in replay { + let finding = &mut runs[index]; let input = dir.join(format!( "stdin-{}-{}-{}-{}.txt", scenario, @@ -548,7 +756,7 @@ fn capture_stacks(scenario: &str, baseline: &[Event], runs: &mut [Finding]) { } } -/// `ALLOC_CHAOS_DEBUGGER=gdb|lldb` forces one; otherwise gdb, then lldb. +/// `ALLOC_CHAOS_DEBUGGER=gdb|lldb` picks one; otherwise gdb, then lldb. fn debugger() -> Option<&'static str> { match std::env::var("ALLOC_CHAOS_DEBUGGER").as_deref() { Ok("gdb") => return command_ok("gdb").then_some("gdb"), diff --git a/libdd-profiling-ffi/tests/allocation_failure/lifecycle.rs b/libdd-profiling-ffi/tests/allocation_failure/lifecycle.rs new file mode 100644 index 0000000000..c934e52aad --- /dev/null +++ b/libdd-profiling-ffi/tests/allocation_failure/lifecycle.rs @@ -0,0 +1,838 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Tier 1 lifecycle scenarios: the profile and exporter calls a .NET profiler makes on every +//! upload cycle. +//! +//! They go through the C ABI, like a native caller. The FFI modules of this crate are private, +//! so [`ffi`] mirrors their `#[repr(C)]` types and imports the exported symbols. Every input is +//! built in the setup, outside the scope, so the scope only sees the call under test. +//! +//! Each scenario checks a post-failure contract once the scope has ended: +//! - a failed constructor publishes no handle; +//! - a profile is still usable after a failed add or serialize; +//! - `send_blocking` takes the encoded profile even when it fails, and the exporter can send again. +//! +//! The exporter sends to a local agent served by a thread of the worker. The file endpoint is +//! not used: each exporter starts a dump server whose socket has a random name in `/tmp`, which +//! makes the trace nondeterministic and leaves one socket file per worker. + +use crate::scenarios::{Scenario, ScenarioError}; +use libdd_common::tag::Tag; +use libdd_common_ffi::{CharSlice, Handle, Slice, ToInner}; +use std::any::Any; +use std::num::NonZeroI64; + +#[allow(improper_ctypes, dead_code)] +mod ffi { + use libdd_common::tag::Tag; + use libdd_common_ffi::{CharSlice, Error, Handle, Result, Slice, Timespec, VoidResult}; + pub use libdd_profiling::api::{ManagedStringId, Period, SampleType}; + pub use libdd_profiling::exporter::ProfileExporter; + pub use libdd_profiling::internal::EncodedProfile; + use std::num::NonZeroI64; + + #[repr(C)] + pub struct Profile { + pub inner: *mut std::ffi::c_void, + } + + #[repr(C)] + pub enum ProfileNewResult { + Ok(Profile), + Err(Error), + } + + #[repr(C)] + pub enum ProfileResult { + Ok(bool), + Err(Error), + } + + #[repr(C)] + pub enum SerializeResult { + Ok(Handle), + Err(Error), + } + + #[repr(C)] + #[derive(Copy, Clone, Default)] + pub struct Mapping<'a> { + pub memory_start: u64, + pub memory_limit: u64, + pub file_offset: u64, + pub filename: CharSlice<'a>, + pub filename_id: ManagedStringId, + pub build_id: CharSlice<'a>, + pub build_id_id: ManagedStringId, + } + + #[repr(C)] + #[derive(Copy, Clone, Default)] + pub struct Function<'a> { + pub name: CharSlice<'a>, + pub name_id: ManagedStringId, + pub system_name: CharSlice<'a>, + pub system_name_id: ManagedStringId, + pub filename: CharSlice<'a>, + pub filename_id: ManagedStringId, + } + + #[repr(C)] + #[derive(Copy, Clone, Default)] + pub struct Location<'a> { + pub mapping: Mapping<'a>, + pub function: Function<'a>, + pub address: u64, + pub line: i64, + } + + #[repr(C)] + #[derive(Copy, Clone, Default)] + pub struct Label<'a> { + pub key: CharSlice<'a>, + pub key_id: ManagedStringId, + pub str: CharSlice<'a>, + pub str_id: ManagedStringId, + pub num: i64, + pub num_unit: CharSlice<'a>, + pub num_unit_id: ManagedStringId, + } + + #[repr(C)] + #[derive(Copy, Clone)] + pub struct Sample<'a> { + pub locations: Slice<'a, Location<'a>>, + pub values: Slice<'a, i64>, + pub labels: Slice<'a, Label<'a>>, + } + + #[repr(C)] + #[derive(Copy, Clone)] + pub enum ProfilingEndpoint<'a> { + Agent(CharSlice<'a>, u64, bool), + Agentless(CharSlice<'a>, CharSlice<'a>, u64, bool), + File(CharSlice<'a>), + } + + #[repr(C)] + pub struct File<'a> { + pub name: CharSlice<'a>, + pub file: Slice<'a, u8>, + } + + #[repr(C)] + pub struct HttpStatus(pub u16); + + unsafe extern "C" { + pub fn ddog_prof_Profile_new( + sample_types: Slice, + period: Option<&Period>, + ) -> ProfileNewResult; + pub fn ddog_prof_Profile_drop(profile: *mut Profile); + pub fn ddog_prof_Profile_add( + profile: *mut Profile, + sample: Sample, + timestamp: Option, + ) -> ProfileResult; + pub fn ddog_prof_Profile_set_endpoint( + profile: *mut Profile, + local_root_span_id: u64, + endpoint: CharSlice, + ) -> ProfileResult; + pub fn ddog_prof_Profile_add_endpoint_count( + profile: *mut Profile, + endpoint: CharSlice, + value: i64, + ) -> ProfileResult; + pub fn ddog_prof_Profile_add_upscaling_rule_proportional( + profile: *mut Profile, + offset_values: Slice, + label_name: CharSlice, + label_value: CharSlice, + total_sampled: u64, + total_real: u64, + ) -> ProfileResult; + pub fn ddog_prof_Profile_serialize( + profile: *mut Profile, + start_time: Option<&Timespec>, + end_time: Option<&Timespec>, + ) -> SerializeResult; + pub fn ddog_prof_EncodedProfile_drop(profile: *mut Handle); + pub fn ddog_prof_Exporter_new( + profiling_library_name: CharSlice, + profiling_library_version: CharSlice, + family: CharSlice, + tags: Option<&libdd_common_ffi::Vec>, + endpoint: ProfilingEndpoint, + ) -> Result>; + pub fn ddog_prof_Exporter_init_runtime( + exporter: *mut Handle, + ) -> VoidResult; + pub fn ddog_prof_Exporter_send_blocking( + exporter: *mut Handle, + profile: *mut Handle, + files_to_compress_and_export: Slice, + optional_additional_tags: Option<&libdd_common_ffi::Vec>, + optional_process_tags: Option<&CharSlice>, + optional_internal_metadata_json: Option<&CharSlice>, + optional_info_json: Option<&CharSlice>, + cancel: *mut std::ffi::c_void, + ) -> Result; + pub fn ddog_prof_Exporter_drop(exporter: *mut Handle); + } +} + +use ffi::{EncodedProfile, ProfileExporter}; + +pub const PROFILE_NEW: &str = "profile_new"; +pub const PROFILE_ADD_FIRST: &str = "profile_add_first"; +pub const PROFILE_ADD_INTERNED: &str = "profile_add_interned"; +pub const PROFILE_ADD_NEW_STRINGS: &str = "profile_add_new_strings"; +pub const PROFILE_ADD_TIMESTAMPED: &str = "profile_add_timestamped"; +pub const PROFILE_ADD_GROWTH: &str = "profile_add_growth"; +pub const PROFILE_SERIALIZE_EMPTY: &str = "profile_serialize_empty"; +pub const PROFILE_SERIALIZE_POPULATED: &str = "profile_serialize_populated"; +pub const PROFILE_SERIALIZE_ENDPOINTS: &str = "profile_serialize_endpoints"; +pub const PROFILE_SERIALIZE_UPSCALING: &str = "profile_serialize_upscaling"; +pub const PROFILE_SERIALIZE_LARGE: &str = "profile_serialize_large"; +pub const EXPORTER_NEW: &str = "exporter_new"; +pub const EXPORTER_SEND_COLD: &str = "exporter_send_cold"; +pub const EXPORTER_SEND_WARM: &str = "exporter_send_warm"; + +/// Distinct samples already in the profile before the growth sample is added. +pub const GROWTH_SAMPLES: u32 = 28; +/// Distinct samples in the large profile. +pub const LARGE_SAMPLES: u32 = 256; +/// Distinct samples in the populated profiles. +const POPULATED_SAMPLES: u32 = 8; + +const SAMPLE_TYPES: [ffi::SampleType; 4] = [ + ffi::SampleType::CpuTime, + ffi::SampleType::CpuSamples, + ffi::SampleType::WallTime, + ffi::SampleType::WallSamples, +]; +const PERIOD: ffi::Period = ffi::Period { + sample_type: ffi::SampleType::WallTime, + value: 10_000_000, +}; +const ENDPOINT: &str = "GET /orders/{id}"; + +fn leak(s: String) -> &'static str { + Box::leak(s.into_boxed_str()) +} + +fn error_text(mut error: libdd_common_ffi::Error) -> String { + let text = error.to_string(); + // SAFETY: `error` is a valid error returned by the FFI. + unsafe { libdd_common_ffi::ddog_Error_drop(Some(&mut error)) }; + text +} + +fn drop_error(error: libdd_common_ffi::Error) -> ScenarioError { + let mut error = error; + // SAFETY: `error` is a valid error returned by the FFI. + unsafe { libdd_common_ffi::ddog_Error_drop(Some(&mut error)) }; + ScenarioError::Oom +} + +/// A .NET-shaped sample: managed frames in the `|lm: |ns: |ct: |fn:` format, wall and CPU +/// values, and the labels the .NET profiler sets (thread, app domain, span IDs). +struct DotnetSample { + locations: Vec>, + values: Vec, + labels: Vec>, +} + +impl DotnetSample { + /// The runtime frames and some label values are shared by every variant. The application + /// frames and the thread and span labels are unique to `variant`: new strings for a profile. + fn new(variant: u32) -> Self { + let mapping = |filename: &'static str| ffi::Mapping { + filename: CharSlice::from(filename), + ..Default::default() + }; + let location = + |mapping: ffi::Mapping<'static>, name: &'static str, line: i64| ffi::Location { + mapping, + function: ffi::Function { + name: CharSlice::from(name), + ..Default::default() + }, + line, + ..Default::default() + }; + let app = mapping("MyApp.dll"); + let corelib = mapping("System.Private.CoreLib.dll"); + let mut locations = Vec::new(); + for depth in 0..4 { + let name = leak(format!( + "|lm:MyApp |ns:MyApp.Orders |ct:OrderService{variant} |fn:Step{depth}" + )); + locations.push(location(app, name, 10 + depth)); + } + for name in [ + "|lm:System.Private.CoreLib |ns:System.Threading |ct:ExecutionContext |fn:RunInternal", + "|lm:System.Private.CoreLib |ns:System.Threading |ct:ThreadPoolWorkQueue |fn:Dispatch", + "|lm:System.Private.CoreLib |ns:System.Threading |ct:PortableThreadPool+WorkerThread \ + |fn:WorkerThreadStart", + "|lm:System.Private.CoreLib |ns:System.Threading |ct:Thread |fn:StartCallback", + ] { + locations.push(location(corelib, name, 0)); + } + let str_label = |key: &'static str, value: &'static str| ffi::Label { + key: CharSlice::from(key), + str: CharSlice::from(value), + ..Default::default() + }; + let num_label = |key: &'static str, num: i64| ffi::Label { + key: CharSlice::from(key), + num, + ..Default::default() + }; + let labels = vec![ + str_label( + "thread id", + leak(format!("<0x{:x}> [#{variant}]", 0x1a2b + variant)), + ), + str_label("thread name", ".NET ThreadPool Worker"), + str_label("appdomain name", "MyApp"), + num_label("pid", 4242), + num_label("local root span id", local_root_span_id(variant) as i64), + num_label("span id", 2_000 + i64::from(variant)), + ]; + DotnetSample { + locations, + values: vec![9_000_000, 1, 10_000_000, 1], + labels, + } + } + + fn as_ffi(&self) -> ffi::Sample<'_> { + ffi::Sample { + locations: Slice::from(self.locations.as_slice()), + values: Slice::from(self.values.as_slice()), + labels: Slice::from(self.labels.as_slice()), + } + } +} + +fn local_root_span_id(variant: u32) -> u64 { + 1_000 + u64::from(variant) +} + +fn new_profile() -> Result { + // SAFETY: the slice and the period are valid for the duration of the call. + match unsafe { ffi::ddog_prof_Profile_new(Slice::from(&SAMPLE_TYPES[..]), Some(&PERIOD)) } { + ffi::ProfileNewResult::Ok(profile) => Ok(profile), + ffi::ProfileNewResult::Err(error) => Err(drop_error(error)), + } +} + +fn add( + profile: &mut ffi::Profile, + sample: &DotnetSample, + timestamp: Option, +) -> Result<(), String> { + // SAFETY: `profile` comes from `ddog_prof_Profile_new`, and `sample` borrows live data. + match unsafe { ffi::ddog_prof_Profile_add(profile, sample.as_ffi(), timestamp) } { + ffi::ProfileResult::Ok(_) => Ok(()), + ffi::ProfileResult::Err(error) => Err(error_text(error)), + } +} + +fn serialize(profile: &mut ffi::Profile) -> Result, String> { + // SAFETY: `profile` comes from `ddog_prof_Profile_new`. + match unsafe { ffi::ddog_prof_Profile_serialize(profile, None, None) } { + ffi::SerializeResult::Ok(encoded) => Ok(encoded), + ffi::SerializeResult::Err(error) => Err(error_text(error)), + } +} + +fn setup_ok(result: ffi::ProfileResult, what: &str) { + if let ffi::ProfileResult::Err(error) = result { + panic!("setup {what}: {}", error_text(error)); + } +} + +struct ProfileFixture { + profile: ffi::Profile, + /// Sample added by the scenario's call. + sample: DotnetSample, + timestamp: Option, + /// Sample added by the post-failure check. + probe: DotnetSample, + encoded: Handle, +} + +impl Drop for ProfileFixture { + fn drop(&mut self) { + // SAFETY: both come from this API, and the drop functions accept empty handles. + unsafe { + ffi::ddog_prof_EncodedProfile_drop(&mut self.encoded); + ffi::ddog_prof_Profile_drop(&mut self.profile); + } + } +} + +/// A profile that already holds `variants` distinct samples (variant 0 first). +fn profile_with(variants: u32) -> ProfileFixture { + let mut profile = new_profile().expect("setup profile"); + for variant in 0..variants { + add(&mut profile, &DotnetSample::new(variant), None).expect("setup add"); + } + ProfileFixture { + profile, + sample: DotnetSample::new(variants), + timestamp: None, + probe: DotnetSample::new(1_000_000), + encoded: Handle::empty(), + } +} + +fn profile_fixture(fixture: &mut Box) -> &mut ProfileFixture { + fixture.downcast_mut::().unwrap() +} + +fn run_add(fixture: &mut Box) -> Result<(), ScenarioError> { + let fx = profile_fixture(fixture); + add(&mut fx.profile, &fx.sample, fx.timestamp).map_err(|_| ScenarioError::Oom) +} + +fn run_serialize(fixture: &mut Box) -> Result<(), ScenarioError> { + let fx = profile_fixture(fixture); + fx.encoded = serialize(&mut fx.profile).map_err(|_| ScenarioError::Oom)?; + Ok(()) +} + +/// The profile is still usable: a new sample can be added, and the profile serialized. +fn check_profile_usable(fixture: &mut Box, _failed: bool) -> Result<(), String> { + let fx = profile_fixture(fixture); + if fx.profile.inner.is_null() { + return Err("the profile handle is null".to_string()); + } + add(&mut fx.profile, &fx.probe, None).map_err(|e| format!("add after the call: {e}"))?; + let mut encoded = + serialize(&mut fx.profile).map_err(|e| format!("serialize after the call: {e}"))?; + // SAFETY: `encoded` comes from `ddog_prof_Profile_serialize`. + unsafe { ffi::ddog_prof_EncodedProfile_drop(&mut encoded) }; + Ok(()) +} + +struct NewProfileFixture { + profile: Option, + probe: DotnetSample, +} + +impl Drop for NewProfileFixture { + fn drop(&mut self) { + if let Some(profile) = &mut self.profile { + // SAFETY: the profile comes from `ddog_prof_Profile_new`. + unsafe { ffi::ddog_prof_Profile_drop(profile) }; + } + } +} + +/// A failed `Profile_new` publishes no handle; a profile it returns is usable. +fn check_new_profile(fixture: &mut Box, failed: bool) -> Result<(), String> { + let fx = fixture.downcast_mut::().unwrap(); + match (&mut fx.profile, failed) { + (Some(_), true) => Err("an error was returned with a profile".to_string()), + (None, false) => Err("success without a profile".to_string()), + (None, true) => Ok(()), + (Some(profile), false) => { + if profile.inner.is_null() { + return Err("Profile_new returned Ok with a null profile".to_string()); + } + add(profile, &fx.probe, None).map_err(|e| format!("add on the new profile: {e}"))?; + let mut encoded = + serialize(profile).map_err(|e| format!("serialize the new profile: {e}"))?; + // SAFETY: `encoded` comes from `ddog_prof_Profile_serialize`. + unsafe { ffi::ddog_prof_EncodedProfile_drop(&mut encoded) }; + Ok(()) + } + } +} + +/// A serialized profile with a few samples, endpoints included. +fn encoded_profile() -> Handle { + let mut fx = profile_with(POPULATED_SAMPLES); + add_endpoints(&mut fx.profile, POPULATED_SAMPLES); + serialize(&mut fx.profile).expect("setup serialize") +} + +fn add_endpoints(profile: &mut ffi::Profile, variants: u32) { + for variant in 0..variants { + // SAFETY: `profile` comes from `ddog_prof_Profile_new`; the slice is live. + setup_ok( + unsafe { + ffi::ddog_prof_Profile_set_endpoint( + profile, + local_root_span_id(variant), + CharSlice::from(ENDPOINT), + ) + }, + "set_endpoint", + ); + } + // SAFETY: as above. + setup_ok( + unsafe { ffi::ddog_prof_Profile_add_endpoint_count(profile, CharSlice::from(ENDPOINT), 3) }, + "add_endpoint_count", + ); +} + +/// Reads one HTTP/1.1 request (`Content-Length` or chunked body) and answers 202. +fn serve_request(mut stream: std::net::TcpStream) -> std::io::Result<()> { + use std::io::{Read, Write}; + let mut request = Vec::new(); + let mut buf = [0u8; 16 * 1024]; + let mut body_start = None; + loop { + let n = stream.read(&mut buf)?; + if n == 0 { + break; + } + request.extend_from_slice(&buf[..n]); + if body_start.is_none() { + body_start = request + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map(|p| p + 4); + } + let Some(start) = body_start else { + continue; + }; + let headers = String::from_utf8_lossy(&request[..start]).to_ascii_lowercase(); + let content_length = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse::().ok()); + let done = match content_length { + Some(len) => request.len() - start >= len, + None if headers.contains("transfer-encoding: chunked") => { + request.ends_with(b"0\r\n\r\n") + } + None => true, + }; + if done { + break; + } + } + stream.write_all(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") +} + +/// A local agent on a loopback port, answering every request on its own thread. Its +/// allocations happen on that thread: the harness reports them, it does not enumerate them. +fn local_agent() -> &'static str { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind the local agent"); + let port = listener.local_addr().expect("local agent address").port(); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let _ = serve_request(stream); + } + }); + leak(format!("http://127.0.0.1:{port}")) +} + +fn agent_endpoint(url: &'static str) -> ffi::ProfilingEndpoint<'static> { + ffi::ProfilingEndpoint::Agent(CharSlice::from(url), 10_000, false) +} + +fn dotnet_tags() -> libdd_common_ffi::Vec { + let tags: Vec = [ + ("service", "my-app"), + ("env", "prod"), + ("version", "1.2.3"), + ("host", "web-1"), + ("runtime-id", "8f2b6c1e-6a4b-4f7e-9d3c-2b1a0e9f8d7c"), + ("profiler_version", "3.30.0"), + ] + .into_iter() + .map(|(k, v)| Tag::new(k, v).expect("tag")) + .collect(); + tags.into() +} + +struct ExporterFixture { + exporter: Handle, + endpoint: ffi::ProfilingEndpoint<'static>, + tags: libdd_common_ffi::Vec, + encoded: Handle, + files: Vec>, + additional_tags: libdd_common_ffi::Vec, + process_tags: CharSlice<'static>, + internal_metadata: CharSlice<'static>, + info: CharSlice<'static>, + status: Option, +} + +impl Drop for ExporterFixture { + fn drop(&mut self) { + // SAFETY: both come from this API, and the drop functions accept empty handles. + unsafe { + ffi::ddog_prof_EncodedProfile_drop(&mut self.encoded); + ffi::ddog_prof_Exporter_drop(&mut self.exporter); + } + } +} + +fn exporter_fixture(endpoint: ffi::ProfilingEndpoint<'static>) -> ExporterFixture { + let attachment: &'static [u8] = Box::leak( + br#"{"runtime":"dotnet","events":[{"type":"gc","duration_ms":3}]}"# + .to_vec() + .into_boxed_slice(), + ); + let files = vec![ffi::File { + name: CharSlice::from("metrics.json"), + file: Slice::from(attachment), + }]; + let additional_tags: Vec = vec![Tag::new("cycle", "42").expect("tag")]; + ExporterFixture { + exporter: Handle::empty(), + endpoint, + tags: dotnet_tags(), + encoded: Handle::empty(), + files, + additional_tags: additional_tags.into(), + process_tags: CharSlice::from("entrypoint.name:MyApp,entrypoint.type:dotnet"), + internal_metadata: CharSlice::from(r#"{"no_signals_workaround_enabled":"false"}"#), + info: CharSlice::from(r#"{"application":{"start_time":"2026-11-20T00:00:00Z"}}"#), + status: None, + } +} + +fn new_exporter(fx: &ExporterFixture) -> Result, String> { + // SAFETY: every slice and the tag vector are live for the duration of the call. + match unsafe { + ffi::ddog_prof_Exporter_new( + CharSlice::from("dd-trace-dotnet"), + CharSlice::from("3.30.0"), + CharSlice::from("dotnet"), + Some(&fx.tags), + fx.endpoint, + ) + } { + libdd_common_ffi::Result::Ok(exporter) => Ok(exporter), + libdd_common_ffi::Result::Err(error) => Err(error_text(error)), + } +} + +fn send(fx: &mut ExporterFixture) -> Result { + // SAFETY: the handles come from this API, and every borrowed input is live. + match unsafe { + ffi::ddog_prof_Exporter_send_blocking( + &mut fx.exporter, + &mut fx.encoded, + Slice::from(fx.files.as_slice()), + Some(&fx.additional_tags), + Some(&fx.process_tags), + Some(&fx.internal_metadata), + Some(&fx.info), + std::ptr::null_mut(), + ) + } { + libdd_common_ffi::Result::Ok(status) => Ok(status.0), + libdd_common_ffi::Result::Err(error) => Err(error_text(error)), + } +} + +fn exporter_ready(endpoint: ffi::ProfilingEndpoint<'static>, warm: bool) -> ExporterFixture { + let mut fx = exporter_fixture(endpoint); + fx.exporter = new_exporter(&fx).expect("setup exporter"); + if warm { + // SAFETY: the exporter comes from `ddog_prof_Exporter_new`. + let result = unsafe { ffi::ddog_prof_Exporter_init_runtime(&mut fx.exporter) }; + if let libdd_common_ffi::VoidResult::Err(error) = result { + panic!("setup init_runtime: {}", error_text(error)); + } + fx.encoded = encoded_profile(); + send(&mut fx).expect("setup send"); + } + fx.encoded = encoded_profile(); + fx +} + +fn run_new_exporter(fixture: &mut Box) -> Result<(), ScenarioError> { + let fx = fixture.downcast_mut::().unwrap(); + fx.exporter = new_exporter(fx).map_err(|_| ScenarioError::Oom)?; + Ok(()) +} + +fn run_send(fixture: &mut Box) -> Result<(), ScenarioError> { + let fx = fixture.downcast_mut::().unwrap(); + fx.status = Some(send(fx).map_err(|_| ScenarioError::Oom)?); + Ok(()) +} + +/// The exporter can send a new profile, and the endpoint accepts it. +fn check_exporter_sends(fx: &mut ExporterFixture) -> Result<(), String> { + fx.encoded = encoded_profile(); + let status = send(fx).map_err(|e| format!("send after the call: {e}"))?; + if !(200..300).contains(&status) { + return Err(format!("send after the call returned HTTP {status}")); + } + Ok(()) +} + +/// A failed `Exporter_new` publishes no handle; an exporter it returns can send. +fn check_new_exporter(fixture: &mut Box, failed: bool) -> Result<(), String> { + let fx = fixture.downcast_mut::().unwrap(); + let mut handle: *mut Handle = &mut fx.exporter; + // SAFETY: the handle is either empty or holds the exporter from `ddog_prof_Exporter_new`. + let published = unsafe { handle.to_inner_mut() }.is_ok(); + match (published, failed) { + (true, true) => Err("an error was returned with an exporter".to_string()), + (false, false) => Err("success without an exporter".to_string()), + (false, true) => Ok(()), + (true, false) => check_exporter_sends(fx), + } +} + +/// `send_blocking` takes the encoded profile whatever the result, and the exporter can send +/// again. +fn check_send(fixture: &mut Box, failed: bool) -> Result<(), String> { + let fx = fixture.downcast_mut::().unwrap(); + let mut handle: *mut Handle = &mut fx.encoded; + // SAFETY: the handle is either empty or holds the profile from `ddog_prof_Profile_serialize`. + if let Ok(profile) = unsafe { handle.take() } { + drop(profile); + return Err("the encoded profile handle still owns the profile".to_string()); + } + if !failed && !fx.status.is_some_and(|s| (200..300).contains(&s)) { + return Err(format!("success with HTTP status {:?}", fx.status)); + } + check_exporter_sends(fx) +} + +pub const SCENARIOS: &[Scenario] = &[ + Scenario { + name: PROFILE_NEW, + setup: || { + Box::new(NewProfileFixture { + profile: None, + probe: DotnetSample::new(0), + }) + }, + run: |fixture| { + let fx = fixture.downcast_mut::().unwrap(); + fx.profile = Some(new_profile()?); + Ok(()) + }, + check: Some(check_new_profile), + }, + Scenario { + name: PROFILE_ADD_FIRST, + setup: || Box::new(profile_with(0)), + run: run_add, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_ADD_INTERNED, + setup: || { + let mut fx = profile_with(1); + fx.sample = DotnetSample::new(0); + Box::new(fx) + }, + run: run_add, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_ADD_NEW_STRINGS, + setup: || Box::new(profile_with(1)), + run: run_add, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_ADD_TIMESTAMPED, + setup: || { + let mut fx = profile_with(1); + fx.sample = DotnetSample::new(0); + fx.timestamp = NonZeroI64::new(1_763_600_000_000_000_000); + Box::new(fx) + }, + run: run_add, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_ADD_GROWTH, + setup: || Box::new(profile_with(GROWTH_SAMPLES)), + run: run_add, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_SERIALIZE_EMPTY, + setup: || Box::new(profile_with(0)), + run: run_serialize, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_SERIALIZE_POPULATED, + setup: || Box::new(profile_with(POPULATED_SAMPLES)), + run: run_serialize, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_SERIALIZE_ENDPOINTS, + setup: || { + let mut fx = profile_with(POPULATED_SAMPLES); + add_endpoints(&mut fx.profile, POPULATED_SAMPLES); + Box::new(fx) + }, + run: run_serialize, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_SERIALIZE_UPSCALING, + setup: || { + let mut fx = profile_with(POPULATED_SAMPLES); + let offsets = [0usize, 1]; + // SAFETY: the profile comes from `ddog_prof_Profile_new`; the slices are live. + setup_ok( + unsafe { + ffi::ddog_prof_Profile_add_upscaling_rule_proportional( + &mut fx.profile, + Slice::from(&offsets[..]), + CharSlice::from("thread name"), + CharSlice::from(".NET ThreadPool Worker"), + 10, + 100, + ) + }, + "add_upscaling_rule_proportional", + ); + Box::new(fx) + }, + run: run_serialize, + check: Some(check_profile_usable), + }, + Scenario { + name: PROFILE_SERIALIZE_LARGE, + setup: || { + let mut fx = profile_with(LARGE_SAMPLES); + add_endpoints(&mut fx.profile, LARGE_SAMPLES); + Box::new(fx) + }, + run: run_serialize, + check: Some(check_profile_usable), + }, + Scenario { + name: EXPORTER_NEW, + setup: || Box::new(exporter_fixture(agent_endpoint(local_agent()))), + run: run_new_exporter, + check: Some(check_new_exporter), + }, + Scenario { + name: EXPORTER_SEND_COLD, + setup: || Box::new(exporter_ready(agent_endpoint(local_agent()), false)), + run: run_send, + check: Some(check_send), + }, + Scenario { + name: EXPORTER_SEND_WARM, + setup: || Box::new(exporter_ready(agent_endpoint(local_agent()), true)), + run: run_send, + check: Some(check_send), + }, +]; diff --git a/libdd-profiling-ffi/tests/allocation_failure/main.rs b/libdd-profiling-ffi/tests/allocation_failure/main.rs index 3ffc0a87cb..71a2832227 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/main.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/main.rs @@ -12,15 +12,21 @@ //! the controller (see `controller.rs`). //! //! Contract and toy-scenario tests are strict. Discovery scenarios record each outcome, including -//! a crash, and fail only on a harness error. For those, a debugger stopped on -//! `alloc_chaos_injection_hook` records the stack and the owned call site. +//! a crash, and fail only on a harness error. For those, the worker writes the return addresses +//! of its first injected failure, and the controller symbolizes them to find the owned call site +//! (`ALLOC_CHAOS_DEBUGGER=gdb|lldb` uses a debugger instead). Workers run in parallel +//! (`ALLOC_CHAOS_JOBS`). mod allocator; mod arena; mod contract; mod controller; +mod lifecycle; mod scenarios; +// The lifecycle scenarios call the C symbols of this crate: link its library. +use datadog_profiling_ffi as _; + use allocator::{Domain, ScopedFailingAllocator}; use controller::{Mode, Outcome, Report}; use std::alloc::System; @@ -31,12 +37,18 @@ static ALLOCATOR: ScopedFailingAllocator = ScopedFailingAllocator(System /// Discovery scenario: every ordinal in both modes is recorded. A crash is a finding. /// A nondeterministic trace is a harness error. fn discover(name: &'static str) { - let _ = load_discovery(name); + let _ = load_discovery(name, false); +} + +/// Same as [`discover`] for a call whose allocations depend on I/O timing (a request to a +/// local endpoint). An unstable baseline and nondeterministic runs are reported, not errors. +fn discover_io(name: &'static str) { + let _ = load_discovery(name, true); } /// Same as [`discover`], then checks which allocation domains the baseline contains. fn discover_arena(name: &'static str, required: &[Domain], absent: &[Domain]) -> Report { - let report = load_discovery(name); + let report = load_discovery(name, false); let present: Vec = report.baseline.iter().map(|e| e.domain).collect(); for domain in required { assert!( @@ -62,9 +74,18 @@ fn profiling_arena_domains() -> &'static [Domain] { } } -fn load_discovery(name: &'static str) -> Report { - let report = - controller::enumerate_with_stacks(name).unwrap_or_else(|e| panic!("harness error: {e}")); +/// The request domain alone: a call whose items fit in the chunks the profile already has. +fn profiling_arena_request() -> &'static [Domain] { + if arena::PROFILING_ARENAS_OBSERVED { + &[Domain::ArenaRequest] + } else { + &[] + } +} + +fn load_discovery(name: &'static str, io_bound: bool) -> Report { + let report = controller::enumerate_with_stacks(name, io_bound) + .unwrap_or_else(|e| panic!("harness error: {e}")); let nondeterministic: Vec<_> = report .runs .iter() @@ -79,7 +100,7 @@ fn load_discovery(name: &'static str) -> Report { }) .collect(); assert!( - nondeterministic.is_empty(), + io_bound || nondeterministic.is_empty(), "{name}: nondeterministic trace: {nondeterministic:?}" ); let missing: Vec = report @@ -145,6 +166,12 @@ const TESTS: &[(&str, fn())] = &[ ("toy::string_format", || { toy(scenarios::TOY_STRING_FORMAT, Outcome::Abort) }), + ("toy::contract_violation", || { + toy( + scenarios::TOY_CONTRACT_VIOLATION, + Outcome::ContractViolation, + ) + }), ("scenario::tag_push_empty", || { discover(scenarios::TAG_PUSH_EMPTY) }), @@ -210,6 +237,76 @@ const TESTS: &[(&str, fn())] = &[ ("scenario::arena_slice_set", || { discover_arena(scenarios::ARENA_SLICE_SET, profiling_arena_domains(), &[]); }), + ("lifecycle::profile_new", || { + discover_arena(lifecycle::PROFILE_NEW, profiling_arena_domains(), &[]); + }), + ("lifecycle::profile_add_first", || { + discover_arena(lifecycle::PROFILE_ADD_FIRST, profiling_arena_request(), &[]); + }), + ("lifecycle::profile_add_interned", || { + discover(lifecycle::PROFILE_ADD_INTERNED) + }), + ("lifecycle::profile_add_new_strings", || { + discover_arena( + lifecycle::PROFILE_ADD_NEW_STRINGS, + profiling_arena_request(), + &[], + ); + }), + ("lifecycle::profile_add_timestamped", || { + discover(lifecycle::PROFILE_ADD_TIMESTAMPED) + }), + ("lifecycle::profile_add_growth", || { + discover_arena( + lifecycle::PROFILE_ADD_GROWTH, + profiling_arena_request(), + &[], + ); + }), + ("lifecycle::profile_serialize_empty", || { + discover_arena( + lifecycle::PROFILE_SERIALIZE_EMPTY, + profiling_arena_domains(), + &[], + ); + }), + ("lifecycle::profile_serialize_populated", || { + discover_arena( + lifecycle::PROFILE_SERIALIZE_POPULATED, + profiling_arena_domains(), + &[], + ); + }), + ("lifecycle::profile_serialize_endpoints", || { + discover_arena( + lifecycle::PROFILE_SERIALIZE_ENDPOINTS, + profiling_arena_domains(), + &[], + ); + }), + ("lifecycle::profile_serialize_upscaling", || { + discover_arena( + lifecycle::PROFILE_SERIALIZE_UPSCALING, + profiling_arena_domains(), + &[], + ); + }), + ("lifecycle::profile_serialize_large", || { + discover_arena( + lifecycle::PROFILE_SERIALIZE_LARGE, + profiling_arena_domains(), + &[], + ); + }), + ("lifecycle::exporter_new", || { + discover(lifecycle::EXPORTER_NEW) + }), + ("lifecycle::exporter_send_cold", || { + discover_io(lifecycle::EXPORTER_SEND_COLD) + }), + ("lifecycle::exporter_send_warm", || { + discover_io(lifecycle::EXPORTER_SEND_WARM) + }), ]; fn main() { diff --git a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs index f96a07c801..3faf586f25 100644 --- a/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs +++ b/libdd-profiling-ffi/tests/allocation_failure/scenarios.rs @@ -18,8 +18,13 @@ pub struct Scenario { pub name: &'static str, pub setup: fn() -> Box, pub run: fn(&mut Box) -> Result<(), ScenarioError>, + /// Post-failure contract, checked once the scope has ended (allocations work again). The + /// flag is `true` when `run` returned an error. `Err` describes the violation. + pub check: Option, } +pub type PostCheck = fn(&mut Box, bool) -> Result<(), String>; + fn no_fixture() -> Box { Box::new(()) } @@ -29,6 +34,16 @@ pub const TOY_BOX_NEW: &str = "toy_box_new"; pub const TOY_VEC_PUSH: &str = "toy_vec_push"; pub const TOY_VEC_TRY_RESERVE: &str = "toy_vec_try_reserve"; pub const TOY_STRING_FORMAT: &str = "toy_string_format"; +/// Like [`TOY_VEC_TRY_RESERVE`], with a post-failure check that rejects every failed call. +pub const TOY_CONTRACT_VIOLATION: &str = "toy_contract_violation"; + +fn run_try_reserve(_: &mut Box) -> Result<(), ScenarioError> { + let mut v: Vec = Vec::new(); + v.try_reserve(black_box(1024)) + .map_err(|_| ScenarioError::Oom)?; + black_box(v); + Ok(()) +} pub const SCENARIOS: &[Scenario] = &[ Scenario { @@ -38,6 +53,7 @@ pub const SCENARIOS: &[Scenario] = &[ black_box(Box::new(black_box([7u8; 64]))); Ok(()) }, + check: None, }, Scenario { name: TOY_VEC_PUSH, @@ -50,17 +66,25 @@ pub const SCENARIOS: &[Scenario] = &[ black_box(v); Ok(()) }, + check: None, }, Scenario { name: TOY_VEC_TRY_RESERVE, setup: no_fixture, - run: |_| { - let mut v: Vec = Vec::new(); - v.try_reserve(black_box(1024)) - .map_err(|_| ScenarioError::Oom)?; - black_box(v); - Ok(()) - }, + run: run_try_reserve, + check: None, + }, + Scenario { + name: TOY_CONTRACT_VIOLATION, + setup: no_fixture, + run: run_try_reserve, + check: Some(|_, failed| { + if failed { + Err("rejected by the toy check".to_string()) + } else { + Ok(()) + } + }), }, Scenario { name: TOY_STRING_FORMAT, @@ -69,6 +93,7 @@ pub const SCENARIOS: &[Scenario] = &[ black_box(format!("{}-{}", black_box(42), black_box("forty-two"))); Ok(()) }, + check: None, }, ]; @@ -147,6 +172,7 @@ pub fn find(name: &str) -> Option<&'static Scenario> { .iter() .chain(TAG_SCENARIOS) .chain(ARENA_SCENARIOS) + .chain(crate::lifecycle::SCENARIOS) .find(|s| s.name == name) } @@ -161,11 +187,13 @@ const TAG_SCENARIOS: &[Scenario] = &[ )) }, run: run_tag_push, + check: None, }, Scenario { name: TAG_PUSH_AT_CAPACITY, setup: || Box::new(tag_push("env", "prod", fill_to_capacity())), run: run_tag_push, + check: None, }, Scenario { name: TAG_PUSH_LONG, @@ -177,6 +205,7 @@ const TAG_SCENARIOS: &[Scenario] = &[ }) }, run: run_tag_push, + check: None, }, ]; @@ -249,26 +278,31 @@ const ARENA_SCENARIOS: &[Scenario] = &[ name: ARENA_FIRST, setup: || Box::new(request_arena()), run: run_arena_8, + check: None, }, Scenario { name: ARENA_SPARE, setup: || Box::new(arena_leaving(1024)), run: run_arena_8, + check: None, }, Scenario { name: ARENA_EXHAUSTED, setup: || Box::new(arena_leaving(32)), run: run_arena_32, + check: None, }, Scenario { name: ARENA_GROW, setup: || Box::new(arena_leaving(0)), run: run_arena_1, + check: None, }, Scenario { name: ARENA_OVERSIZED, setup: || Box::new(request_arena()), run: run_arena_oversized, + check: None, }, Scenario { name: ARENA_STRING_TABLE, @@ -282,6 +316,7 @@ const ARENA_SCENARIOS: &[Scenario] = &[ .map(|_| ()) .map_err(|_| ScenarioError::Oom) }, + check: None, }, Scenario { name: ARENA_SET, @@ -298,6 +333,7 @@ const ARENA_SCENARIOS: &[Scenario] = &[ .map(|_| ()) .map_err(|_| ScenarioError::Oom) }, + check: None, }, Scenario { name: ARENA_SLICE_SET, @@ -315,5 +351,6 @@ const ARENA_SCENARIOS: &[Scenario] = &[ .map(|_| ()) .map_err(|_| ScenarioError::Oom) }, + check: None, }, ]; From db2c66cba0454c49a55cb73714d208229a181fc5 Mon Sep 17 00:00:00 2001 From: Gregory LEOCADIE Date: Fri, 20 Nov 2026 15:12:10 +0000 Subject: [PATCH 6/6] test(profiling-ffi): let cargo-machete ignore the zstd dev-dependency The dev-dependency only enables zstd's with-rust-allocator feature in test builds; no code imports it. Jira: PROF-16808 --- libdd-profiling-ffi/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index d033ed84ee..223e6244f0 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -109,4 +109,5 @@ path = "tests/allocation_failure/main.rs" harness = false [package.metadata.cargo-machete] -ignored = ["libdd-regex"] +# zstd: a dev-dependency only for its with-rust-allocator feature. +ignored = ["libdd-regex", "zstd"]