diff --git a/cli/src/services/agent_trace_db/lock_contention_tests.rs b/cli/src/services/agent_trace_db/lock_contention_tests.rs new file mode 100644 index 000000000..6e3e225b5 --- /dev/null +++ b/cli/src/services/agent_trace_db/lock_contention_tests.rs @@ -0,0 +1,1355 @@ +use std::{ + fs, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Barrier, + }, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::Result; + +use crate::services::db::{ + count_write_contention, count_write_statements, record_write_contention_timeline, + WriteContentionCounts, WriteContentionTimelineEvent, +}; + +use super::repository::RepositoryAgentTraceDb; +use super::{InsertMessageInsert, InsertPartInsert, MessageRole, PartType}; + +const DATABASE_LOCKED_ERROR: &str = "database is locked"; +const WRITE_CONTENTION_ERROR: &str = "under write contention"; +const LOCK_HOLD_DURATIONS_MS: &[u64] = &[ + 100, 250, 500, 750, 1_000, 1_500, 1_750, 2_000, 2_250, 2_500, 3_000, +]; +const RELIABLY_WITHIN_CONTENTION_BUDGET_MS: u64 = 1_000; +const RELIABLY_BEYOND_CONTENTION_BUDGET_MS: u64 = 3_000; +const WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; +const BUSY_TIMEOUT_PRODUCTION_HOLD_MS: u64 = 100; +const ROUNDS_ENV: &str = "SCE_LOCK_CONTENTION_ROUNDS"; +const WRITERS_ENV: &str = "SCE_LOCK_CONTENTION_WRITERS"; +const STRICT_ENV: &str = "SCE_LOCK_CONTENTION_STRICT"; +const DEFAULT_ROUNDS: usize = 200; +const DEFAULT_DUPLICATE_WRITER_COUNTS: &[usize] = &[2, 3, 4, 8]; +const DEFAULT_DISTINCT_WRITER_COUNTS: &[usize] = &[2, 3, 4, 8]; +const MEASUREMENT_PREFIX: &str = "SCE_MEAS"; +const SLOW_OPERATION_MS: f64 = 500.0; +const STALL_MONITOR_TICK: Duration = Duration::from_millis(5); +const STALL_MONITOR_REPORT_MS: f64 = 50.0; + +fn unix_ms_now() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after Unix epoch") + .as_secs_f64() + * 1_000.0 +} + +fn millis(duration: Duration) -> f64 { + duration.as_secs_f64() * 1_000.0 +} + +fn emit_measurement(record: &serde_json::Value) { + eprintln!("{MEASUREMENT_PREFIX} {record}"); +} + +fn timeline_json( + started_at: Instant, + timeline: &[(Instant, WriteContentionTimelineEvent)], +) -> serde_json::Value { + serde_json::Value::Array( + timeline + .iter() + .map(|(at, event)| { + let at_ms = millis(at.saturating_duration_since(started_at)); + match event { + WriteContentionTimelineEvent::AttemptStart => { + serde_json::json!({"event": "attempt_start", "at_ms": at_ms}) + } + WriteContentionTimelineEvent::AttemptEnd => { + serde_json::json!({"event": "attempt_end", "at_ms": at_ms}) + } + WriteContentionTimelineEvent::BackoffRequested(backoff) => serde_json::json!({ + "event": "backoff_requested", + "at_ms": at_ms, + "backoff_ms": millis(*backoff), + }), + WriteContentionTimelineEvent::BackoffSlept => { + serde_json::json!({"event": "backoff_slept", "at_ms": at_ms}) + } + } + }) + .collect(), + ) +} + +struct StallMonitor { + stop: Arc, + handle: thread::JoinHandle>, +} + +impl StallMonitor { + fn start() -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let handle = { + let stop = Arc::clone(&stop); + thread::spawn(move || { + let mut gaps = Vec::new(); + while !stop.load(Ordering::Relaxed) { + let before = Instant::now(); + thread::sleep(STALL_MONITOR_TICK); + let late_ms = millis(before.elapsed()) - millis(STALL_MONITOR_TICK); + if late_ms >= STALL_MONITOR_REPORT_MS { + gaps.push((unix_ms_now(), late_ms)); + } + } + gaps + }) + }; + Self { stop, handle } + } + + fn finish(self) -> Vec<(f64, f64)> { + self.stop.store(true, Ordering::Relaxed); + self.handle.join().expect("stall monitor should not panic") + } +} + +fn stall_gaps_json(gaps: &[(f64, f64)]) -> serde_json::Value { + serde_json::Value::Array( + gaps.iter() + .map(|(ended_unix_ms, late_ms)| { + serde_json::json!({"ended_unix_ms": ended_unix_ms, "late_ms": late_ms}) + }) + .collect(), + ) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum WriteOutcome { + Inserted, + AlreadyPresent, + Locked(String), + Other(String), +} + +impl WriteOutcome { + fn from_result(result: Result) -> Self { + match result { + Ok(true) => Self::Inserted, + Ok(false) => Self::AlreadyPresent, + Err(error) => { + let message = format!("{error:#}"); + if message.contains(DATABASE_LOCKED_ERROR) { + Self::Locked(message) + } else { + Self::Other(message) + } + } + } + } + + fn label(&self) -> &'static str { + match self { + Self::Inserted => "Ok(true)", + Self::AlreadyPresent => "Ok(false)", + Self::Locked(_) => "database is locked", + Self::Other(_) => "other error", + } + } +} + +fn unique_test_db_path(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after Unix epoch") + .as_nanos(); + std::env::temp_dir() + .join(format!( + "sce-lock-contention-{label}-{}-{nonce}", + std::process::id() + )) + .join("agent-trace.db") +} + +fn remove_test_db(db_path: &Path) { + if let Some(parent) = db_path.parent() { + fs::remove_dir_all(parent).expect("test DB directory should be removed"); + } +} + +fn create_repository_db(db_path: &Path) { + RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should be created up front"); +} + +fn open_production_connection(db_path: &Path) -> RepositoryAgentTraceDb { + RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) + .expect("production hook connection should open") +} + +fn conversation_text_event( + session_id: &str, + message_id: &str, +) -> (InsertMessageInsert, InsertPartInsert) { + ( + InsertMessageInsert { + session_id: session_id.to_string(), + message_id: message_id.to_string(), + role: MessageRole::User, + generated_at_unix_ms: 1_000, + }, + InsertPartInsert { + part_type: PartType::Text, + text: format!("text for {message_id}"), + session_id: session_id.to_string(), + message_id: message_id.to_string(), + generated_at_unix_ms: 1_000, + }, + ) +} + +fn session_row_count(db: &RepositoryAgentTraceDb, table: &str, session_id: &str) -> i64 { + db.query_map( + &format!("SELECT COUNT(*) FROM {table} WHERE session_id = ?1"), + (session_id,), + |row| row.get::(0).map_err(Into::into), + ) + .expect("count query should succeed") + .into_iter() + .next() + .expect("count row should exist") +} + +fn env_usize(name: &str, default: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(default) +} + +fn env_writer_counts(default: &[usize]) -> Vec { + std::env::var(WRITERS_ENV) + .ok() + .map(|value| { + value + .split(',') + .filter_map(|count| count.trim().parse().ok()) + .collect::>() + }) + .filter(|counts| !counts.is_empty()) + .unwrap_or_else(|| default.to_vec()) +} + +fn strict_mode() -> bool { + std::env::var(STRICT_ENV).is_ok_and(|value| value == "1") +} + +struct LockBoundarySample { + hold: Duration, + outcome: WriteOutcome, + elapsed: Duration, + holder_released_after: Duration, + messages: i64, + parts: i64, + contention: WriteContentionCounts, + timeline: serde_json::Value, +} + +fn insert_while_write_lock_is_held(hold: Duration) -> LockBoundarySample { + let db_path = unique_test_db_path(&format!("boundary-{}ms", hold.as_millis())); + create_repository_db(&db_path); + let session_id = "cx_lock-boundary"; + + let lock_acquired = Arc::new(Barrier::new(2)); + let holder = { + let db_path = db_path.clone(); + let lock_acquired = Arc::clone(&lock_acquired); + thread::spawn(move || { + let holder = open_production_connection(&db_path); + holder + .execute("BEGIN IMMEDIATE", ()) + .expect("holder should acquire the write lock"); + lock_acquired.wait(); + let held_since = Instant::now(); + thread::sleep(hold); + holder + .execute("COMMIT", ()) + .expect("holder should release the write lock"); + held_since.elapsed() + }) + }; + + let writer = open_production_connection(&db_path); + lock_acquired.wait(); + let (message, part) = conversation_text_event(session_id, "cx:turn-1:user"); + let started_at = Instant::now(); + let ((result, contention), timeline) = record_write_contention_timeline(|| { + count_write_contention(|| writer.insert_conversation_text_event(message, part)) + }); + let elapsed = started_at.elapsed(); + let outcome = WriteOutcome::from_result(result); + let timeline = timeline_json(started_at, &timeline); + + let holder_released_after = holder.join().expect("holder thread should not panic"); + + let verifier = open_production_connection(&db_path); + let messages = session_row_count(&verifier, "messages", session_id); + let parts = session_row_count(&verifier, "parts", session_id); + drop((writer, verifier)); + remove_test_db(&db_path); + + LockBoundarySample { + hold, + outcome, + elapsed, + holder_released_after, + messages, + parts, + contention, + timeline, + } +} + +#[test] +fn lock_budget_boundary_characterizes_single_writer_blocked_by_begin_immediate_holder() { + let samples: Vec = LOCK_HOLD_DURATIONS_MS + .iter() + .map(|hold_ms| insert_while_write_lock_is_held(Duration::from_millis(*hold_ms))) + .collect(); + + eprintln!( + "\nlock-budget boundary (Agent Trace busy_timeout + write-contention retry + contention deadline)" + ); + eprintln!( + "hold_ms | outcome | elapsed_ms | holder_released_ms | attempts | outer_retries | exhaustions | messages | parts" + ); + for sample in &samples { + eprintln!( + "{:>7} | {:<18} | {:>10} | {:>18} | {:>8} | {:>13} | {:>11} | {:>8} | {:>5}", + sample.hold.as_millis(), + sample.outcome.label(), + sample.elapsed.as_millis(), + sample.holder_released_after.as_millis(), + sample.contention.attempts, + sample.contention.outer_retries, + sample.contention.exhaustions, + sample.messages, + sample.parts, + ); + if let WriteOutcome::Locked(message) | WriteOutcome::Other(message) = &sample.outcome { + eprintln!(" error: {message}"); + } + emit_measurement(&serde_json::json!({ + "kind": "boundary_sample", + "hold_ms": sample.hold.as_millis(), + "outcome": sample.outcome.label(), + "error": match &sample.outcome { + WriteOutcome::Locked(message) | WriteOutcome::Other(message) => Some(message), + WriteOutcome::Inserted | WriteOutcome::AlreadyPresent => None, + }, + "elapsed_ms": millis(sample.elapsed), + "holder_released_ms": millis(sample.holder_released_after), + "attempts": sample.contention.attempts, + "outer_retries": sample.contention.outer_retries, + "exhaustions": sample.contention.exhaustions, + "messages": sample.messages, + "parts": sample.parts, + "timeline": sample.timeline, + })); + } + + for sample in &samples { + let hold_ms = u64::try_from(sample.hold.as_millis()).expect("hold should fit in u64"); + assert_eq!( + sample.messages, sample.parts, + "hold {hold_ms}ms left a partial message/part pair" + ); + match &sample.outcome { + WriteOutcome::Inserted => assert_eq!(sample.messages, 1), + WriteOutcome::Locked(_) | WriteOutcome::Other(_) => assert_eq!(sample.messages, 0), + WriteOutcome::AlreadyPresent => panic!("hold {hold_ms}ms reported a phantom replay"), + } + assert!( + (1..=WRITE_CONTENTION_MAX_ATTEMPTS).contains(&sample.contention.attempts), + "hold {hold_ms}ms made {} attempt(s), outside the contention policy", + sample.contention.attempts + ); + assert_eq!( + sample.contention.outer_retries + 1, + sample.contention.attempts, + "hold {hold_ms}ms: every attempt after the first must be an admitted outer retry" + ); + if hold_ms <= RELIABLY_WITHIN_CONTENTION_BUDGET_MS { + assert_eq!( + sample.outcome, + WriteOutcome::Inserted, + "a {hold_ms}ms lock is well inside the contention budget" + ); + assert_eq!( + sample.contention.exhaustions, 0, + "a {hold_ms}ms lock must not exhaust the contention policy" + ); + } + if hold_ms >= RELIABLY_BEYOND_CONTENTION_BUDGET_MS { + match &sample.outcome { + WriteOutcome::Locked(message) => assert!( + message.contains(WRITE_CONTENTION_ERROR), + "a {hold_ms}ms lock should fail with a contention-exhaustion error, got {message}" + ), + outcome => panic!( + "a {hold_ms}ms lock is expected to exhaust the contention policy, got {outcome:?}" + ), + } + assert_eq!( + sample.contention.exhaustions, 1, + "a {hold_ms}ms lock should exhaust the contention policy exactly once" + ); + } + } +} + +#[test] +fn busy_timeout_production_insert_waits_for_begin_immediate_holder() { + let hold = Duration::from_millis(BUSY_TIMEOUT_PRODUCTION_HOLD_MS); + + let sample = insert_while_write_lock_is_held(hold); + + assert_eq!( + sample.outcome, + WriteOutcome::Inserted, + "insert_conversation_text_event should wait out a {}ms holder", + hold.as_millis() + ); + assert_eq!((sample.messages, sample.parts), (1, 1)); + assert!( + sample.elapsed >= hold / 2, + "the insert should have waited for the holder, took {:?}", + sample.elapsed + ); +} + +#[test] +fn agent_trace_db_write_contention_retry_hundred_ms_hold_succeeds_on_the_first_attempt() { + let hold = Duration::from_millis(BUSY_TIMEOUT_PRODUCTION_HOLD_MS); + + let (sample, counts) = count_write_contention(|| insert_while_write_lock_is_held(hold)); + + assert_eq!(sample.outcome, WriteOutcome::Inserted); + assert_eq!((sample.messages, sample.parts), (1, 1)); + assert_eq!( + (counts.attempts, counts.outer_retries, counts.exhaustions), + (1, 0, 0), + "Turso's busy timeout should absorb a {}ms hold without an outer retry", + hold.as_millis() + ); +} + +struct LatencyPercentiles { + p50: Duration, + p95: Duration, + p99: Duration, + max: Duration, +} + +fn nearest_rank(sorted: &[Duration], percentile: usize) -> Duration { + let rank = (percentile * sorted.len()).div_ceil(100).max(1); + sorted[rank - 1] +} + +fn latency_percentiles(latencies: &[Duration]) -> LatencyPercentiles { + if latencies.is_empty() { + return LatencyPercentiles { + p50: Duration::ZERO, + p95: Duration::ZERO, + p99: Duration::ZERO, + max: Duration::ZERO, + }; + } + let mut sorted = latencies.to_vec(); + sorted.sort_unstable(); + LatencyPercentiles { + p50: nearest_rank(&sorted, 50), + p95: nearest_rank(&sorted, 95), + p99: nearest_rank(&sorted, 99), + max: sorted[sorted.len() - 1], + } +} + +#[test] +fn lock_contention_latency_percentiles_use_nearest_rank() { + let latencies: Vec = (1..=100).rev().map(Duration::from_millis).collect(); + let percentiles = latency_percentiles(&latencies); + assert_eq!(percentiles.p50, Duration::from_millis(50)); + assert_eq!(percentiles.p95, Duration::from_millis(95)); + assert_eq!(percentiles.p99, Duration::from_millis(99)); + assert_eq!(percentiles.max, Duration::from_millis(100)); + + let single = latency_percentiles(&[Duration::from_millis(7)]); + assert_eq!(single.p50, Duration::from_millis(7)); + assert_eq!(single.p99, Duration::from_millis(7)); + + let empty = latency_percentiles(&[]); + assert_eq!(empty.max, Duration::ZERO); +} + +struct SlowOperation { + writer_index: usize, + started_unix_ms: f64, + elapsed: Duration, + outcome: WriteOutcome, + contention: WriteContentionCounts, + timeline: serde_json::Value, +} + +struct RoundResult { + slow_operations: Vec, + outcomes: Vec, + latencies: Vec, + contention: Vec, + max_elapsed: Duration, + messages: i64, + parts: i64, +} + +fn run_concurrent_round( + db_path: &Path, + writers: usize, + session_id: &str, + distinct_events: bool, +) -> RoundResult { + let start_together = Arc::new(Barrier::new(writers)); + let handles: Vec<_> = (0..writers) + .map(|writer_index| { + let db_path = db_path.to_path_buf(); + let start_together = Arc::clone(&start_together); + let session_id = session_id.to_string(); + thread::spawn(move || { + let db = open_production_connection(&db_path); + let message_id = if distinct_events { + format!("cx:turn-{writer_index}:user") + } else { + String::from("cx:turn-shared:user") + }; + let (message, part) = conversation_text_event(&session_id, &message_id); + start_together.wait(); + let started_unix_ms = unix_ms_now(); + let started_at = Instant::now(); + let ((result, contention), timeline) = record_write_contention_timeline(|| { + count_write_contention(|| db.insert_conversation_text_event(message, part)) + }); + let elapsed = started_at.elapsed(); + let outcome = WriteOutcome::from_result(result); + let slow = (millis(elapsed) >= SLOW_OPERATION_MS).then(|| SlowOperation { + writer_index, + started_unix_ms, + elapsed, + outcome: outcome.clone(), + contention, + timeline: timeline_json(started_at, &timeline), + }); + (outcome, elapsed, contention, slow) + }) + }) + .collect(); + + let mut slow_operations = Vec::new(); + let mut outcomes = Vec::with_capacity(writers); + let mut latencies = Vec::with_capacity(writers); + let mut contention = Vec::with_capacity(writers); + let mut max_elapsed = Duration::ZERO; + for handle in handles { + let (outcome, elapsed, counts, slow) = + handle.join().expect("writer thread should not panic"); + slow_operations.extend(slow); + outcomes.push(outcome); + latencies.push(elapsed); + contention.push(counts); + max_elapsed = max_elapsed.max(elapsed); + } + + let verifier = open_production_connection(db_path); + RoundResult { + slow_operations, + outcomes, + latencies, + contention, + max_elapsed, + messages: session_row_count(&verifier, "messages", session_id), + parts: session_row_count(&verifier, "parts", session_id), + } +} + +#[derive(Default)] +struct LevelSummary { + writers: usize, + rounds: usize, + clean_rounds: usize, + rounds_with_lock_exhaustion: usize, + inserted: usize, + already_present: usize, + lock_errors: usize, + other_errors: usize, + lost_events: i64, + orphan_rows: i64, + duplicate_rows: i64, + persisted_rows: i64, + attempts: u64, + outer_retries: u64, + exhaustions: u64, + max_elapsed: Duration, + latencies: Vec, + first_lock_error: Option, + first_other_error: Option, +} + +impl LevelSummary { + fn total_errors(&self) -> usize { + self.lock_errors + self.other_errors + } +} + +#[allow(clippy::too_many_lines)] +fn run_contention_level(writers: usize, rounds: usize, distinct_events: bool) -> LevelSummary { + let mode = if distinct_events { + "distinct" + } else { + "duplicate" + }; + let db_path = unique_test_db_path(&format!("{mode}-{writers}w")); + create_repository_db(&db_path); + + let mut summary = LevelSummary { + writers, + rounds, + ..LevelSummary::default() + }; + + let monitor = StallMonitor::start(); + let level_started_unix_ms = unix_ms_now(); + for round in 0..rounds { + let session_id = format!("cx_{mode}-{writers}w-round-{round}"); + let result = run_concurrent_round(&db_path, writers, &session_id, distinct_events); + + for slow in &result.slow_operations { + emit_measurement(&serde_json::json!({ + "kind": "slow_operation", + "mode": mode, + "writers": writers, + "round": round, + "writer_index": slow.writer_index, + "started_unix_ms": slow.started_unix_ms, + "elapsed_ms": millis(slow.elapsed), + "outcome": slow.outcome.label(), + "attempts": slow.contention.attempts, + "outer_retries": slow.contention.outer_retries, + "exhaustions": slow.contention.exhaustions, + "timeline": slow.timeline, + })); + } + summary.orphan_rows += (result.messages - result.parts).abs(); + let expected_max_rows = if distinct_events { writers } else { 1 }; + summary.duplicate_rows += + (result.messages - i64::try_from(expected_max_rows).expect("count fits i64")).max(0); + summary.persisted_rows += result.messages; + + assert_eq!( + result.messages, result.parts, + "{mode} round {round} with {writers} writers left orphaned rows" + ); + + let inserted = result + .outcomes + .iter() + .filter(|outcome| **outcome == WriteOutcome::Inserted) + .count(); + let already_present = result + .outcomes + .iter() + .filter(|outcome| **outcome == WriteOutcome::AlreadyPresent) + .count(); + let mut round_lock_errors = 0; + let mut round_other_errors = 0; + for outcome in &result.outcomes { + match outcome { + WriteOutcome::Locked(message) => { + round_lock_errors += 1; + summary + .first_lock_error + .get_or_insert_with(|| message.clone()); + } + WriteOutcome::Other(message) => { + round_other_errors += 1; + summary + .first_other_error + .get_or_insert_with(|| message.clone()); + } + WriteOutcome::Inserted | WriteOutcome::AlreadyPresent => {} + } + } + + assert_eq!( + i64::try_from(inserted).expect("count fits i64"), + result.messages, + "{mode} round {round}: Ok(true) count must equal persisted message rows" + ); + + if distinct_events { + assert_eq!( + already_present, 0, + "distinct events must never report Ok(false)" + ); + summary.lost_events += + i64::try_from(writers).expect("count fits i64") - result.messages; + } else { + assert!(inserted <= 1, "duplicate delivery inserted more than once"); + if round_lock_errors + round_other_errors < writers { + assert_eq!( + inserted, 1, + "a duplicate round with at least one completed writer must persist the event" + ); + } + } + + let expected_rows = if distinct_events { writers } else { 1 }; + if round_lock_errors + round_other_errors == 0 + && usize::try_from(result.messages).expect("count fits usize") == expected_rows + { + summary.clean_rounds += 1; + } + if round_lock_errors > 0 { + summary.rounds_with_lock_exhaustion += 1; + } + summary.inserted += inserted; + summary.already_present += already_present; + summary.lock_errors += round_lock_errors; + summary.other_errors += round_other_errors; + for counts in &result.contention { + assert!( + counts.attempts <= WRITE_CONTENTION_MAX_ATTEMPTS, + "{mode} round {round}: a writer made {} attempt(s), beyond the contention policy", + counts.attempts + ); + summary.attempts += u64::from(counts.attempts); + summary.outer_retries += u64::from(counts.outer_retries); + summary.exhaustions += u64::from(counts.exhaustions); + } + summary.max_elapsed = summary.max_elapsed.max(result.max_elapsed); + summary.latencies.extend(result.latencies); + } + + let stall_gaps = monitor.finish(); + let latency = latency_percentiles(&summary.latencies); + emit_measurement(&serde_json::json!({ + "kind": "concurrent_level", + "mode": mode, + "writers": writers, + "rounds": rounds, + "started_unix_ms": level_started_unix_ms, + "ended_unix_ms": unix_ms_now(), + "total_writes": writers * rounds, + "expected_rows": if distinct_events { writers * rounds } else { rounds }, + "persisted_rows": summary.persisted_rows, + "lost_events": summary.lost_events, + "lock_errors": summary.lock_errors, + "other_errors": summary.other_errors, + "orphan_rows": summary.orphan_rows, + "duplicate_rows": summary.duplicate_rows, + "inserted": summary.inserted, + "already_present": summary.already_present, + "attempts": summary.attempts, + "outer_retries": summary.outer_retries, + "exhaustions": summary.exhaustions, + "p50_ms": millis(latency.p50), + "p95_ms": millis(latency.p95), + "p99_ms": millis(latency.p99), + "max_ms": millis(latency.max), + "first_lock_error": summary.first_lock_error, + "first_other_error": summary.first_other_error, + "stall_gaps": stall_gaps_json(&stall_gaps), + })); + + remove_test_db(&db_path); + summary +} + +fn report_levels(title: &str, summaries: &[LevelSummary]) { + eprintln!("\n{title}"); + eprintln!( + "writers | rounds | clean rounds | rounds w/ lock exhaustion | lock errors | other errors | Ok(true) | Ok(false) | lost events | attempts | outer retries | exhaustions | max elapsed ms | p50 ms | p95 ms | p99 ms | max ms" + ); + for summary in summaries { + let latency = latency_percentiles(&summary.latencies); + eprintln!( + "{:>7} | {:>6} | {:>12} | {:>25} | {:>11} | {:>12} | {:>8} | {:>9} | {:>11} | {:>8} | {:>13} | {:>11} | {:>14} | {:>6} | {:>6} | {:>6} | {:>6}", + summary.writers, + summary.rounds, + summary.clean_rounds, + summary.rounds_with_lock_exhaustion, + summary.lock_errors, + summary.other_errors, + summary.inserted, + summary.already_present, + summary.lost_events, + summary.attempts, + summary.outer_retries, + summary.exhaustions, + summary.max_elapsed.as_millis(), + latency.p50.as_millis(), + latency.p95.as_millis(), + latency.p99.as_millis(), + latency.max.as_millis(), + ); + } + for summary in summaries { + if let Some(message) = &summary.first_lock_error { + eprintln!("{} writers first lock error: {message}", summary.writers); + } + if let Some(message) = &summary.first_other_error { + eprintln!("{} writers first other error: {message}", summary.writers); + } + } +} + +fn assert_strict_if_requested(summaries: &[LevelSummary]) { + if !strict_mode() { + return; + } + for summary in summaries { + assert_eq!( + summary.total_errors(), + 0, + "{} concurrent writers produced {} lock error(s) and {} other error(s)", + summary.writers, + summary.lock_errors, + summary.other_errors + ); + assert_eq!( + summary.lost_events, 0, + "{} concurrent writers lost {} distinct event(s)", + summary.writers, summary.lost_events + ); + assert_eq!( + summary.exhaustions, 0, + "{} concurrent writers exhausted the contention policy {} time(s)", + summary.writers, summary.exhaustions + ); + } +} + +#[test] +#[ignore = "lock-contention characterization; run with --ignored --nocapture"] +fn concurrent_duplicate_delivery_persists_each_event_once_under_write_contention() { + let rounds = env_usize(ROUNDS_ENV, DEFAULT_ROUNDS); + let summaries: Vec = env_writer_counts(DEFAULT_DUPLICATE_WRITER_COUNTS) + .into_iter() + .map(|writers| run_contention_level(writers, rounds, false)) + .collect(); + + report_levels( + "concurrent duplicate delivery (same logical event, Agent Trace write-contention policy)", + &summaries, + ); + assert_strict_if_requested(&summaries); +} + +#[test] +#[ignore = "lock-contention characterization; run with --ignored --nocapture"] +fn concurrent_distinct_events_persist_every_event_under_write_contention() { + let rounds = env_usize(ROUNDS_ENV, DEFAULT_ROUNDS); + let summaries: Vec = env_writer_counts(DEFAULT_DISTINCT_WRITER_COUNTS) + .into_iter() + .map(|writers| run_contention_level(writers, rounds, true)) + .collect(); + + report_levels( + "concurrent distinct events (one unique event per writer, Agent Trace write-contention policy)", + &summaries, + ); + assert_strict_if_requested(&summaries); +} + +const SCE_BIN_ENV: &str = "SCE_BIN"; + +struct HookLevelSummary { + writers: usize, + expected: i64, + persisted_messages: i64, + persisted_parts: i64, + nonzero_exits: usize, +} + +fn assert_hook_levels(levels: &[HookLevelSummary], strict: bool) { + for level in levels { + assert!( + level.persisted_messages <= level.expected, + "{} hook writers persisted {} messages for {} distinct events", + level.writers, + level.persisted_messages, + level.expected + ); + assert!( + level.persisted_parts <= level.expected, + "{} hook writers persisted {} parts for {} distinct events", + level.writers, + level.persisted_parts, + level.expected + ); + if strict { + assert_eq!( + level.persisted_messages, level.expected, + "{} hook writers lost message rows", + level.writers + ); + assert_eq!( + level.persisted_parts, level.expected, + "{} hook writers lost part rows", + level.writers + ); + assert_eq!( + level.nonzero_exits, 0, + "{} hook writers had non-zero exits", + level.writers + ); + } + } + + if strict { + let total_lost: i64 = levels + .iter() + .map(|level| level.expected - level.persisted_messages) + .sum(); + let total_nonzero_exits: usize = levels.iter().map(|level| level.nonzero_exits).sum(); + assert_eq!(total_lost, 0, "real hook processes lost distinct events"); + assert_eq!( + total_nonzero_exits, 0, + "real hook processes exited with a non-zero status" + ); + } +} + +#[test] +fn lock_contention_hook_level_assertions_reject_over_persistence() { + let over_persisted = [HookLevelSummary { + writers: 2, + expected: 4, + persisted_messages: 5, + persisted_parts: 5, + nonzero_exits: 0, + }]; + let result = std::panic::catch_unwind(|| assert_hook_levels(&over_persisted, false)); + assert!( + result.is_err(), + "over-persistence must fail even outside strict mode" + ); +} + +#[test] +fn lock_contention_hook_level_assertions_enforce_strict_loss_and_exit_status() { + let lost = [HookLevelSummary { + writers: 2, + expected: 4, + persisted_messages: 3, + persisted_parts: 3, + nonzero_exits: 0, + }]; + assert_hook_levels(&lost, false); + assert!(std::panic::catch_unwind(|| assert_hook_levels(&lost, true)).is_err()); + + let nonzero_exit = [HookLevelSummary { + writers: 2, + expected: 4, + persisted_messages: 4, + persisted_parts: 4, + nonzero_exits: 1, + }]; + assert_hook_levels(&nonzero_exit, false); + assert!(std::panic::catch_unwind(|| assert_hook_levels(&nonzero_exit, true)).is_err()); + + let clean = [HookLevelSummary { + writers: 3, + expected: 6, + persisted_messages: 6, + persisted_parts: 6, + nonzero_exits: 0, + }]; + assert_hook_levels(&clean, true); +} +const DEFAULT_PROCESS_ROUNDS: usize = 50; +const DEFAULT_PROCESS_WRITER_COUNTS: &[usize] = &[2, 3, 4]; + +struct HookRound { + nonzero_exits: usize, + stderr_lines: Vec, + latencies: Vec, + slow_processes: Vec<(usize, f64, Duration)>, +} + +struct HookProcessHarness { + sce: PathBuf, + work: PathBuf, + repo: PathBuf, + db_path: PathBuf, +} + +impl HookProcessHarness { + fn command(&self) -> std::process::Command { + let mut command = std::process::Command::new(&self.sce); + command + .current_dir(&self.repo) + .env("XDG_STATE_HOME", self.work.join("state")) + .env("XDG_CONFIG_HOME", self.work.join("config")) + .env("XDG_DATA_HOME", self.work.join("data")); + command + } + + fn create(sce: PathBuf, label: &str) -> Self { + let work = unique_test_db_path(label) + .parent() + .expect("unique path has a parent") + .to_path_buf(); + let repo = work.join("repo"); + fs::create_dir_all(&repo).expect("repo dir should be created"); + for args in [ + vec!["init", "-q"], + vec![ + "remote", + "add", + "origin", + "https://example.invalid/sce/lock-contention.git", + ], + vec![ + "-c", + "user.name=t", + "-c", + "user.email=t@example.invalid", + "commit", + "-q", + "--allow-empty", + "-m", + "init", + ], + ] { + let status = std::process::Command::new("git") + .args(&args) + .current_dir(&repo) + .status() + .expect("git should spawn"); + assert!(status.success(), "git {args:?} should succeed"); + } + + let mut harness = Self { + sce, + work, + repo, + db_path: PathBuf::new(), + }; + + let setup = harness + .command() + .args(["setup", "--codex", "--non-interactive", "--hooks"]) + .output() + .expect("sce setup should spawn"); + assert!( + setup.status.success(), + "sce setup failed: {}", + String::from_utf8_lossy(&setup.stderr) + ); + + let doctor = harness + .command() + .args(["doctor", "--format", "json"]) + .output() + .expect("sce doctor should spawn"); + let report: serde_json::Value = + serde_json::from_slice(&doctor.stdout).expect("doctor should print JSON"); + harness.db_path = PathBuf::from( + report["agent_trace_db"]["path"] + .as_str() + .expect("doctor should report agent_trace_db.path"), + ); + assert!( + harness.db_path.is_file(), + "repository Agent Trace DB should exist" + ); + harness + } + + fn run_round(&self, writers: usize, session_id: &str) -> HookRound { + use std::io::Write; + use std::process::Stdio; + + let children: Vec<_> = (0..writers) + .map(|_| { + self.command() + .args(["hooks", "codex"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("sce hooks codex should spawn") + }) + .collect(); + + let release = Arc::new(Barrier::new(writers)); + let feeders: Vec<_> = children + .into_iter() + .enumerate() + .map(|(writer_index, mut child)| { + let mut stdin = child.stdin.take().expect("stdin should be piped"); + let release = Arc::clone(&release); + let payload = serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "session_id": session_id, + "turn_id": format!("w{writer_index}"), + "prompt": format!("prompt {session_id} w{writer_index}"), + }) + .to_string(); + thread::spawn(move || { + release.wait(); + let started_unix_ms = unix_ms_now(); + let started_at = Instant::now(); + stdin + .write_all(payload.as_bytes()) + .expect("payload should be written"); + drop(stdin); + let output = child + .wait_with_output() + .expect("hook process should finish"); + (output, started_unix_ms, started_at.elapsed()) + }) + }) + .collect(); + + let mut nonzero_exits = 0; + let mut stderr_lines = Vec::new(); + let mut latencies = Vec::with_capacity(writers); + let mut slow_processes = Vec::new(); + for (writer_index, feeder) in feeders.into_iter().enumerate() { + let (output, started_unix_ms, elapsed) = + feeder.join().expect("feeder thread should not panic"); + latencies.push(elapsed); + if millis(elapsed) >= SLOW_OPERATION_MS { + slow_processes.push((writer_index, started_unix_ms, elapsed)); + } + if !output.status.success() { + nonzero_exits += 1; + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if !stderr.is_empty() { + stderr_lines.push(stderr); + } + } + HookRound { + nonzero_exits, + stderr_lines, + latencies, + slow_processes, + } + } +} + +#[test] +#[ignore = "end-to-end hook-process lock contention; set SCE_BIN and run with --ignored --nocapture"] +#[allow(clippy::too_many_lines)] +fn concurrent_real_codex_hook_processes_persist_every_distinct_event() { + let Some(sce) = std::env::var_os(SCE_BIN_ENV).map(PathBuf::from) else { + eprintln!("{SCE_BIN_ENV} is not set; skipping end-to-end hook-process reproduction"); + return; + }; + let rounds = env_usize(ROUNDS_ENV, DEFAULT_PROCESS_ROUNDS); + + eprintln!("\nreal `sce hooks codex` UserPromptSubmit processes (distinct events)"); + eprintln!( + "writers | rounds | expected | persisted msgs | persisted parts | lost | rounds w/ loss | non-zero exits | stderr lines | p50 ms | p95 ms | p99 ms | max ms" + ); + let mut levels = Vec::new(); + for writers in env_writer_counts(DEFAULT_PROCESS_WRITER_COUNTS) { + let harness = HookProcessHarness::create(sce.clone(), &format!("hooks-{writers}w")); + let mut persisted_messages = 0; + let mut persisted_parts = 0; + let mut rounds_with_loss = 0; + let mut nonzero_exits = 0; + let mut stderr_samples = Vec::new(); + let mut latencies = Vec::with_capacity(writers * rounds); + let mut fail_open_lost = 0; + let monitor = StallMonitor::start(); + let level_started_unix_ms = unix_ms_now(); + + for round in 0..rounds { + let session_id = format!("lock-contention-{writers}w-r{round}"); + let HookRound { + nonzero_exits: round_nonzero, + stderr_lines: round_stderr, + latencies: round_latencies, + slow_processes, + } = harness.run_round(writers, &session_id); + for (writer_index, started_unix_ms, elapsed) in slow_processes { + emit_measurement(&serde_json::json!({ + "kind": "slow_hook_process", + "writers": writers, + "round": round, + "writer_index": writer_index, + "started_unix_ms": started_unix_ms, + "elapsed_ms": millis(elapsed), + })); + } + for stderr in &round_stderr { + emit_measurement(&serde_json::json!({ + "kind": "hook_stderr", + "writers": writers, + "round": round, + "stderr": stderr, + })); + } + nonzero_exits += round_nonzero; + stderr_samples.extend(round_stderr); + latencies.extend(round_latencies); + + let verifier = open_production_connection(&harness.db_path); + let prefixed = format!("cx_{session_id}"); + let messages = session_row_count(&verifier, "messages", &prefixed); + let parts = session_row_count(&verifier, "parts", &prefixed); + assert_eq!(messages, parts, "hook round left orphaned rows"); + if usize::try_from(messages).expect("count fits usize") != writers { + rounds_with_loss += 1; + let round_lost = i64::try_from(writers).expect("count fits i64") - messages; + let round_fail_open = + (round_lost - i64::try_from(round_nonzero).expect("count fits i64")).max(0); + fail_open_lost += round_fail_open; + emit_measurement(&serde_json::json!({ + "kind": "hook_round_loss", + "writers": writers, + "round": round, + "lost": round_lost, + "nonzero_exits": round_nonzero, + "fail_open_lost": round_fail_open, + "ended_unix_ms": unix_ms_now(), + })); + } + persisted_messages += messages; + persisted_parts += parts; + } + + let expected = i64::try_from(writers * rounds).expect("count fits i64"); + let lost = expected - persisted_messages; + let latency = latency_percentiles(&latencies); + eprintln!( + "{:>7} | {:>6} | {:>8} | {:>14} | {:>15} | {:>4} | {:>14} | {:>14} | {:>12} | {:>6} | {:>6} | {:>6} | {:>6}", + writers, + rounds, + expected, + persisted_messages, + persisted_parts, + lost, + rounds_with_loss, + nonzero_exits, + stderr_samples.len(), + latency.p50.as_millis(), + latency.p95.as_millis(), + latency.p99.as_millis(), + latency.max.as_millis(), + ); + if let Some(sample) = stderr_samples.first() { + eprintln!(" first stderr: {sample}"); + } + let stall_gaps = monitor.finish(); + emit_measurement(&serde_json::json!({ + "kind": "hook_level", + "writers": writers, + "rounds": rounds, + "started_unix_ms": level_started_unix_ms, + "ended_unix_ms": unix_ms_now(), + "expected": expected, + "persisted_messages": persisted_messages, + "persisted_parts": persisted_parts, + "lost": lost, + "fail_open_lost": fail_open_lost, + "rounds_with_loss": rounds_with_loss, + "nonzero_exits": nonzero_exits, + "stderr_outputs": stderr_samples.len(), + "stderr_lines": stderr_samples.iter().map(|sample| sample.lines().count()).sum::(), + "p50_ms": millis(latency.p50), + "p95_ms": millis(latency.p95), + "p99_ms": millis(latency.p99), + "max_ms": millis(latency.max), + "stall_gaps": stall_gaps_json(&stall_gaps), + })); + fs::remove_dir_all(&harness.work).expect("hook harness dir should be removed"); + levels.push(HookLevelSummary { + writers, + expected, + persisted_messages, + persisted_parts, + nonzero_exits, + }); + } + + assert_hook_levels(&levels, strict_mode()); +} + +#[test] +fn initialized_hook_open_succeeds_while_write_lock_is_held() { + let db_path = unique_test_db_path("hook-open-metadata"); + create_repository_db(&db_path); + let repository_id = "lock-contention-repository"; + let initialized = open_production_connection(&db_path) + .verify_or_initialize_repository_metadata(repository_id) + .expect("metadata should initialize before contention"); + + let hold = Duration::from_millis(RELIABLY_BEYOND_CONTENTION_BUDGET_MS); + let lock_acquired = Arc::new(Barrier::new(2)); + let holder = { + let db_path = db_path.clone(); + let lock_acquired = Arc::clone(&lock_acquired); + thread::spawn(move || { + let holder = open_production_connection(&db_path); + holder + .execute("BEGIN IMMEDIATE", ()) + .expect("holder should acquire the write lock"); + lock_acquired.wait(); + thread::sleep(hold); + holder + .execute("COMMIT", ()) + .expect("holder should release the write lock"); + }) + }; + + let hook = open_production_connection(&db_path); + lock_acquired.wait(); + let started_at = Instant::now(); + let ((schema_ready, metadata), writes) = count_write_statements(|| { + ( + hook.ensure_schema_ready_for_hooks(), + hook.verify_or_initialize_repository_metadata(repository_id), + ) + }); + let elapsed = started_at.elapsed(); + holder.join().expect("holder thread should not panic"); + + eprintln!( + "\nhook-open under a {}ms write lock: schema_ready={:?} metadata={} writes={writes} after {}ms", + hold.as_millis(), + schema_ready.as_ref().map_err(|error| format!("{error:#}")), + match &metadata { + Ok(_) => String::from("Ok"), + Err(error) => format!("Err({error:#})"), + }, + elapsed.as_millis() + ); + + assert!( + schema_ready.is_ok(), + "the read-only schema check should not need the write lock" + ); + let metadata = metadata.expect("an initialized hook open should not need the write lock"); + assert_eq!(metadata, initialized); + assert_eq!(writes, 0, "an initialized hook open must issue no writes"); + assert!( + elapsed < hold, + "the initialized hook open must not wait for the held write lock" + ); + drop(hook); + remove_test_db(&db_path); +} diff --git a/cli/src/services/agent_trace_db/mod.rs b/cli/src/services/agent_trace_db/mod.rs index bfdd4a81e..c2c293b17 100644 --- a/cli/src/services/agent_trace_db/mod.rs +++ b/cli/src/services/agent_trace_db/mod.rs @@ -12,6 +12,8 @@ use crate::services::{ use serde_json::Value; pub mod lifecycle; +#[cfg(test)] +mod lock_contention_tests; pub mod repository; /// Payload type discriminator for diff trace source payloads. diff --git a/cli/src/services/agent_trace_db/repository.rs b/cli/src/services/agent_trace_db/repository.rs index 74c4419d5..dbc9e176f 100644 --- a/cli/src/services/agent_trace_db/repository.rs +++ b/cli/src/services/agent_trace_db/repository.rs @@ -76,6 +76,16 @@ pub fn is_valid_source_instance_id(value: &str) -> bool { !value.trim().is_empty() } +fn ensure_repository_id_matches(stored_repository_id: &str, repository_id: &str) -> Result<()> { + if stored_repository_id != repository_id { + anyhow::bail!( + "repository Agent Trace DB metadata mismatch: stored repository ID \ + {stored_repository_id} does not match resolved repository ID {repository_id}" + ); + } + Ok(()) +} + /// Repository-scoped Agent Trace database configuration. pub struct RepositoryAgentTraceDbSpec; @@ -157,7 +167,19 @@ impl RepositoryAgentTraceDb { &self, repository_id: &str, ) -> Result { - self.execute(INSERT_REPOSITORY_METADATA_SQL, (repository_id,))?; + if let Some((stored_repository_id, source_instance_id)) = + self.select_repository_metadata_row()? + { + ensure_repository_id_matches(&stored_repository_id, repository_id)?; + if is_valid_source_instance_id(&source_instance_id) { + return Ok(RepositoryMetadata { + repository_id: stored_repository_id, + source_instance_id, + }); + } + } + + self.execute_idempotent_write(INSERT_REPOSITORY_METADATA_SQL, (repository_id,))?; let Some((stored_repository_id, source_instance_id)) = self.select_repository_metadata_row()? @@ -168,12 +190,7 @@ impl RepositoryAgentTraceDb { ); }; - if stored_repository_id != repository_id { - anyhow::bail!( - "repository Agent Trace DB metadata mismatch: stored repository ID \ - {stored_repository_id} does not match resolved repository ID {repository_id}" - ); - } + ensure_repository_id_matches(&stored_repository_id, repository_id)?; if is_valid_source_instance_id(&source_instance_id) { return Ok(RepositoryMetadata { @@ -183,7 +200,7 @@ impl RepositoryAgentTraceDb { } let candidate = generate_source_instance_id(); - self.execute(CLAIM_SOURCE_INSTANCE_ID_SQL, (candidate.as_str(),))?; + self.execute_idempotent_write(CLAIM_SOURCE_INSTANCE_ID_SQL, (candidate.as_str(),))?; let (final_repository_id, final_source_instance_id) = self.select_repository_metadata_row()?.ok_or_else(|| { @@ -906,6 +923,95 @@ mod tests { remove_test_db(&db_path); } + #[test] + fn initialized_repository_metadata_hook_runtime_open_issues_no_writes() { + let db_path = unique_test_db_path("metadata-hook-open-no-writes"); + let repository_id = "a".repeat(64); + + let setup = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let (first, first_writes) = crate::services::db::count_write_statements(|| { + setup.verify_or_initialize_repository_metadata(&repository_id) + }); + let first = first.expect("first metadata initialization should succeed"); + assert!( + first_writes > 0, + "first initialization must seed and claim metadata" + ); + drop(setup); + + let (reopened, writes) = crate::services::db::count_write_statements(|| { + let hook = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path)?; + hook.ensure_schema_ready_for_hooks()?; + hook.verify_or_initialize_repository_metadata(&repository_id) + }); + let reopened = reopened.expect("initialized hook-runtime open should succeed"); + assert_eq!( + writes, 0, + "initialized hook-runtime open must issue no writes" + ); + assert_eq!(reopened, first); + + remove_test_db(&db_path); + } + + #[test] + fn mismatched_repository_metadata_errors_without_writes() { + let db_path = unique_test_db_path("metadata-mismatch-no-writes"); + let stored_repository_id = "a".repeat(64); + let other_repository_id = "b".repeat(64); + + let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + db.verify_or_initialize_repository_metadata(&stored_repository_id) + .expect("first metadata initialization should succeed"); + + let (result, writes) = crate::services::db::count_write_statements(|| { + db.verify_or_initialize_repository_metadata(&other_repository_id) + }); + let message = result + .expect_err("mismatched repository ID should fail validation") + .to_string(); + assert!( + message.contains("metadata mismatch"), + "unexpected error: {message}" + ); + assert_eq!(writes, 0, "a rejected mismatch must issue no writes"); + + remove_test_db(&db_path); + } + + #[test] + fn repository_metadata_with_empty_source_instance_id_is_claimed_once() { + let db_path = unique_test_db_path("metadata-empty-source-instance"); + let repository_id = "a".repeat(64); + + let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + db.verify_or_initialize_repository_metadata(&repository_id) + .expect("first metadata initialization should succeed"); + db.execute( + "UPDATE repository_metadata SET source_instance_id = '' WHERE id = 1", + (), + ) + .expect("source-instance ID should reset to the empty placeholder"); + + let claimed = db + .verify_or_initialize_repository_metadata(&repository_id) + .expect("an empty source-instance ID should be claimed"); + assert_eq!(claimed.repository_id, repository_id); + assert!(is_valid_source_instance_id(&claimed.source_instance_id)); + + let (reopened, writes) = crate::services::db::count_write_statements(|| { + db.verify_or_initialize_repository_metadata(&repository_id) + }); + assert_eq!( + reopened.expect("claimed metadata should validate"), + claimed, + "a claimed source-instance ID must not be overwritten" + ); + assert_eq!(writes, 0); + + remove_test_db(&db_path); + } + #[test] fn mismatched_repository_metadata_errors_on_open() { let db_path = unique_test_db_path("mismatch"); diff --git a/cli/src/services/config/render.rs b/cli/src/services/config/render.rs index 571a7db8b..3b1ce40cb 100644 --- a/cli/src/services/config/render.rs +++ b/cli/src/services/config/render.rs @@ -7,7 +7,7 @@ use super::resolver::{ AuthConfigKeySpec, RuntimeConfig, CONTROL_PLANE_BASE_URL_KEY, PRECEDENCE_DESCRIPTION, WORKOS_CLIENT_ID_KEY, }; -use super::types::DatabaseRetryConfig; +use super::types::{AgentTraceDbRetryConfig, DatabaseRetryConfig}; use super::{ConfigPathSource, ReportFormat, ResolvedOptionalValue, ValueSource}; #[allow(clippy::too_many_lines)] @@ -429,6 +429,26 @@ fn format_per_db_retry_text( lines } +fn format_agent_trace_db_retry_text(config: &AgentTraceDbRetryConfig) -> Vec { + let db_label = "agent_trace_db"; + let mut lines = format_per_db_retry_text(&config.retry, db_label); + if let Some(busy_timeout_ms) = config.busy_timeout_ms { + lines.push(format!( + " {}: {} (busy_timeout_ms)", + style::label(db_label), + style::value(&format!("{busy_timeout_ms}ms")) + )); + } + if let Some(contention_deadline_ms) = config.contention_deadline_ms { + lines.push(format!( + " {}: {} (contention_deadline_ms)", + style::label(db_label), + style::value(&format!("{contention_deadline_ms}ms")) + )); + } + lines +} + fn format_database_retry_text(value: &ResolvedOptionalValue) -> String { match (value.value.as_ref(), value.source) { (Some(config), Some(source)) => { @@ -436,8 +456,8 @@ fn format_database_retry_text(value: &ResolvedOptionalValue if let Some(ref per_db) = config.local_db { lines.extend(format_per_db_retry_text(per_db, "local_db")); } - if let Some(ref per_db) = config.agent_trace_db { - lines.extend(format_per_db_retry_text(per_db, "agent_trace_db")); + if let Some(ref agent_trace_db) = config.agent_trace_db { + lines.extend(format_agent_trace_db_retry_text(agent_trace_db)); } if let Some(ref per_db) = config.auth_db { lines.extend(format_per_db_retry_text(per_db, "auth_db")); @@ -492,6 +512,22 @@ fn format_per_db_retry_json(config: &super::types::PerDbRetryConfig) -> Value { Value::Object(obj) } +fn format_agent_trace_db_retry_json(config: &AgentTraceDbRetryConfig) -> Value { + let mut value = format_per_db_retry_json(&config.retry); + if let Value::Object(ref mut obj) = value { + if let Some(busy_timeout_ms) = config.busy_timeout_ms { + obj.insert("busy_timeout_ms".to_string(), json!(busy_timeout_ms)); + } + if let Some(contention_deadline_ms) = config.contention_deadline_ms { + obj.insert( + "contention_deadline_ms".to_string(), + json!(contention_deadline_ms), + ); + } + } + value +} + fn format_database_retry_json(value: &ResolvedOptionalValue) -> Value { let config = value.value.as_ref(); let mut resolved = serde_json::Map::new(); @@ -499,10 +535,10 @@ fn format_database_retry_json(value: &ResolvedOptionalValue if let Some(ref per_db) = c.local_db { resolved.insert("local_db".to_string(), format_per_db_retry_json(per_db)); } - if let Some(ref per_db) = c.agent_trace_db { + if let Some(ref agent_trace_db) = c.agent_trace_db { resolved.insert( "agent_trace_db".to_string(), - format_per_db_retry_json(per_db), + format_agent_trace_db_retry_json(agent_trace_db), ); } if let Some(ref per_db) = c.auth_db { @@ -515,3 +551,116 @@ fn format_database_retry_json(value: &ResolvedOptionalValue "config_source": value.source.and_then(ValueSource::config_source).map(ConfigPathSource::as_str), }) } + +#[cfg(test)] +mod database_retry_render_tests { + use serde_json::json; + + use super::{format_database_retry_json, format_database_retry_text}; + use crate::services::config::{ + AgentTraceDbRetryConfig, ConfigPathSource, DatabaseRetryConfig, PerDbRetryConfig, + ResolvedOptionalValue, ValueSource, + }; + use crate::services::resilience::RetryPolicy; + + fn query_policy() -> RetryPolicy { + RetryPolicy { + max_attempts: 3, + timeout_ms: 150, + initial_backoff_ms: 10, + max_backoff_ms: 50, + } + } + + fn resolved( + busy_timeout_ms: Option, + contention_deadline_ms: Option, + ) -> ResolvedOptionalValue { + ResolvedOptionalValue { + value: Some(DatabaseRetryConfig { + local_db: Some(PerDbRetryConfig { + connection_open: None, + query: Some(query_policy()), + }), + agent_trace_db: Some(AgentTraceDbRetryConfig { + retry: PerDbRetryConfig { + connection_open: None, + query: Some(query_policy()), + }, + busy_timeout_ms, + contention_deadline_ms, + }), + auth_db: None, + }), + source: Some(ValueSource::ConfigFile(ConfigPathSource::Flag)), + } + } + + #[test] + fn database_retry_json_renders_contention_keys_for_agent_trace_db_only() { + let rendered = format_database_retry_json(&resolved(Some(750), Some(2_000))); + + assert_eq!( + rendered["resolved"], + json!({ + "local_db": { + "query": { + "max_attempts": 3, + "timeout_ms": 150, + "initial_backoff_ms": 10, + "max_backoff_ms": 50, + }, + }, + "agent_trace_db": { + "query": { + "max_attempts": 3, + "timeout_ms": 150, + "initial_backoff_ms": 10, + "max_backoff_ms": 50, + }, + "busy_timeout_ms": 750, + "contention_deadline_ms": 2000, + }, + }) + ); + } + + #[test] + fn database_retry_json_omits_unset_contention_keys() { + let rendered = format_database_retry_json(&resolved(None, None)); + + let agent_trace_db = rendered["resolved"]["agent_trace_db"].as_object().unwrap(); + assert!(!agent_trace_db.contains_key("busy_timeout_ms")); + assert!(!agent_trace_db.contains_key("contention_deadline_ms")); + assert_eq!( + rendered["resolved"]["agent_trace_db"]["query"]["timeout_ms"], + 150 + ); + } + + #[test] + fn database_retry_text_renders_contention_keys_for_agent_trace_db() { + let rendered = format_database_retry_text(&resolved(Some(750), Some(0))); + + let busy_line = rendered + .lines() + .find(|line| line.contains("(busy_timeout_ms)")) + .unwrap(); + assert!(busy_line.contains("agent_trace_db"), "{rendered}"); + assert!(busy_line.contains("750ms"), "{rendered}"); + let deadline_line = rendered + .lines() + .find(|line| line.contains("(contention_deadline_ms)")) + .unwrap(); + assert!(deadline_line.contains("agent_trace_db"), "{rendered}"); + assert!(deadline_line.contains("0ms"), "{rendered}"); + assert_eq!( + rendered + .lines() + .filter(|line| line.contains("3 attempts, 150ms timeout, 10..50ms backoff (query)")) + .count(), + 2, + "{rendered}" + ); + } +} diff --git a/cli/src/services/config/schema.rs b/cli/src/services/config/schema.rs index c3723f0fa..c646a3807 100644 --- a/cli/src/services/config/schema.rs +++ b/cli/src/services/config/schema.rs @@ -18,8 +18,9 @@ use serde_json::Value; use super::policy::{parse_bash_policy_presets, parse_custom_bash_policies, CustomBashPolicyEntry}; use super::types::{ - parse_optional_workflow_id, ConfigPathSource, DatabaseRetryConfig, IntegrationTargetId, - IntegrationsConfig, LogFormat, LogLevel, + parse_optional_workflow_id, AgentTraceDbRetryConfig, ConfigPathSource, DatabaseRetryConfig, + IntegrationTargetId, IntegrationsConfig, LogFormat, LogLevel, PerDbRetryConfig, + AGENT_TRACE_DB_BUSY_TIMEOUT_MAX_MS, AGENT_TRACE_DB_CONTENTION_DEADLINE_MAX_MS, }; use crate::services::resilience::RetryPolicy; @@ -47,6 +48,17 @@ pub(crate) const TOP_LEVEL_CONFIG_KEYS: &[&str] = &[ pub(crate) const TOP_LEVEL_CONFIG_KEYS_DESCRIPTION: &str = "$schema, log_level, log_format, log_to_file, workos_client_id, control_plane_base_url, agent_trace, policies, integrations, log_dir, log_file_retention_limit"; +const PER_DB_RETRY_KEYS: &[&str] = &["connection_open", "query"]; +const PER_DB_RETRY_KEYS_DESCRIPTION: &str = "connection_open, query"; +const AGENT_TRACE_DB_RETRY_KEYS: &[&str] = &[ + "connection_open", + "busy_timeout_ms", + "contention_deadline_ms", + "query", +]; +const AGENT_TRACE_DB_RETRY_KEYS_DESCRIPTION: &str = + "connection_open, busy_timeout_ms, contention_deadline_ms, query"; + static CONFIG_SCHEMA_VALIDATOR: OnceLock = OnceLock::new(); pub(crate) fn config_schema_validator() -> &'static Validator { @@ -130,10 +142,18 @@ pub(crate) struct ParsedCustomBashPolicyMatchDocument { #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub(crate) struct ParsedDatabaseRetryConfigDocument { pub(crate) local_db: Option, - pub(crate) agent_trace_db: Option, + pub(crate) agent_trace_db: Option, pub(crate) auth_db: Option, } +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub(crate) struct ParsedAgentTraceDbRetryConfigDocument { + pub(crate) connection_open: Option, + pub(crate) query: Option, + pub(crate) busy_timeout_ms: Option, + pub(crate) contention_deadline_ms: Option, +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub(crate) struct ParsedPerDbRetryConfigDocument { pub(crate) connection_open: Option, @@ -538,7 +558,10 @@ pub(crate) fn map_database_retry_config( }) }; - let build_per_db = |db_key: &str| -> Result> { + let per_db_object = |db_key: &str, + allowed_keys: &[&str], + allowed_keys_description: &str| + -> Result>> { let Some(db_value) = database_retry_object.get(db_key) else { return Ok(None); }; @@ -554,56 +577,133 @@ pub(crate) fn map_database_retry_config( db_object, path, Some(&format!("policies.database_retry.{db_key}")), - &["connection_open", "query"], - "connection_open, query", + allowed_keys, + allowed_keys_description, )?; + Ok(Some(db_object)) + }; + + let build_policy = |db_key: &str, + db_object: &serde_json::Map, + op_key: &str, + typed_policy: Option<&ParsedRetryPolicyDocument>| + -> Result> { + let Some(op_value) = db_object.get(op_key) else { + return Ok(None); + }; + + let _op_object = op_value.as_object().with_context(|| { + format!( + "Config key 'policies.database_retry.{db_key}.{op_key}' in '{}' must be an object.", + path.display() + ) + })?; + + let parsed = typed_policy.with_context(|| { + format!( + "Config key 'policies.database_retry.{db_key}.{op_key}' in '{}' could not be parsed.", + path.display() + ) + })?; + + let context = format!("policies.database_retry.{db_key}.{op_key}"); + build_retry_policy(parsed, &context).map(Some) + }; + + let build_per_db = |db_key: &str| -> Result> { + let Some(db_object) = + per_db_object(db_key, PER_DB_RETRY_KEYS, PER_DB_RETRY_KEYS_DESCRIPTION)? + else { + return Ok(None); + }; + let typed_db = typed.and_then(|doc| match db_key { "local_db" => doc.local_db.as_ref(), - "agent_trace_db" => doc.agent_trace_db.as_ref(), "auth_db" => doc.auth_db.as_ref(), _ => None, }); - let build_policy = |op_key: &str| -> Result> { - let Some(op_value) = db_object.get(op_key) else { - return Ok(None); - }; + Ok(Some(PerDbRetryConfig { + connection_open: build_policy( + db_key, + db_object, + "connection_open", + typed_db.and_then(|db| db.connection_open.as_ref()), + )?, + query: build_policy( + db_key, + db_object, + "query", + typed_db.and_then(|db| db.query.as_ref()), + )?, + })) + }; - let _op_object = op_value.as_object().with_context(|| { - format!( - "Config key 'policies.database_retry.{db_key}.{op_key}' in '{}' must be an object.", - path.display() - ) - })?; + let build_agent_trace_db = || -> Result> { + let db_key = "agent_trace_db"; + let Some(db_object) = per_db_object( + db_key, + AGENT_TRACE_DB_RETRY_KEYS, + AGENT_TRACE_DB_RETRY_KEYS_DESCRIPTION, + )? + else { + return Ok(None); + }; - let typed_policy = typed_db.and_then(|db| match op_key { - "connection_open" => db.connection_open.as_ref(), - "query" => db.query.as_ref(), - _ => None, - }); + let typed_db = typed.and_then(|doc| doc.agent_trace_db.as_ref()); - let parsed = typed_policy.with_context(|| { - format!( - "Config key 'policies.database_retry.{db_key}.{op_key}' in '{}' could not be parsed.", + let bounded_millis = |key: &str, value: Option, max: u64| -> Result> { + let Some(value) = value else { + if db_object.contains_key(key) { + bail!( + "Config key 'policies.database_retry.{db_key}.{key}' in '{}' could not be parsed.", + path.display() + ); + } + return Ok(None); + }; + if value > max { + bail!( + "Config key 'policies.database_retry.{db_key}.{key}' in '{}' must be <= {max}.", path.display() - ) - })?; - - let context = format!("policies.database_retry.{db_key}.{op_key}"); - build_retry_policy(parsed, &context).map(Some) + ); + } + Ok(Some(value)) }; - Ok(Some(super::types::PerDbRetryConfig { - connection_open: build_policy("connection_open")?, - query: build_policy("query")?, + Ok(Some(AgentTraceDbRetryConfig { + retry: PerDbRetryConfig { + connection_open: build_policy( + db_key, + db_object, + "connection_open", + typed_db.and_then(|db| db.connection_open.as_ref()), + )?, + query: build_policy( + db_key, + db_object, + "query", + typed_db.and_then(|db| db.query.as_ref()), + )?, + }, + busy_timeout_ms: bounded_millis( + "busy_timeout_ms", + typed_db.and_then(|db| db.busy_timeout_ms), + AGENT_TRACE_DB_BUSY_TIMEOUT_MAX_MS, + )?, + contention_deadline_ms: bounded_millis( + "contention_deadline_ms", + typed_db.and_then(|db| db.contention_deadline_ms), + AGENT_TRACE_DB_CONTENTION_DEADLINE_MAX_MS, + )?, })) }; Ok(Some(FileConfigValue { value: DatabaseRetryConfig { local_db: build_per_db("local_db")?, - agent_trace_db: build_per_db("agent_trace_db")?, + agent_trace_db: build_agent_trace_db()?, auth_db: build_per_db("auth_db")?, }, source, @@ -855,3 +955,210 @@ mod agent_trace_config_tests { assert!(error.contains("failed schema validation"), "{error}"); } } + +#[cfg(test)] +mod database_retry_config_tests { + use std::path::Path; + + use serde_json::Value; + + use super::{parse_file_config, ConfigPathSource, FileConfig, SCE_CONFIG_SCHEMA_JSON}; + use crate::services::resilience::RetryPolicy; + + fn parse(raw: &str) -> anyhow::Result { + parse_file_config( + raw, + Path::new("/tmp/sce-config.json"), + ConfigPathSource::Flag, + ) + } + + fn parse_error(raw: &str) -> String { + parse(raw).unwrap_err().to_string() + } + + fn generated_database_retry_object(db_key: &str) -> Value { + let schema: Value = serde_json::from_str(SCE_CONFIG_SCHEMA_JSON).unwrap(); + schema["properties"]["policies"]["properties"]["database_retry"]["properties"][db_key] + .clone() + } + + fn property_keys(object: &Value) -> Vec { + let mut keys = object["properties"] + .as_object() + .unwrap() + .keys() + .cloned() + .collect::>(); + keys.sort(); + keys + } + + #[test] + fn database_retry_agent_trace_db_accepts_contention_keys() { + let config = parse( + r#"{"policies":{"database_retry":{"agent_trace_db":{"busy_timeout_ms":750,"contention_deadline_ms":2000}}}}"#, + ) + .unwrap(); + + let agent_trace_db = config.database_retry.unwrap().value.agent_trace_db.unwrap(); + assert_eq!(agent_trace_db.busy_timeout_ms, Some(750)); + assert_eq!(agent_trace_db.contention_deadline_ms, Some(2000)); + assert_eq!(agent_trace_db.retry.connection_open, None); + assert_eq!(agent_trace_db.retry.query, None); + } + + #[test] + fn database_retry_agent_trace_db_accepts_zero_and_upper_bounds() { + for (busy_timeout_ms, contention_deadline_ms) in [(0, 0), (10_000, 30_000)] { + let config = parse(&format!( + r#"{{"policies":{{"database_retry":{{"agent_trace_db":{{"busy_timeout_ms":{busy_timeout_ms},"contention_deadline_ms":{contention_deadline_ms}}}}}}}}}"# + )) + .unwrap(); + let agent_trace_db = config.database_retry.unwrap().value.agent_trace_db.unwrap(); + assert_eq!(agent_trace_db.busy_timeout_ms, Some(busy_timeout_ms)); + assert_eq!( + agent_trace_db.contention_deadline_ms, + Some(contention_deadline_ms) + ); + } + } + + #[test] + fn database_retry_agent_trace_db_omitted_contention_keys_stay_unset() { + let config = parse( + r#"{"policies":{"database_retry":{"agent_trace_db":{"query":{"max_attempts":3,"timeout_ms":150,"initial_backoff_ms":10,"max_backoff_ms":50}}}}}"#, + ) + .unwrap(); + + let agent_trace_db = config.database_retry.unwrap().value.agent_trace_db.unwrap(); + assert_eq!(agent_trace_db.busy_timeout_ms, None); + assert_eq!(agent_trace_db.contention_deadline_ms, None); + } + + #[test] + fn database_retry_agent_trace_db_rejects_out_of_range_values() { + for raw in [ + r#"{"policies":{"database_retry":{"agent_trace_db":{"busy_timeout_ms":10001}}}}"#, + r#"{"policies":{"database_retry":{"agent_trace_db":{"busy_timeout_ms":-1}}}}"#, + r#"{"policies":{"database_retry":{"agent_trace_db":{"contention_deadline_ms":30001}}}}"#, + r#"{"policies":{"database_retry":{"agent_trace_db":{"contention_deadline_ms":-5}}}}"#, + ] { + let error = parse_error(raw); + assert!(error.contains("failed schema validation"), "{error}"); + } + } + + #[test] + fn database_retry_agent_trace_db_rejects_wrong_type_values() { + for raw in [ + r#"{"policies":{"database_retry":{"agent_trace_db":{"busy_timeout_ms":"500"}}}}"#, + r#"{"policies":{"database_retry":{"agent_trace_db":{"busy_timeout_ms":1.5}}}}"#, + r#"{"policies":{"database_retry":{"agent_trace_db":{"contention_deadline_ms":true}}}}"#, + r#"{"policies":{"database_retry":{"agent_trace_db":{"contention_deadline_ms":null}}}}"#, + ] { + let error = parse_error(raw); + assert!(error.contains("failed schema validation"), "{error}"); + } + } + + #[test] + fn database_retry_local_and_auth_db_reject_contention_keys() { + for db_key in ["local_db", "auth_db"] { + for key in ["busy_timeout_ms", "contention_deadline_ms"] { + let error = parse_error(&format!( + r#"{{"policies":{{"database_retry":{{"{db_key}":{{"{key}":500}}}}}}}}"# + )); + assert!( + error.starts_with("Config file '/tmp/sce-config.json' failed schema validation against generated schema"), + "{error}" + ); + assert!(error.contains(key), "{error}"); + assert!( + error.contains(&format!("/policies/database_retry/{db_key}")), + "{error}" + ); + } + } + } + + #[test] + fn database_retry_generated_schema_publishes_contention_keys_only_on_agent_trace_db() { + for db_key in ["local_db", "auth_db"] { + let object = generated_database_retry_object(db_key); + assert_eq!(property_keys(&object), vec!["connection_open", "query"]); + assert_eq!(object["additionalProperties"], Value::Bool(false)); + } + + let agent_trace_db = generated_database_retry_object("agent_trace_db"); + assert_eq!( + property_keys(&agent_trace_db), + vec![ + "busy_timeout_ms", + "connection_open", + "contention_deadline_ms", + "query" + ] + ); + assert_eq!(agent_trace_db["additionalProperties"], Value::Bool(false)); + let busy_timeout = &agent_trace_db["properties"]["busy_timeout_ms"]; + assert_eq!(busy_timeout["minimum"], 0); + assert_eq!(busy_timeout["maximum"], 10_000); + assert_eq!(busy_timeout["default"], 1_000); + let contention_deadline = &agent_trace_db["properties"]["contention_deadline_ms"]; + assert_eq!(contention_deadline["minimum"], 0); + assert_eq!(contention_deadline["maximum"], 30_000); + assert_eq!(contention_deadline["default"], 2_250); + } + + #[test] + fn database_retry_per_db_key_check_rejects_contention_keys_as_backstop() { + let object = serde_json::json!({"busy_timeout_ms": 500}); + let error = super::validate_object_keys( + object.as_object().unwrap(), + Path::new("/tmp/sce-config.json"), + Some("policies.database_retry.local_db"), + super::PER_DB_RETRY_KEYS, + super::PER_DB_RETRY_KEYS_DESCRIPTION, + ) + .unwrap_err() + .to_string(); + assert_eq!( + error, + "Config key 'policies.database_retry.local_db' in '/tmp/sce-config.json' contains unknown key 'busy_timeout_ms'. Allowed keys: connection_open, query." + ); + } + + #[test] + fn database_retry_query_timeout_ms_parsing_is_unchanged() { + let config = parse( + r#"{"policies":{"database_retry":{"local_db":{"query":{"max_attempts":4,"timeout_ms":300,"initial_backoff_ms":20,"max_backoff_ms":80}},"agent_trace_db":{"query":{"max_attempts":3,"timeout_ms":150,"initial_backoff_ms":10,"max_backoff_ms":50},"busy_timeout_ms":250}}}}"#, + ) + .unwrap(); + + let database_retry = config.database_retry.unwrap().value; + assert_eq!( + database_retry.local_db.unwrap().query, + Some(RetryPolicy { + max_attempts: 4, + timeout_ms: 300, + initial_backoff_ms: 20, + max_backoff_ms: 80, + }) + ); + assert_eq!( + database_retry.agent_trace_db.unwrap().retry.query, + Some(RetryPolicy { + max_attempts: 3, + timeout_ms: 150, + initial_backoff_ms: 10, + max_backoff_ms: 50, + }) + ); + + let error = parse_error( + r#"{"policies":{"database_retry":{"agent_trace_db":{"query":{"max_attempts":3,"timeout_ms":0,"initial_backoff_ms":10,"max_backoff_ms":50}}}}}"#, + ); + assert!(error.contains("failed schema validation"), "{error}"); + } +} diff --git a/cli/src/services/config/types.rs b/cli/src/services/config/types.rs index 3ee843c38..7de321d86 100644 --- a/cli/src/services/config/types.rs +++ b/cli/src/services/config/types.rs @@ -230,10 +230,20 @@ use crate::services::resilience::RetryPolicy; #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct DatabaseRetryConfig { pub(crate) local_db: Option, - pub(crate) agent_trace_db: Option, + pub(crate) agent_trace_db: Option, pub(crate) auth_db: Option, } +pub(crate) const AGENT_TRACE_DB_BUSY_TIMEOUT_MAX_MS: u64 = 10_000; +pub(crate) const AGENT_TRACE_DB_CONTENTION_DEADLINE_MAX_MS: u64 = 30_000; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AgentTraceDbRetryConfig { + pub(crate) retry: PerDbRetryConfig, + pub(crate) busy_timeout_ms: Option, + pub(crate) contention_deadline_ms: Option, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct PerDbRetryConfig { pub(crate) connection_open: Option, diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 11d14eac7..4629c29c2 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -13,6 +13,7 @@ use std::{ use anyhow::{Context, Result}; use turso::Value as TursoValue; +use crate::services::config::{AgentTraceDbRetryConfig, DatabaseRetryConfig}; use crate::services::lifecycle::{ HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, }; @@ -39,6 +40,11 @@ const QUERY_RETRY_POLICY: RetryPolicy = RetryPolicy { max_backoff_ms: 100, }; const QUERY_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; +const AGENT_TRACE_DB_CONFIG_KEY: &str = "agent_trace_db"; +const AGENT_TRACE_DB_BUSY_TIMEOUT_MS: u64 = 1_000; +const AGENT_TRACE_DB_CONTENTION_DEADLINE_MS: u64 = 2_250; +const AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; +const AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS: u64 = 100; pub mod encryption_key; @@ -291,15 +297,24 @@ async fn execute_insert_pair_if_absent_body( second_sql: &str, second_params: turso::params::Params, fail_before_second: bool, -) -> Result { - let mut rows = tx - .query(exists_sql, exists_params) - .await - .map_err(|e| anyhow::anyhow!("{db_name} existence check failed: {exists_sql}: {e}"))?; +) -> std::result::Result { + let mut rows = tx.query(exists_sql, exists_params).await.map_err(|e| { + classify_turso_error( + db_name, + &format!("existence check failed: {exists_sql}"), + &e, + ) + })?; let already_exists = rows .next() .await - .map_err(|e| anyhow::anyhow!("{db_name} existence row fetch failed: {exists_sql}: {e}"))? + .map_err(|e| { + classify_turso_error( + db_name, + &format!("existence row fetch failed: {exists_sql}"), + &e, + ) + })? .is_some(); if already_exists { @@ -308,15 +323,17 @@ async fn execute_insert_pair_if_absent_body( tx.execute(first_sql, first_params) .await - .map_err(|e| anyhow::anyhow!("{db_name} execute failed: {first_sql}: {e}"))?; + .map_err(|e| classify_turso_error(db_name, &format!("execute failed: {first_sql}"), &e))?; if fail_before_second { - anyhow::bail!("{db_name} injected failure before second statement (test-only)"); + return Err(WriteAttemptFailure::Deterministic(anyhow::anyhow!( + "{db_name} injected failure before second statement (test-only)" + ))); } tx.execute(second_sql, second_params) .await - .map_err(|e| anyhow::anyhow!("{db_name} execute failed: {second_sql}: {e}"))?; + .map_err(|e| classify_turso_error(db_name, &format!("execute failed: {second_sql}"), &e))?; Ok(true) } @@ -348,25 +365,30 @@ impl<'a> TransactionStatement<'a> { } } -#[allow(dead_code)] fn is_retryable_turso_error(error: &turso::Error) -> bool { matches!(error, turso::Error::Busy(_) | turso::Error::BusySnapshot(_)) } -#[allow(dead_code)] -enum CasBatchFailure { +enum WriteAttemptFailure { Retryable(anyhow::Error), Deterministic(anyhow::Error), } -#[allow(dead_code)] -fn classify_turso_error(db_name: &str, action: &str, error: &turso::Error) -> CasBatchFailure { +impl WriteAttemptFailure { + fn into_error(self) -> anyhow::Error { + match self { + Self::Retryable(err) | Self::Deterministic(err) => err, + } + } +} + +fn classify_turso_error(db_name: &str, action: &str, error: &turso::Error) -> WriteAttemptFailure { let wrapped = anyhow::anyhow!("{db_name} {action}: {error}"); if is_retryable_turso_error(error) { - CasBatchFailure::Retryable(wrapped) + WriteAttemptFailure::Retryable(wrapped) } else { - CasBatchFailure::Deterministic(wrapped) + WriteAttemptFailure::Deterministic(wrapped) } } @@ -378,11 +400,11 @@ enum CasBatchAttemptOutcome { #[allow(dead_code)] fn cas_batch_failure_into_attempt_result( - failure: CasBatchFailure, + failure: WriteAttemptFailure, ) -> Result { match failure { - CasBatchFailure::Retryable(err) => Err(err), - CasBatchFailure::Deterministic(err) => Ok(CasBatchAttemptOutcome::Deterministic(err)), + WriteAttemptFailure::Retryable(err) => Err(err), + WriteAttemptFailure::Deterministic(err) => Ok(CasBatchAttemptOutcome::Deterministic(err)), } } @@ -392,7 +414,7 @@ async fn execute_cas_batch_body( db_name: &str, guard: &TransactionStatement<'_>, statements: &[TransactionStatement<'_>], -) -> std::result::Result { +) -> std::result::Result { let guard_rows_affected = tx .execute(guard.sql, guard.params.clone()) .await @@ -404,7 +426,7 @@ async fn execute_cas_batch_body( 0 => return Ok(false), 1 => {} n => { - return Err(CasBatchFailure::Deterministic(anyhow::anyhow!( + return Err(WriteAttemptFailure::Deterministic(anyhow::anyhow!( "{db_name} CAS guard affected {n} rows; expected 0 or 1: {}", guard.sql ))); @@ -421,7 +443,7 @@ async fn execute_cas_batch_body( if let Some(expected) = statement.expected_rows_affected { if rows_affected != expected { - return Err(CasBatchFailure::Deterministic(anyhow::anyhow!( + return Err(WriteAttemptFailure::Deterministic(anyhow::anyhow!( "{db_name} statement affected {rows_affected} rows; expected {expected}: {}", statement.sql ))); @@ -456,7 +478,7 @@ fn resolve_connection_open_retry_policy() -> RetryPolicy { if let Some(config) = crate::services::config::get_database_retry_config() { let per_db = match M::db_config_key() { "local_db" => config.local_db.as_ref(), - "agent_trace_db" => config.agent_trace_db.as_ref(), + "agent_trace_db" => config.agent_trace_db.as_ref().map(|db| &db.retry), "auth_db" => config.auth_db.as_ref(), _ => None, }; @@ -473,7 +495,7 @@ fn resolve_query_retry_policy() -> RetryPolicy { if let Some(config) = crate::services::config::get_database_retry_config() { let per_db = match M::db_config_key() { "local_db" => config.local_db.as_ref(), - "agent_trace_db" => config.agent_trace_db.as_ref(), + "agent_trace_db" => config.agent_trace_db.as_ref().map(|db| &db.retry), "auth_db" => config.auth_db.as_ref(), _ => None, }; @@ -486,6 +508,246 @@ fn resolve_query_retry_policy() -> RetryPolicy { QUERY_RETRY_POLICY } +/// Resolve the Turso busy timeout applied to every connection opened for `M`. +/// +/// Multiprocess WAL provides cross-process correctness and locking; it does +/// not wait for a contended lock. The busy timeout is Turso's own wait policy +/// for a `Busy` result: while another process holds the write lock, Turso +/// sleeps in short phases until the lock clears or the timeout elapses, and +/// only then returns `Busy`. Because `Transaction::new_unchecked(.., Immediate)` +/// runs `BEGIN IMMEDIATE` through `Connection::execute` on the same connection, +/// the wait also covers writer-lock acquisition. The timeout is connection-wide. +/// +/// Only the Agent Trace DB has a busy timeout, taken from +/// `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured; +/// every other database resolves to zero, which leaves Turso's busy handler +/// unset. +fn resolve_busy_timeout() -> std::time::Duration { + busy_timeout_from_config::(crate::services::config::get_database_retry_config()) +} + +fn busy_timeout_from_config( + config: Option<&DatabaseRetryConfig>, +) -> std::time::Duration { + agent_trace_db_millis::( + config, + |db| db.busy_timeout_ms, + AGENT_TRACE_DB_BUSY_TIMEOUT_MS, + ) +} + +/// Resolve the Agent Trace write-contention deadline for `M`. +/// +/// The deadline decides whether another outer write-contention retry may +/// start; it does not interrupt a running Turso operation. It is taken from +/// `policies.database_retry.agent_trace_db.contention_deadline_ms` when +/// configured. Every other database resolves to zero. +fn resolve_contention_deadline() -> std::time::Duration { + contention_deadline_from_config::(crate::services::config::get_database_retry_config()) +} + +fn contention_deadline_from_config( + config: Option<&DatabaseRetryConfig>, +) -> std::time::Duration { + agent_trace_db_millis::( + config, + |db| db.contention_deadline_ms, + AGENT_TRACE_DB_CONTENTION_DEADLINE_MS, + ) +} + +fn agent_trace_db_millis( + config: Option<&DatabaseRetryConfig>, + select: impl Fn(&AgentTraceDbRetryConfig) -> Option, + default_ms: u64, +) -> std::time::Duration { + if M::db_config_key() != AGENT_TRACE_DB_CONFIG_KEY { + return std::time::Duration::ZERO; + } + let configured = config + .and_then(|config| config.agent_trace_db.as_ref()) + .and_then(select); + std::time::Duration::from_millis(configured.unwrap_or(default_ms)) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct WriteContentionPolicy { + db_name: &'static str, + max_attempts: u32, + backoff_cap: std::time::Duration, + busy_timeout: std::time::Duration, + contention_deadline: std::time::Duration, +} + +fn write_contention_policy() -> Option { + if M::db_config_key() != AGENT_TRACE_DB_CONFIG_KEY { + return None; + } + Some(WriteContentionPolicy { + db_name: M::db_name(), + max_attempts: AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS, + backoff_cap: std::time::Duration::from_millis( + AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS, + ), + busy_timeout: resolve_busy_timeout::(), + contention_deadline: resolve_contention_deadline::(), + }) +} + +fn write_contention_backoff( + jitter: &mut impl rand::Rng, + cap: std::time::Duration, +) -> std::time::Duration { + let cap_ms = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX); + std::time::Duration::from_millis(jitter.gen_range(0..=cap_ms)) +} + +fn write_contention_retry_may_sleep( + policy: WriteContentionPolicy, + elapsed: std::time::Duration, + backoff: std::time::Duration, +) -> bool { + let Some(remaining) = policy.contention_deadline.checked_sub(elapsed) else { + return false; + }; + !remaining.is_zero() && remaining >= backoff.saturating_add(policy.busy_timeout) +} + +fn write_contention_retry_may_start_now( + policy: WriteContentionPolicy, + elapsed: std::time::Duration, +) -> bool { + write_contention_retry_may_sleep(policy, elapsed, std::time::Duration::ZERO) +} + +fn run_with_write_contention_retry( + policy: WriteContentionPolicy, + operation_name: &str, + retry_hint: &str, + attempt: impl FnMut(u32) -> std::result::Result, +) -> Result { + let mut jitter = rand::thread_rng(); + let started_at = std::time::Instant::now(); + run_with_write_contention_retry_using( + policy, + &mut || write_contention_backoff(&mut jitter, policy.backoff_cap), + &mut std::thread::sleep, + &mut || started_at.elapsed(), + operation_name, + retry_hint, + attempt, + ) +} + +fn run_with_write_contention_retry_using( + policy: WriteContentionPolicy, + draw_backoff: &mut impl FnMut() -> std::time::Duration, + sleep: &mut impl FnMut(std::time::Duration), + elapsed: &mut impl FnMut() -> std::time::Duration, + operation_name: &str, + retry_hint: &str, + mut attempt: impl FnMut(u32) -> std::result::Result, +) -> Result { + let mut attempt_number = 0; + + loop { + attempt_number += 1; + #[cfg(test)] + note_write_contention(|counts| counts.attempts += 1); + + #[cfg(test)] + note_write_contention_timeline(WriteContentionTimelineEvent::AttemptStart); + let outcome = attempt(attempt_number); + #[cfg(test)] + note_write_contention_timeline(WriteContentionTimelineEvent::AttemptEnd); + + let error = match outcome { + Ok(value) => return Ok(value), + Err(WriteAttemptFailure::Deterministic(error)) => return Err(error), + Err(WriteAttemptFailure::Retryable(error)) => error, + }; + + if attempt_number < policy.max_attempts { + let backoff = draw_backoff(); + if write_contention_retry_may_sleep(policy, elapsed(), backoff) { + #[cfg(test)] + note_write_contention_timeline(WriteContentionTimelineEvent::BackoffRequested( + backoff, + )); + sleep(backoff); + #[cfg(test)] + note_write_contention_timeline(WriteContentionTimelineEvent::BackoffSlept); + if write_contention_retry_may_start_now(policy, elapsed()) { + #[cfg(test)] + note_write_contention(|counts| counts.outer_retries += 1); + continue; + } + } + } + + return Err(contention_exhausted_error( + policy, + operation_name, + attempt_number, + elapsed(), + &error, + retry_hint, + )); + } +} + +const CONTENTION_EXHAUSTED_EVENT_ID: &str = "sce.agent_trace_db.contention_exhausted"; +const CONTENTION_EXHAUSTED_CAUSE: &str = "database busy (busy timeout exhausted)"; + +fn contention_exhausted_error( + policy: WriteContentionPolicy, + operation_name: &str, + attempts: u32, + elapsed: std::time::Duration, + last_error: &anyhow::Error, + retry_hint: &str, +) -> anyhow::Error { + let busy_timeout_ms = u64::try_from(policy.busy_timeout.as_millis()).unwrap_or(u64::MAX); + let contention_deadline_ms = + u64::try_from(policy.contention_deadline.as_millis()).unwrap_or(u64::MAX); + let elapsed_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX); + + #[cfg(test)] + note_write_contention(|counts| counts.exhaustions += 1); + + tracing::warn!( + target: "sce", + event_id = CONTENTION_EXHAUSTED_EVENT_ID, + db_name = policy.db_name, + operation = operation_name, + attempts, + busy_timeout_ms, + contention_deadline_ms, + elapsed_ms, + cause = CONTENTION_EXHAUSTED_CAUSE, + last_error = %last_error, + "Agent Trace DB write contention retries exhausted" + ); + + anyhow::anyhow!( + "Operation '{operation_name}' failed after {attempts} attempt(s) under write contention (db_name={}, operation={operation_name}, attempts={attempts}, busy_timeout_ms={busy_timeout_ms}, contention_deadline_ms={contention_deadline_ms} [no retry is scheduled past this cutoff], elapsed_ms={elapsed_ms}, cause={CONTENTION_EXHAUSTED_CAUSE}). Last error: {last_error}. Try: {retry_hint}", + policy.db_name, + ) +} + +/// Install `busy_timeout` on `conn`; a zero timeout leaves the handler unset. +fn apply_busy_timeout( + conn: &turso::Connection, + db_name: &str, + busy_timeout: std::time::Duration, +) -> Result<()> { + if busy_timeout.is_zero() { + return Ok(()); + } + conn.busy_timeout(busy_timeout) + .map_err(|e| anyhow::anyhow!("failed to set {db_name} database busy timeout: {e}")) +} + #[cfg(test)] thread_local! { static READ_STATEMENTS_ISSUED: std::cell::Cell = const { std::cell::Cell::new(0) }; @@ -517,6 +779,86 @@ pub(crate) fn count_read_statements(body: impl FnOnce() -> T) -> (T, usize) { (result, issued) } +#[cfg(test)] +thread_local! { + static WRITE_STATEMENTS_ISSUED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +fn note_write_statement_issued() { + WRITE_STATEMENTS_ISSUED.with(|count| count.set(count.get() + 1)); +} + +#[cfg(test)] +pub(crate) fn count_write_statements(body: impl FnOnce() -> T) -> (T, usize) { + WRITE_STATEMENTS_ISSUED.with(|count| count.set(0)); + let result = body(); + let issued = WRITE_STATEMENTS_ISSUED.with(std::cell::Cell::get); + (result, issued) +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct WriteContentionCounts { + pub(crate) attempts: u32, + pub(crate) outer_retries: u32, + pub(crate) exhaustions: u32, +} + +#[cfg(test)] +thread_local! { + static WRITE_CONTENTION_COUNTS: std::cell::Cell = + const { std::cell::Cell::new(WriteContentionCounts { attempts: 0, outer_retries: 0, exhaustions: 0 }) }; +} + +#[cfg(test)] +fn note_write_contention(update: impl FnOnce(&mut WriteContentionCounts)) { + WRITE_CONTENTION_COUNTS.with(|cell| { + let mut counts = cell.get(); + update(&mut counts); + cell.set(counts); + }); +} + +#[cfg(test)] +pub(crate) fn count_write_contention(body: impl FnOnce() -> T) -> (T, WriteContentionCounts) { + WRITE_CONTENTION_COUNTS.with(|cell| cell.set(WriteContentionCounts::default())); + let result = body(); + let counts = WRITE_CONTENTION_COUNTS.with(std::cell::Cell::get); + (result, counts) +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum WriteContentionTimelineEvent { + AttemptStart, + AttemptEnd, + BackoffRequested(std::time::Duration), + BackoffSlept, +} + +#[cfg(test)] +thread_local! { + static WRITE_CONTENTION_TIMELINE: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; +} + +#[cfg(test)] +fn note_write_contention_timeline(event: WriteContentionTimelineEvent) { + let now = std::time::Instant::now(); + WRITE_CONTENTION_TIMELINE.with(|timeline| timeline.borrow_mut().push((now, event))); +} + +#[cfg(test)] +pub(crate) fn record_write_contention_timeline( + body: impl FnOnce() -> T, +) -> (T, Vec<(std::time::Instant, WriteContentionTimelineEvent)>) { + WRITE_CONTENTION_TIMELINE.with(|timeline| timeline.borrow_mut().clear()); + let result = body(); + let timeline = WRITE_CONTENTION_TIMELINE.with(std::cell::RefCell::take); + (result, timeline) +} + /// Generic Turso database adapter. /// /// Wraps a Turso connection with a tokio current-thread runtime so callers can @@ -598,6 +940,7 @@ impl TursoDb { let runtime = build_current_thread_runtime(db_name)?; let retry_policy = resolve_connection_open_retry_policy::(); + let busy_timeout = resolve_busy_timeout::(); let operation_name = format!("open {db_name} database connection"); let conn = run_with_retry_sync( @@ -619,9 +962,11 @@ impl TursoDb { db_path.display() ) })?; - db.connect().map_err(|e| { + let conn = db.connect().map_err(|e| { anyhow::anyhow!("failed to connect to {db_name} database: {e}") - }) + })?; + apply_busy_timeout(&conn, db_name, busy_timeout)?; + Ok(conn) }) }, )?; @@ -645,6 +990,9 @@ impl TursoDb { })?; let operation_name = format!("execute {} database query", M::db_name()); + #[cfg(test)] + note_write_statement_issued(); + run_with_retry_sync( resolve_query_retry_policy::(), &operation_name, @@ -661,6 +1009,35 @@ impl TursoDb { ) } + pub fn execute_idempotent_write( + &self, + sql: &str, + params: impl turso::params::IntoParams, + ) -> Result { + let Some(policy) = write_contention_policy::() else { + return self.execute(sql, params); + }; + let db_name = M::db_name(); + let params = turso::params::IntoParams::into_params(params) + .map_err(|e| anyhow::anyhow!("{db_name} parameter conversion failed: {sql}: {e}"))?; + let operation_name = format!("execute {db_name} database query"); + + #[cfg(test)] + note_write_statement_issued(); + + run_with_write_contention_retry(policy, &operation_name, QUERY_RETRY_HINT, |_| { + block_on_isolated(&self.core.runtime, async { + self.core + .conn + .execute(sql, params.clone()) + .await + .map_err(|e| { + classify_turso_error(db_name, &format!("execute failed: {sql}"), &e) + }) + }) + }) + } + /// Execute a SQL query that returns rows. /// /// # Arguments @@ -789,46 +1166,57 @@ impl TursoDb { anyhow::anyhow!("{db_name} parameter conversion failed: {second_sql}: {e}") })?; + #[cfg(test)] + note_write_statement_issued(); + + let run_attempt = || { + block_on_isolated(&self.core.runtime, async { + let tx = turso::transaction::Transaction::new_unchecked( + &self.core.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .map_err(|e| classify_turso_error(db_name, "failed to begin transaction", &e))?; + + let outcome = execute_insert_pair_if_absent_body( + &tx, + db_name, + exists_sql, + exists_params.clone(), + first_sql, + first_params.clone(), + second_sql, + second_params.clone(), + fail_before_second, + ) + .await; + + match outcome { + Ok(inserted) => { + tx.commit().await.map_err(|e| { + classify_turso_error(db_name, "failed to commit transaction", &e) + })?; + Ok(inserted) + } + Err(failure) => { + let _ = tx.rollback().await; + Err(failure) + } + } + }) + }; + + if let Some(policy) = write_contention_policy::() { + return run_with_write_contention_retry(policy, operation_name, retry_hint, |_| { + run_attempt() + }); + } + run_with_retry_sync( resolve_query_retry_policy::(), operation_name, retry_hint, - |_| { - block_on_isolated(&self.core.runtime, async { - let tx = turso::transaction::Transaction::new_unchecked( - &self.core.conn, - turso::transaction::TransactionBehavior::Immediate, - ) - .await - .map_err(|e| anyhow::anyhow!("{db_name} failed to begin transaction: {e}"))?; - - let outcome = execute_insert_pair_if_absent_body( - &tx, - db_name, - exists_sql, - exists_params.clone(), - first_sql, - first_params.clone(), - second_sql, - second_params.clone(), - fail_before_second, - ) - .await; - - match outcome { - Ok(inserted) => { - tx.commit().await.map_err(|e| { - anyhow::anyhow!("{db_name} failed to commit transaction: {e}") - })?; - Ok(inserted) - } - Err(err) => { - let _ = tx.rollback().await; - Err(err) - } - } - }) - }, + |_| run_attempt().map_err(WriteAttemptFailure::into_error), ) } @@ -899,43 +1287,46 @@ impl TursoDb { ) -> Result { let db_name = M::db_name(); + #[cfg(test)] + note_write_statement_issued(); + + let run_attempt = || { + block_on_isolated(&self.core.runtime, async { + let tx = turso::transaction::Transaction::new_unchecked( + &self.core.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .map_err(|e| classify_turso_error(db_name, "failed to begin transaction", &e))?; + + match execute_cas_batch_body(&tx, db_name, guard, statements).await { + Ok(applied) => { + tx.commit().await.map_err(|e| { + classify_turso_error(db_name, "failed to commit transaction", &e) + })?; + Ok(applied) + } + Err(failure) => { + let _ = tx.rollback().await; + Err(failure) + } + } + }) + }; + + if let Some(policy) = write_contention_policy::() { + return run_with_write_contention_retry(policy, operation_name, retry_hint, |_| { + run_attempt() + }); + } + let outcome = run_with_retry_sync( resolve_query_retry_policy::(), operation_name, retry_hint, - |_| { - block_on_isolated(&self.core.runtime, async { - let tx = match turso::transaction::Transaction::new_unchecked( - &self.core.conn, - turso::transaction::TransactionBehavior::Immediate, - ) - .await - { - Ok(tx) => tx, - Err(e) => { - return cas_batch_failure_into_attempt_result(classify_turso_error( - db_name, - "failed to begin transaction", - &e, - )); - } - }; - - match execute_cas_batch_body(&tx, db_name, guard, statements).await { - Ok(applied) => match tx.commit().await { - Ok(()) => Ok(CasBatchAttemptOutcome::Settled(applied)), - Err(e) => cas_batch_failure_into_attempt_result(classify_turso_error( - db_name, - "failed to commit transaction", - &e, - )), - }, - Err(failure) => { - let _ = tx.rollback().await; - cas_batch_failure_into_attempt_result(failure) - } - } - }) + |_| match run_attempt() { + Ok(applied) => Ok(CasBatchAttemptOutcome::Settled(applied)), + Err(failure) => cas_batch_failure_into_attempt_result(failure), }, )?; @@ -1264,9 +1655,12 @@ impl EncryptedTursoDb { #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + use rand::SeedableRng; + use super::*; const QUERY_RETRY_FAILURE_BUDGET_MS: u64 = 2_000; @@ -1291,6 +1685,190 @@ mod tests { } } + struct AgentTraceTestDbSpec; + + impl DbSpec for AgentTraceTestDbSpec { + fn db_name() -> &'static str { + "agent trace test" + } + + fn db_path() -> Result { + unreachable!("tests always open via TursoDb::new_at with an explicit path") + } + + fn migrations() -> &'static [(&'static str, &'static str)] { + &[] + } + + fn db_config_key() -> &'static str { + AGENT_TRACE_DB_CONFIG_KEY + } + } + + const BUSY_TIMEOUT_LOCK_HOLD_MS: u64 = 300; + + fn begin_immediate_while_write_lock_is_held( + db_path: &Path, + ) -> (std::result::Result, Duration) { + let hold = Duration::from_millis(BUSY_TIMEOUT_LOCK_HOLD_MS); + drop(TursoDb::::new_at(db_path).expect("test DB should be created up front")); + let contender = + TursoDb::::open_without_migrations_at(db_path).expect("contender DB should open"); + let lock_acquired = std::sync::Arc::new(std::sync::Barrier::new(2)); + let holder = { + let db_path = db_path.to_path_buf(); + let lock_acquired = std::sync::Arc::clone(&lock_acquired); + thread::spawn(move || { + let holder = TursoDb::::open_without_migrations_at(&db_path) + .expect("holder DB should open"); + holder + .execute("BEGIN IMMEDIATE", ()) + .expect("holder should acquire the write lock"); + lock_acquired.wait(); + thread::sleep(hold); + holder + .execute("COMMIT", ()) + .expect("holder should release the write lock"); + }) + }; + + lock_acquired.wait(); + let started_at = Instant::now(); + let outcome = block_on_isolated(&contender.core.runtime, async { + contender.core.conn.execute("BEGIN IMMEDIATE", ()).await + }); + let elapsed = started_at.elapsed(); + if outcome.is_ok() { + contender + .execute("ROLLBACK", ()) + .expect("contender should release the write lock"); + } + holder.join().expect("holder thread should not panic"); + drop(contender); + + (outcome, elapsed) + } + + #[test] + fn busy_timeout_resolves_default_for_agent_trace_db_and_zero_for_other_dbs() { + assert_eq!( + resolve_busy_timeout::(), + Duration::from_millis(AGENT_TRACE_DB_BUSY_TIMEOUT_MS) + ); + assert_eq!(resolve_busy_timeout::(), Duration::ZERO); + } + + fn agent_trace_retry_config( + busy_timeout_ms: Option, + contention_deadline_ms: Option, + ) -> DatabaseRetryConfig { + DatabaseRetryConfig { + local_db: None, + agent_trace_db: Some(AgentTraceDbRetryConfig { + retry: crate::services::config::PerDbRetryConfig { + connection_open: None, + query: None, + }, + busy_timeout_ms, + contention_deadline_ms, + }), + auth_db: None, + } + } + + #[test] + fn database_retry_configured_busy_timeout_overrides_agent_trace_default() { + let config = agent_trace_retry_config(Some(1_500), None); + assert_eq!( + busy_timeout_from_config::(Some(&config)), + Duration::from_millis(1_500) + ); + assert_eq!( + busy_timeout_from_config::(Some(&config)), + Duration::ZERO + ); + + let disabled = agent_trace_retry_config(Some(0), None); + assert_eq!( + busy_timeout_from_config::(Some(&disabled)), + Duration::ZERO + ); + + let unset = agent_trace_retry_config(None, None); + assert_eq!( + busy_timeout_from_config::(Some(&unset)), + Duration::from_millis(AGENT_TRACE_DB_BUSY_TIMEOUT_MS) + ); + } + + #[test] + fn database_retry_contention_deadline_resolves_default_and_configured_value() { + assert_eq!( + resolve_contention_deadline::(), + Duration::from_millis(AGENT_TRACE_DB_CONTENTION_DEADLINE_MS) + ); + assert_eq!( + AGENT_TRACE_DB_CONTENTION_DEADLINE_MS, 2_250, + "contention deadline default" + ); + assert_eq!(resolve_contention_deadline::(), Duration::ZERO); + + let config = agent_trace_retry_config(None, Some(3_500)); + assert_eq!( + contention_deadline_from_config::(Some(&config)), + Duration::from_millis(3_500) + ); + assert_eq!( + contention_deadline_from_config::(Some(&config)), + Duration::ZERO + ); + + let zero = agent_trace_retry_config(None, Some(0)); + assert_eq!( + contention_deadline_from_config::(Some(&zero)), + Duration::ZERO + ); + } + + #[test] + fn busy_timeout_unset_connection_returns_busy_promptly_on_begin_immediate() { + let db_path = unique_test_db_path(); + + let (outcome, elapsed) = begin_immediate_while_write_lock_is_held::(&db_path); + + assert!( + matches!(outcome, Err(turso::Error::Busy(_))), + "a connection without a busy handler should get Busy, got {outcome:?}" + ); + assert!( + elapsed < Duration::from_millis(BUSY_TIMEOUT_LOCK_HOLD_MS / 2), + "Busy should be returned without waiting for the holder, took {elapsed:?}" + ); + if let Some(parent) = db_path.parent() { + let _ = fs::remove_dir_all(parent); + } + } + + #[test] + fn busy_timeout_agent_trace_connection_waits_for_begin_immediate_holder() { + let db_path = unique_test_db_path(); + + let (outcome, elapsed) = + begin_immediate_while_write_lock_is_held::(&db_path); + + assert!( + outcome.is_ok(), + "Turso's busy handler should wait out the holder, got {outcome:?}" + ); + assert!( + elapsed >= Duration::from_millis(BUSY_TIMEOUT_LOCK_HOLD_MS / 2), + "BEGIN IMMEDIATE should have waited for the holder, took {elapsed:?}" + ); + if let Some(parent) = db_path.parent() { + let _ = fs::remove_dir_all(parent); + } + } + fn unique_test_db_path() -> PathBuf { let nonce = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -1637,12 +2215,12 @@ mod tests { ); match failure { - CasBatchFailure::Retryable(err) => { + WriteAttemptFailure::Retryable(err) => { let message = err.to_string(); assert!(message.contains("failed to begin transaction")); assert!(message.contains("database is locked")); } - CasBatchFailure::Deterministic(err) => { + WriteAttemptFailure::Deterministic(err) => { panic!("Busy should classify as retryable, got deterministic: {err}") } } @@ -1658,12 +2236,12 @@ mod tests { ); match failure { - CasBatchFailure::Deterministic(err) => { + WriteAttemptFailure::Deterministic(err) => { let message = err.to_string(); assert!(message.contains("failed to commit transaction")); assert!(message.contains("UNIQUE constraint failed")); } - CasBatchFailure::Retryable(err) => { + WriteAttemptFailure::Retryable(err) => { panic!("Constraint should classify as deterministic, got retryable: {err}") } } @@ -1791,4 +2369,608 @@ mod tests { "default query retry failure budget was {budget_ms}ms; expected <= {QUERY_RETRY_FAILURE_BUDGET_MS}ms" ); } + + fn write_contention_test_policy( + busy_timeout_ms: u64, + contention_deadline_ms: u64, + ) -> WriteContentionPolicy { + WriteContentionPolicy { + db_name: "agent trace test", + max_attempts: AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS, + backoff_cap: Duration::from_millis(AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS), + busy_timeout: Duration::from_millis(busy_timeout_ms), + contention_deadline: Duration::from_millis(contention_deadline_ms), + } + } + + fn busy_failure() -> WriteAttemptFailure { + classify_turso_error( + "agent trace test", + "failed to begin transaction", + &turso::Error::Busy(String::from("database is locked")), + ) + } + + fn run_write_contention_test( + policy: WriteContentionPolicy, + attempt: impl FnMut(u32) -> std::result::Result, + ) -> (Result, Vec, WriteContentionCounts) { + let mut jitter = rand::rngs::StdRng::seed_from_u64(7); + let clock = std::cell::Cell::new(Duration::ZERO); + let mut sleeps = Vec::new(); + let (result, counts) = count_write_contention(|| { + run_with_write_contention_retry_using( + policy, + &mut || write_contention_backoff(&mut jitter, policy.backoff_cap), + &mut |backoff| { + sleeps.push(backoff); + clock.set(clock.get() + backoff); + }, + &mut || clock.get(), + "write contention test", + "retry the operation", + attempt, + ) + }); + (result, sleeps, counts) + } + + struct OversleepOutcome { + result: Result<&'static str>, + attempt_calls: u32, + counts: WriteContentionCounts, + } + + fn run_oversleep_scenario( + busy_at: Duration, + backoff: Duration, + oversleep: Duration, + ) -> OversleepOutcome { + let policy = write_contention_test_policy(500, 1_250); + let clock = std::cell::Cell::new(Duration::ZERO); + let mut attempt_calls = 0; + let (result, counts) = count_write_contention(|| { + run_with_write_contention_retry_using( + policy, + &mut || backoff, + &mut |requested| clock.set(clock.get() + requested + oversleep), + &mut || clock.get(), + "write contention test", + "retry the operation", + |_| { + attempt_calls += 1; + if attempt_calls == 1 { + clock.set(busy_at); + Err(busy_failure()) + } else { + Ok("written") + } + }, + ) + }); + OversleepOutcome { + result, + attempt_calls, + counts, + } + } + + #[test] + fn agent_trace_db_write_contention_retry_seeded_jitter_is_reproducible_and_bounded() { + let cap = Duration::from_millis(AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS); + let draw = |seed| { + let mut jitter = rand::rngs::StdRng::seed_from_u64(seed); + (0..64) + .map(|_| write_contention_backoff(&mut jitter, cap)) + .collect::>() + }; + + let first = draw(42); + assert_eq!( + first, + draw(42), + "a seeded jitter source must be reproducible" + ); + assert!(first.iter().all(|backoff| *backoff <= cap)); + assert!( + first.iter().any(|backoff| *backoff != first[0]), + "full jitter should vary across draws" + ); + assert_eq!( + write_contention_backoff(&mut rand::rngs::StdRng::seed_from_u64(1), Duration::ZERO), + Duration::ZERO + ); + } + + #[test] + fn agent_trace_db_write_contention_retry_start_rule_boundaries() { + let policy = write_contention_test_policy(500, 1_250); + let backoff = Duration::from_millis(50); + + assert!(write_contention_retry_may_sleep( + policy, + Duration::from_millis(699), + backoff + )); + assert!(write_contention_retry_may_sleep( + policy, + Duration::from_millis(700), + backoff + )); + assert!(!write_contention_retry_may_sleep( + policy, + Duration::from_millis(701), + backoff + )); + assert!(!write_contention_retry_may_sleep( + policy, + Duration::from_millis(1_250), + backoff + )); + assert!(!write_contention_retry_may_sleep( + policy, + Duration::from_secs(5), + backoff + )); + + let no_wait = write_contention_test_policy(0, 1_250); + assert!(write_contention_retry_may_sleep( + no_wait, + Duration::from_millis(1_249), + Duration::ZERO + )); + assert!( + !write_contention_retry_may_sleep( + no_wait, + Duration::from_millis(1_250), + Duration::ZERO + ), + "nothing starts once the deadline has expired, even with zero backoff and busy timeout" + ); + assert!( + !write_contention_retry_may_sleep( + write_contention_test_policy(0, 0), + Duration::ZERO, + Duration::ZERO + ), + "a zero contention deadline never starts an outer retry" + ); + } + + #[test] + fn agent_trace_db_write_contention_retry_post_sleep_admission_requires_a_full_busy_timeout() { + let policy = write_contention_test_policy(500, 1_250); + + assert!(write_contention_retry_may_start_now( + policy, + Duration::from_millis(749) + )); + assert!( + write_contention_retry_may_start_now(policy, Duration::from_millis(750)), + "remaining == busy_timeout must still admit the next attempt" + ); + assert!(!write_contention_retry_may_start_now( + policy, + Duration::from_millis(751) + )); + assert!(!write_contention_retry_may_start_now( + policy, + Duration::from_millis(1_250) + )); + assert!(!write_contention_retry_may_start_now( + write_contention_test_policy(0, 1_250), + Duration::from_millis(1_250) + )); + } + + #[test] + fn agent_trace_db_write_contention_retry_rejects_a_retry_after_the_backoff_sleep_oversleeps() { + let outcome = run_oversleep_scenario( + Duration::from_millis(690), + Duration::from_millis(50), + Duration::from_millis(70), + ); + + let message = outcome + .result + .expect_err("an oversleep past the admission budget must exhaust") + .to_string(); + assert!( + message.contains("failed after 1 attempt(s) under write contention"), + "{message}" + ); + assert!(message.contains("elapsed_ms=810"), "{message}"); + assert!(message.contains("database is locked"), "{message}"); + assert_eq!(outcome.attempt_calls, 1, "attempt 2 must not run"); + assert_eq!( + outcome.counts, + WriteContentionCounts { + attempts: 1, + outer_retries: 0, + exhaustions: 1 + } + ); + } + + #[derive(Clone, Debug, Default)] + struct CapturedEvent { + target: String, + level: String, + fields: BTreeMap, + } + + struct CapturedEventVisitor<'a>(&'a mut BTreeMap); + + impl tracing::field::Visit for CapturedEventVisitor<'_> { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.0.insert(field.name().to_string(), value.to_string()); + } + + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + self.0.insert(field.name().to_string(), value.to_string()); + } + + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0 + .insert(field.name().to_string(), format!("{value:?}")); + } + } + + #[derive(Clone, Default)] + struct CapturingSubscriber { + events: std::sync::Arc>>, + } + + impl tracing::Subscriber for CapturingSubscriber { + fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + + fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {} + + fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {} + + fn event(&self, event: &tracing::Event<'_>) { + let mut fields = BTreeMap::new(); + event.record(&mut CapturedEventVisitor(&mut fields)); + self.events + .lock() + .expect("captured events mutex should not be poisoned") + .push(CapturedEvent { + target: event.metadata().target().to_string(), + level: event.metadata().level().to_string(), + fields, + }); + } + + fn enter(&self, _span: &tracing::span::Id) {} + + fn exit(&self, _span: &tracing::span::Id) {} + } + + fn capture_tracing_events(body: impl FnOnce() -> T) -> (T, Vec) { + let subscriber = CapturingSubscriber::default(); + let events = std::sync::Arc::clone(&subscriber.events); + let result = tracing::subscriber::with_default(subscriber, body); + let events = events + .lock() + .expect("captured events mutex should not be poisoned") + .clone(); + (result, events) + } + + fn contention_exhausted_events(events: &[CapturedEvent]) -> Vec<&CapturedEvent> { + events + .iter() + .filter(|event| { + event.fields.get("event_id").map(String::as_str) + == Some(CONTENTION_EXHAUSTED_EVENT_ID) + }) + .collect() + } + + #[test] + fn agent_trace_db_contention_exhausted_error_and_event_carry_every_field() { + let (outcome, events) = capture_tracing_events(|| { + run_oversleep_scenario( + Duration::from_millis(690), + Duration::from_millis(50), + Duration::from_millis(70), + ) + }); + + let message = outcome + .result + .expect_err("an oversleep past the admission budget must exhaust") + .to_string(); + assert_eq!( + message, + "Operation 'write contention test' failed after 1 attempt(s) under write contention \ + (db_name=agent trace test, operation=write contention test, attempts=1, \ + busy_timeout_ms=500, contention_deadline_ms=1250 [no retry is scheduled past this cutoff], \ + elapsed_ms=810, cause=database busy (busy timeout exhausted)). \ + Last error: agent trace test failed to begin transaction: database is locked. \ + Try: retry the operation" + ); + assert_eq!(outcome.counts.exhaustions, 1); + + let exhausted = contention_exhausted_events(&events); + assert_eq!( + exhausted.len(), + 1, + "exactly one exhaustion event: {events:?}" + ); + let event = exhausted[0]; + assert_eq!(event.target, "sce"); + assert_eq!(event.level, "WARN"); + let expected = [ + ("db_name", "agent trace test"), + ("operation", "write contention test"), + ("attempts", "1"), + ("busy_timeout_ms", "500"), + ("contention_deadline_ms", "1250"), + ("elapsed_ms", "810"), + ("cause", "database busy (busy timeout exhausted)"), + ( + "last_error", + "agent trace test failed to begin transaction: database is locked", + ), + ]; + for (key, value) in expected { + assert_eq!( + event.fields.get(key).map(String::as_str), + Some(value), + "field {key}: {event:?}" + ); + } + } + + #[test] + fn agent_trace_db_contention_exhausted_event_is_emitted_once_after_the_attempt_cap() { + let (outcome, events) = capture_tracing_events(|| { + run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { + Err::<(), _>(busy_failure()) + }) + }); + let (result, _sleeps, counts) = outcome; + + let message = result + .expect_err("persistent Busy must exhaust") + .to_string(); + assert!(message.contains("attempts=2"), "{message}"); + assert_eq!(counts.exhaustions, 1); + + let exhausted = contention_exhausted_events(&events); + assert_eq!(exhausted.len(), 1, "{events:?}"); + assert_eq!( + exhausted[0].fields.get("attempts").map(String::as_str), + Some("2") + ); + } + + #[test] + fn agent_trace_db_contention_exhausted_event_is_not_emitted_for_success_or_deterministic_errors( + ) { + let ((), events) = capture_tracing_events(|| { + let mut calls = 0; + let _ = run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { + calls += 1; + if calls == 1 { + Err(busy_failure()) + } else { + Ok(()) + } + }); + let _ = run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { + Err::<(), _>(WriteAttemptFailure::Deterministic(anyhow::anyhow!( + "constraint failed" + ))) + }); + }); + + assert!( + contention_exhausted_events(&events).is_empty(), + "{events:?}" + ); + } + + #[test] + fn agent_trace_db_write_contention_retry_admits_a_retry_when_post_sleep_remaining_equals_busy_timeout( + ) { + let outcome = run_oversleep_scenario( + Duration::from_millis(690), + Duration::from_millis(50), + Duration::from_millis(10), + ); + + assert_eq!( + outcome + .result + .expect("remaining == busy_timeout must admit attempt 2"), + "written" + ); + assert_eq!(outcome.attempt_calls, 2); + assert_eq!( + outcome.counts, + WriteContentionCounts { + attempts: 2, + outer_retries: 1, + exhaustions: 0 + } + ); + } + + #[test] + fn agent_trace_db_write_contention_retry_retries_busy_and_busy_snapshot_once() { + for failure in [ + turso::Error::Busy(String::from("database is locked")), + turso::Error::BusySnapshot(String::from("snapshot is stale")), + ] { + let mut failure = Some(failure); + let (result, sleeps, counts) = run_write_contention_test( + write_contention_test_policy(0, 30_000), + |_| match failure.take() { + Some(error) => Err(classify_turso_error("agent trace test", "write", &error)), + None => Ok("written"), + }, + ); + + assert_eq!(result.expect("the retry should succeed"), "written"); + assert_eq!(sleeps.len(), 1); + assert!( + sleeps[0] <= Duration::from_millis(AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS) + ); + assert_eq!( + counts, + WriteContentionCounts { + attempts: 2, + outer_retries: 1, + exhaustions: 0 + } + ); + } + } + + #[test] + fn agent_trace_db_write_contention_retry_fails_deterministic_errors_once_without_sleeping() { + for error in [ + turso::Error::Constraint(String::from("UNIQUE constraint failed")), + turso::Error::Misuse(String::from("misuse")), + turso::Error::Readonly(String::from("readonly")), + ] { + let mut error = Some(error); + let (result, sleeps, counts) = + run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { + Err::<(), _>(classify_turso_error( + "agent trace test", + "write", + &error + .take() + .expect("a deterministic error must not be retried"), + )) + }); + + assert!(result.is_err()); + assert!(sleeps.is_empty(), "a deterministic error must not sleep"); + assert_eq!( + counts, + WriteContentionCounts { + attempts: 1, + outer_retries: 0, + exhaustions: 0 + } + ); + } + } + + #[test] + fn agent_trace_db_write_contention_retry_caps_outer_attempts_at_two() { + let (result, sleeps, counts) = + run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { + Err::<(), _>(busy_failure()) + }); + + let message = result + .expect_err("persistent Busy must exhaust") + .to_string(); + assert!(message.contains("failed after 2 attempt(s)"), "{message}"); + assert!(message.contains("database is locked"), "{message}"); + assert!(message.contains("Try: retry the operation"), "{message}"); + assert_eq!(sleeps.len(), 1); + assert_eq!( + counts, + WriteContentionCounts { + attempts: 2, + outer_retries: 1, + exhaustions: 1 + } + ); + } + + #[test] + fn agent_trace_db_write_contention_retry_starts_no_retry_the_rule_disallows() { + for policy in [ + write_contention_test_policy(0, 0), + write_contention_test_policy(1_251, 1_250), + ] { + let (result, sleeps, counts) = + run_write_contention_test(policy, |_| Err::<(), _>(busy_failure())); + + let message = result.expect_err("Busy must fail").to_string(); + assert!(message.contains("failed after 1 attempt(s)"), "{message}"); + assert!(sleeps.is_empty()); + assert_eq!( + counts, + WriteContentionCounts { + attempts: 1, + outer_retries: 0, + exhaustions: 1 + } + ); + } + } + + #[test] + fn agent_trace_db_write_contention_retry_policy_applies_only_to_agent_trace_db() { + assert_eq!( + write_contention_policy::(), + Some(write_contention_test_policy( + AGENT_TRACE_DB_BUSY_TIMEOUT_MS, + AGENT_TRACE_DB_CONTENTION_DEADLINE_MS + )) + ); + assert_eq!(write_contention_policy::(), None); + } + + #[test] + fn agent_trace_db_write_contention_retry_leaves_reads_and_other_writes_on_the_generic_policy() { + assert_eq!( + resolve_query_retry_policy::(), + QUERY_RETRY_POLICY + ); + + let db_path = unique_test_db_path(); + let db = TursoDb::::new_at(&db_path).expect("test DB should open"); + db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)", ()) + .expect("table should be created"); + + let ((), generic) = count_write_contention(|| { + db.execute("INSERT INTO t (id, v) VALUES (1, 'a')", ()) + .expect("generic execute should succeed"); + db.query("SELECT v FROM t", ()) + .expect("query should succeed"); + db.query_values("SELECT v FROM t", ()) + .expect("query_values should succeed"); + db.query_map("SELECT v FROM t", (), |row| { + row.get::(0).map_err(Into::into) + }) + .expect("query_map should succeed"); + db.passive_checkpoint() + .expect("passive checkpoint should succeed"); + }); + assert_eq!( + generic, + WriteContentionCounts::default(), + "reads, passive_checkpoint, and generic execute must not use the write-contention policy" + ); + + let (affected, opted_in) = count_write_contention(|| { + db.execute_idempotent_write( + "INSERT INTO t (id, v) VALUES (1, 'b') ON CONFLICT (id) DO NOTHING", + (), + ) + .expect("idempotent write should succeed") + }); + assert_eq!(affected, 0); + assert_eq!(opted_in.attempts, 1); + + drop(db); + if let Some(parent) = db_path.parent() { + let _ = fs::remove_dir_all(parent); + } + } } diff --git a/cli/src/services/mutation_trace/store.rs b/cli/src/services/mutation_trace/store.rs index 48306e12b..2b5ce05f7 100644 --- a/cli/src/services/mutation_trace/store.rs +++ b/cli/src/services/mutation_trace/store.rs @@ -536,7 +536,7 @@ impl<'a> MutationTraceStore<'a> { /// `initial_tree`. A no-op when the worktree row already exists — an /// existing cursor, revision, or failure state is never overwritten. pub fn initialize_worktree(&self, worktree: &WorktreeId, initial_tree: &TreeId) -> Result<()> { - self.db.execute( + self.db.execute_idempotent_write( INSERT_WORKTREE_IF_ABSENT_SQL, ( worktree.0.as_str(), @@ -575,7 +575,7 @@ impl<'a> MutationTraceStore<'a> { ); } - self.db.execute( + self.db.execute_idempotent_write( INSERT_SCOPE_IF_ABSENT_SQL, ( scope.0.as_str(), @@ -860,7 +860,7 @@ impl<'a> MutationTraceStore<'a> { ); } - self.db.execute( + self.db.execute_idempotent_write( INSERT_SCOPE_PROVENANCE_IF_ABSENT_SQL, ( provenance.scope_id.0.as_str(), diff --git a/config/pkl/base/sce-config-schema.pkl b/config/pkl/base/sce-config-schema.pkl index 2c7d2191c..89424d1b1 100644 --- a/config/pkl/base/sce-config-schema.pkl +++ b/config/pkl/base/sce-config-schema.pkl @@ -64,6 +64,30 @@ local perDbRetrySchema = new JsonSchema { } } +local agentTraceDbRetrySchema = new JsonSchema { + type = "object" + description = "Retry policy overrides and write-contention settings for the Agent Trace database." + additionalProperties = false + properties { + ["connection_open"] = retryPolicyFieldsSchema + ["query"] = retryPolicyFieldsSchema + ["busy_timeout_ms"] = new JsonSchema { + type = "integer" + description = "Turso busy timeout in milliseconds applied to every Agent Trace database connection. 0 disables the busy handler." + minimum = 0 + maximum = 10000 + default = 1000 + } + ["contention_deadline_ms"] = new JsonSchema { + type = "integer" + description = "Deadline in milliseconds after which no further Agent Trace write-contention retry is started. It schedules retries and does not interrupt a running operation. 0 starts no outer retry." + minimum = 0 + maximum = 30000 + default = 2250 + } + } +} + local sceConfigSchema = new JsonSchema { $schema = "https://json-schema.org/draft/2020-12/schema" $id = configSchemaUrl @@ -160,7 +184,7 @@ local sceConfigSchema = new JsonSchema { additionalProperties = false properties { ["local_db"] = perDbRetrySchema - ["agent_trace_db"] = perDbRetrySchema + ["agent_trace_db"] = agentTraceDbRetrySchema ["auth_db"] = perDbRetrySchema } } diff --git a/context/architecture.md b/context/architecture.md index 4ed19d5d0..9eef4416d 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -124,8 +124,8 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `AppContext` is the CLI's borrowed dependency view in `cli/src/app.rs`: it is generic over logger, telemetry, filesystem, and git capability implementations and stores references plus an optional `repo_root: Option` instead of owning `Arc` trait objects. Because it borrows from `AppRuntime`, `AppContext` is a lightweight, short-lived view and must not be stored long-term (e.g., in structs or across await points). Startup creates a context view over `AppRuntime`'s concrete production dependencies with `repo_root` set to `None`; command paths can derive repo-root-scoped context views through the `ContextWithRepoRoot` accessor trait / `AppContext::with_repo_root(...)`, which reuses the same borrowed dependencies while attaching the resolved root. Narrow accessor traits expose associated concrete capability types for logger, telemetry, fs, and git (`&Self::...`) plus repo-root access, so call sites can express capability requirements without erasing the borrowed dependencies back to trait objects; lifecycle providers consume the repo-root accessor rather than the full context type. - Command parse-time conversion and run-time handling are separated by an internal static `RuntimeCommand` seam. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` enum with variants for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync, plus a deterministic `CommandRegistry` name catalog populated by `build_default_registry()`. `parse_command_phase` in `cli/src/app.rs` delegates clap-output conversion to `cli/src/services/parse/command_runtime.rs`, which owns clap error classification, help rendering bridges, longest-valid-parent traversal for unknown command paths, and parsed-request-to-enum conversion while returning concrete enum values. Service-owned `command.rs` modules define command payload structs and generic execution methods with narrow context requirements: context-free commands accept any context, hooks requires logger access, setup/doctor require repo-root scoping, and central dispatch requires the union of logger plus repo-root-scoping capabilities. `services::app_support::execute_command_phase` emits lifecycle logs around `RuntimeCommand::execute_with_stderr(...)`; the enum performs the only central dispatch match and delegates business behavior to the service-owned command structs, with sync receiving the app-owned stderr writer for format-gated progress. - Startup observability bootstrapping in `cli/src/app.rs` still tolerates invalid default-discovered config files by continuing with degraded defaults plus `sce.config.invalid_config` warn-level logs, but the warning/logging work is now isolated behind the startup-context and runtime-initialization phases rather than one inline startup function. -- `cli/src/services/observability.rs` provides deterministic runtime observability controls and rendering for app lifecycle logs, including shared config-resolved threshold/format, explicit config-file/default `log_to_file`, and `log_dir` inputs with precedence `env > config file > defaults` for non-flag observability keys, stable event identifiers, severity filtering, the forced-emission warning path used for invalid discovered config startup diagnostics, error-specific stderr suppression when file logging is enabled while non-error records and file-write diagnostics remain on stderr, redaction-safe emission through the shared security helper, and log-directory writes with bounded retention. Config resolution also carries a positive config-file/default-only `log_file_retention_limit` (`10` by default) into startup observability config and `sce config show`; the concrete logger stores that resolved value and threads it through primary and v2 cleanup. When `log_dir` resolves from `SCE_LOG_DIR`, config, or the `/sce/logs` default, each enabled or forced log operation selects `/sce-.log` or `/sce--.log` using the machine-local date and optional logger session context, with deterministic percent-encoding for unsafe session filename bytes; after successfully writing a newly created selected file, retention keeps the configured number of newest direct regular `*.log` files by mtime plus path/name tie-break and fails open on cleanup errors. Its `observability::traits` submodule exposes the current `Logger` API with `Option<&str>` session context plus object-safe `Telemetry` trait boundaries and `NoopLogger`; the concrete observability logger and telemetry runtime still own behavior and implement those traits. `services::app_support::render_run_outcome` consumes the logger through that trait boundary when logging classified errors and stdout-write failures. -- `cli/src/services/observability.rs` no longer owns duplicate log enums or parsing helpers; it consumes the canonical primitive seam from `cli/src/services/config/mod.rs` and stays focused on logger and telemetry runtime behavior. +- `cli/src/services/observability.rs` provides deterministic runtime observability controls and rendering for app lifecycle logs, including shared config-resolved threshold/format, explicit config-file/default `log_to_file`, and `log_dir` inputs with precedence `env > config file > defaults` for non-flag observability keys, stable event identifiers, severity filtering, the forced-emission warning path used for invalid discovered config startup diagnostics, error-specific stderr suppression when file logging is enabled while non-error records and file-write diagnostics remain on stderr, redaction-safe emission through the shared security helper, and log-directory writes with bounded retention. Config resolution also carries a positive config-file/default-only `log_file_retention_limit` (`10` by default) into startup observability config and `sce config show`; the concrete logger stores that resolved value and threads it through primary and v2 cleanup. When `log_dir` resolves from `SCE_LOG_DIR`, config, or the `/sce/logs` default, each enabled or forced log operation selects `/sce-.log` or `/sce--.log` using the machine-local date and optional logger session context, with deterministic percent-encoding for unsafe session filename bytes; after successfully writing a newly created selected file, retention keeps the configured number of newest direct regular `*.log` files by mtime plus path/name tie-break and fails open on cleanup errors. Its `observability::traits` submodule exposes the current `Logger` API with `Option<&str>` session context plus object-safe `Telemetry` trait boundaries and `NoopLogger`; the concrete observability `Logger` and `NoopTelemetry` implement those capability traits; `NoopTelemetry` installs no tracing subscriber. `services::app_support::render_run_outcome` consumes the logger through that trait boundary when logging classified errors and stdout-write failures. +- `cli/src/services/observability.rs` no longer owns duplicate log enums or parsing helpers; it consumes the canonical primitive seam from `cli/src/services/config/mod.rs` and stays focused on concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary. - `cli/src/cli_schema.rs` is now the canonical owner for top-level command metadata for the real clap-backed command set (`auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, `completion`), including the slim top-level help purpose text and per-command visibility on `sce`, `sce help`, and `sce --help`; `cli/src/command_surface.rs` remains the custom top-level help renderer and known-command classifier, adding the synthetic `help` row plus the ASCII banner while consuming that shared metadata instead of maintaining a parallel command catalog. - `cli/src/services/default_paths.rs` is the canonical production path catalog for the CLI: it resolves config/state/cache roots with platform-aware XDG or `dirs` fallbacks through an internal `roots` seam, exposes named default paths for current persisted artifacts and database/log files (global config, auth tokens, auth DB, local DB, default observability log directory, and the sole Agent Trace DB path helper `agent_trace_db_path_for_repository` under `repos//agent-trace.db`; the former global-sentinel and per-checkout Agent Trace path helpers were removed by the `retire-legacy-agent-trace-db` plan), and owns canonical repo-relative, embedded-asset, install, hook, and context-path accessors so non-test production path definitions have one shared owner. Compile-time generated payload paths are owned by `build.rs` under `OUT_DIR`, not by the default-path catalog. Current production consumers such as config discovery, observability config resolution, doctor reporting, setup/install flows, database adapters, checkout identity, Agent Trace storage resolution, and local hook runtime path resolution consume this shared catalog rather than defining owned path literals in their own modules. - `cli/src/services/agent_trace.rs` is the Rust CLI owner for the SCE web base URL (`SCE_WEB_BASE_URL`) and exposes helpers for SCE-owned URL construction: Agent Trace conversation lookup URLs, persisted Agent Trace trace URLs, Agent Trace session URLs, and setup-created config schema URLs. Production Rust code should consume those helpers instead of repeating `sce.crocoder.dev` literals. The config resolver separately owns the `control_plane_base_url` runtime seam, whose baked `sce sync` default is `https://sce.crocoderlab.dev`; this control-plane host is not a web URL or schema owner. @@ -136,7 +136,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/lifecycle.rs` defines the current compile-safe lifecycle seam. `ServiceLifecycle` has default no-op generic `diagnose`, `fix`, and `setup` methods over `C: HasRepoRoot`, with lifecycle-owned health, fix, and setup result types so the trait contract is not publicly anchored to doctor/setup module types or the full `AppContext` shape. The same module owns the static `LifecycleProvider` enum and shared `lifecycle_providers(include_hooks)` catalog/factory, returning providers in deterministic order (config → local_db → auth_db → agent_trace_db → hooks when requested); enum dispatch calls each concrete provider through generic context methods without boxed lifecycle-provider allocation or repo-root trait-object context erasure. Hooks exposes a `HooksLifecycle` provider in `cli/src/services/hooks/lifecycle.rs` for hook rollout diagnosis/fix/setup using lifecycle-owned health records plus the canonical required-hook installer. Config exposes a `ConfigLifecycle` provider in `cli/src/services/config/lifecycle.rs` for global/repo-local config validation and repo-local `.sce/config.json` bootstrap. local_db exposes a `LocalDbLifecycle` provider in `cli/src/services/local_db/lifecycle.rs` for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup. auth_db exposes an `AuthDbLifecycle` provider in `cli/src/services/auth_db/lifecycle.rs` for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup. agent_trace_db exposes an `AgentTraceDbLifecycle` provider in `cli/src/services/agent_trace_db/lifecycle.rs` for setup-time repository-scoped Agent Trace storage initialization when a repo root is available and repository Agent Trace DB path health/fix from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path; the former fallback was removed by the `retire-legacy-agent-trace-db` plan). Doctor runtime aggregates the full provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor report/fix records at the orchestration boundary; setup command aggregates the shared catalog for `setup` with hooks included only when requested and adapts hook setup outcomes before rendering setup-owned messages. - Agent Trace lifecycle setup resolves `agent_trace.repository_id` / `agent_trace.repository_remote`, creates/reuses checkout identity for diagnostics, and creates or migrates the repository-scoped DB through `agent_trace_storage::resolve_agent_trace_storage(...)`; hook runtime uses the same storage identity/path resolution and no-migration open path, with missing or stale schema failing open through the existing `Run 'sce setup'.` guidance. - `cli/src/services/auth_command/mod.rs` defines the implemented auth command surface for `sce auth login|logout|whoami`, including device-flow login, stored-credential validation/renewal through login with device-flow fallback, logout, and Control Plane `/me`-backed whoami rendering in text/JSON formats; text mode uses flat `Email`, `First Name`, `Last Name`, `Role`, `Permissions`, and `Organization Name` labels, with optional names and missing role/permissions/workspace values handled deterministically. Logged-out text returns exact login guidance and renewal reports retain the `login` operation label. `cli/src/services/auth_command/command.rs` owns the `AuthCommand` payload used by the static `RuntimeCommand` enum. There is no public renewal or status subcommand. -- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, ordered embedded migrations, and config-file lookup key (`db_config_key()`), while `TursoDb` owns parent-directory creation, `Builder::new_local(...)` initialization (with `experimental_multiprocess_wal(true)` for safe concurrent access), Turso connection setup, tokio current-thread runtime bridging, retry-backed blocking `execute`/`query`/`query_values`/`query_map` wrappers, and generic migration execution with per-database `__sce_migrations` metadata. `TursoDb::new()` and `EncryptedTursoDb::new()` wrap only their local open/connect block in `run_with_retry_sync` using a config-driven connection-open policy resolved from the `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults, while operation methods use a config-driven operation policy from the same source. `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic operator-facing rendering; `query_map()` retries the initial query and row-fetch loop, then applies caller row mapping after retry completion. Migration execution is not retried and uses batch execution so one migration file may contain multiple SQL statements while still recording one migration ID. The same module also provides `EncryptedTursoDb`, a structurally parallel encrypted adapter that resolves the encryption key through `encryption_key::get_or_create_encryption_key()`, enables Turso local encryption with strict `aegis256` cipher selection, and exposes retry-backed synchronous wrappers plus migration execution. `cli/src/services/db/encryption_key.rs` first derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text when present, otherwise falls back to keyring-backed credential-store get-or-create behavior; no plaintext auth DB fallback exists. +- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, ordered embedded migrations, and config-file lookup key (`db_config_key()`), while `TursoDb` owns parent-directory creation, `Builder::new_local(...)` initialization (with `experimental_multiprocess_wal(true)` for safe concurrent access), Turso connection setup, tokio current-thread runtime bridging, retry-backed blocking `execute`/`query`/`query_values`/`query_map` wrappers, and generic migration execution with per-database `__sce_migrations` metadata. `TursoDb::new()` and `EncryptedTursoDb::new()` wrap only their local open/connect block in `run_with_retry_sync` using a config-driven connection-open policy resolved from the `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults, while operation methods use a config-driven operation policy from the same source; on the Agent Trace DB only, replay-safe write units (the transactional insert-pair and CAS-batch primitives and the opt-in `execute_idempotent_write`) instead use a bounded write-contention retry layered over Turso's `busy_timeout` (see `context/sce/shared-turso-db.md`). `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic operator-facing rendering; `query_map()` retries the initial query and row-fetch loop, then applies caller row mapping after retry completion. Migration execution is not retried and uses batch execution so one migration file may contain multiple SQL statements while still recording one migration ID. The same module also provides `EncryptedTursoDb`, a structurally parallel encrypted adapter that resolves the encryption key through `encryption_key::get_or_create_encryption_key()`, enables Turso local encryption with strict `aegis256` cipher selection, and exposes retry-backed synchronous wrappers plus migration execution. `cli/src/services/db/encryption_key.rs` first derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text when present, otherwise falls back to keyring-backed credential-store get-or-create behavior; no plaintext auth DB fallback exists. - `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies retry-backed blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. - `cli/src/services/auth_db/mod.rs` provides the encrypted auth DB spec and `AuthDb` type alias over `EncryptedTursoDb`. `AuthDbSpec` resolves `/sce/auth.db` through the shared default-path seam and embeds ordered auth migrations. Auth DB lifecycle setup/doctor integration is wired through `AuthDbLifecycle`; auth command/token-storage reads/writes are directed through `token_storage.rs`. - `cli/src/services/agent_trace_db/mod.rs` owns the shared Agent Trace insert payloads, SQL constants, and typed row helpers (diff-trace/intersection/Agent Trace/message/part) plus `ensure_schema_ready_for_hooks()` consumed by the repository adapter. `cli/src/services/agent_trace_db/repository.rs` defines the sole `RepositoryAgentTraceDb` adapter over `TursoDb` with one fresh `agent-trace-repository/001_repository_schema.sql` baseline for `diff_traces` (including `payload_type`), `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes, and triggers, `repository_metadata` validation, no trace-table `checkout_id` columns, `agent_traces.agent_trace_id NOT NULL UNIQUE`, and `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` using the inclusive chronological parser without checkout filtering; structured-row reconstruction applies the persisted row `model_id` to every hunk and the persisted canonical `session_id` to every touched line before downstream combination and intersection. Active hook runtime, setup/lifecycle storage, and `sce sync` resolve through `agent_trace_storage` and use `RepositoryAgentTraceDb`. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, the 15-file `cli/migrations/agent-trace/` chain, and the former `sce trace --legacy` surface were removed by the `retire-legacy-agent-trace-db` plan. diff --git a/context/cli/config-precedence-contract.md b/context/cli/config-precedence-contract.md index 8e9bfabdf..75c9f7b88 100644 --- a/context/cli/config-precedence-contract.md +++ b/context/cli/config-precedence-contract.md @@ -106,6 +106,10 @@ When a default-discovered global or repo-local config file exists but fails JSON - `sce setup` writes this key: it records the selection resolved for the run and reads the stored value back when `--workflow` is absent, which is the only consumer of the key today. See [setup local bootstrap](../sce/setup-repo-local-config-bootstrap.md). - `policies` must be an object when present and currently allows `attribution_hooks`, `database_retry`, and `bash`. +- `policies.database_retry` must be an object when present and allows `local_db`, `agent_trace_db`, and `auth_db`. `local_db` and `auth_db` allow only `connection_open` and `query`; `agent_trace_db` additionally allows the Agent Trace-only `busy_timeout_ms` and `contention_deadline_ms` keys. +- `policies.database_retry.agent_trace_db.busy_timeout_ms` must be an integer in `0..=10000`; omitted values resolve to `1000`, and `0` disables the Turso busy handler on Agent Trace DB connections. +- `policies.database_retry.agent_trace_db.contention_deadline_ms` must be an integer in `0..=30000`; omitted values resolve to `2250`, and `0` means no outer write-contention retry is started. It bounds when another retry may start, not how long a running operation takes. +- `busy_timeout_ms` or `contention_deadline_ms` under `local_db`/`auth_db` fails generated-schema validation (`Config file '' failed schema validation against generated schema '': …`, naming the key); the Rust per-DB key check (`contains unknown key`, allowed keys `connection_open, query`) remains as a backstop. `query.timeout_ms` keeps its existing meaning for every database. - `policies.attribution_hooks` must be an object when present and currently allows `enabled`; explicit `enabled: false` remains a valid opt-out alongside the runtime `SCE_ATTRIBUTION_HOOKS_DISABLED` environment opt-out. - `policies.bash` must be an object when present and currently allows only `presets` and `custom`. - `policies.bash.presets` must be an array of unique built-in preset IDs: `forbid-git-all`, `forbid-git-commit`, `use-pnpm-over-npm`, `use-bun-over-npm`, `use-nix-flake-over-cargo`. @@ -127,6 +131,7 @@ When a default-discovered global or repo-local config file exists but fails JSON - `validate` text output is limited to `SCE config validation`, `Validation issues`, and `Validation warnings` lines. - `validate` JSON output is limited to `result.command`, `result.valid`, `result.issues`, and `result.warnings`. - `show` includes resolved Agent Trace configuration under `result.resolved.agent_trace` (JSON: `repository_id` optional-value shape, `repository_remote` and `auto_sync` resolved-value shapes) and as per-key text lines, reporting `(unset)` for a missing `repository_id`, `source: default` for the `origin` remote fallback, and `source: default` for omitted `auto_sync`. +- `show` includes `policies.database_retry` overrides with provenance; for `agent_trace_db` it also renders configured `busy_timeout_ms` and `contention_deadline_ms` (JSON integers under `agent_trace_db`, text lines suffixed `(busy_timeout_ms)` / `(contention_deadline_ms)`). Unset keys are omitted; their defaults apply at resolution. - Doctor consumes the same resolved `agent_trace.auto_sync` value and source metadata; its separate `post_commit_auto_sync` report fact documents hook readiness without launching synchronization. - `show` includes resolved bash-tool policies under `result.resolved.policies.bash`. - Bash-policy output includes resolved preset IDs, expanded custom entries (`id`, `match.argv_prefix`, `message`), and config-file source metadata when present. diff --git a/context/cli/mutation-trace-store.md b/context/cli/mutation-trace-store.md index 88a50646a..50ed63725 100644 --- a/context/cli/mutation-trace-store.md +++ b/context/cli/mutation-trace-store.md @@ -192,10 +192,10 @@ the transaction and propagate out of `commit()` as `Err`, never as `CasResult::Conflict`, and neither is retried unless the underlying error is `Busy`/`BusySnapshot`. -`execute_transactional_cas_batch` keeps three outcomes distinct: a stale -revision is a `Conflict` that is never retried; a transient DB failure -(`Busy`/`BusySnapshot`) retries the whole transaction from a fresh -`BEGIN IMMEDIATE`; and any other deterministic SQL/constraint failure +`execute_transactional_cas_batch` keeps three outcomes distinct: a stale revision is a +`Conflict` that is never retried; a transient failure (`Busy`/`BusySnapshot`) retries the +whole transaction from a fresh `BEGIN IMMEDIATE` under the Agent Trace write-contention +policy ([shared-turso-db.md](../sce/shared-turso-db.md)); any other deterministic failure propagates out of `commit()` as `Err`, never as `CasResult::Conflict`. `DurableTransition`'s six fields are private — `between()` is the only way to @@ -206,8 +206,8 @@ them. ## Initialization -`initialize_worktree`/`register_scope` are idempotent idle-inserts (`INSERT -... ON CONFLICT DO NOTHING`) outside the CAS commit path: +`initialize_worktree`/`register_scope` are idempotent idle-inserts (`INSERT ... ON +CONFLICT DO NOTHING` via `execute_idempotent_write`) outside the CAS commit path: `initialize_worktree` never overwrites an existing cursor, and `register_scope` requires the referenced worktree to already have a durable row and never auto-creates it. An existing scope is returned unchanged only diff --git a/context/context-map.md b/context/context-map.md index 97258ba10..07e001232 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -48,7 +48,7 @@ Feature/domain context: - `context/sce/cli-version-command-contract.md` (implemented `sce version` contract for deterministic human and machine-readable runtime identification) - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) -- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete logger/telemetry runtime behavior plus logger and object-safe telemetry trait boundaries, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, and the trimmed `sce config validate` status-only validation surface) +- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and object-safe telemetry trait boundaries, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, and the trimmed `sce config validate` status-only validation surface) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) - [Context workflow rules](sce/context-workflow-rules.md) (canonical bootstrap, ongoing context maintenance, task synchronization, hygiene, discoverability, and feature-existence rules) @@ -82,7 +82,8 @@ Feature/domain context: - `context/sce/agent-trace-post-rewrite-local-remap-ingestion.md` (current post-rewrite no-op baseline plus historical remap-ingestion reference) - `context/sce/agent-trace-rewrite-trace-transformation.md` (current post-rewrite no-op baseline plus historical rewrite-transformation reference) - `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited retry-backed blocking `execute`/`query`/`query_map` methods using the shared Turso adapter) -- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_values`/`query_map` with a `<= 2_000ms` default query failure budget, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`) +- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_values`/`query_map` (retry-policy budget, not a wall-clock bound), per-`DbSpec` Turso busy timeout on connect (1000 ms for the Agent Trace DB, unset for other databases), the Agent Trace-only outer write-contention retry for replay-safe write units including the opt-in `execute_idempotent_write` and its structured contention-exhaustion error plus `sce.agent_trace_db.contention_exhausted` tracing event, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`) +- `context/sce/agent-trace-db-write-contention-evidence.md` (measured Agent Trace DB write-contention behavior under the busy-timeout/outer-retry/contention-deadline contract: the `lock_contention_tests.rs` suite with its lock-budget boundary, strict N=2–4 in-process and real-hook gates and N=8 stress characterization, policy history from 500 / 1250 to the current 1000 / 2250 defaults, the supported-load failure that motivated the tuning, before-vs-after comparison, and the bounded-policy exhaustion failure mode) - `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, encrypted token-storage persistence, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) - `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by the fresh multi-statement baseline schema plus additive `source_instance_id` and `claude_model_state` migrations, with `repository_metadata` carrying both `repository_id` and a concurrency-safe atomic-claim `source_instance_id` (physical database identity, independent of `repository_id`), narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, parts, and the non-exported exact-scope Claude model-state register, repository-level recent diff-trace reads without checkout filtering including persisted hunk-model plus canonical touched-line-session enrichment for structured rows, on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) - `context/sce/agent-trace-export-readers.md` (implemented `AgentTraceExportReader<'a>` in `cli/src/services/agent_trace_export/mod.rs`: the read-only local export boundary over `RepositoryAgentTraceDb` — `read_messages_after`/`read_parts_after`/`read_diff_traces_after`/`read_agent_traces_after`, each cursor/limit/JS-safe-integer validated, materialized into owned camelCase `serde::Serialize` DTOs; composes directly with `ResolvedAgentTraceStorage` without owning `source_instance_id`; `sce sync` consumes three of the four readers and `read_diff_traces_after` is a retained compatibility surface; no local sync cursor, no `agent-trace-sync.db`, no Turso Sync, no ETL, no DWH) diff --git a/context/glossary.md b/context/glossary.md index 5529d826b..db8d0163a 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -96,9 +96,9 @@ - `TursoConnectionCore`: Internal shared operation core in `cli/src/services/db/mod.rs` used by both `TursoDb` and `EncryptedTursoDb`; owns the Turso connection and tokio current-thread runtime bridging used by the public adapter methods; generic embedded migration execution with per-database `__sce_migrations` metadata is delegated to `run_embedded_migrations` helpers. - `no-migration DB open path`: `TursoDb::open_without_migrations()` / `TursoDb::open_without_migrations_at(path)` plus Agent Trace adapter-specific no-migration seams; opens/connects a local Turso database with parent-directory creation and configured connection-open retry but does not create `__sce_migrations` or run embedded schema migrations. Active Agent Trace hook callers first try the repository-scoped no-migration path and then fall back to migration-running initialization when readiness or repository metadata validation fails. - `TursoDb migration readiness check`: Public methods on `TursoDb` in `cli/src/services/db/mod.rs` for non-mutating schema-readiness verification: `migration_metadata_problems(&self) -> Result>` queries `__sce_migrations` metadata and compares applied IDs against `M::migrations()`, returning problems (missing table, incomplete migrations, unexpected migrations) or an empty list when ready; `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>` calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found. `RepositoryAgentTraceDb::ensure_schema_ready_for_hooks()` delegates to `TursoDb::ensure_schema_ready()` with the Agent Trace–specific `AGENT_TRACE_SCHEMA_SETUP_GUIDANCE` constant. -- `database_retry config namespace`: Nested config namespace under `policies.database_retry` in `sce/config.json`, authored in `config/pkl/base/sce-config-schema.pkl` and parsed/resolved in `cli/src/services/config/mod.rs`. Supports per-database overrides (`local_db`, `agent_trace_db`, `auth_db`) each with optional `connection_open` and `query` objects containing `max_attempts`, `timeout_ms`, `initial_backoff_ms`, `max_backoff_ms`. Validated against JSON Schema at config load and surfaced in `sce config show`/`validate`. Wired into DB adapter constructors and operation methods via config-aware retry resolution with fallback to hardcoded defaults. -- `DatabaseRetryConfig` / `PerDbRetryConfig`: Rust types in `cli/src/services/config/mod.rs` holding parsed, validated per-database retry policy overrides (`local_db`/`agent_trace_db`/`auth_db`, each `Option`) and one database's optional `connection_open`/`query` policies from the `policies.database_retry` config namespace. Initialized at app startup via `DATABASE_RETRY_CONFIG` `OnceLock` and consumed by config-aware retry resolution in DB adapters. -- `DB connection-open/query retry policies`: Connection-open retry applies to `TursoDb::new()` and `EncryptedTursoDb::new()` through `policies.database_retry..connection_open`, with fallback to `3` attempts, `1s` timeout, and `25..200ms` backoff; query retry applies to `execute()`/`query()`/`query_map()` through `policies.database_retry..query`, with fallback to `5` attempts, `200ms` timeout, and `25..100ms` backoff (worst-case `<= 2_000ms`). Both resolve through `DATABASE_RETRY_CONFIG` and `run_with_retry_sync`; embedded migrations stay outside connection-open retry and `query_map()` keeps caller row mapping outside retry. +- `database_retry config namespace`: Nested config namespace under `policies.database_retry` in `sce/config.json`, authored in `config/pkl/base/sce-config-schema.pkl` and parsed/resolved in `cli/src/services/config/`. Supports per-database overrides (`local_db`, `agent_trace_db`, `auth_db`) each with optional `connection_open` and `query` objects containing `max_attempts`, `timeout_ms`, `initial_backoff_ms`, `max_backoff_ms`; `agent_trace_db` alone also accepts `busy_timeout_ms` and `contention_deadline_ms`. Validated against JSON Schema at config load and surfaced in `sce config show`/`validate`. Parsed into `DatabaseRetryConfig` (`local_db`/`auth_db` as `Option`, `agent_trace_db` as `Option` wrapping a `PerDbRetryConfig` plus the two Agent Trace keys), initialized at app startup via the `DATABASE_RETRY_CONFIG` `OnceLock`, and consumed by config-aware retry and busy-timeout resolution in DB adapters with fallback to hardcoded defaults. +- `busy timeout` / `contention deadline`: Agent Trace DB contention settings (`policies.database_retry.agent_trace_db.busy_timeout_ms`, default `1000`, max `10000`; `contention_deadline_ms`, default `2250`, max `30000`). The busy timeout is Turso's connection-wide wait for a contended lock before it reports `Busy` (`0` disables the handler). The contention deadline is the cutoff after which SCE starts no further outer write-contention retry (`0` starts none); it is not a hard timeout on an operation already running. The outer write-contention retry (at most `2` attempts, full-jitter `0..=100ms` backoff, typed `Busy`/`BusySnapshot` only) covers only Agent Trace write units that are safe to replay whole: the transactional insert-pair and CAS-batch primitives and `TursoDb::execute_idempotent_write`. See `context/sce/shared-turso-db.md`. +- `DB connection-open/query retry policies`: Connection-open retry applies to `TursoDb::new()` and `EncryptedTursoDb::new()` through `policies.database_retry..connection_open`, with fallback to `3` attempts, `1s` timeout, and `25..200ms` backoff; query retry applies to `execute()`/`query()`/`query_map()` (and every Agent Trace operation outside the write-contention set) through `policies.database_retry..query`, with fallback to `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff. `timeout_ms` only labels an attempt that ran long after it returns; it never interrupts a synchronous operation, so the policy's configured budget is not a wall-clock bound (Agent Trace attempts can also include Turso busy waiting). Both resolve through `DATABASE_RETRY_CONFIG` and `run_with_retry_sync`; embedded migrations stay outside connection-open retry and `query_map()` keeps caller row mapping outside retry. - `__sce_migrations`: Per-database migration metadata table created by the shared `TursoConnectionCore` migration path behind public adapter `run_migrations()` methods; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. - `CLI generated migration manifest`: Build-time Rust source at `OUT_DIR/generated_migrations.rs` written by `cli/build.rs` from immediate `cli/migrations//*.sql` directories after staging SQL under `OUT_DIR/static/migrations`; constants are named from the database directory (for example `AGENT_TRACE_REPOSITORY_MIGRATIONS`, `AUTH_MIGRATIONS`), sorted by the numeric filename prefix before `_`, and embed staged SQL via `include_str!`. - `sync command deferral` (historical): Former plan/state note that a user-invocable sync command was deferred to `0.4.0`; superseded first by nested `sce trace sync` and now by top-level `sce sync` (see `context/cli/sync-command.md`). Local DB bootstrap and setup-time repository-scoped Agent Trace DB initialization still flow through lifecycle providers aggregated by the setup command, hook runtime still keeps a lazy repository Agent Trace DB fallback for repositories where setup has not run or schema metadata is incomplete, and DB health/repair still flows through the doctor surface. @@ -131,10 +131,10 @@ - `SCE_LOG_FILE_MODE`: Optional runtime env key controlling `SCE_LOG_FILE` write policy; allowed values are `truncate` and `append`, defaults to `truncate`, and requires `SCE_LOG_FILE`. - `SCE_LOG_DIR`: Optional runtime env key for `sce` observability log-directory configuration; when set, it overrides config-file `log_dir` and the `/sce/logs` default and must be non-empty. - `logger trait boundary`: `services::observability::traits::Logger` mirrors the current observability logger API (`info`, `debug`, `warn`, `error`, `log_cli_error`) for generic command/runtime bounds and tests, with each method accepting `Option<&str>` session context for file routing; the concrete `services::observability::Logger` implements it and `NoopLogger` remains available for side-effect-free tests. -- `telemetry trait boundary`: `services::observability::traits::Telemetry` mirrors the current telemetry subscriber API (`with_default_subscriber`) for generic app-runtime bounds and tests, with the concrete `services::observability::TelemetryRuntime` implementing it by delegating to the existing inherent method. +- `telemetry trait boundary`: `services::observability::traits::Telemetry` defines the `with_default_subscriber` command-lifecycle capability used by generic app-runtime bounds and tests. Production currently uses `NoopTelemetry`, which executes the action directly and installs no tracing subscriber; the boundary is retained for future real telemetry/OTEL integration. - `app startup phases`: Current `cli/src/app.rs` execution model that separates dependency checking, startup-context construction, runtime initialization, command parse/execute, and output rendering into named helpers while preserving the CLI's existing exit-code, stderr-diagnostic, and degraded-startup behavior; output rendering and execution-phase logging helpers live in `cli/src/services/app_support.rs`. - `RunOutcome`: Generic final render payload in `cli/src/services/app_support.rs` (`RunOutcome`) carrying a command result, optional startup diagnostic, and optional logger implementing the logger trait boundary. Production construction in `cli/src/app.rs` uses the concrete observability logger, while rendering is not hardcoded to that production type. -- `AppContext`: Generic borrowed dependency view in `cli/src/app.rs` passed through static command dispatch. `AppRuntime` owns the concrete production logger, telemetry, filesystem, and git implementations; `AppContext` stores references to those dependencies plus an optional `repo_root: Option`, not owned `Arc` trait objects. Because it borrows from `AppRuntime`, `AppContext` is a lightweight, short-lived view and must not be stored long-term (e.g., in structs or across await points). The `repo_root` field is `None` at startup and command paths can derive a repo-root-scoped context with `AppContext::with_repo_root(...)` / `ContextWithRepoRoot`, preserving the borrowed runtime dependencies while attaching the resolved root. Narrow accessor traits (`HasLogger`, `HasTelemetry`, `HasFs`, `HasGit`, `HasRepoRoot`) let command and lifecycle call sites express capability requirements without depending on the full production context type; logger/telemetry/fs/git accessors use associated concrete capability types and return `&Self::{Capability}` rather than object-erased `&dyn ...` values. +- `AppContext`: Generic borrowed dependency view in `cli/src/app.rs` passed through static command dispatch. `AppRuntime` owns the concrete production logger, `NoopTelemetry` (no tracing subscriber), filesystem, and git implementations; `AppContext` stores references to those dependencies plus an optional `repo_root: Option`, not owned `Arc` trait objects. Because it borrows from `AppRuntime`, `AppContext` is a lightweight, short-lived view and must not be stored long-term (e.g., in structs or across await points). The `repo_root` field is `None` at startup and command paths can derive a repo-root-scoped context with `AppContext::with_repo_root(...)` / `ContextWithRepoRoot`, preserving the borrowed runtime dependencies while attaching the resolved root. Narrow accessor traits (`HasLogger`, `HasTelemetry`, `HasFs`, `HasGit`, `HasRepoRoot`) let command and lifecycle call sites express capability requirements without depending on the full production context type; logger/telemetry/fs/git accessors use associated concrete capability types and return `&Self::{Capability}` rather than object-erased `&dyn ...` values. - `CommandRegistry`: Static command-name catalog in `cli/src/services/command_registry.rs`; populated by `build_default_registry()` and carried by `AppRuntime` during command dispatch. It exposes deterministic command-name membership for the current top-level command catalog (`help`, `auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, and `completion`) while actual parsed command payloads are represented by the `RuntimeCommand` enum rather than zero-arg boxed constructors. - `ServiceLifecycle`: Compile-safe lifecycle trait seam in `cli/src/services/lifecycle.rs` with default no-op generic `diagnose`, `fix`, and `setup` methods over `C: HasRepoRoot`; it exposes lifecycle-owned health, fix, and setup result types, while doctor/setup adapt those records at orchestration boundaries before rendering command-owned output. The hooks service has `HooksLifecycle` for hook rollout diagnosis/fix/setup, the config service has `ConfigLifecycle` for global/repo-local config validation plus repo-local config bootstrap, local_db has `LocalDbLifecycle` for canonical local DB path health/bootstrap/setup, auth_db has `AuthDbLifecycle` for canonical auth DB path health/bootstrap/setup, and agent_trace_db has `AgentTraceDbLifecycle` for repository identity resolution, checkout identity setup for diagnostics, setup-time repository-scoped Agent Trace DB initialization, and repository Agent Trace DB path health/bootstrap, returning an actionable "requires a Git repository" diagnostic outside repository context (the former global parent fallback was removed by the `retire-legacy-agent-trace-db` plan). Doctor runtime aggregates the static provider catalog for `diagnose` and `fix`; setup command aggregates providers for `setup` in order (config → local_db → auth_db → agent_trace_db → hooks when requested). - `lifecycle provider catalog`: Shared factory in `cli/src/services/lifecycle.rs` (`lifecycle_providers(include_hooks)`) that returns static `LifecycleProvider` enum values in deterministic config → local_db → auth_db → agent_trace_db → hooks order, used by doctor with hooks included and by setup with hooks included only when requested. The enum owns concrete-provider dispatch for `id`, `diagnose`, `fix`, and `setup` without boxed provider trait objects or `&dyn HasRepoRoot` lifecycle context erasure. diff --git a/context/overview.md b/context/overview.md index fbfbb0576..57f262c22 100644 --- a/context/overview.md +++ b/context/overview.md @@ -74,7 +74,7 @@ The `setup` command includes an `inquire`-backed target-selection flow: default For repository generation consumers, `config/pkl/generator-inputs.txt` declares the canonical Pkl/plugin input set and `scripts/produce-cli-generated-input.sh` owns its discovery, two-pass `config/pkl/generate.pkl` evaluation, determinism comparison, payload/input inventories, in-flight input-mutation rejection, atomic handoff publication, and staging cleanup. `scripts/run-cli-cargo.sh` creates a fresh temporary destination, delegates generation to that producer, invokes the requested Cargo workflow with `SCE_CLI_GENERATED_INPUT_DIR`, and removes the handoff after Cargo success, failure, or handled signals. `config/pkl/check-generated.sh` delegates the same production mechanics while retaining contract and path assertions. The root flake also runs `codex-hook-command`, which generates the Codex assets and verifies root, nested-cwd, spaced-path, stdin-forwarding, and fail-open invocation behavior against a stub `sce`. `scripts/prepare-cli-generated-assets.sh` moves the producer-validated Pkl payload and checksums into the unchanged package fallback, adds hooks, migrations, and the Agent Trace schema, and appends only those static checksums to the combined inventory. The root flake's pre-Cargo `cliGeneratedInput` derivation invokes the same producer from a declarative source containing the producer plus its declared inputs. `cli/build.rs` rejects missing, incomplete, modified, or stale repository handoffs, copies the validated payload into Cargo `OUT_DIR/pkl-generated`, stages static inputs under `OUT_DIR/static`, and writes setup-asset, optional-workflow-catalog, and migration Rust manifests into `OUT_DIR`; it never invokes Pkl. Published crates carry the ignored packaging-only fallback, and unpacked downstream builds validate and copy it into their own `OUT_DIR` without requiring Pkl or parent repository paths. The setup service also provides repository-root install orchestration: it resolves the repository root, ensures the additive durable-context baseline, then for normal modes derives a repo-root-scoped `AppContext` from the runtime command context, aggregates `ServiceLifecycle::setup` calls across lifecycle providers (config → local_db → auth_db → agent_trace_db → hooks when requested), handles interactive or flag-based target selection for config asset installation, and reports deterministic completion details (selected target(s) and installed file counts). Setup installs config assets (`.opencode`/`.claude`/`.pi`) per file: each embedded asset is staged and swapped into its own destination path, creating parent directories as needed, without removing or recreating the target directory as a whole, so files a repository owns inside an SCE-managed target directory survive a setup run untouched. Two assets are merge targets rather than verbatim writes: Claude's `.claude/settings.json` and OpenCode's `.opencode/opencode.json`. For each, setup JSON-merges the generated document into the user's existing file rather than overwriting it, and fails deterministically without writing if the existing file is not valid JSON; a missing file is still created from the generated document verbatim. Claude's merge replaces only SCE-owned hook entries (identified by a command containing `run-sce-or-show-install-guidance.sh`) and the `$schema` key while preserving every other key and hook entry untouched. OpenCode's merge replaces the `$schema` key and merges the `plugin` array as a set: any entry shaped like an SCE plugin path (`./plugins/sce-*`) is dropped, structurally, so a path an older or renamed catalog once installed is still recognized and pruned, and the generated document's canonical plugin entries are appended after the surviving user entries. Required-hook install uses the same per-file stage/atomic-swap choreography as config-asset install — the staging file is renamed directly over an existing hook without unlinking it first, so a rename failure leaves the prior hook untouched. Both flows return deterministic recovery guidance (recover from version control) on swap failure, without creating backup artifacts. After installing, config install prunes stale SCE-owned assets: it deletes every path the full embedded catalog for the target claims but the current selection did not install (a deselected optional workflow, or an asset a newer catalog renamed or dropped), then removes any parent directory left empty by that deletion, leaving a directory intact if a user file still lives inside it. The setup command gates all modes on an existing git repository before any writes. Internally, `cli/src/services/setup/mod.rs`now separates install-flow logic from interactive prompt logic through focused support seams. The CLI now also applies baseline security hardening for reliability-driven automation: diagnostics/logging paths use deterministic secret redaction,`sce setup --hooks --repo ` canonicalizes and validates repository paths before execution, and setup write flows run explicit directory write-permission probes before staging/swap operations. -The config service now provides deterministic runtime config resolution with explicit precedence (`flags > env > config file > defaults`), strict config-file validation (`$schema`, `log_level`, `log_format`, `log_to_file`, `log_dir`, `workos_client_id`, and nested `policies.bash`, `policies.attribution_hooks.enabled`, plus `policies.database_retry` with per-DB `connection_open`/`query` retry policy specs), deterministic default discovery/merge of global+local config files (`${config*root}/sce/config.json`then`.sce/config.json`with local override, where`config_root` comes from the shared default-path seam with XDG/`dirs::config_dir()` config-root resolution), defaults for the resolved observability value set (`log_level=error`, `log_format=text`, `log_dir=/sce/logs`), shared auth-key resolution with optional baked defaults starting at `workos_client_id`, first-class bash-policy preset/custom parsing with deterministic conflict and duplicate-prefix validation, custom-policy `satisfied_by`wrapper exemption (a policy does not fire when the matched command was unwrapped from a declared wrapper such as`nix shell nixpkgs#ripgrep`), and a canonical Pkl-authored `sce/config.json`JSON Schema generated beneath Cargo`OUT_DIR`and embedded by`cli/src/services/config/mod.rs`for both`sce config validate`and doctor-time config checks. Runtime startup config loading keeps parity with that schema by accepting its`$schema`declaration in repo-local and global config files, so startup commands such as`sce version`no longer fail before dispatch on that field; the canonical declaration is versioned as`"https://sce.crocoder.dev/v/config.json"` using the CLI release version; this schema URL is separate from the `https://sce.crocoderlab.dev` baked default used by `sce sync` for control-plane ingestion. App-runtime observability now consumes flat logging keys through the shared resolver, so env values still override config-file values while config files provide deterministic fallback for `log_dir`; positive-integer `log_file_retention_limit` uses config-file/default precedence, defaults to `10`, and controls creation-triggered cleanup for primary and v2 log files; `sce config show` reports resolved observability/auth/policy values with provenance, while `sce config validate` is now a trimmed validation surface that reports only pass/fail plus validation errors or warnings in text and JSON modes. The canonical preset catalog and matching contract live in `config/pkl/base/bash-policy-presets.pkl` and `context/sce/bash-tool-policy-enforcement-contract.md`. +The config service now provides deterministic runtime config resolution with explicit precedence (`flags > env > config file > defaults`), strict config-file validation (`$schema`, `log_level`, `log_format`, `log_to_file`, `log_dir`, `workos_client_id`, and nested `policies.bash`, `policies.attribution_hooks.enabled`, plus `policies.database_retry` with per-DB `connection_open`/`query` retry policy specs and Agent Trace-only `agent_trace_db.busy_timeout_ms`/`contention_deadline_ms`), deterministic default discovery/merge of global+local config files (`${config*root}/sce/config.json`then`.sce/config.json`with local override, where`config_root` comes from the shared default-path seam with XDG/`dirs::config_dir()` config-root resolution), defaults for the resolved observability value set (`log_level=error`, `log_format=text`, `log_dir=/sce/logs`), shared auth-key resolution with optional baked defaults starting at `workos_client_id`, first-class bash-policy preset/custom parsing with deterministic conflict and duplicate-prefix validation, custom-policy `satisfied_by`wrapper exemption (a policy does not fire when the matched command was unwrapped from a declared wrapper such as`nix shell nixpkgs#ripgrep`), and a canonical Pkl-authored `sce/config.json`JSON Schema generated beneath Cargo`OUT_DIR`and embedded by`cli/src/services/config/mod.rs`for both`sce config validate`and doctor-time config checks. Runtime startup config loading keeps parity with that schema by accepting its`$schema`declaration in repo-local and global config files, so startup commands such as`sce version`no longer fail before dispatch on that field; the canonical declaration is versioned as`"https://sce.crocoder.dev/v/config.json"` using the CLI release version; this schema URL is separate from the `https://sce.crocoderlab.dev` baked default used by `sce sync` for control-plane ingestion. App-runtime observability now consumes flat logging keys through the shared resolver, so env values still override config-file values while config files provide deterministic fallback for `log_dir`; positive-integer `log_file_retention_limit` uses config-file/default precedence, defaults to `10`, and controls creation-triggered cleanup for primary and v2 log files; `sce config show` reports resolved observability/auth/policy values with provenance, while `sce config validate` is now a trimmed validation surface that reports only pass/fail plus validation errors or warnings in text and JSON modes. The canonical preset catalog and matching contract live in `config/pkl/base/bash-policy-presets.pkl` and `context/sce/bash-tool-policy-enforcement-contract.md`. Invalid default-discovered config files now also degrade gracefully at startup: `sce` keeps running with degraded observability defaults, logs `sce.config.invalid_config` warnings, and reserves hard failures for explicit `--config` / `SCE_CONFIG_FILE` targets or other truly invalid runtime observability inputs. `cli/src/services/config/mod.rs` is now a module facade that declares focused config submodules (`types`, `schema`, `policy`, `resolver`, private `render`, `command`, and `lifecycle`), re-exporting `pub use types::*`and`pub(crate) use schema::validate_config_file`. Shared config primitive ownership is delegated to `cli/src/services/config/types.rs`; schema loading and file parsing to `cli/src/services/config/schema.rs`; bash-policy semantic validation and policy-specific formatting to `cli/src/services/config/policy.rs`; runtime discovery/precedence to `cli/src/services/config/resolver.rs`; and `sce config show`/`sce config validate`text+JSON output construction to`cli/src/services/config/render.rs`. Downstream modules continue importing through `services::config`unchanged. The CLI now has a generic borrowed`AppContext`dependency view in`cli/src/app.rs`; `AppRuntime`owns concrete production logger/telemetry/fs/git dependencies, and command execution receives context views that borrow those dependencies plus an optional`repo_root: Option`. `AppContext::with_repo_root(...)`/`ContextWithRepoRoot`derives repo-root-scoped views while preserving the borrowed runtime dependencies, and command execution is generic over associated-type narrow accessor traits where practical. The broad capability seam lives in`cli/src/services/capabilities.rs`, where `FsOps`/`StdFsOps`wrap filesystem operations and`GitOps`/`ProcessGitOps`wrap git process execution plus repository-root/hooks-directory resolution. The shared default path service in`cli/src/services/default_paths.rs`is now the canonical owner for production CLI path definitions. It resolves per-user config/state/cache roots through a dedicated internal`roots`seam, exposes the current persisted-artifact inventory (global config and auth tokens), and also defines named DB paths (auth DB, local DB, Agent Trace DB) plus the repo-relative, install, hook, and context-path accessors consumed across current CLI production code. Non-test production modules should consume this shared catalog instead of hardcoding owned path literals. No default cache-backed persisted artifact currently exists, so cache-root resolution remains available without speculative cache-path features and no legacy default-path fallback is supported. diff --git a/context/patterns.md b/context/patterns.md index 20523c6d7..ffb4a6893 100644 --- a/context/patterns.md +++ b/context/patterns.md @@ -151,7 +151,7 @@ - Route local Turso access through service adapters so command handlers do not expose low-level `turso` API details. New Turso-backed services should build on `cli/src/services/db/mod.rs` (`DbSpec` + `TursoDb`, or `EncryptedTursoDb` when at-rest encryption is required) for runtime, connection, per-database `__sce_migrations` tracking, and migration infrastructure, then expose domain-specific methods from their own service modules. - For current local DB flows, route initialization through the dedicated adapter (`cli/src/services/local_db/mod.rs`) and invoke it from approved orchestration surfaces such as setup or doctor rather than exposing a partial user command before its contract is approved. - For Turso-backed services with setup/doctor ownership, add service-owned lifecycle providers that reuse shared DB path-health and parent-bootstrap helpers, then register them through `lifecycle_providers()` instead of adding command-local database checks. -- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` for async operations and sync `run_with_retry_sync` for pure blocking contexts where a Tokio sleep/timeout future cannot be awaited. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. +- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` for async operations and sync `run_with_retry_sync` for pure blocking contexts where a Tokio sleep/timeout future cannot be awaited. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. Opt an Agent Trace write into the write-contention retry only when its whole retry unit is replay-safe: a complete `BEGIN IMMEDIATE` → `COMMIT` transaction primitive, or a single statement made idempotent by its SQL (`ON CONFLICT DO NOTHING`, a guarded `UPDATE`) routed through `TursoDb::execute_idempotent_write`; append-only or last-writer statements stay on generic `execute()`. - For SCE operator-health commands, prefer deterministic local diagnostics over implicit pass/fail behavior: report the inspected environment scope, stable problem categories, severity/fixability classes, actionable remediation text, and any path/location facts needed to repair the issue; when repair mode exists, keep outcome vocabulary deterministic and idempotent (`cli/src/services/doctor/mod.rs`, with focused diagnosis/render/fix helpers under `cli/src/services/doctor/`). - For service-owned operator health, keep command modules as thin aggregators over `ServiceLifecycle` providers once a lifecycle slice is wired: providers own diagnosis/fix problem production through narrow capability accessors, while command-specific report builders preserve existing output facts and rendering contracts. - Keep static lifecycle provider-list construction centralized in the lifecycle service layer so doctor/setup choose provider inclusion without maintaining parallel concrete provider lists. diff --git a/context/plans/agent-trace-db-write-contention.md b/context/plans/agent-trace-db-write-contention.md new file mode 100644 index 000000000..acb906708 --- /dev/null +++ b/context/plans/agent-trace-db-write-contention.md @@ -0,0 +1,793 @@ +# Plan: agent-trace-db-write-contention + +## Change summary + +Fixes ingestion loss in the repository-scoped Agent Trace DB under multiprocess-WAL writer contention, found while validating PR #297. Today every Turso connection runs with the default `BusyHandler::None`. A contended `BEGIN IMMEDIATE` (or any write) returns `Busy` immediately, and the shared `run_with_retry_sync` query policy in `cli/src/services/db/mod.rs` (`QUERY_RETRY_POLICY`: 5 attempts, 25..100 ms deterministic backoff, no jitter) is the only thing that serializes writers. The original investigation saw real `sce hooks codex` processes lose distinct events with 2–3 concurrent writers. The T01 baseline on the current host did not reproduce loss at N=2–4. It did reproduce loss deterministically when one writer holds `BEGIN IMMEDIATE` for ≥ ~300 ms, and under N=8 stress (1525 of 4000 distinct events lost). Database integrity is unaffected; the failure is ingestion availability. + +This plan implements this contention contract for the Agent Trace DB: + +``` +multiprocess WAL cross-process correctness and locking + ↓ +Turso busy_timeout Turso waits for the lock while it reports Busy (busy_timeout_ms) + ↓ +typed Busy/BusySnapshot still returned? + ↓ +small jittered SCE outer retry write-capable operations only; at most one outer retry by default + ↓ +contention deadline decides whether another outer retry may start (contention_deadline_ms) +``` + +The final implementation contract: + +``` +Agent Trace connection: + multiprocess WAL + Turso busy_timeout + +Agent Trace write-capable operation (whole retry unit known to be safe): + Turso handles ordinary Busy waiting + ↓ + if Busy/BusySnapshot escapes: + at most one jittered outer retry, subject to contention_deadline_ms + +Agent Trace read-only queries and everything else: + existing SCE outer retry semantics, unchanged +``` + +**Turso's `busy_timeout` is connection-wide, so it naturally applies to any Turso statement on an Agent Trace DB connection that hits `Busy`. SCE's new outer retry policy is intentionally scoped to write-capable Agent Trace operations whose complete retry unit is known to be safe.** Read-only query APIs (`query`, `query_values`, `query_map`), `passive_checkpoint`, non-idempotent single-statement writes, and migrations keep their existing generic retry semantics in this PR. + +**The contention deadline bounds SCE retry scheduling, not the execution time of an already-running Turso operation.** SCE will not begin another contention retry once the configured contention deadline has expired. An individual Turso operation already in progress may complete after that deadline, so this is a retry-scheduling bound, not a hard wall-clock operation timeout. The investigation already saw successful operations take about 1.3 s, 2.8 s and 6.4 s. Turso 0.8.1's Rust binding exposes `Connection::busy_timeout(...)` but no public query-timeout or cancellation API, so this plan cannot enforce a hard wall-clock bound on a running operation and does not claim one. + +The plan also makes the hook-runtime open read-only for repository metadata that is already initialized. That removes one unnecessary write from the hook fast path. + +Two Agent Trace-only settings are added: `policies.database_retry.agent_trace_db.busy_timeout_ms` and `policies.database_retry.agent_trace_db.contention_deadline_ms`. They are not accepted under `local_db` or `auth_db`; config validation rejects them there with the existing unknown-key error, so no ignored setting is silently accepted. The existing `query.timeout_ms` semantics are unchanged, and generic `RetryPolicy.timeout_ms` cleanup is a separate follow-up. `local_db` and `auth_db` keep their current retry configuration and behavior. + +**Branch and base:** PR #299 is currently stacked on `mutation-trace-health-invariant` (PR #297) so it can reuse the contention investigation and test stabilization work. Its base is `mutation-trace-health-invariant` and its head is `agent-trace-db-write-contention`. After #297 merges, rebase the branch onto the updated `main` before final merge if necessary. + +The investigation suite `cli/src/services/agent_trace_db/lock_contention_tests.rs` is version-controlled on this branch (committed by T01) and registered as a `#[cfg(test)]` module. It holds the pre-fix baseline and is the acceptance gate for later tasks. + +**Pre-fix reproduction signals.** The N=2–4 concurrent suites are regression gates on this host, not a deterministic pre-fix reproduction. The deterministic held-lock boundary and the N=8 stress test are the pre-fix reproduction signals. The fix is not weakened because N=2–4 happened to pass on this machine before it. + +## Acceptance criteria + +- [x] AC1: Every Agent Trace DB connection opened by SCE has Turso's busy handler set to the resolved `busy_timeout_ms` (default 1000 ms; 500 ms before T08), and `experimental_multiprocess_wal(true)` stays enabled on every local open path. + - Validate: the T02 busy-timeout tests pass (`nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml busy_timeout`), and inspection of `cli/src/services/db/mod.rs` shows `experimental_multiprocess_wal(true)` and the `busy_timeout` call on every local `TursoDb` open path. +- [x] AC2: Turso's busy handler covers the failing writer-lock acquisition. Turso 0.8.1's `Transaction::new_unchecked(conn, TransactionBehavior::Immediate)` runs `BEGIN IMMEDIATE` through `Connection::execute(...)` on the same connection. With connection A holding `BEGIN IMMEDIATE` for about 100 ms: + - a control connection without a busy handler returns `Busy` promptly; + - the production `insert_conversation_text_event` on a busy-timeout-configured Agent Trace DB connection waits and succeeds, with no test-level retry around the call. + - Validate: the T02 direct busy-handler tests pass. +- [x] AC3: Agent Trace write-capable operations (the enumerated set in T04) use at most two outer attempts, retry only typed `Busy`/`BusySnapshot` failures, use bounded full jitter, and never start another outer attempt after the contention retry-start rule disallows it. A retry starts only when `remaining_deadline >= jittered_backoff + busy_timeout`, and never once the contention deadline has expired. Deterministic errors fail after exactly one attempt with no sleep. A ~100 ms lock hold succeeds with `attempts = 1` and `outer_retries = 0`. Read-only query APIs (`query`, `query_values`, `query_map`), `passive_checkpoint`, and Agent Trace writes outside the enumerated set retain their existing outer retry semantics. + - Validate: the T04 unit tests (`... test --manifest-path cli/Cargo.toml agent_trace_db_write_contention_retry`) pass. They cover: + - a seeded jitter seam; + - retry-start rule boundary cases (remaining just above, equal to, and just below `backoff + busy_timeout`, and an expired deadline); + - Busy/BusySnapshot vs non-Busy classification; + - the attempt cap; + - the 100 ms-hold single-attempt assertion; + - a regression assertion that an Agent Trace `query`/`query_map` call still resolves the existing generic query retry policy. +- [x] AC4: When the contention policy is exhausted, the returned error carries `db_name`, `operation`, `attempts`, `busy_timeout_ms`, `contention_deadline_ms`, `elapsed_ms` and `cause`. Existing hook fail-open paths log that error through the configured SCE `Logger`. Log-file/stderr routing stays as it is today, stdout is unchanged, and hook fail-open behavior is unchanged. + - The DB layer also emits one structured `tracing` event, `sce.agent_trace_db.contention_exhausted`, with the same contention fields. + - This event is a telemetry instrumentation point. It reaches a sink only when a tracing subscriber is installed. Production currently runs with `NoopTelemetry`, and PR #299 does not add a production tracing subscriber. + - Validate: + - the T05 unit tests prove the error text and the shape of the structured `tracing` event, using a test-only capturing subscriber; + - the existing hook/logger tests (`... test --manifest-path cli/Cargo.toml hooks`) and the existing hook fail-open `log.error(...)` paths show the returned error stays observable through the production `Logger` path; + - inspection confirms no new stdout writes on hook paths. +- [x] AC5: Opening an already-initialized repository Agent Trace DB through the hook runtime issues zero write statements. The metadata guarantees still hold: repository-ID mismatch is an error, the source instance ID is stable and never overwritten, and concurrent first initialization converges on one ID. An initialized hook open succeeds while another connection holds `BEGIN IMMEDIATE`. + - Validate: the T06 write-statement-count test passes with 0 writes; the converted hook-open-under-write-lock test passes; existing `verify_or_initialize_repository_metadata` tests pass. +- [x] AC6: `insert_conversation_text_event` and the mutation-trace CAS batch keep their whole-transaction semantics. Message and part commit together. The first delivery returns `Ok(true)` and a duplicate replay returns `Ok(false)`. An injected mid-transaction failure leaves no orphans. Every retry restarts the whole unit from `BEGIN IMMEDIATE`, never an individual statement. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db` and `... mutation_trace` pass. +- [x] AC7: The lock-budget boundary test characterizes busy timeout, outer retry and the contention retry-start deadline together. One connection holds `BEGIN IMMEDIATE` for 100, 250, 500, 750, 1000, 1500, 1750, 2000, 2250, 2500 or 3000 ms while another calls the real production API. Assertions use wide margins and are stated as outcome and retry policy, not wall-clock cutoffs: + - holds ≤ 1000 ms succeed; + - holds ≥ 3000 ms exhaust the configured contention policy and fail cleanly; + - every failure leaves 0 partial rows, every sample makes at most 2 attempts, and `outer_retries + 1 == attempts`; + - the middle holds (1500–2500 ms) are reported (outcome, latency, attempts, retry timeline), not asserted. + - These thresholds come from the T08 10-run held-lock dataset for the 1000 / 2250 defaults. The T07 thresholds (≤ 250 ms succeed, ≥ 2000 ms exhaust) belonged to the superseded 500 / 1250 defaults. + - Validate: `... test --manifest-path cli/Cargo.toml lock_budget_boundary -- --nocapture` passes and prints per-hold outcome, latency and attempts. +- [x] AC8: The Rust production-API suite (`insert_conversation_text_event`) passes the strict levels N=2–4: distinct-event 2×1000, 3×1000, 4×500, and duplicate-delivery 2×500, 3×500, 4×500. These levels are regression gates on the reference host; the T01 baseline already passes them before the fix. The test enforces: + - always, per round: `messages == parts` (no orphan rows); the `Ok(true)` count equals the persisted message rows; distinct events never report `Ok(false)`; a duplicate-delivery round inserts at most once, and exactly once when at least one writer completed (no duplicate persisted events); + - under `SCE_LOCK_CONTENTION_STRICT=1`, per writer level: 0 lock errors, 0 other errors and 0 lost distinct events. + - 8 writers runs as non-strict stress/characterization only. If the fixed system makes N=8 reliable within acceptable latency, that result is reported, but N=8 is not a supported semantic requirement unless deliberately decided. + - Validate: run these, reporting p50/p95/p99/max latency, outer retries and contention exhaustions (from test instrumentation), next to the T01 baseline: + - `SCE_LOCK_CONTENTION_STRICT=1 SCE_LOCK_CONTENTION_WRITERS=2,3 SCE_LOCK_CONTENTION_ROUNDS=1000 nix develop -c ./scripts/run-cli-cargo.sh test --release --manifest-path cli/Cargo.toml concurrent_distinct_events -- --ignored --nocapture`; + - the same with `WRITERS=4 ROUNDS=500`; + - `concurrent_duplicate_delivery` with `WRITERS=2,3,4 ROUNDS=500`; + - a non-strict `WRITERS=8` run. +- [x] AC9: Real release `sce hooks codex` `UserPromptSubmit` processes pass the strict levels 2×500, 3×500 and 4×200. These levels are regression gates on the reference host; the T01 baseline already passes them before the fix. `concurrent_real_codex_hook_processes_persist_every_distinct_event` enforces: + - always, per round: `messages == parts` (no orphan rows); + - always, per writer level: `persisted_messages <= expected` and `persisted_parts <= expected` (no over-persistence or duplicate events); + - under `SCE_LOCK_CONTENTION_STRICT=1`, per writer level: `persisted_messages == expected`, `persisted_parts == expected`, and 0 non-zero hook exits; + - under `SCE_LOCK_CONTENTION_STRICT=1`, across all levels: `total_lost == 0` and `total_nonzero_exits == 0`. + - Hook stderr lines are recorded and reported but are not a strict invariant. Hooks fail open, and the contention fix may legitimately emit diagnostics through the configured logging path. "Lock-exhaustion failures" in hook processes show up as lost events, because the hook fails open; they are covered by the lost-event and persisted-count assertions. + - Validate: `nix build .#default`, then `SCE_BIN=$PWD/result/bin/sce SCE_LOCK_CONTENTION_STRICT=1 SCE_LOCK_CONTENTION_WRITERS=2,3 SCE_LOCK_CONTENTION_ROUNDS=500 nix develop -c ./scripts/run-cli-cargo.sh test --release --manifest-path cli/Cargo.toml concurrent_real_codex_hook_processes -- --ignored --nocapture`, and the same with `WRITERS=4 ROUNDS=200`. Record p50/p95/p99/max per-event latency next to the T01 baseline. Hook processes are separate processes, so in-process counters are unavailable; report persisted-row outcomes and latency only, and no retry counts unless they are derived from the hook log file. +- [x] AC10: `busy_timeout_ms` and `contention_deadline_ms` are Agent Trace-only settings. They are accepted, validated and documented only under `policies.database_retry.agent_trace_db`: + - `policies.database_retry.local_db` and `policies.database_retry.auth_db` keep their existing keys (`connection_open`, `query`) only, and reject `busy_timeout_ms` and `contention_deadline_ms` through the existing config-validation path. The generated-schema check runs first and fails with the existing stable `Config file '' failed schema validation against generated schema '': …` error, naming the offending key. The Rust per-DB key check (`validate_object_keys`, allowed keys `connection_open, query` for `local_db`/`auth_db`) stays as a backstop with its existing `contains unknown key` wording; + - the generated config schema publishes the two keys only on the `agent_trace_db` object; the `local_db`/`auth_db` objects keep `additionalProperties = false` with only `connection_open` and `query`; + - both are non-negative integers with upper bounds; + - `busy_timeout_ms = 0` disables the Turso busy handler; + - `contention_deadline_ms = 0` means no outer retry is started; + - both are rendered by `sce config show` in text and JSON and published in the generated config schema; + - defaults (1000 / 2250; 500 / 1250 before T08) apply when they are unset; + - existing `query.timeout_ms` behavior and rendering are unchanged. + - Validate: the T03 config tests (`... test --manifest-path cli/Cargo.toml database_retry`) pass, including the `local_db`/`auth_db` rejection tests for both keys and an unchanged-`query.timeout_ms` regression assertion; `nix run .#pkl-check-generated` passes. + +### Full validation + +- `nix flake check` +- `nix run .#pkl-check-generated` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml mutation_trace` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml resilience` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml database_retry` +- The strict contention commands from AC8 and AC9, reported as before-vs-after measurements + +### Context sync + +- `context/sce/shared-turso-db.md`: clearly distinguish these layers, and replace the stale "5 attempts / ≤2_000 ms" Agent Trace default text: + - **multiprocess WAL:** cross-process correctness and locking; + - **`busy_timeout_ms`:** Turso's wait/retry policy for `Busy`, covering `BEGIN IMMEDIATE` via `Transaction::new_unchecked` → `Connection::execute`; + - **outer write-contention retry:** SCE-level retry after Turso exhausts its wait, scoped to the enumerated write-capable Agent Trace operations; read-only queries keep the generic retry; + - **`contention_deadline_ms`:** cutoff for launching another outer retry, explicitly not a hard operation timeout; + - **`query.timeout_ms`:** existing generic retry setting, unchanged. +- `context/sce/agent-trace-db.md`: the hook-runtime read-only metadata fast path, the contention contract (including the not-a-hard-timeout statement), and a summary of the measured contention evidence. +- `context/cli/config-precedence-contract.md`: the Agent Trace-only `busy_timeout_ms` and `contention_deadline_ms` keys, their validation, the meaning of zero, and their rejection under `local_db`/`auth_db`. +- `context/glossary.md`: "busy timeout" and "contention deadline" entries, if the glossary covers retry vocabulary. + +## Task context synchronization lifecycle + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/db/mod.rs` (connection open, Agent Trace write-contention retry routing for transactional primitives and the opt-in idempotent write entrypoint, test seams), `cli/src/services/mutation_trace/store.rs` (only switching the three `…_IF_ABSENT` writes to the opt-in entrypoint), `cli/src/services/resilience.rs` (only if a small deadline/jitter helper belongs there), `cli/src/services/agent_trace_db/{repository.rs,mod.rs,lock_contention_tests.rs}`, `cli/src/services/agent_trace_storage/mod.rs` (hook-runtime open path), `cli/src/services/config/{schema.rs,resolver.rs,render.rs}`, `config/pkl/base/sce-config-schema.pkl`, and the durable context listed above. +- **Out of scope:** + - retry behavior and config keys of `local_db` and `auth_db`; + - outer retry semantics of Agent Trace read-only queries (`query`/`query_values`/`query_map`), `passive_checkpoint`, and non-idempotent Agent Trace writes; + - changing `query.timeout_ms` semantics, and generic `RetryPolicy.timeout_ms` redesign for all databases (follow-up); + - Codex/Claude/OpenCode/Pi hook fail-open semantics; + - any change to PR #297's health-invariant behavior; + - production-wide retry/exhaustion metrics. +- **Constraints:** + - keep `experimental_multiprocess_wal(true)`; + - stay on Turso 0.8.1 and use the existing `rand` dependency for jitter (no new crates); + - retries always wrap whole transaction units, restarting from `BEGIN IMMEDIATE`; a single-statement write is retried as the complete statement, and only where replay is safe by its SQL semantics; + - classify retryability only from typed `turso::Error` through the existing `is_retryable_turso_error`, never by `anyhow` string matching; + - no hook diagnostics on stdout; + - keep existing user-facing error prefixes stable; + - run Cargo only through `scripts/run-cli-cargo.sh` under `nix develop`. +- **Non-goal:** + - no spool, daemon, background worker, queue or custom cross-process lock; + - no Turso replacement and no disabling of multiprocess WAL; + - no indefinite hook waits; + - no blind increases to retry counts or budgets to make tests pass; + - no change to mutation-trace semantics; + - no claim of a hard wall-clock operation timeout. + +## Assumptions + +- PR #299 (`agent-trace-db-write-contention`) is stacked on `mutation-trace-health-invariant` (PR #297). After #297 merges, the branch is rebased onto the updated `main` before final merge if necessary. +- Historical (T01, complete): T01 recovered the original investigation harness from a local Nix flake-source snapshot and committed it to the branch. `lock_contention_tests.rs` is now version-controlled and registered as a `#[cfg(test)]` module in `agent_trace_db/mod.rs`. No later task depends on a Nix store path. +- Initial defaults, to be tuned only from measured evidence: + - `busy_timeout_ms = 500`; + - `contention_deadline_ms = 1250`; + - outer `max_attempts = 2`; + - full-jitter backoff `random(0..=100 ms)`. +- Current defaults, tuned in T08 from the failed final validation run and the T08 measurement campaign: + - `busy_timeout_ms = 1000`; + - `contention_deadline_ms = 2250`; + - outer `max_attempts = 2` (unchanged); + - full-jitter backoff `random(0..=100 ms)` (unchanged). +- Retry-start rule: after a typed `Busy`/`BusySnapshot`, compute `backoff = jitter(0..=cap)` and `remaining = contention_deadline - (now - operation_start)`. Launch the next attempt only if attempts remain and `remaining >= backoff + busy_timeout`; otherwise fail. With the current defaults, a second attempt is possible only if the first one returned within about 1150–1250 ms (650–750 ms under the initial 500 / 1250 defaults). +- Upper bounds: `busy_timeout_ms <= 10_000` and `contention_deadline_ms <= 30_000`. +- Config contract (decided): `busy_timeout_ms` and `contention_deadline_ms` are Agent Trace DB contention settings, supported only as `policies.database_retry.agent_trace_db.{busy_timeout_ms,contention_deadline_ms}`, beside that object's existing `connection_open`/`query` keys: + + ```json + { + "policies": { + "database_retry": { + "agent_trace_db": { + "busy_timeout_ms": 1000, + "contention_deadline_ms": 2250 + } + } + } + } + ``` + + `local_db` and `auth_db` keep their existing `connection_open`/`query` keys only and reject the two new keys; ignored configuration is never silently accepted. This specialization is feasible within the current architecture: the Pkl schema adds a dedicated `agentTraceDbRetrySchema` object (the shared `perDbRetrySchema` properties plus the two keys) used only for `agent_trace_db`, and the Rust `build_per_db` closure in `config/schema.rs` passes a per-DB allowed-key list to the existing `validate_object_keys`. If T03 finds this specialization disproportionately complex, it stops and records the blocker; it must not broaden the public config contract to `local_db`/`auth_db`. +- The outer `max_attempts` and backoff cap for the Agent Trace write-contention retry are fixed constants in this plan, not new config keys. `query.*` overrides keep their current meaning for the generic retry path, which remains the retry path for every Agent Trace operation outside the T04 write set. +- Write-contention retry scope (decided). Turso's `busy_timeout` is connection-wide; the SCE outer write-contention retry applies only to write-capable Agent Trace operations whose complete retry unit is known to be safe: + - the transactional insert-pair primitive used by `insert_conversation_text_event`. Retry unit: `BEGIN IMMEDIATE` → check message → insert message → insert part → `COMMIT`; + - the transactional mutation-trace CAS batch. Retry unit: `BEGIN IMMEDIATE` → CAS statements → `COMMIT`; + - single-statement Agent Trace writes whose replay is safe by existing SQL semantics, invoked through an explicit opt-in write entrypoint (not by changing generic `execute`): + - `INSERT_REPOSITORY_METADATA_SQL` (`ON CONFLICT (id) DO NOTHING`); + - `CLAIM_SOURCE_INSTANCE_ID_SQL` (guarded `UPDATE … WHERE source_instance_id = ''`); + - mutation-trace `INSERT_WORKTREE_IF_ABSENT_SQL`, `INSERT_SCOPE_IF_ABSENT_SQL` and `INSERT_SCOPE_PROVENANCE_IF_ABSENT_SQL` (`ON CONFLICT … DO NOTHING`). + + The repository-metadata writes keep the outer retry only on the initialization path that still writes after T06. A retry never re-runs an individual statement inside a transaction unit (for example only the second insert or only `COMMIT`); it restarts the whole unit from `BEGIN IMMEDIATE`. +- Explicitly unchanged in this PR (keep the existing generic `run_with_retry_sync` query policy): + - read-only `query`, `query_values` and `query_map`, because there is no evidence the read paths need the write-contention policy; + - `passive_checkpoint`. It is documented as never blocking on readers or writers, it is best-effort post-commit maintenance whose failure is already logged and swallowed (`sce.agent_trace_db.passive_checkpoint_failed`), and no investigation evidence shows it losing work to writer contention; + - append-only single-statement writes with no conflict clause (`INSERT_DIFF_TRACE_SQL`, `INSERT_POST_COMMIT_PATCH_INTERSECTION_SQL`, `INSERT_AGENT_TRACE_SQL`, multi-row `insert_parts`), the last-writer `UPSERT_CLAUDE_MODEL_STATE_SQL`, the sync/export `insert_messages` batches, and migration statements. Widening the policy to them needs separate proof of replay safety. +- Production observability of contention exhaustion is the returned structured error. Existing hook fail-open handlers log it through the configured SCE `Logger` (log file / stderr per logger config), and the plan does not claim it is otherwise user-visible. The structured `tracing` event `sce.agent_trace_db.contention_exhausted` is kept as a telemetry instrumentation point for future telemetry work. Production runs with `NoopTelemetry` and has no tracing subscriber, so the raw event is not persisted during normal CLI execution. No `Logger` dependency is added to `TursoDb`, and no telemetry runtime or tracing-to-`Logger` bridge is added in this PR. Process-local atomic counters (attempts, outer retries, exhaustions) are test instrumentation only. They are meaningful inside the in-process Rust suite, not across hook processes, and are not production-wide telemetry. +- Turso 0.8.1 does not expose how often or how long its busy handler waited, so no busy-handler wait count is reported. Measurements cover outer attempts and retries, contention exhaustions and latency only. +- The only existing statement-count seam is the `#[cfg(test)]` `count_read_statements`. T06 adds a parallel `count_write_statements` seam with the same thread-local pattern rather than asserting on SQL strings. +- `hook_open_metadata_write_exhausts_retry_budget_while_write_lock_is_held` characterizes the bug being fixed. T06 converts it into a positive test (an initialized hook open succeeds under a held write lock) instead of deleting it. + +## Task stack + +- [x] T01: `Restore and wire the lock-contention suite and record the baseline` (status:done) + - Task ID: T01 + - Scope: In: + - restore `cli/src/services/agent_trace_db/lock_contention_tests.rs` unchanged from the investigation's local Nix flake-source snapshot (historical; done, and the file is now version-controlled); + - register `#[cfg(test)] mod lock_contention_tests;` in `agent_trace_db/mod.rs`; + - extend the suite's reporting with per-write p50/p95/p99/max latency for the Rust API and hook-process tests; + - run the AC8/AC9 commands against current behavior and record the baseline numbers in this task's completion record. + Out — any production code change; relaxing or removing any existing assertion. + - Dependencies: none + - Done when: the suite is tracked and compiles as part of the test build; the non-ignored tests pass on unfixed code; the baseline table (N, rounds, lost events, lock errors, orphans, duplicates, latency percentiles) is recorded for the Rust API and hook-process runs. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml lock_contention`; the AC8/AC9 commands in non-strict mode to capture the baseline. + - Completed: 2026-10-04 + - Files changed: + - `cli/src/services/agent_trace_db/lock_contention_tests.rs` (restored verbatim from the Nix store copy, sha256 `e94912a71fc594af…`, then extended with latency reporting; staged in git so flake builds include it) + - `cli/src/services/agent_trace_db/mod.rs` (`#[cfg(test)] mod lock_contention_tests;`) + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - The suite compiles in the test build and in `nix build .#default` / the flake clippy and fmt checks. + - Added a nearest-rank `latency_percentiles` helper (with the unit test `lock_contention_latency_percentiles_use_nearest_rank`) and p50/p95/p99/max columns to the in-process level report and to the hook-process report. + - Hook-process latency is now measured per process, from stdin release to process exit; each child is awaited on its own feeder thread. + - No production code changed and no existing assertion was changed. + - Deviation: the baseline did **not** reproduce ingestion loss at N=2–4, either in-process or with real hook processes, on this host (release builds, the plan's round counts). Loss reproduces at N=8: 1525 of 4000 distinct events lost. The single-writer lock-budget boundary confirms the root cause: any `BEGIN IMMEDIATE` hold of ≥300 ms exhausts `QUERY_RETRY_POLICY` after about 280 ms with no busy handler. The N=2–4 strict gates in AC8/AC9 already pass before the fix, so they act as regression gates; the 8-writer and boundary rows are the before-fix signal. + - Baseline (before fix, release build, non-strict, 2026-10-04, HEAD `5f77cf2d` + T01 test changes): + - Lock-budget boundary (debug build, single writer vs. a `BEGIN IMMEDIATE` holder): + + | hold ms | outcome | elapsed ms | rows (msg/part) | + | --- | --- | --- | --- | + | 50 | Ok(true) | 91 | 1/1 | + | 100 | Ok(true) | 194 | 1/1 | + | 200 | Ok(true) | 294 | 1/1 | + | 300 | database is locked (5 attempts) | 279 | 0/0 | + | 500 | database is locked (5 attempts) | 280 | 0/0 | + | 1000 | database is locked (5 attempts) | 280 | 0/0 | + + - Hook-open metadata under a 500 ms write lock: the schema check passes, and the metadata `INSERT … ON CONFLICT DO NOTHING` fails with `database is locked` after 5 attempts (276 ms). + - Rust production API (`insert_conversation_text_event`, per-write latency): + + | mode | N | rounds | lost events | lock errors | orphans | duplicates | p50 ms | p95 ms | p99 ms | max ms | + | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | + | distinct | 2 | 1000 | 0 | 0 | 0 | 0 | 37 | 107 | 138 | 228 | + | distinct | 3 | 1000 | 0 | 0 | 0 | 0 | 68 | 207 | 231 | 372 | + | distinct | 4 | 500 | 0 | 0 | 0 | 0 | 87 | 212 | 318 | 384 | + | duplicate | 2 | 500 | 0 | 0 | 0 | 0 | 20 | 33 | 34 | 39 | + | duplicate | 3 | 500 | 0 | 0 | 0 | 0 | 44 | 135 | 207 | 2303 | + | duplicate | 4 | 500 | 0 | 0 | 0 | 0 | 83 | 190 | 286 | 347 | + | distinct (stress) | 8 | 500 | 1525 | 1525 (500/500 rounds) | 0 | 0 | 275 | 288 | 298 | 410 | + | duplicate (stress) | 8 | 500 | 0 | 1545 (500/500 rounds) | 0 | 0 | 275 | 284 | 296 | 351 | + + - Real release `sce hooks codex` processes (`UserPromptSubmit`, distinct events, per-process latency): + + | N | rounds | expected | persisted | lost | orphans | non-zero exits | stderr lines | p50 ms | p95 ms | p99 ms | max ms | + | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | + | 2 | 500 | 1000 | 1000 | 0 | 0 | 0 | 0 | 35 | 45 | 49 | 51 | + | 3 | 500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 45 | 105 | 212 | 245 | + | 4 | 200 | 800 | 800 | 0 | 0 | 0 | 0 | 50 | 196 | 197 | 231 | + + - Orphans and duplicates are enforced by the suite's per-round assertions (messages == parts; Ok(true) count == persisted rows; ≤1 insert per duplicate round). All runs passed them. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml lock_contention`: passed (3 passed, 3 ignored). + - AC8 commands (non-strict, release): `concurrent_distinct_events` with `WRITERS=2,3 ROUNDS=1000` and with `WRITERS=4 ROUNDS=500`; `concurrent_duplicate_delivery` with `WRITERS=2,3,4 ROUNDS=500`; `WRITERS=8 ROUNDS=500` for both. All ran and passed; results are in the table above. + - AC9 commands (non-strict, release): `nix build .#default`, then `concurrent_real_codex_hook_processes` with `WRITERS=2` / `3` and `ROUNDS=500`, and with `WRITERS=4 ROUNDS=200`. All ran and passed; results are in the table above. + - Additional: `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt` passed. + - Post-completion amendment (plan/harness correction before T02): the strict hook-process assertions now match AC9. Per level, `persisted_messages`/`persisted_parts <= expected` is always asserted. Under strict mode, both must equal `expected`, non-zero exits must be 0, and `total_lost == 0` and `total_nonzero_exits == 0` hold across all levels. The in-process strict check now also asserts `lost_events == 0` per level. Two unit tests cover the hook-level assertion helper (`lock_contention_hook_level_assertions_*`). No baseline number changed. + - Context impact: none to durable context. This is a test-only change plus plan evidence. The baseline numbers feed the `context/sce/agent-trace-db.md` measured-evidence summary planned for later tasks. + - Context synchronization: synced + +- [x] T02: `Configure Turso busy timeout on Agent Trace DB connections` (status:done) + - Task ID: T02 + - Scope: In: + - one named default constant (`AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 500`) resolved per `DbSpec` through a single resolver; + - call `connection.busy_timeout(...)` on the connection returned by `connect()` on every local `TursoDb` open path, so the same connection later runs `Transaction::new_unchecked(.., Immediate)` → `BEGIN IMMEDIATE`, keeping `experimental_multiprocess_wal(true)`; + - zero leaves the handler unset; + - a doc comment explaining the distinction between multiprocess WAL and busy timeout. + Behavioral tests (Turso 0.8.1 has no busy-timeout getter): + - (a) connection A holds `BEGIN IMMEDIATE` ~100 ms; a control connection with busy timeout 0 gets `Busy` promptly; + - (b) under the same hold, a production `insert_conversation_text_event` on an Agent Trace DB connection waits and succeeds, with no test-level retry around the call and an elapsed time showing it waited for the holder. + Out — config-file surface (T03); outer-retry redesign and attempt counters (T04); `local_db`/`auth_db` defaults. + - Dependencies: T01 + - Done when: AC1 and AC2 hold; Agent Trace DB writes wait for transient lock contention inside Turso; the other DBs are unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml busy_timeout`; `... test --manifest-path cli/Cargo.toml agent_trace_db`. + - Completed: 2026-10-04 + - Files changed: + - `cli/src/services/db/mod.rs` + - `cli/src/services/agent_trace_db/lock_contention_tests.rs` + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - Added `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 500` and the single resolver `resolve_busy_timeout::()`. It returns 500 ms for `agent_trace_db` and zero for every other `DbSpec`. T03 feeds config into this resolver. + - `apply_busy_timeout` calls `Connection::busy_timeout(...)` on the connection returned by `connect()` inside `TursoDb::open_without_migrations_at`, the only local `TursoDb` open path (`new`, `new_at` and `open_without_migrations` all delegate to it). A zero timeout leaves the handler unset. `experimental_multiprocess_wal(true)` is unchanged. + - The resolver's doc comment explains how multiprocess WAL differs from the busy timeout. + - The encrypted `EncryptedTursoDb::new` path is `auth_db` only and does not use multiprocess WAL; it is untouched. `local_db` and `auth_db` resolve to zero, so their behavior is unchanged. + - Tests: + - `busy_timeout_resolves_default_for_agent_trace_db_and_zero_for_other_dbs`. + - `busy_timeout_unset_connection_returns_busy_promptly_on_begin_immediate` (control): a raw `BEGIN IMMEDIATE` with no SCE retry gets `turso::Error::Busy` in under half the hold. + - `busy_timeout_agent_trace_connection_waits_for_begin_immediate_holder`: a raw `BEGIN IMMEDIATE` on an Agent Trace connection waits for the holder and succeeds. + - `busy_timeout_production_insert_waits_for_begin_immediate_holder`: under a 100 ms hold, `insert_conversation_text_event` succeeds with no test-level retry, writes 1/1 rows, and its elapsed time is at least half the hold. + - Deviation: the two raw-connection tests hold the lock for 300 ms instead of ~100 ms. 300 ms is beyond the pre-fix generic retry budget, so the waiting test fails without the busy handler. + - Deviation (needed to keep existing non-ignored tests valid): the T01 characterization tests assumed the ~280 ms pre-fix budget. With a 500 ms busy timeout under the unchanged generic 5-attempt query retry, the interim worst case is about 2.8 s. So `LOCK_HOLD_DURATIONS_MS` gains a 4 000 ms hold and `RELIABLY_BEYOND_RETRY_BUDGET_MS` becomes 4 000. `hook_open_metadata_write_exhausts_retry_budget_while_write_lock_is_held` therefore holds for 4 s. Both tests are still recalibrated or converted by T07/T06. No assertion was removed or weakened in kind. + - Interim lock-budget boundary (debug build): + + | hold ms | outcome | elapsed ms | rows (msg/part) | + | --- | --- | --- | --- | + | 50 | Ok(true) | 70 | 1/1 | + | 100 | Ok(true) | 119 | 1/1 | + | 200 | Ok(true) | 244 | 1/1 | + | 300 | Ok(true) | 344 | 1/1 | + | 500 | Ok(true) | 521 | 1/1 | + | 1000 | Ok(true) | 1043 | 1/1 | + | 4000 | database is locked (5 attempts) | 2784 | 0/0 | + + Before the fix, 300, 500 and 1000 ms holds failed at about 280 ms. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml busy_timeout`: passed (4 passed). + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db`: passed (49 passed, 3 ignored). + - Additional: `... test --manifest-path cli/Cargo.toml lock_contention -- --nocapture` passed (6 passed, 3 ignored) and produced the table above. `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt` passed. + - Context impact: localized behavior change. Agent Trace DB connections now carry a 500 ms Turso busy timeout. The planned `context/sce/shared-turso-db.md` layering text (multiprocess WAL vs. `busy_timeout_ms`) applies; the config key and outer-retry layers arrive in T03/T04. + - Context synchronization: synced + +- [x] T03: `Expose busy_timeout_ms and contention_deadline_ms in database_retry config` (status:done) + - Task ID: T03 + - Scope: In: + - add `busy_timeout_ms` and `contention_deadline_ms` to the `agent_trace_db` database_retry config document and resolved config only (for example Agent Trace-only optional fields, so `local_db`/`auth_db` documents cannot carry them); + - in `config/pkl/base/sce-config-schema.pkl`, add a dedicated `agentTraceDbRetrySchema` (the `perDbRetrySchema` `connection_open`/`query` properties plus the two keys, `additionalProperties = false`) used only for `["agent_trace_db"]`; `local_db`/`auth_db` keep `perDbRetrySchema` unchanged; + - in `config/schema.rs` `build_per_db`, pass a per-DB allowed-key list to the existing `validate_object_keys`: `connection_open, busy_timeout_ms, contention_deadline_ms, query` for `agent_trace_db`, and `connection_open, query` for `local_db`/`auth_db`; + - validate the values (non-negative integers, upper bounds 10_000 / 30_000, documented zero semantics) with stable error text; + - feed `busy_timeout_ms` into the T02 resolver; + - put the `contention_deadline_ms` default constant (`1250`) in one place for T04 to consume; + - render both in `sce config show` text and JSON for `agent_trace_db` only; + - resolver/schema/render tests: + - `agent_trace_db` accepts both keys and overrides the defaults; + - out-of-range and wrong-type values are rejected; + - `local_db.busy_timeout_ms`, `local_db.contention_deadline_ms`, `auth_db.busy_timeout_ms` and `auth_db.contention_deadline_ms` are each rejected with the existing `failed schema validation` error naming the key; + - the generated schema's `local_db`/`auth_db` objects list only `connection_open`/`query`; + - `query.timeout_ms` parsing and rendering are unchanged (regression). + Out — any change to `query.timeout_ms`/`connection_open` semantics; new top-level config keys; outer-retry behavior; any `local_db`/`auth_db` key additions. If per-DB specialization proves disproportionately complex, stop and record the blocker instead of broadening the contract. + - Dependencies: T02 + - Done when: AC10 holds; configured values override the defaults for Agent Trace DB connections; invalid values are rejected; `local_db`/`auth_db` reject both keys. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml database_retry`; `nix run .#pkl-check-generated`. + - Completed: 2026-10-04 + - Files changed: + - `config/pkl/base/sce-config-schema.pkl` + - `cli/src/services/config/types.rs` + - `cli/src/services/config/schema.rs` + - `cli/src/services/config/render.rs` + - `cli/src/services/db/mod.rs` + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - Pkl: a new `agentTraceDbRetrySchema` (`connection_open`, `query`, `busy_timeout_ms` 0..=10000 default 500, `contention_deadline_ms` 0..=30000 default 1250, `additionalProperties = false`) is used only for `agent_trace_db`. `local_db`/`auth_db` keep `perDbRetrySchema` unchanged. + - Types: `DatabaseRetryConfig.agent_trace_db` is now `AgentTraceDbRetryConfig { retry: PerDbRetryConfig, busy_timeout_ms, contention_deadline_ms }`. `PerDbRetryConfig` (used by `local_db`/`auth_db`) cannot carry the new keys. Upper-bound constants are `AGENT_TRACE_DB_BUSY_TIMEOUT_MAX_MS = 10_000` and `AGENT_TRACE_DB_CONTENTION_DEADLINE_MAX_MS = 30_000`. + - Parsing: `map_database_retry_config` passes a per-DB allowed-key list to `validate_object_keys`: `connection_open, busy_timeout_ms, contention_deadline_ms, query` for `agent_trace_db`, and `connection_open, query` for `local_db`/`auth_db`. A Rust bounds backstop rejects values above the maximum with `Config key 'policies.database_retry.agent_trace_db.' in '' must be <= .`. Generated-schema validation runs first and catches out-of-range, negative and wrong-type values with the existing `failed schema validation` error. + - Resolution (`db/mod.rs`): `resolve_busy_timeout` now reads the configured `busy_timeout_ms` and falls back to 500; 0 leaves the busy handler unset. Added `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS = 1_250` and `resolve_contention_deadline::()` (zero for other DBs) for T04 to consume. It is `#[cfg_attr(not(test), allow(dead_code))]` until T04 wires it in. Pure `*_from_config` helpers make the override testable without the global `OnceLock`. + - Rendering: `sce config show` text and JSON output include `busy_timeout_ms` / `contention_deadline_ms` for `agent_trace_db` when they are configured. Like the existing per-DB overrides, they are omitted when unset. `query`/`connection_open` rendering is unchanged. + - Tests (all prefixed `database_retry`): accepts both keys; accepts 0 and the upper bounds; omitted keys stay unset; rejects out-of-range values; rejects wrong-type values; `local_db`/`auth_db` reject both keys with the `failed schema validation` error naming the key and path; the generated schema lists only `connection_open`/`query` for `local_db`/`auth_db`; the Rust per-DB key backstop keeps its `contains unknown key` wording; a regression test shows `query.timeout_ms` parsing is unchanged; resolver override, zero and default cases; JSON/text render for `agent_trace_db` only. + - Assumption: the versioned release schema snapshots under `schema/v*/config.json` are written at release bump and are left untouched. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml database_retry`: passed (14 passed). + - `nix run .#pkl-check-generated`: passed (ephemeral Pkl generation passed, 142 files). + - Additional: `... test --manifest-path cli/Cargo.toml busy_timeout` passed (5 passed); `... test --manifest-path cli/Cargo.toml config` passed (116 passed); `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt` passed. + - Context impact: localized config-contract change. There are two new Agent Trace-only keys under `policies.database_retry.agent_trace_db`, and `local_db`/`auth_db` reject them. This affects `context/cli/config-precedence-contract.md` and `context/sce/shared-turso-db.md` (config layer for `busy_timeout_ms`/`contention_deadline_ms`). Outer-retry behavior is not changed yet (T04). + - Context synchronization: synced + +- [x] T04: `Add Agent Trace DB write-contention retry for safe write units` (status:done) + - Task ID: T04 + - Scope: In — an Agent Trace DB write-contention retry seam, applied only when `M::db_config_key() == "agent_trace_db"` and only to the write-capable operations enumerated under "Write-contention retry scope" in Assumptions: + - `execute_transactional_insert_pair_if_absent` (used by `insert_conversation_text_event`); the retry unit is the whole `BEGIN IMMEDIATE` → check message → insert message → insert part → `COMMIT` transaction; + - `execute_transactional_cas_batch`; the retry unit is the whole `BEGIN IMMEDIATE` → CAS statements → `COMMIT` transaction; + - a new explicit opt-in single-statement write entrypoint (for example `execute_idempotent_write`) that retries the complete statement. Only the enumerated replay-safe writes switch to it: repository-metadata `INSERT … ON CONFLICT DO NOTHING` and the guarded source-instance claim `UPDATE`, plus the mutation-trace `INSERT_WORKTREE_IF_ABSENT_SQL`, `INSERT_SCOPE_IF_ABSENT_SQL` and `INSERT_SCOPE_PROVENANCE_IF_ABSENT_SQL`. Generic `execute` keeps the generic retry for every other caller. + + Policy: + - default `max_attempts = 2`; + - full-jitter backoff `random(0..=cap)` behind an injectable jitter source (seeded/fixed in tests); + - the retry-start rule from Assumptions (`remaining >= backoff + busy_timeout`, never after deadline expiry), measured from operation start so time spent in Turso's busy wait counts; + - only typed `Busy`/`BusySnapshot` retried via `is_retryable_turso_error`, with the opt-in single-statement wrapper keeping the typed `turso::Error` long enough to classify before converting to `anyhow`; + - deterministic errors fail once with no sleep; + - a retry never re-runs an individual statement of a transaction unit (never only the second insert, never only `COMMIT`); + - `#[cfg(test)]`-oriented process-local counters for attempts, outer retries and exhaustions. + + Doc comments state that Turso's `busy_timeout` is connection-wide while the outer write-contention retry is scoped to safe write units, and that the deadline schedules retries and does not interrupt a running operation. + + Unit tests (prefix `agent_trace_db_write_contention_retry`): + - each policy property; + - the retry-start boundary cases; + - the ~100 ms-hold case asserting `attempts = 1` and `outer_retries = 0`; + - a regression assertion that Agent Trace `query`/`query_map` still use the generic query retry policy. + + Out: + - `local_db`/`auth_db` retry behavior; + - generic `run_with_retry_sync` semantics; + - read-only `query`/`query_values`/`query_map`; + - `passive_checkpoint` (it stays on its existing behavior unless concrete writer/checkpoint-lock evidence appears, which would need its own justification and test); + - append-only or last-writer single-statement writes; + - the exhaustion error/event shape (T05). + - Dependencies: T03 + - Done when: AC3 holds; the default never yields more than 2 attempts on the enumerated write operations and never starts an attempt in violation of the retry-start rule; reads and non-enumerated writes keep their existing retry policy; existing `agent_trace_db` and `mutation_trace` tests pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db_write_contention_retry`; `... agent_trace_db`; `... mutation_trace`; `... resilience`. + - Completed: 2026-10-04 + - Files changed: + - `cli/src/services/db/mod.rs` + - `cli/src/services/agent_trace_db/repository.rs` + - `cli/src/services/mutation_trace/store.rs` + - `cli/src/services/agent_trace_db/lock_contention_tests.rs` + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - Policy: `WriteContentionPolicy` holds `max_attempts`, `backoff_cap`, `busy_timeout` and `contention_deadline`. `write_contention_policy::()` returns `Some` only for `agent_trace_db`. It uses the fixed constants `AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS = 2` and `AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS = 100`, plus the T02/T03 resolvers for `busy_timeout` and `contention_deadline`. `resolve_contention_deadline` is now live, and its `dead_code` allowances are removed. + - `run_with_write_contention_retry` is the production wrapper. It uses `rand::thread_rng()` for full-jitter backoff `0..=cap`, `std::thread::sleep`, and a monotonic `std::time::Instant` started at the operation start. It delegates to the private `run_with_write_contention_retry_using`, which takes the backoff source, the sleep function and an elapsed-time clock (`FnMut() -> Duration`) as injected parameters. This seam is not public. + - Retry admission is checked at two points, with `remaining = contention_deadline - elapsed` and `elapsed` measured from the operation start: + 1. Before the backoff sleep, `write_contention_retry_may_sleep` requires enough budget for the backoff plus a full busy-timeout wait: `remaining >= backoff + busy_timeout`. + 2. After the backoff sleep has actually returned, with elapsed time recomputed, `write_contention_retry_may_start_now` requires enough budget for a full busy-timeout wait: `remaining >= busy_timeout`. Equality still admits the attempt. + + Neither check admits anything once the deadline has expired, and failing either one exhausts the policy. Scheduler oversleep therefore cannot start a new attempt outside the admission contract. A running attempt is never interrupted. + - `outer_retries` is incremented only after the post-sleep check succeeds, so it counts additional attempts actually admitted. + - retries only `WriteAttemptFailure::Retryable`, which is typed `Busy`/`BusySnapshot` via `is_retryable_turso_error`; deterministic failures return after one attempt with no sleep. + - `CasBatchFailure` is renamed `WriteAttemptFailure` and gains `into_error`. + - Retry units: + - `execute_insert_pair_if_absent_body` now classifies each typed `turso::Error` with `classify_turso_error`, keeping the existing message wording. + - `execute_transactional_insert_pair_if_absent` and `execute_transactional_cas_batch` build one attempt closure that runs the whole `BEGIN IMMEDIATE` → `COMMIT` unit. They route it through the contention retry for Agent Trace; every other DB stays on the unchanged generic `run_with_retry_sync` path. + - New `TursoDb::execute_idempotent_write` runs the complete statement under the contention policy on Agent Trace and delegates to `execute` everywhere else. + - Only the five enumerated writes switch to `execute_idempotent_write`: `INSERT_REPOSITORY_METADATA_SQL` and `CLAIM_SOURCE_INSTANCE_ID_SQL` in `repository.rs`, and `INSERT_WORKTREE_IF_ABSENT_SQL`, `INSERT_SCOPE_IF_ABSENT_SQL` and `INSERT_SCOPE_PROVENANCE_IF_ABSENT_SQL` in `store.rs`. + - Reads, `passive_checkpoint`, generic `execute` and migrations are unchanged. + - Deviation (user request): the code carries no comments. The planned doc comments were removed, and the `busy_timeout`-scope and deadline-semantics statements live in `context/sce/shared-turso-db.md` instead. + - Interim exhaustion error (T05 reshapes it): `Operation '' failed after attempt(s) under write contention (busy_timeout=…ms, contention_deadline=…ms, elapsed=…ms). Last error: . Try: `. The cause keeps Turso's `database is locked` text. + - Test instrumentation: `#[cfg(test)]` `WriteContentionCounts { attempts, outer_retries, exhaustions }` read through `count_write_contention`. + - Tests (prefix `agent_trace_db_write_contention_retry`): + - seeded jitter is reproducible and stays within the cap; + - retry-start boundaries: remaining just above, equal to and just below `backoff + busy_timeout`, plus an expired deadline and a zero deadline; + - post-sleep admission boundaries: remaining just above, equal to and just below `busy_timeout`, plus an expired deadline; + - an oversleep regression on a fake clock (`busy_timeout = 500`, `contention_deadline = 1250`, Busy at 690 ms, backoff 50 ms, sleep ends at 810 ms): attempt 2 never runs, giving `attempts = 1`, `outer_retries = 0`, `exhaustions = 1` and a contention-exhaustion error; + - the adjacent case where the sleep ends at 750 ms, so post-sleep remaining equals `busy_timeout`: attempt 2 is admitted, giving `attempts = 2` and `outer_retries = 1`; + - `Busy` and `BusySnapshot` are each retried once; + - deterministic errors (`Constraint`, `Misuse`, `Readonly`) fail once and never sleep; + - the attempt cap of 2 under persistent `Busy`; + - no retry where the rule disallows one; + - the policy applies only to `agent_trace_db`; + - regression: Agent Trace `query`/`query_values`/`query_map`/`passive_checkpoint`/`execute` resolve `QUERY_RETRY_POLICY` and record zero contention attempts; + - the 100 ms-hold production `insert_conversation_text_event` succeeds with `attempts = 1`, `outer_retries = 0`, `exhaustions = 0`. + - Deviation: the counters are thread-local, following the `count_read_statements` pattern, not global atomics. Parallel tests cannot skew each other, and T07 writer threads can read their own counts. + - Interim lock-budget boundary (debug build; T07 recalibrates it): + + | hold ms | outcome | elapsed ms | rows (msg/part) | + | --- | --- | --- | --- | + | 50 | Ok(true) | 81 | 1/1 | + | 100 | Ok(true) | 128 | 1/1 | + | 200 | Ok(true) | 260 | 1/1 | + | 300 | Ok(true) | 350 | 1/1 | + | 500 | Ok(true) | 533 | 1/1 | + | 1000 | Ok(true) | 1081 | 1/1 | + | 4000 | database is locked (2 attempts) | 1007 | 0/0 | + + The 4 000 ms hold now fails cleanly after 2 attempts in about 1.0 s; after T02 it took about 2.8 s and 5 attempts. The hook-open metadata write under a 4 000 ms hold now fails after 2 attempts in 1 054 ms. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db_write_contention_retry`: passed (9 passed). + - `... agent_trace_db`: passed (66 passed, 3 ignored). + - `... mutation_trace`: passed (389 passed). + - `... resilience`: passed (6 passed). + - Additional: `... lock_contention -- --nocapture` passed (7 passed, 3 ignored); `... services::db::` passed (35 passed); `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt` passed. + - Context impact: localized behavior change in the shared Turso DB layer. Agent Trace write units now have the outer write-contention retry layer, and there is a new opt-in `execute_idempotent_write` entrypoint. This affects `context/sce/shared-turso-db.md` (the outer-retry layer and its scope; the stale "5 attempts" Agent Trace text) and `context/sce/agent-trace-db.md` (the contention contract). + - Context synchronization: synced + +- [x] T05: `Make exhausted Agent Trace DB contention failures observable` (status:done) + - Task ID: T05 + - Scope: In — when the T04 policy is exhausted: + - return an error carrying `db_name`, `operation`, `attempts`, `busy_timeout_ms`, `contention_deadline_ms`, `elapsed_ms` and `cause` (database busy / busy timeout exhausted), worded so the deadline is described as a retry-scheduling cutoff; + - emit one structured `tracing::warn!` event `sce.agent_trace_db.contention_exhausted` with the same fields, as a telemetry instrumentation point (never stdout; it has no production sink until a tracing subscriber is installed); + - increment the test exhaustion counter. + Add a test asserting the error fields and the captured event. Out — changing hook fail-open behavior; a metrics system; claims of user visibility when logging is not configured. + - Dependencies: T04 + - Done when: AC4 holds; the stdout of the hook commands is unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db_contention_exhausted`; `... test --manifest-path cli/Cargo.toml hooks`. + - Completed: 2026-10-04 + - Files changed: + - `cli/src/services/db/mod.rs` + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - `WriteContentionPolicy` now carries `db_name` (from `M::db_name()`). + - Exhaustion goes through a new `contention_exhausted_error` helper. It computes `elapsed` once, increments the `#[cfg(test)]` exhaustion counter, emits the event, and returns the error. + - Error text: `Operation '' failed after attempt(s) under write contention (db_name=, operation=, attempts=, busy_timeout_ms=, contention_deadline_ms= [no retry is scheduled past this cutoff], elapsed_ms=, cause=database busy (busy timeout exhausted)). Last error: . Try: `. + - The T04 prefix is unchanged. + - The deadline is worded as a retry-scheduling cutoff. + - Turso's `database is locked` text stays in `Last error`. + - Event: one `tracing::warn!(target: "sce", event_id = "sce.agent_trace_db.contention_exhausted", …)` with the fields `db_name`, `operation`, `attempts`, `busy_timeout_ms`, `contention_deadline_ms`, `elapsed_ms`, `cause` and `last_error`, following the `sce.resilience.retry` pattern. There are no stdout writes, and hook fail-open behavior is unchanged. + - The constants `CONTENTION_EXHAUSTED_EVENT_ID` and `CONTENTION_EXHAUSTED_CAUSE` hold the stable strings. + - Tests (prefix `agent_trace_db_contention_exhausted`). They use a minimal in-test capturing `tracing::Subscriber` installed with `tracing::subscriber::with_default`; no new crates. + - The exact error text and every event field, plus target `sce` and level `WARN`, on the fake-clock oversleep scenario (`elapsed_ms=810`). + - Exactly one event, with `attempts=2`, after the attempt cap. + - No event for success-after-retry or for deterministic errors. + - The T04 oversleep test now asserts `elapsed_ms=810`; it previously asserted the interim `elapsed=810ms`. + - Observability scope boundary (amended 2026-10-05): + - The `tracing` event is kept as an instrumentation point for future telemetry/OTEL integration. + - Production currently uses `NoopTelemetry`, so the raw event itself is not persisted during normal CLI execution. `sce.resilience.retry` behaves the same way. + - The returned error carries every AC4 field. Existing hook fail-open handlers already pass it to the configured SCE `Logger` through `log.error(...)` (for example `sce.hooks.codex.error` and the conversation-trace hook). + - No `Logger` dependency was added to `TursoDb`, and no telemetry runtime was added in this PR. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db_contention_exhausted`: passed (3 passed). + - `... test --manifest-path cli/Cargo.toml hooks`: passed (806 passed, 1 ignored). + - Additional: + - `... agent_trace_db`: passed (72 passed, 3 ignored). + - `... mutation_trace`: passed (389 passed). + - `... services::db::`: passed (41 passed). + - `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt`: passed. + - Inspection: the diff adds no `print!`/`println!`/stdout writes. + - Amendment (2026-10-05): AC4, the observability assumption and this task's scope were reworded to match the production observability above. T05 runtime code is unchanged. `context/sce/cli-observability-contract.md` was corrected so it no longer claims that app runtime installs a production tracing subscriber. + - Context impact: localized observability change. The exhaustion error shape and the `sce.agent_trace_db.contention_exhausted` event affect `context/sce/shared-turso-db.md` and `context/sce/agent-trace-db.md` (the contention contract: what gets reported on exhaustion and where it is visible). + - Context synchronization: synced + +- [x] T06: `Make hook-runtime repository metadata open read-only when initialized` (status:done) + - Task ID: T06 + - Scope: In: + - restructure `verify_or_initialize_repository_metadata` to `SELECT` first: when the row exists with a matching `repository_id` and a valid `source_instance_id`, return with no write; a repository-ID mismatch stays an error; + - only a missing row or an empty/invalid `source_instance_id` runs the existing `INSERT … ON CONFLICT DO NOTHING` / atomic-claim `UPDATE … WHERE source_instance_id = ''` path, followed by a re-read; + - add a `#[cfg(test)]` `count_write_statements` seam in `db/mod.rs` that mirrors `count_read_statements`. + Tests: + - an initialized hook-runtime open issues 0 writes; + - a mismatch still errors; + - an existing valid ID is never overwritten; + - concurrent first opens converge; + - `hook_open_metadata_write_exhausts_retry_budget_while_write_lock_is_held` is converted into a test that an initialized hook open succeeds while another connection holds `BEGIN IMMEDIATE`. + Out — schema/migration changes; setup-path behavior beyond what the shared method changes. + - Dependencies: T01 + - Done when: AC5 holds; all existing metadata tests pass. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml repository_metadata`; `... agent_trace_db`; `... agent_trace_storage`. + - Completed: 2026-10-05 + - Files changed: + - `cli/src/services/agent_trace_db/repository.rs` + - `cli/src/services/agent_trace_db/lock_contention_tests.rs` + - `cli/src/services/db/mod.rs` + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - `verify_or_initialize_repository_metadata` now reads the metadata row first. If the row exists, a `repository_id` mismatch errors with no write. If the `source_instance_id` is valid, the method returns with no write. + - Only a missing row or an empty/invalid `source_instance_id` falls through to the unchanged path: `INSERT … ON CONFLICT DO NOTHING` → re-read → mismatch check → atomic claim → re-read. + - The mismatch check moved into a private `ensure_repository_id_matches` helper. The error text is unchanged. + - `db/mod.rs` gains a `#[cfg(test)]` `count_write_statements` seam that mirrors `count_read_statements`. It counts once per logical call, before any retry wrapper, in `TursoDb::execute`, `execute_idempotent_write`, `execute_transactional_insert_pair_if_absent` and `execute_transactional_cas_batch`. When `execute_idempotent_write` falls back to `execute`, the call is still counted once. `EncryptedTursoDb` is not instrumented. + - New tests in `repository.rs`: + - `initialized_repository_metadata_hook_runtime_open_issues_no_writes`: a hook-runtime open (`open_for_hooks_without_migrations_at` → `ensure_schema_ready_for_hooks` → verify) issues 0 writes and returns the original metadata; the first initialization issues more than 0 writes. + - `mismatched_repository_metadata_errors_without_writes` + - `repository_metadata_with_empty_source_instance_id_is_claimed_once`: an empty placeholder is claimed, and the claimed ID is not overwritten afterwards (0 writes on the next open). + - The existing tests still pass: stable ID across reopen, mismatch, concurrent convergence, and baseline-only migration. + - `hook_open_metadata_write_exhausts_retry_budget_while_write_lock_is_held` became `initialized_hook_open_succeeds_while_write_lock_is_held`. With another connection holding `BEGIN IMMEDIATE` for 4000 ms, the schema check and the metadata open both succeed, return the initialized metadata, issue 0 writes, and finish before the lock is released (observed: 1 ms). + - Per user instruction, the generated code carries no new comments. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml repository_metadata`: passed (5 passed). + - `... agent_trace_db`: passed (75 passed, 3 ignored). + - `... agent_trace_storage`: passed (14 passed). + - Additional: + - `... initialized_hook_open_succeeds -- --nocapture`: passed (`writes=0 after 1ms` under a 4000 ms lock). + - `... concurrent_initialization_converges`: passed. + - `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt`: passed. + - Context impact: localized behavior change. Opening an already-initialized repository Agent Trace DB is now read-only, so it no longer contends for the write lock; this affects `context/sce/agent-trace-db.md` (hook-runtime read-only metadata fast path). There is no config, schema or public CLI contract change. + - Context synchronization: synced + +- [x] T07: `Recalibrate the lock-budget boundary test to the new contention contract` (status:done) + - Task ID: T07 + - Scope: In: + - Deterministic contention contract (primary proof): set `LOCK_HOLD_DURATIONS_MS` to 100/250/500/750/1000/1500/2000 and align the budget constants with the T03/T04 contract. Assert by outcome and retry policy with wide margins: holds ≤ 250 ms succeed, holds ≥ 2000 ms exhaust the contention policy and fail cleanly, and every failure leaves zero message/part rows. Middle holds are characterization only. + - Concurrent regression/stress: N=2–4 strict levels (AC8/AC9) must show zero loss and zero exhaustion. They are regression gates, not the pre-fix reproduction. N=8 is reported as stress characterization. If the fix makes N=8 reliable within acceptable latency, report it, but do not turn N=8 into a supported requirement without a deliberate decision. + - make the strict in-process concurrent tests report outer retries and contention exhaustions from the T04/T05 test counters next to the latency percentiles; + - run the full AC8/AC9 strict matrix plus the N=8 stress runs and record the after-fix measurements next to the T01 baseline in this task's completion record; + - tune the defaults only if the evidence requires it, recording any tuning and its reason. + Out — weakening, skipping or deleting any strict assertion; wall-clock cutoff assertions that assume a running operation is interrupted; raising attempts or budgets beyond what the measurements justify. + - Dependencies: T03, T04, T05, T06 + - Done when: AC7 holds deterministically; the strict N=2–4 matrix passes every AC8/AC9 strict assertion; before-vs-after measurements (p50, p95, p99, max, outer retries and exhaustions where measurable, plus the 8-writer stress run) are recorded. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml lock_budget_boundary -- --nocapture`; the AC8 and AC9 strict commands. + - Completed: 2026-10-05 + - Files changed: + - `cli/src/services/agent_trace_db/lock_contention_tests.rs` + - `context/plans/agent-trace-db-write-contention.md` (this record) + - Result: + - Lock-budget boundary: + - `LOCK_HOLD_DURATIONS_MS` is now 100/250/500/750/1000/1500/2000. The budget constants are `RELIABLY_WITHIN_CONTENTION_BUDGET_MS = 250` and `RELIABLY_BEYOND_CONTENTION_BUDGET_MS = 2_000`, and `WRITE_CONTENTION_MAX_ATTEMPTS = 2` mirrors the T04 cap. + - Each sample wraps the production insert in `count_write_contention`. It reports and asserts attempts, outer retries and exhaustions. + - For every hold: messages equal parts; a failure leaves 0/0 rows; `1 <= attempts <= 2`; `outer_retries + 1 == attempts`. + - Holds of 250 ms or less: `Ok(true)` with 0 exhaustions. + - Holds of 2000 ms or more: a `database is locked` contention-exhaustion error (text contains `under write contention`) with exactly 1 exhaustion. + - The previous `elapsed < hold` wall-clock assertion is removed. + - Middle holds are reported only. + - Concurrent in-process suites: + - Each writer's insert is wrapped in `count_write_contention`. The level report adds attempts, outer-retry and exhaustion columns next to the latency percentiles. + - Every round asserts that no writer exceeds the attempt cap. + - Strict mode additionally asserts 0 exhaustions per level. + - The stale "no outer retry" titles are replaced, and the test functions are renamed to `concurrent_distinct_events_persist_every_event_under_write_contention`, `concurrent_duplicate_delivery_persists_each_event_once_under_write_contention`. The AC8 filters still match. + - The stale `T05` prefix is dropped from the hook-process report title. + - `initialized_hook_open_succeeds_while_write_lock_is_held` now uses a 2000 ms hold through the renamed constant (previously 4000 ms). + - No strict assertion was weakened, skipped or deleted. + - Defaults are not tuned in T07: `busy_timeout_ms = 500`, `contention_deadline_ms = 1250`, max attempts = 2, backoff cap = 100 ms. The T07 measurement campaign below did not justify a policy change. **Superseded by T08:** final validation then failed AC8 under these defaults, and T08 tuned them to 1000 / 2250. + - After-fix measurement campaign (authoritative T07 evidence): + - Detailed environment, methodology, raw run tables, aggregate statistics, retry timelines, host correlation and policy conclusions are recorded in `context/sce/agent-trace-db-write-contention-evidence.md`. That doc is the canonical measurement source; this record only summarizes it. + - Method: frozen release artifacts plus test-only retry-timeline instrumentation (`record_write_contention_timeline`) and scheduler-gap monitors. + - Contract (asserted by tests): + - lock-budget boundary: holds `<= 250 ms` succeed; holds `>= 2000 ms` exhaust cleanly with 0/0 partial rows; no writer exceeds 2 attempts. Holds of 500–1500 ms are characterization only; + - strict N=2–4 in-process and real-hook gates (AC8/AC9): 0 lock errors, 0 exhaustions, 0 lost distinct events, no orphan or duplicate rows, 0 non-zero hook exits. + - Observed on reference host (not guarantees): + - 10 full held-lock boundary runs passed the contract (70/70 samples). + - 5/5 complete supported N=2–4 strict in-process matrices passed: 57,500 writes, 0 lock errors, 0 contention exhaustions, 0 lost distinct events. + - Real release hooks, N=2–4: 16,500 events, 0 lost events, 0 fail-open persistence losses, 0 non-zero exits. + - N=8 remains stress characterization only, not a supported requirement: + - distinct: 20,000 writes, 0 exhaustions, 0 loss; + - duplicate: 20,000 writes, 16 contention exhaustions in one of five runs, 0 logical-event loss because another concurrent writer persisted each affected duplicate. + - Retry timelines show those N=8 failures were genuine bounded DB-policy exhaustion while another transaction held the writer lock for more than about 1 s. No scheduler stall was observed during the failure. It correlated with a device-level IO burst; the source of that burst was not proven. + - Conclusion at T07: no policy change was recommended from this dataset; the policy looked adequate for supported N=2–4 and borderline only at unsupported N=8 stress. A later validation run contradicted the supported-load part of that conclusion (see T08). + - Preliminary / superseded measurement: an earlier single-run campaign on the same day (HEAD `c513278d` + T07 test changes) recorded a strict distinct-event failure that it attributed to host-wide stalls, and reported N=8 as clean. It is not the final T07 evidence. Its host-stall attribution is withdrawn because the full campaign above observed no scheduler stall and no strict N=2–4 failure. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml lock_budget_boundary -- --nocapture`: passed in every run, printing per-hold outcome, latency, attempts, outer retries and exhaustions. + - AC8 strict `concurrent_distinct_events` (`WRITERS=2,3 ROUNDS=1000`, `WRITERS=4 ROUNDS=500`) and `concurrent_duplicate_delivery` (`WRITERS=2,3,4 ROUNDS=500`): 5/5 complete matrices passed in the measurement campaign. + - AC8 N=8 stress (distinct and duplicate, 500 rounds each, 5 runs each): results above; reported as characterization only. + - AC9: `nix build .#default` passed; strict `concurrent_real_codex_hook_processes` with `WRITERS=2,3 ROUNDS=500` and `WRITERS=4 ROUNDS=200` passed in every campaign run. + - Additional: `... lock_contention` passed (7 passed, 3 ignored); `nix build .#checks.x86_64-linux.cli-clippy .#checks.x86_64-linux.cli-fmt` passed. + - Context impact: none to code contracts or durable architecture; this is a test-only change plus plan evidence. The measured evidence lives in `context/sce/agent-trace-db-write-contention-evidence.md`. + - Context synchronization: synced + +- [x] T08: `Tune Agent Trace contention defaults after the supported-load validation failure` (status:done) + - Task ID: T08 + - Trigger: final validation run 1 (500 / 1250 defaults) failed AC8. Strict distinct N=4×500 lost 1 event, and strict duplicate N=3×500 recorded 2 lock exhaustions. In both failing rounds a writer held the writer lock for 1616–1667 ms, and waiters exhausted after 1006–1090 ms. + - Scope: In: + - test the larger bounded two-attempt policy `busy_timeout_ms = 1000`, `contention_deadline_ms = 2250`, max attempts 2, backoff cap 100 ms; + - extend the held-lock experiment to 100–3000 ms and set the AC7 thresholds from a 10-run dataset; + - rerun the full strict AC8/AC9 campaign (5 matrices each) and N=8 stress (5 runs each) on frozen release artifacts with host evidence; + - update config defaults, schema, tests and durable context. + Out — weakening AC8; a third attempt; a queue, spool or daemon; any change to retry classification, retry units, attempt cap, jitter, admission re-check, deterministic-error handling, hook fail-open, the metadata fast path or exhaustion observability. + - Dependencies: T07, validation run 1 + - Done when: the candidate is either adopted with all supported strict matrices passing, or rejected with the architectural conclusion recorded. + - Completed: 2026-10-05 + - Files changed: + - `cli/src/services/db/mod.rs`: `AGENT_TRACE_DB_BUSY_TIMEOUT_MS` 500 → 1_000; `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS` 1_250 → 2_250; the default-pinning test assertion. + - `config/pkl/base/sce-config-schema.pkl`: `busy_timeout_ms` default 500 → 1000; `contention_deadline_ms` default 1250 → 2250. + - `cli/src/services/config/schema.rs`: generated-schema default assertions. + - `cli/src/services/agent_trace_db/lock_contention_tests.rs`: `LOCK_HOLD_DURATIONS_MS` = 100/250/500/750/1000/1500/1750/2000/2250/2500/3000; `RELIABLY_WITHIN_CONTENTION_BUDGET_MS` 250 → 1_000; `RELIABLY_BEYOND_CONTENTION_BUDGET_MS` 2_000 → 3_000 (also lengthens the `initialized_hook_open_succeeds_while_write_lock_is_held` hold to 3000 ms). + - `context/sce/agent-trace-db-write-contention-evidence.md`, `context/sce/shared-turso-db.md`, `context/cli/config-precedence-contract.md`, `context/glossary.md`, `context/context-map.md`, this plan. + - Result (decision A, candidate passes; detailed tables, timelines and host evidence in `context/sce/agent-trace-db-write-contention-evidence.md`): + - Contract (asserted): holds ≤ 1000 ms succeed, holds ≥ 3000 ms exhaust cleanly; failure leaves 0/0 rows; ≤ 2 attempts; `outer_retries + 1 == attempts`; 1500–2500 ms is characterization only. + - Observed on reference host (65 runs on frozen release artifacts, all exit 0, nothing rerun or dropped): + - Held lock, 10 runs × 11 holds: 110/110 samples met the contract with 0 invariant violations. Holds of 100–1000 ms succeeded on attempt 1 and 1500–2000 ms on attempt 2; 2250–3000 ms exhausted at 2001–2094 ms. The transition lies between a holder release of 2013 ms (succeeded) and 2254 ms (exhausted). + - Strict N=2–4 in-process: 5/5 matrices passed; 57,500 writes, 0 lock errors, 0 exhaustions, 0 outer retries, 0 lost of 35,000 distinct events, 0 orphan or duplicate rows, 0 operations ≥ 500 ms. + - Strict real hooks: 5/5 matrices passed; 16,500 events, 0 lost, 0 fail-open losses, 0 non-zero exits, 0 stderr records. + - N=8 stress (characterization only): distinct 20,000 writes and duplicate 20,000 writes, 0 exhaustions, 0 loss; worst max 859 ms (1834 ms under 500 / 1250). + - Host: 0 external sleep gaps ≥ 50 ms in 50.6 minutes; no post-failure snapshot was needed. + - Latency cost: an exhausting write now blocks its hook for about 2.0–2.1 s instead of 1.0–1.1 s (about +1 s). Supported-load percentiles did not get worse (distinct p99 ranges 102–135 / 134–202 / 99–167 ms for N=2 / 3 / 4). + - Caveat: the T08 campaign never produced a supported-load holder of 1.6–1.7 s. Coverage of that holder rests on the held-lock experiment (1500–2000 ms holds succeeded 30/30), with about 300 ms of slack. + - The 500 / 1250 T07 campaign and validation run 1 remain in the evidence doc and this plan as historical evidence. + - Verify: + - `nix run .#pkl-check-generated`: passed. + - `... test --manifest-path cli/Cargo.toml` with `database_retry` (14), `busy_timeout` (7), `agent_trace_db_write_contention_retry` (12), `agent_trace_db` (75, 3 ignored), `agent_trace_storage` (14), `resilience` (6): passed. + - `... lock_budget_boundary -- --nocapture` with the final thresholds: passed; `initialized_hook_open` passed under the 3000 ms hold. + - Campaign A–D on frozen release artifacts: every run exit 0 (results above). + - Context impact: changes the default Agent Trace contention budget. Durable context (`shared-turso-db.md`, `config-precedence-contract.md`, `glossary.md`, the evidence doc) now states 1000 / 2250. `agent-trace-db.md` does not mention the defaults and needed no change. + - Context synchronization: synced + +## Open questions + +- The current 1000 / 2250 / 2 policy exhausts when a transaction holds the writer lock for more than about 2 s. Neither the T08 campaign (supported N=2–4 and N=8 stress) nor the held-lock experiment showed such a holder at supported load. If one is observed, do not tune timeouts further: bounded synchronous hook latency, no durable queue or spool, and unbounded writer-lock duration together mean zero-loss ingestion cannot be guaranteed. That needs a product/architecture decision between accepting occasional fail-open ingestion loss and introducing eventual persistence (spool or queue). Why the 1.6–1.7 s holders occurred in the failed validation run is also still unexplained (no host evidence was captured). + +- PR #297's branch carries test-only stabilizers for this same contention (`c6cbc94a` retries locked SQLite writes in concurrent repository tests; `94dd0937` stabilizes convergence checks). Once this fix lands, those test-side retries may be redundant and could mask a regression. Should they be revisited in a follow-up after both PRs merge? This plan leaves them alone. +- Follow-up, not in this plan: proper CLI telemetry / OTEL integration, as a separate future telemetry effort. It would: + - replace production `NoopTelemetry` with a real telemetry runtime; + - install a tracing subscriber during command execution; + - export structured tracing events through OTEL, covering `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted`; + - define trace/session/repository correlation; + - avoid `Logger` → tracing → `Logger` feedback or duplicate events; + - keep the existing file/stderr `Logger` behavior during migration. +- Follow-up, not in this plan: generic `RetryPolicy.timeout_ms` is still documented and rendered as a per-attempt timeout for every database, even though `run_with_retry_sync` only checks elapsed time after a synchronous call returns. Should the generic cleanup become its own plan? + +## Validation history + +Earlier `/validate` runs, kept as evidence. The next `/validate` writes a fresh `## Validation Report` below. + +### Validation run 1 (2026-10-05, 500 / 1250 defaults): failed + +**Status:** failed +**Date:** 2026-10-05 + +#### Commands run + +- `nix flake check` -> exit 0 (all checks passed) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 142 files) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db` -> exit 0 (75 passed, 3 ignored) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml mutation_trace` -> exit 0 (389 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml resilience` -> exit 0 (6 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml database_retry` -> exit 0 (14 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml busy_timeout` -> exit 0 (7 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db_write_contention_retry` -> exit 0 (12 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml hooks` -> exit 0 (806 passed, 1 ignored) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml contention_exhausted` -> exit 0 (3 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml repository_metadata` -> exit 0 (5 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml initialized_hook_open` -> exit 0 (1 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml lock_budget_boundary -- --nocapture` -> exit 0 (100/250/500 ms attempt 1; 750/1000 ms attempt 2; 1500/2000 ms exhausted after 2 attempts at about 1.0 s with 0/0 rows) +- AC8 strict `concurrent_distinct_events`, `WRITERS=2,3 ROUNDS=1000` -> exit 0 (0 lost, 0 exhaustions; p50/p95/p99/max N=2 30/93/135/1138 ms, N=3 45/105/151/236 ms) +- AC8 strict `concurrent_distinct_events`, `WRITERS=4 ROUNDS=500` -> exit 101 (1 lock error, 1 exhaustion, 1 lost distinct event in round 384; p50/p95/p99/max 45/173/254/1666 ms) +- AC8 strict `concurrent_duplicate_delivery`, `WRITERS=2,3,4 ROUNDS=500` -> exit 101 (N=3: 2 lock errors, 2 exhaustions in round 338, 0 lost logical events; N=2 and N=4 clean) +- AC8 non-strict `concurrent_distinct_events`, `WRITERS=8 ROUNDS=500` -> exit 0 (stress: 2 exhaustions, 2 lost of 4000, max 2134 ms) +- AC8 non-strict `concurrent_duplicate_delivery`, `WRITERS=8 ROUNDS=500` -> exit 0 (stress: 0 exhaustions, 0 lost) +- `nix build .#default` -> exit 0 +- AC9 strict `concurrent_real_codex_hook_processes`, `WRITERS=2,3 ROUNDS=500` -> exit 0 (2500/2500 persisted, 0 non-zero exits; p99 43/166 ms) +- AC9 strict `concurrent_real_codex_hook_processes`, `WRITERS=4 ROUNDS=200` -> exit 0 (800/800 persisted, 0 non-zero exits; p99 73 ms) + +#### Success-criteria verification + +- [x] AC1: busy handler on every Agent Trace DB connection, multiprocess WAL kept -> `busy_timeout` tests passed; inspection of `cli/src/services/db/mod.rs`: the single `TursoDb::open` local path calls `.experimental_multiprocess_wal(true)` then `apply_busy_timeout`; the other `new_local` call is `EncryptedTursoDb` (auth DB), outside AC1 +- [x] AC2: busy handler covers `BEGIN IMMEDIATE` -> T02 direct busy-handler tests passed within the `busy_timeout` filter +- [x] AC3: bounded write-contention retry -> `agent_trace_db_write_contention_retry` 12 passed +- [x] AC4: exhausted contention is observable -> `contention_exhausted` 3 passed; `hooks` 806 passed; inspection of `git diff main...HEAD -- cli/src` found no new stdout writes (only `eprintln!` in the test-only `lock_contention_tests.rs`) +- [x] AC5: initialized hook open issues zero writes -> `initialized_repository_metadata_hook_runtime_open_issues_no_writes`, `initialized_hook_open_succeeds_while_write_lock_is_held` and metadata tests passed +- [x] AC6: whole-transaction semantics -> `agent_trace_db` and `mutation_trace` passed +- [x] AC7: lock-budget boundary -> `lock_budget_boundary` passed, printing per-hold outcome, latency and attempts +- [ ] AC8: strict N=2–4 production-API levels -> failed: strict distinct N=4×500 lost 1 event to contention exhaustion, and strict duplicate N=3×500 recorded 2 lock errors +- [x] AC9: strict real-hook levels -> 2×500, 3×500 and 4×200 passed with 0 lost and 0 non-zero exits +- [x] AC10: Agent Trace-only config keys -> `database_retry` 14 passed; `pkl-check-generated` passed + +#### Failed checks and follow-ups + +- AC8 strict distinct `WRITERS=4 ROUNDS=500`: 1 lost distinct event; evidence: round 384, one writer's single attempt held the writer lock for 1666 ms (`Ok(true)`), and a waiter exhausted after 2 attempts at 1006 ms (`database is locked`, `busy_timeout_ms=500`, `contention_deadline_ms=1250`); required: decide whether the 500/1250/2-attempt policy must cover a lock holder of about 1.6 s at supported N=4, or whether AC8's strict gate needs a different acceptance basis, then fix in a normal work session. +- AC8 strict duplicate `WRITERS=2,3,4 ROUNDS=500`: 2 lock errors at N=3 (0 logical events lost); evidence: round 338, one holder's attempt took 1616 ms, and two waiters exhausted at 1064 and 1089 ms; required: same decision as above. +- The failure shape matches the documented N=8 exhaustion mode in `context/sce/agent-trace-db-write-contention-evidence.md` (holder transaction above about 1.05 s), but here it occurred at supported N=3 and N=4. The evidence doc's claim of 0 strict N=2–4 failures holds only for its recorded campaign. + +#### Residual risks + +- Supported N=2–4 writes can lose a distinct event when one transaction holds the writer lock for more than about 1.05 s. Host IO state during this validation run was not captured. +- Real hook processes passed, but they ran at lower round counts than the in-process suite. + +#### Retry + +After repairs, rerun: + +`/validate context/plans/agent-trace-db-write-contention.md` + +## Validation Report + +**Status:** validated +**Date:** 2026-10-05 + +### Commands run + +- `nix flake check` -> exit 0 (all checks passed) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 142 files) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db` -> exit 0 (75 passed, 3 ignored) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml mutation_trace` -> exit 0 (389 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml resilience` -> exit 0 (6 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml database_retry` -> exit 0 (14 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml busy_timeout` -> exit 0 (7 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_db_write_contention_retry` -> exit 0 (12 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml hooks` -> exit 0 (806 passed, 1 ignored) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml contention_exhausted` -> exit 0 (3 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml repository_metadata` -> exit 0 (5 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml initialized_hook_open` -> exit 0 (1 passed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml lock_budget_boundary -- --nocapture` -> exit 0 (100–1000 ms holds succeeded on attempt 1; 1500/1750/2000 ms succeeded on attempt 2; 2250/2500/3000 ms exhausted after 2 attempts at 2047–2063 ms with 0/0 rows) +- AC8 strict `concurrent_distinct_events`, `WRITERS=2,3 ROUNDS=1000` -> exit 0 (0 lock errors, 0 lost, 0 exhaustions, 0 outer retries; p50/p95/p99/max N=2 31/95/134/806 ms, N=3 52/119/175/285 ms) +- AC8 strict `concurrent_distinct_events`, `WRITERS=4 ROUNDS=500` -> exit 0 (0 lock errors, 0 lost, 0 exhaustions, 1 outer retry; p50/p95/p99/max 45/157/232/1246 ms) +- AC8 strict `concurrent_duplicate_delivery`, `WRITERS=2,3,4 ROUNDS=500` -> exit 0 (0 lock errors, 0 exhaustions, exactly 500 inserts per level; p99/max N=2 28/38, N=3 154/1412, N=4 191/299 ms) +- AC8 non-strict `concurrent_distinct_events`, `WRITERS=8 ROUNDS=500` -> exit 0 (stress: 0 lost of 4000, 0 exhaustions, 2 outer retries; p99/max 505/1067 ms) +- AC8 non-strict `concurrent_duplicate_delivery`, `WRITERS=8 ROUNDS=500` -> exit 0 (stress: 0 exhaustions, 500 inserts; p99/max 345/844 ms) +- `nix build .#default` -> exit 0 +- AC9 strict `concurrent_real_codex_hook_processes`, `WRITERS=2,3 ROUNDS=500` -> exit 0 (1000/1000 and 1500/1500 persisted, 0 non-zero exits, 0 stderr lines; p50/p95/p99/max N=2 26/38/40/46 ms, N=3 46/108/155/629 ms) +- AC9 strict `concurrent_real_codex_hook_processes`, `WRITERS=4 ROUNDS=200` -> exit 0 (800/800 persisted, 0 non-zero exits, 0 stderr lines; p50/p95/p99/max 44/74/77/81 ms) + +### Success-criteria verification + +- [x] AC1: busy handler on every Agent Trace DB connection, multiprocess WAL kept -> `busy_timeout` 7 passed; inspection of `cli/src/services/db/mod.rs`: the single `TursoDb` local open path calls `.experimental_multiprocess_wal(true)` and then `apply_busy_timeout`; the only other `new_local` call is `EncryptedTursoDb` (auth DB), outside AC1 +- [x] AC2: busy handler covers `BEGIN IMMEDIATE` -> T02 direct busy-handler tests passed within the `busy_timeout` filter +- [x] AC3: bounded write-contention retry -> `agent_trace_db_write_contention_retry` 12 passed; the boundary run showed the 100 ms hold at `attempts = 1`, `outer_retries = 0` +- [x] AC4: exhausted contention is observable -> `contention_exhausted` 3 passed; `hooks` 806 passed; inspection of `git diff main...HEAD -- cli/src` found no new stdout writes (new `eprintln!` lines appear only in the test-only `lock_contention_tests.rs`) +- [x] AC5: initialized hook open issues zero writes -> `repository_metadata` 5 passed and `initialized_hook_open` passed under the 3000 ms held write lock +- [x] AC6: whole-transaction semantics -> `agent_trace_db` 75 passed and `mutation_trace` 389 passed; strict runs showed no orphan or duplicate rows +- [x] AC7: lock-budget boundary -> `lock_budget_boundary` passed: ≤ 1000 ms succeeded, ≥ 3000 ms exhausted cleanly, every sample ≤ 2 attempts with `outer_retries + 1 == attempts` and 0/0 rows on failure +- [x] AC8: strict N=2–4 production-API levels -> distinct 2×1000, 3×1000, 4×500 and duplicate 2×500, 3×500, 4×500 passed with 0 lock errors, 0 other errors and 0 lost events; N=8 stress reported with 0 loss +- [x] AC9: strict real-hook levels -> 2×500, 3×500 and 4×200 passed with 0 lost and 0 non-zero exits +- [x] AC10: Agent Trace-only config keys -> `database_retry` 14 passed; `pkl-check-generated` passed + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Writer-lock holders longer than about 2 s still exhaust the 1000 / 2250 / 2-attempt policy and lose the event through hook fail-open; no such holder occurred at supported load in this run. The open question on accepting occasional loss vs. eventual persistence still applies. +- The unexplained 1.6–1.7 s holders from validation run 1 did not recur here, but this run captured no host IO evidence. Coverage of such holders rests on the held-lock boundary results (1500–2000 ms holds succeeded on attempt 2). +- Strict contention gates are timing-sensitive and were measured on one reference host. diff --git a/context/sce/agent-trace-db-write-contention-evidence.md b/context/sce/agent-trace-db-write-contention-evidence.md new file mode 100644 index 000000000..67c155472 --- /dev/null +++ b/context/sce/agent-trace-db-write-contention-evidence.md @@ -0,0 +1,913 @@ +# Agent Trace DB write-contention evidence + +This doc records the measured behavior of the Agent Trace DB contention contract and the test suite that measures it. The contract is: Turso `busy_timeout`, then at most one jittered outer retry, cut off by `contention_deadline_ms`. The contract itself is defined in [shared-turso-db.md](shared-turso-db.md). The repository adapter is described in [agent-trace-db.md](agent-trace-db.md). + +This doc keeps **Contract** (what tests assert) apart from **Observed on reference host** (what one campaign measured on one machine). Nothing under "Observed" is a guarantee. + +This is the canonical detailed measurement source. Plans and other context docs summarize it and link here. + +## Policy history + +| Policy (`busy_timeout_ms` / `contention_deadline_ms` / max attempts / backoff cap) | Status | Evidence | +| --- | --- | --- | +| 1000 / 2250 / 2 / 100 ms | **current defaults** | [Current campaign](#current-campaign-1000--2250--2--100) | +| 500 / 1250 / 2 / 100 ms | superseded | [Supported-load failure](#supported-load-failure-under-the-500--1250-policy) and [historical campaign](#historical-campaign-500--1250--2--100) | + +Why the defaults were tuned: + +- Under 500 / 1250, a final validation run of the strict N=2–4 suite failed. A supported-load writer held the Agent Trace writer lock for about 1.6–1.7 s, and the waiting writers exhausted the policy after about 1.0–1.1 s. One distinct event was lost. +- The 500 / 1250 policy cannot cover a lock hold above about 1.05 s. Its held-lock transition sat between a holder release of 1026 ms (succeeded) and 1503 ms (exhausted). +- 1000 / 2250 keeps the same two-attempt architecture and raises only the time budget. It was measured first; a third attempt was not tried, because this candidate passed. +- Only the default time budget changed. Retry classification (typed `Busy`/`BusySnapshot`), whole-transaction retry units, the 2-attempt cap, the full-jitter algorithm, the post-backoff admission re-check, deterministic-error behavior, hook fail-open behavior, the metadata read-only fast path and exhaustion observability are unchanged. + +## Contractual assertions (current) + +These are the only assertions the suite makes. + +- Held-lock boundary (`lock_budget_boundary_characterizes_single_writer_blocked_by_begin_immediate_holder`), holds of 100, 250, 500, 750, 1000, 1500, 1750, 2000, 2250, 2500 and 3000 ms: + - holds ≤ 1000 ms must succeed with 0 exhaustions; + - holds ≥ 3000 ms must fail with the `under write contention` error and exactly 1 exhaustion; + - every failure leaves 0 message and 0 part rows; + - every sample makes 1–2 attempts, and `outer_retries + 1 == attempts`. + - 1500–2500 ms is characterization only. + - Margin basis: an exhausting writer cannot give up before about 2000 ms (two 1000 ms busy waits), so a ≤ 1000 ms hold has about 1000 ms of slack. The latest exhaustion observed in 110 samples was 2094 ms, so a ≥ 3000 ms hold has about 900 ms of slack. The earlier contract (≤ 250 ms succeeds, ≥ 2000 ms exhausts) used comparable slack against the 500 / 1250 policy. +- Every in-process round: + - `messages == parts`; + - the `Ok(true)` count equals the persisted rows; + - no writer exceeds 2 attempts; + - every duplicate round inserts at most once. +- `SCE_LOCK_CONTENTION_STRICT=1` adds, per level: + - 0 lock errors, 0 other errors, 0 lost events and 0 exhaustions; + - for hook processes: every expected message and part persisted, and 0 non-zero exits. +- Strict supported levels (unchanged): + + | Suite | Levels (N × rounds) | + | --- | --- | + | distinct events | 2×1000, 3×1000, 4×500 | + | duplicate delivery | 2×500, 3×500, 4×500 | + | hook processes | 2×500, 3×500, 4×200 | + + N=8 is stress characterization, not a supported requirement. + +## Current campaign: 1000 / 2250 / 2 / 100 + +### Environment and artifacts + +- Same reference host as the historical campaign (see [Test environment](#test-environment)): bare-metal AMD Ryzen 9 5900X, 24 threads, 46 GiB RAM, Linux 6.18.37, `/tmp` on ext4 over LUKS dm-crypt on NVMe, Turso crate 0.8.1, rustc 1.95.0. The same io_uring PSI/iowait caveat applies. +- Campaign date 2026-10-05, Unix ms 1791206344424–1791209382718 (50.6 minutes of measured runs). Load average (1 min) before each run ranged 0.77–5.78. +- Commit: `HEAD` `e72116239fa8f05befe9fb8e1220c79100914490` plus the working-tree policy change. The `cli/` + `config/` diff at build time had sha256 `3f78fe790fb2e9d10e070e2ab903be256634e8461baf8a23e0515b1f74498c91`. + - `AGENT_TRACE_DB_BUSY_TIMEOUT_MS` 500 → 1000 and `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS` 1250 → 2250 in `cli/src/services/db/mod.rs`, with matching Pkl schema defaults. + - The boundary hold set was extended to 100–3000 ms, with a provisional exhaustion threshold of 3000 ms and the old success threshold of 250 ms. + - After the campaign, the success threshold was raised to 1000 ms from this data. That is a test-only change; the measured binaries predate it. +- The policy ran with defaults, with no `policies.database_retry.agent_trace_db` override. +- Frozen release artifacts, built once before the first run: + - test binary sha256 `d257dda55923bec809953e84489c927e2a410fb24d15119c1419eca03f917cf1`; + - `sce` sha256 `c90b428162f60ef374374d3dcc5272a72bec837b0f079c3d9c482e0e5e2f526e`. +- Methodology and run order are identical to the historical campaign ([Measurement methodology](#measurement-methodology)): A = 10 boundary runs, B = 5 strict matrices, C = 5 N=8 stress runs, D = 5 strict real-hook matrices. Every level ran as its own strict-mode process with `uptime`, `/proc/pressure/*`, `vmstat 1 5` and `iostat -xz 1 5` captured before it, and the external host monitor ran for the whole campaign. +- 65 runs, all exit 0. No post-failure snapshot was needed. Nothing was rerun, replaced or dropped. The raw `SCE_MEAS` logs, host snapshots and monitor JSONL were kept in the session scratchpad and are not committed. + +### Held-lock results + +Contract: ≤ 1000 ms succeeds, ≥ 3000 ms exhausts, failure leaves 0/0 rows, ≤ 2 attempts. All 10 runs satisfied it: 110/110 samples, 0 invariant violations. + +Observed on reference host (10 runs × 11 holds): + +- **The success/exhaust transition lies between 2000 and 2250 ms of requested hold.** Actual holder release was 2004–2013 ms for the 2000 ms hold (10/10 succeeded) versus 2254–2265 ms for the 2250 ms hold (10/10 exhausted). +- **Exhausted writers gave up at 2001–2094 ms**: a `1000.1–1000.2 ms` busy wait, the drawn backoff (≤ 0.1 ms oversleep), and a second `1000.1–1000.2 ms` busy wait. No admission rejection occurred; every exhaustion made both attempts. +- **Holds of 1500–2000 ms succeeded on attempt 2** (30/30). This covers the 1.6–1.7 s holder that failed the 500 / 1250 validation run. The 2000 ms hold is borderline: it succeeded because the second busy wait ended a few ms after the release, and a smaller backoff draw could tip it to exhaustion. It is characterization only. +- **Holds up to 1000 ms succeeded on attempt 1** (50/50), including 750 ms holds that needed an outer retry under 500 / 1250. +- One 2500 ms sample released at 2977 ms, a delayed holder release. It exhausted at about 2.0 s as specified. + +#### Held-lock raw samples + +| run | hold ms | holder released ms | writer elapsed ms | result | attempts | outer retries | exhaustions | messages | parts | attempt1 ms | backoff req/actual ms | attempt2 ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 100 | 104.1 | 115.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.9 | - | - | +| 1 | 250 | 254.3 | 341.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.8 | - | - | +| 1 | 500 | 512.1 | 541.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 541.8 | - | - | +| 1 | 750 | 754.4 | 841.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 841.4 | - | - | +| 1 | 1000 | 1004.5 | 1018.0 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1018.0 | - | - | +| 1 | 1500 | 1504.5 | 1583.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 34/34.1 | 549.3 | +| 1 | 1750 | 1762.5 | 1862.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 14/14.1 | 848.3 | +| 1 | 2000 | 2004.4 | 2085.2 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 81/81.1 | 1004.0 | +| 1 | 2250 | 2254.4 | 2056.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 56/56.1 | 1000.2 | +| 1 | 2500 | 2976.9 | 2015.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 15/15.1 | 1000.2 | +| 1 | 3000 | 3004.4 | 2008.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 8/8.1 | 1000.2 | +| 2 | 100 | 103.9 | 115.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.5 | - | - | +| 2 | 250 | 262.2 | 340.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.7 | - | - | +| 2 | 500 | 515.5 | 540.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 540.8 | - | - | +| 2 | 750 | 762.6 | 840.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 840.7 | - | - | +| 2 | 1000 | 1012.7 | 1023.3 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1023.3 | - | - | +| 2 | 1500 | 1512.7 | 1562.1 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 20/20.1 | 541.8 | +| 2 | 1750 | 1762.6 | 1848.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 7/7.1 | 841.3 | +| 2 | 2000 | 2012.5 | 2052.4 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.1 | 39/39.1 | 1013.2 | +| 2 | 2250 | 2262.6 | 2042.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 42/42.1 | 1000.2 | +| 2 | 2500 | 2509.8 | 2003.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 3/3.1 | 1000.2 | +| 2 | 3000 | 3005.3 | 2014.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 14/14.1 | 1000.2 | +| 3 | 100 | 104.2 | 116.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 116.7 | - | - | +| 3 | 250 | 253.9 | 341.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.7 | - | - | +| 3 | 500 | 504.3 | 541.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 541.2 | - | - | +| 3 | 750 | 754.4 | 842.3 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 842.3 | - | - | +| 3 | 1000 | 1004.4 | 1038.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1038.4 | - | - | +| 3 | 1500 | 1504.6 | 1523.4 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 75/75.1 | 448.1 | +| 3 | 1750 | 1754.7 | 1771.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.1 | 26/26.1 | 745.4 | +| 3 | 2000 | 2004.2 | 2065.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 53/53.0 | 1012.4 | +| 3 | 2250 | 2254.3 | 2070.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 70/70.1 | 1000.2 | +| 3 | 2500 | 2513.1 | 2010.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 10/10.1 | 1000.2 | +| 3 | 3000 | 3012.4 | 2008.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 8/8.1 | 1000.2 | +| 4 | 100 | 103.9 | 115.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.1 | - | - | +| 4 | 250 | 269.8 | 341.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.5 | - | - | +| 4 | 500 | 512.0 | 540.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 540.1 | - | - | +| 4 | 750 | 762.5 | 841.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 841.9 | - | - | +| 4 | 1000 | 1012.5 | 1020.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1020.4 | - | - | +| 4 | 1500 | 1512.2 | 1550.8 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 9/9.1 | 541.6 | +| 4 | 1750 | 1765.8 | 1844.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 3/3.1 | 841.4 | +| 4 | 2000 | 2004.1 | 2017.2 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.1 | 78/78.1 | 939.0 | +| 4 | 2250 | 2256.5 | 2043.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 43/43.1 | 1000.2 | +| 4 | 2500 | 2504.1 | 2030.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 30/30.1 | 1000.2 | +| 4 | 3000 | 3008.3 | 2086.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 86/86.0 | 1000.2 | +| 5 | 100 | 104.2 | 134.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 134.5 | - | - | +| 5 | 250 | 258.0 | 346.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 346.7 | - | - | +| 5 | 500 | 505.3 | 547.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 547.5 | - | - | +| 5 | 750 | 754.3 | 847.0 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 847.0 | - | - | +| 5 | 1000 | 1004.4 | 1022.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1022.1 | - | - | +| 5 | 1500 | 1511.9 | 1590.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 41/41.1 | 549.4 | +| 5 | 1750 | 1754.1 | 1828.7 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 73/73.1 | 755.5 | +| 5 | 2000 | 2004.3 | 2059.4 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 41/41.1 | 1018.2 | +| 5 | 2250 | 2264.9 | 2073.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 73/73.1 | 1000.3 | +| 5 | 2500 | 2512.4 | 2053.6 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 53/53.1 | 1000.3 | +| 5 | 3000 | 3012.5 | 2038.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 38/38.1 | 1000.2 | +| 6 | 100 | 104.0 | 115.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.9 | - | - | +| 6 | 250 | 270.3 | 341.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.6 | - | - | +| 6 | 500 | 514.9 | 540.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 540.5 | - | - | +| 6 | 750 | 762.2 | 841.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 841.4 | - | - | +| 6 | 1000 | 1012.5 | 1020.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1020.6 | - | - | +| 6 | 1500 | 1515.1 | 1523.3 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 79/79.1 | 444.1 | +| 6 | 1750 | 1762.5 | 1779.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 38/38.1 | 741.3 | +| 6 | 2000 | 2012.2 | 2060.9 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 44/44.1 | 1016.7 | +| 6 | 2250 | 2253.9 | 2036.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 36/36.1 | 1000.2 | +| 6 | 2500 | 2509.1 | 2054.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 54/54.1 | 1000.2 | +| 6 | 3000 | 3013.8 | 2053.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 53/53.1 | 1000.2 | +| 7 | 100 | 104.3 | 116.3 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 116.3 | - | - | +| 7 | 250 | 262.4 | 340.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.6 | - | - | +| 7 | 500 | 512.1 | 540.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 540.9 | - | - | +| 7 | 750 | 761.9 | 840.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 840.2 | - | - | +| 7 | 1000 | 1004.5 | 1016.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1016.2 | - | - | +| 7 | 1500 | 1504.5 | 1568.8 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 27/27.1 | 541.6 | +| 7 | 1750 | 1754.6 | 1797.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 57/57.1 | 740.3 | +| 7 | 2000 | 2004.2 | 2024.0 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 83/83.1 | 940.7 | +| 7 | 2250 | 2254.2 | 2027.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 27/27.1 | 1000.3 | +| 7 | 2500 | 2513.5 | 2020.7 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.3 | 20/20.1 | 1000.3 | +| 7 | 3000 | 3004.3 | 2086.6 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 86/86.1 | 1000.3 | +| 8 | 100 | 104.0 | 115.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.8 | - | - | +| 8 | 250 | 271.1 | 340.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.6 | - | - | +| 8 | 500 | 512.3 | 540.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 540.2 | - | - | +| 8 | 750 | 762.1 | 840.3 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 840.3 | - | - | +| 8 | 1000 | 1019.0 | 1027.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1027.6 | - | - | +| 8 | 1500 | 1512.0 | 1570.4 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 30/30.1 | 540.2 | +| 8 | 1750 | 1763.4 | 1775.0 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 31/31.1 | 743.8 | +| 8 | 2000 | 2012.4 | 2023.7 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 83/83.1 | 940.5 | +| 8 | 2250 | 2262.2 | 2094.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 94/94.0 | 1000.2 | +| 8 | 2500 | 2512.9 | 2076.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 76/76.1 | 1000.2 | +| 8 | 3000 | 3013.0 | 2018.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 18/18.1 | 1000.2 | +| 9 | 100 | 103.8 | 115.0 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.0 | - | - | +| 9 | 250 | 254.3 | 340.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.9 | - | - | +| 9 | 500 | 512.3 | 540.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 540.6 | - | - | +| 9 | 750 | 762.4 | 841.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 841.1 | - | - | +| 9 | 1000 | 1012.5 | 1024.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1024.3 | - | - | +| 9 | 1500 | 1512.6 | 1593.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 52/52.1 | 541.4 | +| 9 | 1750 | 1762.4 | 1805.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 63/63.1 | 742.4 | +| 9 | 2000 | 2012.7 | 2021.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 5/5.1 | 1016.2 | +| 9 | 2250 | 2262.0 | 2055.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 55/55.1 | 1000.2 | +| 9 | 2500 | 2505.0 | 2001.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 1/1.0 | 1000.2 | +| 9 | 3000 | 3012.8 | 2073.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 73/73.1 | 1000.2 | +| 10 | 100 | 104.2 | 116.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 116.5 | - | - | +| 10 | 250 | 262.4 | 341.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.8 | - | - | +| 10 | 500 | 512.1 | 541.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 541.2 | - | - | +| 10 | 750 | 767.0 | 841.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 841.1 | - | - | +| 10 | 1000 | 1012.3 | 1020.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 1020.2 | - | - | +| 10 | 1500 | 1512.0 | 1557.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 17/17.1 | 540.3 | +| 10 | 1750 | 1762.1 | 1775.0 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 34/34.1 | 740.8 | +| 10 | 2000 | 2012.6 | 2027.2 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 1000.2 | 11/11.1 | 1016.0 | +| 10 | 2250 | 2262.1 | 2067.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 67/67.1 | 1000.2 | +| 10 | 2500 | 2512.3 | 2030.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 30/30.1 | 1000.2 | +| 10 | 3000 | 3016.1 | 2039.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 1000.2 | 39/39.1 | 1000.2 | + +#### Held-lock per-hold aggregate (10 runs) + +| hold ms | success | exhausted | attempt-1 success | attempt-2 success | writer p50 ms | writer p95 ms | writer max ms | holder release p50 ms | holder release max ms | +|---|---|---|---|---|---|---|---|---|---| +| 100 | 10/10 | 0/10 | 10 | 0 | 115.9 | 134.6 | 134.6 | 104.0 | 104.3 | +| 250 | 10/10 | 0/10 | 10 | 0 | 341.5 | 346.7 | 346.7 | 262.2 | 271.1 | +| 500 | 10/10 | 0/10 | 10 | 0 | 540.8 | 547.5 | 547.5 | 512.1 | 515.5 | +| 750 | 10/10 | 0/10 | 10 | 0 | 841.1 | 847.0 | 847.0 | 762.1 | 767.0 | +| 1000 | 10/10 | 0/10 | 10 | 0 | 1020.6 | 1038.4 | 1038.4 | 1012.3 | 1019.0 | +| 1500 | 10/10 | 0/10 | 0 | 10 | 1562.1 | 1593.6 | 1593.6 | 1512.0 | 1515.1 | +| 1750 | 10/10 | 0/10 | 0 | 10 | 1797.5 | 1862.5 | 1862.5 | 1762.4 | 1765.8 | +| 2000 | 10/10 | 0/10 | 0 | 10 | 2027.2 | 2085.2 | 2085.2 | 2004.4 | 2012.7 | +| 2250 | 0/10 | 10/10 | 0 | 0 | 2055.4 | 2094.5 | 2094.5 | 2256.5 | 2264.9 | +| 2500 | 0/10 | 10/10 | 0 | 0 | 2020.7 | 2076.5 | 2076.5 | 2512.3 | 2976.9 | +| 3000 | 0/10 | 10/10 | 0 | 0 | 2038.5 | 2086.6 | 2086.6 | 3012.4 | 3016.1 | + +### Strict N=2–4 results + +| run | mode | N | rounds | total writes | expected rows | persisted rows | lost | lock errors | other errors | orphan rows | duplicate rows | attempts | outer retries | exhaustions | p50 ms | p95 ms | p99 ms | max ms | ops ≥500 ms | in-proc stall gaps ≥50 ms (max) | exit | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 30.1 | 93.0 | 128.9 | 180.2 | 0 | 0 (0) | 0 | +| 2 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 29.1 | 61.6 | 110.7 | 254.7 | 0 | 0 (0) | 0 | +| 3 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 29.6 | 63.6 | 101.8 | 241.6 | 0 | 0 (0) | 0 | +| 4 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 21.7 | 60.7 | 113.5 | 189.1 | 0 | 0 (0) | 0 | +| 5 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 30.1 | 92.3 | 135.3 | 292.9 | 0 | 0 (0) | 0 | +| 1 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 50.2 | 126.8 | 201.8 | 303.7 | 0 | 0 (0) | 0 | +| 2 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 36.1 | 111.1 | 171.3 | 292.8 | 0 | 0 (0) | 0 | +| 3 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 41.8 | 102.6 | 147.6 | 212.7 | 0 | 0 (0) | 0 | +| 4 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 31.4 | 67.2 | 134.2 | 236.2 | 0 | 0 (0) | 0 | +| 5 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 34.4 | 106.2 | 148.6 | 230.1 | 0 | 0 (0) | 0 | +| 1 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 45.2 | 87.4 | 132.3 | 250.8 | 0 | 0 (0) | 0 | +| 2 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 31.3 | 65.8 | 98.9 | 221.4 | 0 | 0 (0) | 0 | +| 3 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 32.1 | 97.1 | 144.4 | 399.4 | 0 | 0 (0) | 0 | +| 4 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 33.5 | 84.3 | 167.2 | 230.9 | 0 | 0 (0) | 0 | +| 5 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 45.3 | 99.4 | 164.5 | 273.4 | 0 | 0 (0) | 0 | +| 1 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 29.7 | 66.3 | 130.7 | 181.4 | 0 | 0 (0) | 0 | +| 2 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 17.0 | 26.9 | 28.1 | 32.8 | 0 | 0 (0) | 0 | +| 3 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 25.9 | 37.0 | 54.5 | 99.0 | 0 | 0 (0) | 0 | +| 4 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 25.5 | 43.2 | 52.9 | 117.7 | 0 | 0 (0) | 0 | +| 5 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 23.2 | 42.5 | 76.5 | 147.4 | 0 | 0 (0) | 0 | +| 1 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 16.7 | 40.1 | 41.4 | 42.0 | 0 | 0 (0) | 0 | +| 2 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 26.2 | 41.8 | 45.1 | 74.1 | 0 | 0 (0) | 0 | +| 3 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 26.3 | 41.7 | 42.8 | 46.3 | 0 | 0 (0) | 0 | +| 4 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 26.0 | 41.5 | 41.9 | 44.9 | 0 | 0 (0) | 0 | +| 5 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 22.2 | 41.3 | 41.7 | 42.4 | 0 | 0 (0) | 0 | +| 1 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 26.1 | 61.3 | 61.7 | 65.5 | 0 | 0 (0) | 0 | +| 2 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 40.2 | 69.3 | 93.7 | 258.0 | 0 | 0 (0) | 0 | +| 3 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 40.7 | 63.9 | 136.9 | 205.8 | 0 | 0 (0) | 0 | +| 4 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 26.2 | 61.4 | 61.6 | 86.3 | 0 | 0 (0) | 0 | +| 5 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 39.6 | 62.5 | 105.3 | 168.8 | 0 | 0 (0) | 0 | + +**Aggregate across runs** (latency percentiles are of per-run values; pooled raw latencies are not retained, so the pooled column is the worst per-run value) + +| mode | N | runs passing strict | total writes | expected rows | persisted | lost | lock errors | other errors | exhaustions | exhaustion rate | outer retries | attempts | p50 range ms | p95 range ms | p99 range ms | worst max ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| distinct | 2 | 5/5 | 10000 | 10000 | 10000 | 0 | 0 | 0 | 0 | 0 / 10000 = 0.000% | 0 | 10000 | 21.7–30.1 | 60.7–93.0 | 101.8–135.3 | 292.9 | +| distinct | 3 | 5/5 | 15000 | 15000 | 15000 | 0 | 0 | 0 | 0 | 0 / 15000 = 0.000% | 0 | 15000 | 31.4–50.2 | 67.2–126.8 | 134.2–201.8 | 303.7 | +| distinct | 4 | 5/5 | 10000 | 10000 | 10000 | 0 | 0 | 0 | 0 | 0 / 10000 = 0.000% | 0 | 10000 | 31.3–45.3 | 65.8–99.4 | 98.9–167.2 | 399.4 | +| duplicate | 2 | 5/5 | 5000 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 5000 = 0.000% | 0 | 5000 | 17.0–29.7 | 26.9–66.3 | 28.1–130.7 | 181.4 | +| duplicate | 3 | 5/5 | 7500 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 7500 = 0.000% | 0 | 7500 | 16.7–26.3 | 40.1–41.8 | 41.4–45.1 | 74.1 | +| duplicate | 4 | 5/5 | 10000 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 10000 = 0.000% | 0 | 10000 | 26.1–40.7 | 61.3–69.3 | 61.6–136.9 | 258.0 | + +Strict matrices passing: 5/5. Writes: 57500. Distinct events requested: 35000, distinct events lost: 0. Exhaustions: 0. Lock errors: 0. + +### N=8 stress results + +**Stress characterization — not a supported requirement** + +| run | mode | N | rounds | total writes | expected rows | persisted rows | lost | lock errors | other errors | orphan rows | duplicate rows | attempts | outer retries | exhaustions | p50 ms | p95 ms | p99 ms | max ms | ops ≥500 ms | in-proc stall gaps ≥50 ms (max) | exit | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 90.4 | 251.9 | 454.1 | 780.9 | 27 | 0 (0) | 0 | +| 2 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 89.6 | 241.2 | 443.6 | 672.7 | 14 | 0 (0) | 0 | +| 3 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 79.6 | 192.7 | 274.9 | 573.2 | 3 | 0 (0) | 0 | +| 4 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 96.5 | 260.8 | 546.4 | 859.1 | 47 | 0 (0) | 0 | +| 5 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 91.0 | 243.9 | 440.8 | 745.9 | 23 | 0 (0) | 0 | +| 1 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 86.6 | 218.4 | 349.0 | 841.2 | 13 | 0 (0) | 0 | +| 2 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 86.3 | 239.5 | 351.1 | 661.4 | 9 | 0 (0) | 0 | +| 3 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 86.3 | 237.0 | 436.8 | 847.9 | 28 | 0 (0) | 0 | +| 4 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 86.4 | 196.9 | 352.0 | 683.1 | 13 | 0 (0) | 0 | +| 5 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 86.1 | 187.5 | 295.9 | 637.0 | 4 | 0 (0) | 0 | + +**Aggregate across runs** (latency percentiles are of per-run values; pooled raw latencies are not retained, so the pooled column is the worst per-run value) + +| mode | N | runs passing strict | total writes | expected rows | persisted | lost | lock errors | other errors | exhaustions | exhaustion rate | outer retries | attempts | p50 range ms | p95 range ms | p99 range ms | worst max ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| distinct | 8 | 5/5 | 20000 | 20000 | 20000 | 0 | 0 | 0 | 0 | 0 / 20000 = 0.000% | 0 | 20000 | 79.6–96.5 | 192.7–260.8 | 274.9–546.4 | 859.1 | +| duplicate | 8 | 5/5 | 20000 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 20000 = 0.000% | 0 | 20000 | 86.1–86.6 | 187.5–239.5 | 295.9–436.8 | 847.9 | + +N=8 writes: 40000. Distinct events requested: 20000, lost: 0. Exhaustions: 0. Lock errors: 0. + +### Real-hook results + +| run | N | rounds | expected | persisted msgs | persisted parts | lost | fail-open lost | non-zero exits | stderr outputs | stderr lines | p50 ms | p95 ms | p99 ms | max ms | procs ≥500 ms | in-proc stall gaps ≥50 ms (max) | exit | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 25.2 | 37.0 | 37.5 | 41.6 | 0 | 0 (0) | 0 | +| 2 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 25.7 | 38.0 | 51.9 | 104.3 | 0 | 0 (0) | 0 | +| 3 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 29.8 | 74.0 | 127.2 | 221.1 | 0 | 0 (0) | 0 | +| 4 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 27.5 | 37.5 | 40.1 | 44.5 | 0 | 0 (0) | 0 | +| 5 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 26.8 | 37.4 | 41.1 | 44.6 | 0 | 0 (0) | 0 | +| 1 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 36.6 | 60.8 | 135.1 | 191.7 | 0 | 0 (0) | 0 | +| 2 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 36.6 | 69.6 | 98.5 | 211.6 | 0 | 0 (0) | 0 | +| 3 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 36.9 | 53.9 | 77.6 | 185.7 | 0 | 0 (0) | 0 | +| 4 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 37.7 | 80.6 | 114.7 | 235.8 | 0 | 0 (0) | 0 | +| 5 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 36.8 | 58.1 | 96.7 | 225.4 | 0 | 0 (0) | 0 | +| 1 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 57.4 | 127.3 | 193.3 | 262.9 | 0 | 0 (0) | 0 | +| 2 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 63.9 | 154.3 | 227.7 | 388.3 | 0 | 0 (0) | 0 | +| 3 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 51.6 | 97.1 | 107.5 | 260.7 | 0 | 0 (0) | 0 | +| 4 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 51.0 | 87.4 | 114.3 | 247.4 | 0 | 0 (0) | 0 | +| 5 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 50.7 | 78.3 | 103.1 | 196.6 | 0 | 0 (0) | 0 | + +**Aggregate** + +| N | runs passing strict | expected | persisted | lost | fail-open lost | non-zero exits | stderr lines | p50 range ms | p95 range ms | p99 range ms | worst max ms | +|---|---|---|---|---|---|---|---|---|---|---|---| +| 2 | 5/5 | 5000 | 5000 | 0 | 0 | 0 | 0 | 25.2–29.8 | 37.0–74.0 | 37.5–127.2 | 221.1 | +| 3 | 5/5 | 7500 | 7500 | 0 | 0 | 0 | 0 | 36.6–37.7 | 53.9–80.6 | 77.6–135.1 | 235.8 | +| 4 | 5/5 | 4000 | 4000 | 0 | 0 | 0 | 0 | 50.7–63.9 | 78.3–154.3 | 103.1–227.7 | 388.3 | + +### Slow operations (≥ 500 ms) + +Attempt duration is the time inside one Turso attempt (busy-handler wait included). Backoff actual vs requested shows scheduler oversleep. 'Outside' is time not inside an attempt or backoff. + + +#### B (strict N=2–4): 0 slow ops + + +#### C (N=8 stress): 181 slow ops + +- Ok(false), 1 attempt(s): 67 +- Ok(true), 1 attempt(s): 114 +- longest single attempt: 859.1 ms; worst backoff oversleep: 0.0 ms; worst time outside attempts/backoff: 0.02 ms + +### Every exhausted operation + +None. No operation exhausted the policy in this campaign. + +### Host monitor summary (whole campaign) + +- campaign window: 1791206344424–1791209382718 (50.6 min), 3037 1 s samples +- external monitor sleep-gap events ≥50 ms: 0; max 0.0 ms +- psi_cpu_some_ms: p50 1.1, p99 2.9, max 5.7 +- psi_mem_some_ms: p50 0.0, p99 0.0, max 0.0 +- psi_io_full_ms: p50 398.2, p99 945.6, max 974.4 — io_uring idle-wait artifact, not usable (see Test environment) +- cpu_iowait_pct: p50 9.8, p99 18.6, max 20.0 — same artifact +- disk busy ms per 1 s (max of nvme0n1/dm-0): p50 593, p99 950, max 1004 +- procs_blocked: p50 3, max 7 + +### Current-campaign conclusions + +Correctness: no correctness violation in any of the 114,110 measured write operations (A 110 boundary samples, B 57,500, C 40,000, D 16,500 hook events). There were 0 orphan rows and 0 duplicate logical events across all 10,000 duplicate rounds (7,500 at N=2–4, 2,500 at N=8). Every failure left 0/0 rows, and no writer exceeded 2 attempts. + +Availability (observed on reference host): + +| Scope | Exhaustions | Lost distinct events | +| --- | --- | --- | +| Strict N=2–4 in-process | 0 / 57,500 writes; 5/5 strict matrices passed | 0 / 35,000 | +| Real hooks N=2–4 | not countable (no cross-process counters) | 0 / 16,500 events; 0 fail-open losses; 0 non-zero exits; 0 stderr records; 15/15 level-runs passed | +| N=8 stress, distinct | 0 / 20,000 | 0 / 20,000 | +| N=8 stress, duplicate | 0 / 20,000 | 0 (no logical-event loss) | +| Held lock, actual release ≤ 2013 ms | 0 / 80 samples | n/a | +| Held lock, actual release ≥ 2254 ms | 30 / 30 samples (by design) | n/a | + +Supported-load slow operations: none. No strict N=2–4 operation took ≥ 500 ms, so this campaign did not reproduce a supported-load lock holder of 1.6–1.7 s. Coverage of such a holder rests on the held-lock experiment (1500–2000 ms holds succeeded 30/30), not on a reproduced concurrent incident. + +N=8 stress: the former exhaustion mode did not appear. There were 181 operations ≥ 500 ms, all of them single-attempt (`Ok(true)` 114, `Ok(false)` 67). The longest single attempt was 859 ms; backoff oversleep was 0.0 ms and time outside attempts/backoff was ≤ 0.02 ms. + +Host: the external monitor recorded 0 sleep-gap events ≥ 50 ms in 50.6 minutes. CPU PSI stayed at p99 2.9 ms/s. Device busy time had the same profile as the historical campaign (p50 593, max 1004 ms/s). As before, IO PSI and iowait are not usable on this host. + +## Supported-load failure under the 500 / 1250 policy + +During final validation of the 500 / 1250 policy (2026-10-05, release build from `HEAD` `e72116239fa8f05befe9fb8e1220c79100914490`, run through `nix develop -c ./scripts/run-cli-cargo.sh test --release`), the strict AC8 suite failed at supported levels. The run was a single pass, not part of the protocol above. **No host IO evidence (`vmstat`, `iostat`, monitor) was captured for it**, so the cause of the long holder transactions is not known. + +| Level | Rounds | Result | Lock errors | Exhaustions | Lost distinct events | p50 / p95 / p99 / max ms | +| --- | --- | --- | --- | --- | --- | --- | +| distinct N=2 | 1000 | pass | 0 | 0 | 0 | 30 / 93 / 135 / 1138 | +| distinct N=3 | 1000 | pass | 0 | 0 | 0 | 45 / 105 / 151 / 236 | +| distinct N=4 | 500 | **fail** | 1 | 1 | **1** | 45 / 173 / 254 / 1666 | +| duplicate N=2 | 500 | pass | 0 | 0 | 0 | 18 / 26 / 26 / 34 | +| duplicate N=3 | 500 | **fail** | 2 | 2 | 0 (another writer persisted the event) | 40 / 119 / 168 / 1616 | +| duplicate N=4 | 500 | pass | 0 | 0 | 0 | 62 / 123 / 177 / 400 | +| distinct N=8 (stress) | 500 | non-strict | 2 | 2 | 2 | 105 / 360 / 530 / 2134 | +| duplicate N=8 (stress) | 500 | non-strict | 0 | 0 | 0 | 86 / 245 / 451 / 650 | + +Retry timelines of the failing supported rounds (test-only instrumentation): + +| Level | Round | Operation | Result | Elapsed ms | Attempt durations ms | Backoff requested / actual ms | Outside attempts + backoff ms | +| --- | --- | --- | --- | --- | --- | --- | --- | +| distinct N=4 | 384 | holder | `Ok(true)` | 1666.8 | 1666.8 | - | 0.00 | +| distinct N=4 | 384 | waiter | `database is locked` | 1006.5 | 500.2 + 500.2 | 6 / 6.1 | 0.01 | +| duplicate N=3 | 338 | holder | `Ok(true)` | 1616.1 | 1616.1 | - | 0.00 | +| duplicate N=3 | 338 | waiter | `database is locked` | 1089.5 | 500.2 + 500.2 | 89 / 89.1 | 0.01 | +| duplicate N=3 | 338 | waiter | `database is locked` | 1064.5 | 500.2 + 500.2 | 64 / 64.1 | 0.02 | + +Why the waiters exhausted: + +- **The holder kept the writer lock beyond the waiter's budget.** The holding transaction's single attempt took 1616–1667 ms; the waiters gave up at 1006–1090 ms. +- **Retry admission did not reject attempt 2.** Every waiter made both attempts. +- **Turso did not return `Busy` early.** Each busy wait lasted the full 500.2 ms. +- **No scheduler delay was observed.** Backoff matched the request within 0.1 ms, time outside attempts and backoff was ≤ 0.02 ms, and the in-process gap monitor recorded 0 gaps ≥ 50 ms. +- Why the holder transaction took 1.6–1.7 s is not established; no host evidence was captured. + +This failure stays in the dataset. It is the reason the defaults were tuned. + +## Policy comparison: 500 / 1250 / 2 vs 1000 / 2250 / 2 + +All figures are observed on the reference host, not guarantees. The 500 / 1250 column combines the historical campaign and the failed validation run. + +| Measure | 500 / 1250 / 2 | 1000 / 2250 / 2 | +| --- | --- | --- | +| Supported N=2–4 in-process loss | 0 / 35,000 distinct (campaign); **1 / 7,000** distinct (validation run) | 0 / 35,000 distinct | +| Supported N=2–4 in-process exhaustions | 0 / 57,500 (campaign); **3 / 11,500** (validation run) | 0 / 57,500 | +| Supported strict matrices passed | 5/5 (campaign); validation run failed | 5/5 | +| Real hooks N=2–4 | 0 / 16,500 lost | 0 / 16,500 lost | +| In-process p50 range (distinct N=2 / 3 / 4) | 22.5–42.6 / 36.7–69.0 / 43.6–54.4 ms | 21.7–30.1 / 31.4–50.2 / 31.3–45.3 ms | +| In-process p95 range (distinct N=2 / 3 / 4) | 44–95 / 92–141 / 69–145 ms | 61–93 / 67–127 / 66–99 ms | +| In-process p99 range (distinct N=2 / 3 / 4) | 87–137 / 138–216 / 156–220 ms | 102–135 / 134–202 / 99–167 ms | +| In-process worst max, supported | 404 ms (campaign); 1666 ms (validation run) | 399 ms | +| Real-hook worst max (N=2 / 3 / 4) | 223 / 105 / 543 ms | 221 / 236 / 388 ms | +| Held-lock transition (actual holder release) | succeeded ≤ 1026 ms; exhausted ≥ 1503 ms | succeeded ≤ 2013 ms; exhausted ≥ 2254 ms | +| Exhausting writer latency (blocked write gives up) | 1.0–1.1 s (observed 1003–1099 ms) | 2.0–2.1 s (observed 2001–2094 ms) | +| N=8 stress exhaustions | 16 / 40,000 (campaign, 1 of 5 duplicate runs); 2 / 8,000 (validation run) | 0 / 40,000 | +| N=8 stress worst max | 1834 ms (campaign); 2134 ms (validation run) | 859 ms | + +Latency cost of the larger budget: + +- **Worst blocked-hook latency roughly doubles: about +1.0 s.** A write that exhausts now blocks its hook for about 2.0–2.1 s instead of 1.0–1.1 s. The bounded-wait property is unchanged: no hook waits indefinitely, and no wait exceeds two busy timeouts plus at most 100 ms backoff. +- **Writes blocked 0.5–1.0 s no longer pay for an outer retry.** A 750 ms hold completed at 841–847 ms on attempt 1, versus 771–860 ms through a retry before. The cost is similar. +- **The uncontended and normally contended path is unchanged.** Neither policy produced supported-load slow operations in its campaign, and the per-run percentile ranges overlap. The busy timeout only adds latency when a lock is held longer than the old 500 ms. + +### Policy decision + +Adopt 1000 / 2250 / 2 / 100 ms as the defaults. + +- All 5 supported strict in-process matrices and all 5 strict real-hook matrices passed, with 0 lock errors, 0 exhaustions and 0 lost events. +- The policy covers the 1.6–1.7 s lock holder that failed the 500 / 1250 validation run, with about 300 ms of slack before the observed 2.0–2.1 s exhaustion point. +- The former N=8 exhaustion mode did not occur (0 / 40,000). +- The price is about +1 s of worst-case blocked-hook latency on exhausting writes. +- Limit: a supported-load holder above about 2 s would still exhaust this policy. If that is observed, the next step is not further timeout tuning. Bounded synchronous hook latency, no durable queue or spool, and unbounded writer-lock duration together mean zero-loss ingestion cannot be guaranteed. That would need a product/architecture decision between accepting occasional fail-open ingestion loss and introducing eventual persistence (spool or queue). + +## Historical campaign: 500 / 1250 / 2 / 100 + +**Superseded.** The sections below are the full measurement campaign for the earlier 500 / 1250 defaults, kept unchanged as evidence. Their contract, conclusions and policy decision describe that policy, not the current defaults. + +### Test environment + +| Item | Value | +| --- | --- | +| Host | bare metal (`systemd-detect-virt` → `none`, no `hypervisor` CPU flag), not CI (`CI` unset) | +| Kernel | `Linux nixos 6.18.37 #1-NixOS SMP PREEMPT_DYNAMIC Sat Jun 27 10:06:50 UTC 2026 x86_64 GNU/Linux` | +| CPU | AMD Ryzen 9 5900X, 12 cores / 24 threads, 1 socket | +| RAM | 46 GiB total, about 41 GiB available at start, swap unused | +| Test workspace | `std::env::temp_dir()` = `/tmp`, ext4 on LUKS dm-crypt (`/dev/mapper/cryptroot`, `dm-0`) on NVMe (`nvme0n1`); `/tmp` is not tmpfs | +| Turso | `turso` crate `0.8.1` (`cli/Cargo.lock`), linked into both test and `sce` binaries. The repo-pinned Turso CLI (0.7.0) was not used. | +| Rust | `rustc 1.95.0 (59807616e 2026-04-14)`, `cargo 1.95.0`, via `nix develop` | +| Other load | Desktop session left running: Brave, Discord, Ghostty, and an idle Claude Code session. No builds, package managers or other benchmarks ran during the campaign. | +| Campaign date | 2026-10-05, about 12:29–13:18 local; 49.0 minutes of measured runs | + +Host-health caveat: on this host `/proc/pressure/io` reads about 40–95% "full" even when idle. The disk had 0 requests in flight and flat `io_ms` counters at the same time. The cause is Ghostty's renderer and IO threads parked in `io_cqring_wait`: the kernel accounts io_uring CQ waits as iowait. Two "blocked" tasks therefore appear in `vmstat`'s `b` column, and about 8–9% shows as `wa`, permanently. IO PSI, `vmstat wa`/`b` and `cpu_iowait` are **not** usable as host-stall evidence here. This doc uses scheduler-gap monitors, CPU PSI, per-device busy time, and the operation timelines. + +### Exact commit/config + +- `git rev-parse HEAD` → `a33797054ae774783124c4b312be209dd94e6e36` (branch `agent-trace-db-write-contention`); `git status --short` was clean before the campaign. +- Measurement build = HEAD + a **test-only** instrumentation diff, sha256 `61e92a853fc9d6e03703bbcc0e8bc82c93dbabf620d43b1bad445a0caaf241a9`: + - `cli/src/services/db/mod.rs`: `#[cfg(test)]` `record_write_contention_timeline` with `Instant` stamps for attempt start/end, backoff requested, and backoff slept. + - `cli/src/services/agent_trace_db/lock_contention_tests.rs`: JSON `SCE_MEAS` records, per-operation timelines, slow-operation (≥ 500 ms) records, and an in-process scheduler-gap monitor (5 ms sleep loop; reports wake-ups ≥ 50 ms late). + - No non-test code path changed. + - After the campaign, the working tree took three clippy-only refactors: `RefCell::take` method reference, a `HookRound` struct for the round return type, and `#[allow(clippy::too_many_lines)]` on two test functions. None of them changes behavior. The measured binaries predate them. + - The raw `SCE_MEAS` logs, host snapshots and monitor JSONL were kept in the session scratchpad and are not committed. Every per-run figure below is copied from them. +- The policy ran with defaults, with no `policies.database_retry.agent_trace_db` override: + + | Setting | Value | + | --- | --- | + | `busy_timeout_ms` | 500 | + | `contention_deadline_ms` | 1250 | + | max attempts | 2 | + | full-jitter backoff | `0..=100` ms | +- All measurements used one frozen pair of release artifacts, built once before the first run. Build time is not in any latency. + - test binary sha256 `3d5445513b25accd34473898864f556d6c4f61c34a64ffdbc5d5a7cc9f90a312`; + - `sce` sha256 `c044fd62cc0d8baae60fcb07734d3c1d8850eb34ecddf8fc832104a0339ce29a`. + +### Contractual assertions + +These are the only assertions the suite makes. + +- Held-lock boundary (`lock_budget_boundary_characterizes_single_writer_blocked_by_begin_immediate_holder`): + - holds ≤ 250 ms must succeed with 0 exhaustions; + - holds ≥ 2000 ms must fail with the `under write contention` error and exactly 1 exhaustion; + - every failure leaves 0 message and 0 part rows; + - every sample makes 1–2 attempts, and `outer_retries + 1 == attempts`. + - 500–1500 ms is characterization only. +- Every in-process round: + - `messages == parts`; + - the `Ok(true)` count equals the persisted rows; + - no writer exceeds 2 attempts; + - every duplicate round inserts at most once. +- `SCE_LOCK_CONTENTION_STRICT=1` adds, per level: + - 0 lock errors, 0 other errors, 0 lost events and 0 exhaustions; + - for hook processes: every expected message and part persisted, and 0 non-zero exits. +- Strict supported levels: + + | Suite | Levels (N × rounds) | + | --- | --- | + | distinct events | 2×1000, 3×1000, 4×500 | + | duplicate delivery | 2×500, 3×500, 4×500 | + | hook processes | 2×500, 3×500, 4×200 | + + N=8 is stress characterization, not a supported requirement. + +### Measurement methodology + +Artifacts were built once and frozen: + +```sh +nix develop -c ./scripts/run-cli-cargo.sh test --release --manifest-path cli/Cargo.toml --no-run +nix develop -c ./scripts/run-cli-cargo.sh build --release --manifest-path cli/Cargo.toml +cp cli/target/release/deps/sce- $S/bin/sce-tests && cp cli/target/release/sce $S/bin/sce +``` + +Each measured run was a separate process of the frozen test binary. Test path prefix: `services::agent_trace_db::lock_contention_tests::`. + +```sh +# A (×10) +sce-tests --exact lock_budget_boundary_characterizes_single_writer_blocked_by_begin_immediate_holder --nocapture --test-threads=1 +# B/C: one process per level; D adds SCE_BIN=$S/bin/sce +SCE_LOCK_CONTENTION_STRICT=1 SCE_LOCK_CONTENTION_WRITERS= SCE_LOCK_CONTENTION_ROUNDS= \ + sce-tests --exact --ignored --nocapture --test-threads=1 +``` + +- `` is one of: + - `concurrent_distinct_events_persist_every_event_under_write_contention`; + - `concurrent_duplicate_delivery_persists_each_event_once_under_write_contention`; + - `concurrent_real_codex_hook_processes_persist_every_distinct_event`. +- Run order: + 1. A: 10 runs; + 2. B: matrices 1–5, each in the order d2, d3, d4, dup2, dup3, dup4; + 3. C: runs 1–5, each distinct then duplicate N=8; + 4. D: matrices 1–5, each in the order N=2, N=3, N=4. +- Every level ran in strict mode as its own process, so a strict failure could not stop later levels. +- Before every run: `uptime`, `/proc/pressure/*`, `vmstat 1 5`, and `iostat -xz 1 5` (sysstat 12.7.7 via a pre-resolved Nix store path). +- After every non-zero exit: `uptime`, `/proc/pressure/*`, `vmstat 1 10`, and `iostat -xz 1 10`. +- For the whole campaign, an external monitor process (`hostmon.py`) logged two things: + - per second: CPU, memory and IO PSI deltas, iowait and steal, per-device busy ms and in-flight IO for `nvme0n1`/`dm-0`, `procs_running` and `procs_blocked`; + - every wake-up of a 5 ms sleep loop that was ≥ 50 ms late, with Unix ms. +- Latency: + - in-process: a monotonic clock around `insert_conversation_text_event`, from barrier release; + - hooks: from the payload write to the child's exit (includes process startup). + - Percentiles are nearest-rank per level-run. Raw per-write latencies were not retained, so cross-run aggregates report the range of per-run percentiles and the worst max. +- Two smoke runs preceded the protocol and are not in the dataset: one boundary run, which passed, and hook N=2×5, which passed. Nothing was rerun or replaced, and no run was dropped. + +### Held-lock results + +Contract: ≤ 250 ms succeeds, ≥ 2000 ms exhausts, failure leaves 0/0 rows, ≤ 2 attempts. All 10 runs satisfied it: 70/70 samples, 10/10 test passes. + +Observed on reference host (10 runs × 7 holds): + +| hold ms | success | exhausted | attempt-1 success | attempt-2 success | writer p50 ms | writer p95 ms | writer max ms | holder release p50 ms | holder release max ms | +|---|---|---|---|---|---|---|---|---|---| +| 100 | 10/10 | 0/10 | 10 | 0 | 116.5 | 155.4 | 155.4 | 104.2 | 114.5 | +| 250 | 10/10 | 0/10 | 10 | 0 | 341.2 | 369.5 | 369.5 | 262.3 | 262.4 | +| 500 | 10/10 | 0/10 | 9 | 1 | 520.8 | 597.5 | 597.5 | 512.3 | 515.1 | +| 750 | 10/10 | 0/10 | 0 | 10 | 799.2 | 859.5 | 859.5 | 762.3 | 766.4 | +| 1000 | 10/10 | 0/10 | 0 | 10 | 1035.9 | 1102.0 | 1102.0 | 1012.2 | 1025.7 | +| 1500 | 0/10 | 10/10 | 0 | 0 | 1052.4 | 1098.4 | 1098.4 | 1511.6 | 1515.9 | +| 2000 | 0/10 | 10/10 | 0 | 0 | 1021.5 | 1099.4 | 1099.4 | 2012.3 | 2035.5 | + +What the timelines show: + +- **Attempt 1 always lasts about 500.1–500.2 ms when blocked.** That is the busy timeout. +- **Backoff sleeps matched the requested jitter within ≤ 0.1 ms** in every sample. +- **The empirical success/exhaust transition lies between 1000 and 1500 ms of requested hold.** In this run set, actual holder release was 1012–1026 ms (all succeeded) versus 1503–1516 ms (all exhausted). + - A blocked writer exhausts at about 1.0–1.1 s: 500 + jitter + 500. + - A write succeeds only if the holder releases before attempt 2's busy wait ends, at about 1.0–1.1 s after start, depending on the drawn backoff. + - Requested holds tracked actual release closely (+2–36 ms), so no host-delayed releases occurred in this campaign. +- **500 ms is not a first-attempt guarantee.** 9/10 succeeded on attempt 1. In 1/10 the holder released 5–15 ms after the 500 ms busy timeout expired, which forced an outer retry. +- **Turso's busy wait polls; it is not woken on release.** + - 250 ms holds completed at 341 ms (p50), about 80 ms after release. + - 100 ms holds completed at 116 ms (p50). + +Raw samples: + +| run | hold ms | holder released ms | writer elapsed ms | result | attempts | outer retries | exhaustions | messages | parts | attempt1 ms | backoff req/actual ms | attempt2 ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 100 | 104.2 | 117.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 117.1 | - | - | +| 1 | 250 | 262.3 | 342.0 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 342.0 | - | - | +| 1 | 500 | 512.7 | 520.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 520.8 | - | - | +| 1 | 750 | 762.4 | 853.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 12/12.1 | 341.3 | +| 1 | 1000 | 1012.2 | 1041.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 28/28.1 | 513.2 | +| 1 | 1500 | 1513.0 | 1055.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 55/55.1 | 500.2 | +| 1 | 2000 | 2013.1 | 1084.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 84/84.0 | 500.2 | +| 2 | 100 | 104.0 | 115.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.5 | - | - | +| 2 | 250 | 262.2 | 340.9 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.9 | - | - | +| 2 | 500 | 512.6 | 520.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 520.8 | - | - | +| 2 | 750 | 762.1 | 793.9 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 53/53.0 | 240.7 | +| 2 | 1000 | 1012.3 | 1024.8 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 11/11.1 | 513.6 | +| 2 | 1500 | 1513.3 | 1095.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.1 | 95/95.0 | 500.2 | +| 2 | 2000 | 2012.3 | 1015.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 15/15.1 | 500.1 | +| 3 | 100 | 104.0 | 115.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.5 | - | - | +| 3 | 250 | 262.4 | 341.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.1 | - | - | +| 3 | 500 | 512.3 | 520.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 520.8 | - | - | +| 3 | 750 | 762.1 | 854.6 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 13/13.1 | 341.3 | +| 3 | 1000 | 1012.5 | 1027.3 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 86/86.0 | 441.1 | +| 3 | 1500 | 1511.6 | 1052.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 52/52.1 | 500.2 | +| 3 | 2000 | 2012.2 | 1032.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 32/32.1 | 500.2 | +| 4 | 100 | 104.4 | 116.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 116.7 | - | - | +| 4 | 250 | 262.4 | 341.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.2 | - | - | +| 4 | 500 | 512.2 | 521.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 521.5 | - | - | +| 4 | 750 | 763.0 | 778.0 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 87/87.1 | 190.8 | +| 4 | 1000 | 1012.5 | 1035.9 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 95/95.1 | 440.7 | +| 4 | 1500 | 1512.4 | 1048.3 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.1 | 48/48.1 | 500.2 | +| 4 | 2000 | 2012.9 | 1099.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 99/99.0 | 500.2 | +| 5 | 100 | 104.4 | 116.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 116.5 | - | - | +| 5 | 250 | 262.4 | 340.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.8 | - | - | +| 5 | 500 | 512.6 | 520.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 520.4 | - | - | +| 5 | 750 | 762.4 | 859.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 18/18.1 | 341.3 | +| 5 | 1000 | 1012.2 | 1066.1 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.1 | 53/53.1 | 512.9 | +| 5 | 1500 | 1515.9 | 1048.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 48/48.1 | 500.2 | +| 5 | 2000 | 2012.4 | 1029.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 29/29.1 | 500.2 | +| 6 | 100 | 103.9 | 115.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.1 | - | - | +| 6 | 250 | 262.4 | 340.2 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 340.2 | - | - | +| 6 | 500 | 515.1 | 522.8 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 522.8 | - | - | +| 6 | 750 | 762.3 | 771.4 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 78/78.1 | 193.2 | +| 6 | 1000 | 1012.0 | 1023.4 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 11/11.1 | 512.2 | +| 6 | 1500 | 1512.5 | 1043.3 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.1 | 43/43.1 | 500.1 | +| 6 | 2000 | 2012.7 | 1020.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 20/20.1 | 500.2 | +| 7 | 100 | 103.9 | 115.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 115.6 | - | - | +| 7 | 250 | 262.3 | 341.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 341.4 | - | - | +| 7 | 500 | 512.5 | 520.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 520.7 | - | - | +| 7 | 750 | 766.4 | 799.2 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.1 | 59/59.1 | 240.0 | +| 7 | 1000 | 1004.9 | 1020.0 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.1 | 8/8.1 | 511.8 | +| 7 | 1500 | 1506.6 | 1090.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 90/90.0 | 500.2 | +| 7 | 2000 | 2004.2 | 1048.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 48/48.1 | 500.1 | +| 8 | 100 | 105.2 | 138.7 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 138.7 | - | - | +| 8 | 250 | 254.6 | 346.1 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 346.1 | - | - | +| 8 | 500 | 504.7 | 597.5 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 76/76.1 | 21.3 | +| 8 | 750 | 754.5 | 787.1 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 72/72.0 | 214.9 | +| 8 | 1000 | 1008.3 | 1089.7 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 59/59.1 | 530.4 | +| 8 | 1500 | 1505.0 | 1066.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.1 | 66/66.1 | 500.2 | +| 8 | 2000 | 2005.2 | 1009.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 9/9.1 | 500.2 | +| 9 | 100 | 104.2 | 132.0 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 132.0 | - | - | +| 9 | 250 | 254.1 | 348.6 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 348.6 | - | - | +| 9 | 500 | 504.4 | 541.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 541.5 | - | - | +| 9 | 750 | 754.6 | 801.8 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 56/56.0 | 245.6 | +| 9 | 1000 | 1014.7 | 1102.0 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.1 | 71/71.1 | 530.8 | +| 9 | 1500 | 1505.2 | 1005.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.1 | 5/5.1 | 500.2 | +| 9 | 2000 | 2004.4 | 1003.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 3/3.1 | 500.2 | +| 10 | 100 | 114.5 | 155.4 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 155.4 | - | - | +| 10 | 250 | 254.7 | 369.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 369.4 | - | - | +| 10 | 500 | 504.6 | 541.5 | Ok(true) | 1 | 0 | 0 | 1 | 1 | 541.5 | - | - | +| 10 | 750 | 764.6 | 802.7 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.2 | 26/26.1 | 276.5 | +| 10 | 1000 | 1025.7 | 1085.1 | Ok(true) | 2 | 1 | 0 | 1 | 1 | 500.1 | 33/33.1 | 551.9 | +| 10 | 1500 | 1505.9 | 1098.4 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.1 | 98/98.0 | 500.2 | +| 10 | 2000 | 2035.5 | 1021.5 | database is locked | 2 | 1 | 1 | 0 | 0 | 500.2 | 21/21.1 | 500.2 | + +### Strict N=2–4 results + +| run | mode | N | rounds | total writes | expected rows | persisted rows | lost | lock errors | other errors | orphan rows | duplicate rows | attempts | outer retries | exhaustions | p50 ms | p95 ms | p99 ms | max ms | ops ≥500 ms | in-proc stall gaps ≥50 ms (max) | exit | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 42.6 | 92.7 | 136.5 | 186.4 | 0 | 0 (0) | 0 | +| 2 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 29.7 | 70.8 | 100.2 | 241.2 | 0 | 0 (0) | 0 | +| 3 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 29.7 | 94.7 | 136.0 | 192.1 | 0 | 0 (0) | 0 | +| 4 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 24.3 | 70.9 | 102.0 | 151.1 | 0 | 0 (0) | 0 | +| 5 | distinct | 2 | 1000 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 22.5 | 44.4 | 87.3 | 150.1 | 0 | 0 (0) | 0 | +| 1 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 69.0 | 140.7 | 215.6 | 315.7 | 0 | 0 (0) | 0 | +| 2 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 44.4 | 114.5 | 167.0 | 264.3 | 0 | 0 (0) | 0 | +| 3 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 38.0 | 100.2 | 137.8 | 261.9 | 0 | 0 (0) | 0 | +| 4 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 36.7 | 92.3 | 151.6 | 236.2 | 0 | 0 (0) | 0 | +| 5 | distinct | 3 | 1000 | 3000 | 3000 | 3000 | 0 | 0 | 0 | 0 | 0 | 3000 | 0 | 0 | 44.9 | 119.1 | 184.0 | 304.5 | 0 | 0 (0) | 0 | +| 1 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 46.1 | 127.8 | 175.4 | 252.6 | 0 | 0 (0) | 0 | +| 2 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 54.4 | 139.6 | 220.1 | 404.2 | 0 | 0 (0) | 0 | +| 3 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 45.1 | 144.5 | 210.9 | 360.7 | 0 | 0 (0) | 0 | +| 4 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 44.4 | 69.2 | 156.0 | 239.9 | 0 | 0 (0) | 0 | +| 5 | distinct | 4 | 500 | 2000 | 2000 | 2000 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 43.6 | 97.5 | 178.4 | 341.3 | 0 | 0 (0) | 0 | +| 1 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 16.9 | 32.0 | 40.1 | 91.5 | 0 | 0 (0) | 0 | +| 2 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 18.4 | 26.7 | 45.5 | 123.0 | 0 | 0 (0) | 0 | +| 3 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 27.0 | 45.7 | 93.7 | 150.7 | 0 | 0 (0) | 0 | +| 4 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 18.5 | 27.2 | 31.1 | 222.9 | 0 | 0 (0) | 0 | +| 5 | duplicate | 2 | 500 | 1000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1000 | 0 | 0 | 15.4 | 26.5 | 26.8 | 32.7 | 0 | 0 (0) | 0 | +| 1 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 22.4 | 41.3 | 41.6 | 45.9 | 0 | 0 (0) | 0 | +| 2 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 26.1 | 41.5 | 41.7 | 42.0 | 0 | 0 (0) | 0 | +| 3 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 17.6 | 40.0 | 41.0 | 41.7 | 0 | 0 (0) | 0 | +| 4 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 19.1 | 41.2 | 41.5 | 44.9 | 0 | 0 (0) | 0 | +| 5 | duplicate | 3 | 500 | 1500 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 1500 | 0 | 0 | 24.4 | 41.3 | 41.5 | 45.4 | 0 | 0 (0) | 0 | +| 1 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 26.9 | 61.8 | 66.7 | 111.7 | 0 | 0 (0) | 0 | +| 2 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 26.4 | 61.4 | 61.7 | 65.4 | 0 | 0 (0) | 0 | +| 3 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 26.3 | 61.5 | 68.8 | 111.4 | 0 | 0 (0) | 0 | +| 4 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 38.7 | 86.5 | 127.9 | 237.3 | 0 | 0 (0) | 0 | +| 5 | duplicate | 4 | 500 | 2000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 2000 | 0 | 0 | 26.6 | 61.6 | 62.2 | 87.1 | 0 | 0 (0) | 0 | + +Aggregate across runs: (latency percentiles are of per-run values; pooled raw latencies are not retained, so the pooled column is the worst per-run value) + +| mode | N | runs passing strict | total writes | expected rows | persisted | lost | lock errors | other errors | exhaustions | exhaustion rate | outer retries | attempts | p50 range ms | p95 range ms | p99 range ms | worst max ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| distinct | 2 | 5/5 | 10000 | 10000 | 10000 | 0 | 0 | 0 | 0 | 0 / 10000 = 0.000% | 0 | 10000 | 22.5–42.6 | 44.4–94.7 | 87.3–136.5 | 241.2 | +| distinct | 3 | 5/5 | 15000 | 15000 | 15000 | 0 | 0 | 0 | 0 | 0 / 15000 = 0.000% | 0 | 15000 | 36.7–69.0 | 92.3–140.7 | 137.8–215.6 | 315.7 | +| distinct | 4 | 5/5 | 10000 | 10000 | 10000 | 0 | 0 | 0 | 0 | 0 / 10000 = 0.000% | 0 | 10000 | 43.6–54.4 | 69.2–144.5 | 156.0–220.1 | 404.2 | +| duplicate | 2 | 5/5 | 5000 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 5000 = 0.000% | 0 | 5000 | 15.4–27.0 | 26.5–45.7 | 26.8–93.7 | 222.9 | +| duplicate | 3 | 5/5 | 7500 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 7500 = 0.000% | 0 | 7500 | 17.6–26.1 | 40.0–41.5 | 41.0–41.7 | 45.9 | +| duplicate | 4 | 5/5 | 10000 | 2500 | 2500 | 0 | 0 | 0 | 0 | 0 / 10000 = 0.000% | 0 | 10000 | 26.3–38.7 | 61.4–86.5 | 61.7–127.9 | 237.3 | + +Strict matrices passing: 5/5. Writes: 57500. Distinct events requested: 35000, distinct events lost: 0. Exhaustions: 0. Lock errors: 0. + +Every strict write in B committed on its first attempt: 57,500 writes, 0 outer retries, and no operation ≥ 500 ms. The in-process scheduler-gap monitor recorded 0 gaps ≥ 50 ms in all 30 level-runs. + +### N=8 stress results + +**Stress characterization — not a supported requirement.** + +| run | mode | N | rounds | total writes | expected rows | persisted rows | lost | lock errors | other errors | orphan rows | duplicate rows | attempts | outer retries | exhaustions | p50 ms | p95 ms | p99 ms | max ms | ops ≥500 ms | in-proc stall gaps ≥50 ms (max) | exit | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4001 | 1 | 0 | 89.7 | 206.6 | 365.5 | 575.7 | 8 | 0 (0) | 0 | +| 2 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 89.8 | 191.5 | 280.1 | 576.9 | 6 | 0 (0) | 0 | +| 3 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4002 | 2 | 0 | 89.3 | 193.9 | 354.0 | 662.5 | 9 | 0 (0) | 0 | +| 4 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4001 | 1 | 0 | 87.8 | 192.0 | 256.6 | 577.8 | 2 | 0 (0) | 0 | +| 5 | distinct | 8 | 500 | 4000 | 4000 | 4000 | 0 | 0 | 0 | 0 | 0 | 4007 | 7 | 0 | 89.7 | 193.1 | 354.0 | 677.6 | 18 | 0 (0) | 0 | +| 1 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4000 | 0 | 0 | 86.1 | 235.9 | 346.7 | 522.4 | 4 | 0 (0) | 0 | +| 2 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 16 | 0 | 0 | 0 | 4022 | 22 | 16 | 86.4 | 229.2 | 445.0 | 1834.4 | 34 | 0 (0) | 101 | +| 3 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4001 | 1 | 0 | 86.1 | 188.7 | 337.8 | 559.4 | 3 | 0 (0) | 0 | +| 4 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4003 | 3 | 0 | 86.1 | 192.2 | 346.1 | 615.1 | 12 | 0 (0) | 0 | +| 5 | duplicate | 8 | 500 | 4000 | 500 | 500 | 0 | 0 | 0 | 0 | 0 | 4003 | 3 | 0 | 85.3 | 194.0 | 350.6 | 602.2 | 11 | 0 (0) | 0 | + +Aggregate across runs: (latency percentiles are of per-run values; pooled raw latencies are not retained, so the pooled column is the worst per-run value) + +| mode | N | runs passing strict | total writes | expected rows | persisted | lost | lock errors | other errors | exhaustions | exhaustion rate | outer retries | attempts | p50 range ms | p95 range ms | p99 range ms | worst max ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| distinct | 8 | 5/5 | 20000 | 20000 | 20000 | 0 | 0 | 0 | 0 | 0 / 20000 = 0.000% | 11 | 20011 | 87.8–89.8 | 191.5–206.6 | 256.6–365.5 | 677.6 | +| duplicate | 8 | 4/5 | 20000 | 2500 | 2500 | 0 | 16 | 0 | 16 | 16 / 20000 = 0.080% | 29 | 20029 | 85.3–86.4 | 188.7–235.9 | 337.8–445.0 | 1834.4 | + +N=8 writes: 40000. Distinct events requested: 20000, lost: 0. Exhaustions: 16. Lock errors: 16. + +### Real-hook results + +| run | N | rounds | expected | persisted msgs | persisted parts | lost | fail-open lost | non-zero exits | stderr outputs | stderr lines | p50 ms | p95 ms | p99 ms | max ms | procs ≥500 ms | in-proc stall gaps ≥50 ms (max) | exit | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 36.2 | 77.0 | 123.9 | 223.1 | 0 | 0 (0) | 0 | +| 2 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 27.5 | 38.0 | 45.2 | 64.1 | 0 | 0 (0) | 0 | +| 3 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 29.6 | 51.4 | 72.3 | 134.9 | 0 | 0 (0) | 0 | +| 4 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 28.7 | 42.6 | 75.7 | 173.3 | 0 | 0 (0) | 0 | +| 5 | 2 | 500 | 1000 | 1000 | 1000 | 0 | 0 | 0 | 0 | 0 | 24.5 | 37.1 | 38.3 | 46.9 | 0 | 0 (0) | 0 | +| 1 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 34.2 | 52.2 | 53.1 | 60.2 | 0 | 0 (0) | 0 | +| 2 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 30.7 | 52.0 | 52.8 | 59.2 | 0 | 0 (0) | 0 | +| 3 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 29.1 | 52.1 | 52.7 | 72.2 | 0 | 0 (0) | 0 | +| 4 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 36.5 | 52.6 | 54.1 | 72.5 | 0 | 0 (0) | 0 | +| 5 | 3 | 500 | 1500 | 1500 | 1500 | 0 | 0 | 0 | 0 | 0 | 36.1 | 52.5 | 58.0 | 105.2 | 0 | 0 (0) | 0 | +| 1 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 37.3 | 72.5 | 73.5 | 79.4 | 0 | 0 (0) | 0 | +| 2 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 51.3 | 97.5 | 129.1 | 410.9 | 0 | 0 (0) | 0 | +| 3 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 51.4 | 99.8 | 160.8 | 281.1 | 0 | 0 (0) | 0 | +| 4 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 61.8 | 139.8 | 203.0 | 542.5 | 1 | 0 (0) | 0 | +| 5 | 4 | 200 | 800 | 800 | 800 | 0 | 0 | 0 | 0 | 0 | 51.1 | 77.2 | 109.3 | 160.9 | 0 | 0 (0) | 0 | + +Aggregate: + +| N | runs passing strict | expected | persisted | lost | fail-open lost | non-zero exits | stderr lines | p50 range ms | p95 range ms | p99 range ms | worst max ms | +|---|---|---|---|---|---|---|---|---|---|---|---| +| 2 | 5/5 | 5000 | 5000 | 0 | 0 | 0 | 0 | 24.5–36.2 | 37.1–77.0 | 38.3–123.9 | 223.1 | +| 3 | 5/5 | 7500 | 7500 | 0 | 0 | 0 | 0 | 29.1–36.5 | 52.0–52.6 | 52.7–58.0 | 105.2 | +| 4 | 5/5 | 4000 | 4000 | 0 | 0 | 0 | 0 | 37.3–61.8 | 72.5–139.8 | 73.5–203.0 | 542.5 | + +There was no fail-open persistence loss: in no round was an event lost while the hook exited 0. One hook process took ≥ 500 ms: 542.5 ms, D-m4 N=4. Process-local contention counters do not exist across processes, so the table shows no attempt or exhaustion figures for hooks. + +### Observed failures + +Only one run exited non-zero: **`C-r2-duplicate-n8`** (N=8 stress, duplicate delivery, run 2 of 5, exit 101). The strict assertion failed with `8 concurrent writers produced 16 lock error(s) and 0 other error(s)`. + +- 16 contention exhaustions out of 4,000 writes (0.40% of that run). They fell in 4 of 500 rounds: 272 (5), 273 (6), 274 (4) and 453 (1). +- 0 lost logical events. In every affected round at least one writer persisted or found the event (`Ok(true)`/`Ok(false)`), so `persisted rows = 500/500`. In **distinct** mode, the same exhaustions would have lost events. +- 0 orphan rows, 0 duplicate rows, no writer above 2 attempts. +- The same level ran 3 more times afterwards (C-r3, C-r4, C-r5). Each run had 0 exhaustions. The failure was not reproduced in those 3 runs, and it stays in the dataset. +- Totals for all strict-mode runs across A–D: 1 failing level-run out of 55 contention level-runs, and 0 out of 45 at supported N=2–4. + +Anatomy of the failure (from the test-only timelines): + +| run | round | writer | started (unix ms) | elapsed ms | attempt durations ms | backoff req/actual ms | in-proc gaps in window | host-monitor gaps in window | host psi_cpu some ms/s (max) | disk busy ms/s (max) | +|---|---|---|---|---|---|---|---|---|---|---| +| C-r2-duplicate-n8 | 272 | 2 | 1791197989232 | 1052.4 | 500.2 + 500.1 | 52/52.1 | 0 | 0 | 1.1 | 773 | +| C-r2-duplicate-n8 | 272 | 4 | 1791197989232 | 1085.4 | 500.2 + 500.2 | 85/85.1 | 0 | 0 | 1.1 | 773 | +| C-r2-duplicate-n8 | 272 | 5 | 1791197989232 | 1014.4 | 500.2 + 500.2 | 14/14.1 | 0 | 0 | 1.1 | 773 | +| C-r2-duplicate-n8 | 272 | 6 | 1791197989232 | 1039.4 | 500.2 + 500.1 | 39/39.1 | 0 | 0 | 1.1 | 773 | +| C-r2-duplicate-n8 | 272 | 7 | 1791197989232 | 1024.4 | 500.2 + 500.2 | 24/24.1 | 0 | 0 | 1.1 | 773 | +| C-r2-duplicate-n8 | 273 | 0 | 1791197991003 | 1099.5 | 500.2 + 500.1 | 99/99.1 | 0 | 0 | 0.4 | 995 | +| C-r2-duplicate-n8 | 273 | 2 | 1791197991003 | 1025.5 | 500.2 + 500.2 | 25/25.1 | 0 | 0 | 0.4 | 995 | +| C-r2-duplicate-n8 | 273 | 3 | 1791197991003 | 1062.5 | 500.2 + 500.2 | 62/62.1 | 0 | 0 | 0.4 | 995 | +| C-r2-duplicate-n8 | 273 | 4 | 1791197991003 | 1024.5 | 500.2 + 500.2 | 24/24.1 | 0 | 0 | 0.4 | 995 | +| C-r2-duplicate-n8 | 273 | 6 | 1791197991003 | 1089.5 | 500.2 + 500.2 | 89/89.0 | 0 | 0 | 0.4 | 995 | +| C-r2-duplicate-n8 | 273 | 7 | 1791197991003 | 1018.5 | 500.2 + 500.2 | 18/18.1 | 0 | 0 | 0.4 | 995 | +| C-r2-duplicate-n8 | 274 | 2 | 1791197992535 | 1050.4 | 500.2 + 500.2 | 50/50.1 | 0 | 0 | 0.4 | 993 | +| C-r2-duplicate-n8 | 274 | 3 | 1791197992535 | 1038.4 | 500.2 + 500.2 | 38/38.1 | 0 | 0 | 0.4 | 993 | +| C-r2-duplicate-n8 | 274 | 5 | 1791197992535 | 1081.4 | 500.2 + 500.2 | 81/81.0 | 0 | 0 | 0.4 | 993 | +| C-r2-duplicate-n8 | 274 | 6 | 1791197992535 | 1000.4 | 500.2 + 500.2 | 0/0.0 | 0 | 0 | 0.4 | 993 | +| C-r2-duplicate-n8 | 453 | 2 | 1791198031559 | 1093.4 | 500.2 + 500.2 | 93/93.0 | 0 | 0 | 1.4 | 515 | + +- Every exhausted writer's time breaks down as `500.1–500.2 ms` attempt 1, backoff equal to the drawn jitter (≤ 0.1 ms oversleep), and `500.1–500.2 ms` attempt 2. The time outside attempts and backoff was ≤ 0.05 ms. This is Turso busy wait plus one retry. It is **not** a thread that was not scheduled. +- The writers holding the lock in those rounds spent **far longer than the busy timeout inside a single attempt** (transaction body plus commit): + + | Round | Writer | Result | Attempt durations | + | --- | --- | --- | --- | + | 272 | 0 | `Ok(false)` | 500.2 + 1208.5 ms | + | 273 | 1 | `Ok(true)` | 827.6 ms | + | 273 | 5 | `Ok(false)` | 1527.3 ms | + | 274 | 1 | `Ok(false)` | 500.2 + 1312.2 ms | + | 453 | 1 | `Ok(false)` | 1156.6 ms | + | 453 | 3 | `Ok(false)` | 500.2 + 930.5 ms | + + Normal attempts in the same run were tens of ms. The waiters exhausted because some connection held the write lock (or the commit path) for more than about 1.05 s. + +### Host correlation + +- campaign window (Unix ms): 1791196162002–1791199102331 (49.0 min), 2938 1 s samples +- external monitor sleep-gap events ≥50 ms: 0; max 0.0 ms +- psi_cpu_some_ms: p50 0.9, p99 2.4, max 4.4 +- psi_mem_some_ms: p50 0.0, p99 0.0, max 0.0 +- psi_io_full_ms: p50 529.0, p99 971.3, max 976.0 — io_uring idle-wait artifact, not usable (see Test environment) +- cpu_iowait_pct: p50 9.9, p99 12.8, max 13.5 — same artifact +- disk busy ms per 1 s (max of nvme0n1/dm-0): p50 600, p99 952, max 1001 +- procs_blocked: p50 3, max 5 — baseline 2 are the parked Ghostty io_uring threads + +Slow-operation anatomy (all operations ≥ 500 ms): + +Attempt duration is the time inside one Turso attempt (busy-handler wait included). Backoff actual vs requested shows scheduler oversleep. 'Outside' is time not inside an attempt or backoff. + + +##### B (strict N=2–4): 0 slow ops + + +##### C (N=8 stress): 107 slow ops + +- Ok(false), 1 attempt(s): 34 +- Ok(false), 2 attempt(s): 13 +- Ok(true), 1 attempt(s): 33 +- Ok(true), 2 attempt(s): 11 +- database is locked, 2 attempt(s): 16 +- longest single attempt: 1527.3 ms; worst backoff oversleep: 0.1 ms; worst time outside attempts/backoff: 0.05 ms + +Around the failure: + +- **No scheduler stall was observed.** + - The in-process gap monitor: 0 gaps ≥ 50 ms in `C-r2-duplicate-n8`. + - The external gap monitor: 0 events ≥ 50 ms in the failure windows, and 0 across the whole 49-minute campaign. + - CPU PSI stayed at 0.3–1.5 ms/s, no higher than baseline. +- **Disk busy time rose sharply.** `dm-0` busy time went from about 350–390 ms/s, the run's baseline, to 773, 1000, 995, 993 and 998 ms/s. That lasted about 1791197989.9–1791197993.9 (about 4–5 s), exactly covering rounds 272–274. It then fell back to about 400 ms/s. + - `nvme0n1` showed the same shape: 606 → 986 → 840 → 981 → 724 ms/s. + - Round 453 coincided with a smaller rise to 515 ms/s. +- Busy ms/s on NVMe behind dm-crypt is a weak saturation signal. B-m1 distinct runs sat at a 915 ms/s median without failing, because commit fsyncs at N=2–3 keep the device permanently busy. +- The since-boot `iostat` averages show `dm-0` with `w_await ≈ 203 ms` and `aqu-sz ≈ 32`, versus `nvme0n1` with `w_await ≈ 41 ms`. This host's dm-crypt write path has heavy write-latency tails. +- Per-process IO attribution was not captured. It is therefore unknown whether the burst came from this test's own WAL/checkpoint/fsync traffic or from another writer on the same device. + +Classification of `C-r2-duplicate-n8`: + +- **Confirmed DB-policy exhaustion.** The policy behaved as specified, exhausting after 2 attempts at about 1.0–1.1 s against a lock held more than 1 s by a slow in-transaction attempt. +- **Correlated with a device-level IO burst.** `dm-0`/`nvme0n1` were near 100% busy for about 4–5 s. +- **No host-wide scheduler stall observed.** +- **Insufficient evidence** about the IO burst's source. + +None of these waives the strict failure. The failure is outside the supported N=2–4 range. + +The previous version of this doc said that 1–2 s whole-host stalls had made strict N=2–4 runs fail, and that such failures should be treated as host noise. Neither claim is supported by this campaign. No strict N=2–4 run failed, and no scheduler stall was observed. That guidance is withdrawn. + +### Conclusions + +#### Correctness + +No correctness violation in any of the 114,070 measured write operations (A 70 boundary samples, B 57,500, C 40,000, D 16,500 hook events). + +- 0 orphan message/part rows (`messages == parts` in every round and every hook round). +- 0 duplicate logical events (duplicate delivery inserted at most once in all 10,000 duplicate rounds: 7,500 at N=2–4 and 2,500 at N=8). +- 0 partial commits (every failed write left 0/0 rows). +- 0 writers above 2 attempts. + +#### Availability (observed on reference host) + +| Scope | Exhaustions | Lost distinct events | +| --- | --- | --- | +| Strict N=2–4 in-process | 0 / 57,500 writes (0 / 35,000 distinct events lost); 5/5 strict matrices passed | 0 | +| Real hooks N=2–4 | not countable (no cross-process counters) | 0 / 16,500 events; 0 non-zero exits; 0 fail-open losses; 15/15 level-runs passed | +| N=8 stress, distinct | 0 / 20,000 | 0 / 20,000 | +| N=8 stress, duplicate | 16 / 20,000 = 0.080%, all in 1 of 5 runs (4/5 runs passed strict) | 0 (another writer persisted each affected event) | +| Held lock, actual release ≤ 1026 ms | 0 / 50 samples | n/a | +| Held lock, actual release ≥ 1503 ms | 20 / 20 samples (by design) | n/a | + +#### Latency (observed on reference host) + +| Load | p50 | p95 | p99 | Worst max | +| --- | --- | --- | --- | --- | +| N=2 distinct / duplicate | 22.5–42.6 / 15.4–27.0 ms | 44–95 / 27–46 ms | 87–137 / 27–94 ms | 241 / 223 ms | +| N=3 distinct / duplicate | 36.7–69.0 / 17.6–26.1 ms | 92–141 / 40–42 ms | 138–216 / 41–42 ms | 316 / 46 ms | +| N=4 distinct / duplicate | 43.6–54.4 / 26.3–38.7 ms | 69–145 / 61–87 ms | 156–220 / 62–128 ms | 404 / 237 ms | +| N=8 stress distinct / duplicate | 87.8–89.8 / 85.3–86.4 ms | 192–207 / 189–236 ms | 257–366 / 338–445 ms | 678 / 1834 ms | +| Real hooks N=2 / 3 / 4 (incl. process start) | 24–36 / 29–37 / 37–62 ms | 37–77 / 52–53 / 73–140 ms | 38–124 / 53–58 / 74–203 ms | 223 / 105 / 543 ms | + +Ranges are across the 5 runs. The worst max is the largest single write across all runs. + +#### Policy decision + +Overall verdict: current defaults (500 ms busy timeout / 1250 ms deadline / 2 attempts / 100 ms backoff cap) are **adequate** for the supported N=2–4 range, and **borderline at N=8 stress**. + +- **Supported range.** Across 57,500 strict in-process writes and 16,500 real hook events, there was no loss and no exhaustion. B needed no outer retries at all. Tails stayed below about 0.55 s. +- **N=8 stress.** 16 exhaustions in 40,000 writes were concentrated in one about 4–5 s episode. During it, lock-holding transactions themselves took 0.8–1.5 s, coincident with a device-level IO burst. +- **No policy change is recommended from this dataset.** + - The only failure mode observed is a holder whose transaction or commit runs more than about 1.05 s. + - Raising `contention_deadline_ms` alone would not fix it. The exhausted writers stopped at the 2-attempt cap, not at the deadline. + - Fixing it would need, for example, `busy_timeout_ms` ≈ 1000 with `contention_deadline_ms` ≥ about 2100, or a third attempt. + - Either would roughly double worst-case blocked-hook latency, from about 1.1 s to about 2.1 s, for every exhausting write. The benefit would be on a load level that is not supported. + - Whether to pay that cost is a product decision that this evidence does not force. + +## Reproducing + +The suite lives in `cli/src/services/agent_trace_db/lock_contention_tests.rs`, which is `#[cfg(test)]`. It drives the real production API (`insert_conversation_text_event` on hook-runtime connections) and the real release `sce hooks codex` binary against a temporary repository Agent Trace DB. + +- Configuration: + - The ignored tests take `SCE_LOCK_CONTENTION_WRITERS` (comma-separated) and `SCE_LOCK_CONTENTION_ROUNDS`. + - The hook test also needs `SCE_BIN`. + - `SCE_LOCK_CONTENTION_STRICT=1` enables the strict gates. +- How to run: + - Run with `--release` and `-- --ignored --nocapture` through `nix develop -c ./scripts/run-cli-cargo.sh test`. + - Or build once with `--no-run` and invoke the test binary directly, as above. +- Output: + - Each test prints a human table and machine-readable `SCE_MEAS {json}` lines: `boundary_sample`, `concurrent_level`, `slow_operation`, `hook_level`, `slow_hook_process`, `hook_round_loss` and `hook_stderr`. + - Those lines carry per-operation timelines (`record_write_contention_timeline`, test-only) and scheduler-gap monitor results. +- `count_write_contention` and the timelines are thread-local test instrumentation. They are unavailable for the hook-process test. diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index 3ec843382..4811232f8 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -100,7 +100,7 @@ The shared `TursoDb` runner records applied IDs in the database-local `__sce_mig Repository-scoped storage resolution first resolves `agent_trace.repository_id` / `agent_trace.repository_remote` through config, then splits by caller into two resolution paths in `agent_trace_storage`, both sharing the same identity setup and returning the same `ResolvedAgentTraceStorage { metadata: RepositoryMetadata, .. } `: - **Setup/lifecycle** (`resolve_agent_trace_storage(...)`): tries `RepositoryAgentTraceDb::open_without_migrations_at(path)` + `ensure_schema_ready_for_hooks()` + repository metadata validation first. If the repository DB has not been initialized, metadata is absent, or migrations are incomplete, it falls back to migration-running `RepositoryAgentTraceDb::new_at(path)` and validates/seeds `repository_metadata.repository_id` before returning. This is the only path that may run migration `002` (or any migration). It retries the fast-path/migration sequence for a bounded window during concurrent first opens; if another opener completed the one-file schema but the baseline migration record is missing, the repository adapter records that metadata only after verifying all required repository schema tables already exist. When the fallback also fails, the error context includes the fast-path failure reason (`(fast-path attempt: {fast_error})`) so both failure causes are visible in diagnostics. -- **Hook runtime** (`resolve_agent_trace_storage_for_hook_runtime(...)`): opens with `RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(path)` and never falls back to `new_at`/migrations. If `ensure_schema_ready_for_hooks()` fails, it applies only the same narrow concurrent-first-open migration-metadata repair the setup path uses, then re-checks readiness; a missing database or a baseline-only (pre-`002`) database surfaces the existing `Run 'sce setup'.` guidance instead of migrating. Once readiness passes, it calls the same `verify_or_initialize_repository_metadata(repository_id)` to initialize `source_instance_id` when needed. `sce hooks conversation-trace`/`diff-trace`/`commit-msg` (and any other high-frequency hook caller) use this path exclusively via `open_agent_trace_db_for_hook_runtime` in `cli/src/services/hooks/mod.rs`. +- **Hook runtime** (`resolve_agent_trace_storage_for_hook_runtime(...)`): opens with `RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(path)` and never falls back to `new_at`/migrations. If `ensure_schema_ready_for_hooks()` fails, it applies only the same narrow concurrent-first-open migration-metadata repair the setup path uses, then re-checks readiness; a missing database or a baseline-only (pre-`002`) database surfaces the existing `Run 'sce setup'.` guidance instead of migrating. Once readiness passes, it calls the same `verify_or_initialize_repository_metadata(repository_id)` to initialize `source_instance_id` when needed. That method reads the `repository_metadata` row first: when the row already holds the matching `repository_id` and a valid `source_instance_id`, it returns without any write statement, so an initialized open never needs the writer lock and succeeds while another connection holds `BEGIN IMMEDIATE`; a mismatch still errors without writing. Only a missing row or an empty/invalid `source_instance_id` runs the idempotent seed insert and atomic claim, then re-reads. `sce hooks conversation-trace`/`diff-trace`/`commit-msg` (and any other high-frequency hook caller) use this path exclusively via `open_agent_trace_db_for_hook_runtime` in `cli/src/services/hooks/mod.rs`. Normal readiness for both paths is based on exact migration metadata parity with `AGENT_TRACE_REPOSITORY_MIGRATIONS`; table introspection is used only by the narrow concurrent-first-open metadata repair seam. diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index 43139fce4..6ed58cfa6 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -53,7 +53,7 @@ Runtime observability consumes the shared resolved observability config from `cl - All `CliError` instances are logged via `Logger::log_cli_error()` before user-facing stderr diagnostics are written; observability retains full technical detail independently of what is rendered to the terminal, and never writes a second competing stderr diagnostic for the same error. - Event records include deterministic metadata keys used by automation (`command`, `failure_class`, `component` when applicable). - Error log records include `error_code` and `error_class` fields for structured observability, plus `error_surface` (`user` for `CliError::User`, `internal` for `CliError::Internal`), `user_error` (the catalog `UserError::key()`, present only for `CliError::User`), and `error_source` (the full technical source chain, present whenever one was preserved — always for `CliError::Internal`, and for `CliError::User` only when constructed with `CliError::user_with_source`). -- App runtime initializes tracing subscriber context before parse/dispatch and shuts down tracer provider on process exit. +- App runtime wraps parse/dispatch in `Telemetry::with_default_subscriber`. Production uses `NoopTelemetry`, which installs no tracing subscriber and no tracer provider. Raw `tracing` events emitted outside `Logger`, such as `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted`, are instrumentation points with no production sink. `Logger` records still follow the file/stderr routing below. - Tracing event emission checks the `sce` target and requested tracing level before constructing serialized `fields` payloads; disabled or filtered tracing events return without building field JSON while enabled events preserve the same `event_id`, `event_message`, and `fields` payload shape. ## Format contract @@ -71,8 +71,8 @@ Runtime observability consumes the shared resolved observability config from `cl - The concrete `services::observability::Logger` implements the trait while retaining the existing inherent methods and behavior. - `NoopLogger` is available from the same traits module for tests and future dependency-injected services that need a logger without side effects. - The same traits module exposes object-safe `services::observability::traits::Telemetry` with the current app subscriber boundary: `with_default_subscriber` for command-lifecycle execution. -- The concrete `services::observability::TelemetryRuntime` implements the telemetry trait by delegating to its existing inherent method. -- `cli/src/app.rs` stores the production logger and telemetry runtime as concrete `AppRuntime` fields, creates borrowed `AppContext` views for command execution, and exposes logger/telemetry access through associated-type context accessors instead of owned `Arc` fields or object-erased accessor return values. +- Production currently uses `NoopTelemetry` from the same traits module. Its `with_default_subscriber` implementation invokes the action directly and installs no tracing subscriber or tracer provider; the boundary is retained for future telemetry/OTEL integration. +- `cli/src/app.rs` stores the concrete production `Logger` and `NoopTelemetry` as `AppRuntime` fields, creates borrowed `AppContext` views for command execution, and exposes logger/telemetry access through associated-type context accessors instead of owned `Arc` fields or object-erased accessor return values. - Final stream rendering uses `RunOutcome` in `cli/src/services/app_support.rs`, so classified-error and stdout-write-failure logging depends on the logger trait boundary rather than the concrete production logger type. - `run_command_lifecycle` expects the telemetry subscriber action to execute command dispatch at most once; if a telemetry implementation invokes the action again, the app returns a `SCE-ERR-RUNTIME` classified error rather than panicking or reparsing consumed arguments. diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index dcb898dc1..4b5507b0b 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -11,9 +11,12 @@ - `cli/build.rs` scans immediate `cli/migrations//*.sql` directories at compile time, copies them into `OUT_DIR/static/migrations`, and writes `OUT_DIR/generated_migrations.rs` with database-named migration constants (`AGENT_TRACE_REPOSITORY_MIGRATIONS`, `AUTH_MIGRATIONS`, etc.). Generated entries use the filename stem as the migration ID, embed the staged SQL with `include_str!`, and are sorted by the numeric prefix before the first `_`. - `TursoDb`: generic unencrypted adapter that owns: - tokio current-thread runtime creation - - Turso local database open/connect flow using `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)` so concurrent `sce` processes can safely access the same local database without WAL lock contention + - Turso local database open/connect flow using `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)`, which gives concurrent `sce` processes cross-process correctness and locking on the same local database; it does not make a contended writer wait + - per-`DbSpec` Turso busy timeout, resolved by `resolve_busy_timeout::()` and installed with `Connection::busy_timeout(...)` on the connection returned by `connect()` in `open_without_migrations_at` (the open path behind `new`, `new_at`, and `open_without_migrations`). It is Turso's wait policy for `Busy`: Turso sleeps in short phases until the lock clears or the timeout elapses, and only then returns `Busy`. It is connection-wide, so it also covers `BEGIN IMMEDIATE` issued by `Transaction::new_unchecked(.., Immediate)` on the same connection. The Agent Trace DB (`db_config_key() == "agent_trace_db"`) uses `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured (`0` leaves the handler unset), falling back to `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 1_000`; every other database resolves to zero, which leaves the busy handler unset, so `local_db` behavior is unchanged. `EncryptedTursoDb` (auth DB) never installs a busy handler. A single attempt may therefore spend up to about 1000 ms waiting inside Turso before returning `Busy`; the Agent Trace write-contention retry below decides whether SCE tries once more + - Agent Trace write-contention deadline, resolved by `resolve_contention_deadline::()` from `policies.database_retry.agent_trace_db.contention_deadline_ms` with fallback to `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS = 2_250` (zero for every other database). It is the cutoff for starting another outer write-contention retry, not a hard operation timeout + - Agent Trace write-contention retry (`write_contention_policy::()`, `run_with_write_contention_retry`), returned only for `agent_trace_db` and applied only to write units whose whole replay is safe: the two transactional primitives below (each attempt is a complete `BEGIN IMMEDIATE` → `COMMIT` unit) and `execute_idempotent_write(sql, params)`, the opt-in single-statement entrypoint used only by the repository-metadata `INSERT … ON CONFLICT DO NOTHING`, the guarded source-instance claim `UPDATE … WHERE source_instance_id = ''`, and the mutation-trace worktree/scope/scope-provenance `…_IF_ABSENT` inserts. It retries only typed `Busy`/`BusySnapshot` (classified before conversion to `anyhow`), allows at most `2` attempts, sleeps a full-jitter backoff drawn from `0..=100ms`, and admits the second attempt at two points: before the backoff sleep it requires `remaining contention deadline >= backoff + busy_timeout`, and after the sleep actually returns it re-checks `remaining >= busy_timeout`, so scheduler oversleep can never start an attempt outside the admission contract (time is measured on a monotonic clock from the operation start, so Turso's busy wait counts; nothing starts after the deadline expires). Deterministic failures return after one attempt without sleeping. On every other database these entrypoints keep the generic query retry, and `execute_idempotent_write` is plain `execute()` - config-driven connection-open retry around only the `build().await.connect()` block using `run_with_retry_sync` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) - - config-driven operation retry for `execute()`, `query()`, and `query_map()` using `run_with_retry_sync` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` timeout, `25ms..100ms` backoff, with default worst-case failure budget `<= 2_000ms`) + - config-driven operation retry for `execute()`, `query()`, and `query_map()` using `run_with_retry_sync` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). This is the outer retry for every operation outside the Agent Trace write-contention set, including Agent Trace reads (`query`/`query_values`/`query_map`), `passive_checkpoint`, generic `execute()`, and migrations. Its retry-policy budget is not a wall-clock bound; see the retry-budget note below - parent-directory creation - retry-backed synchronous `execute()`, `query()`, raw-value `query_values()`, and row-mapping `query_map()` wrappers via the public adapter methods, with config-driven query retry resolved from `policies.database_retry..query` - migration-running initialization through `new()` and generic embedded migration execution through `run_migrations()` delegated to the shared internal `TursoConnectionCore` with per-database `__sce_migrations` metadata @@ -23,6 +26,8 @@ - `migration_metadata_problems(&self) -> Result>`: non-mutating readiness check that queries `__sce_migrations` metadata and compares applied migration IDs against `M::migrations()`; returns a list of problems (missing metadata table, incomplete applied migrations, unexpected extra migrations) or an empty list when the schema is ready - `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>`: non-mutating hook-readiness gate that calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found; returns `Ok(())` when the schema is ready - `passive_checkpoint(&self) -> Result<()>`: runs `PRAGMA wal_checkpoint(PASSIVE)` through the same query/runtime/retry path as `execute()`/`query()` (config-driven query retry, `block_on_isolated`), draining the checkpoint result row without exposing its busy/log/checkpointed statistics. PASSIVE checkpoints only what is currently safe to move from the WAL into the main database file and never blocks on active readers or writers, so it does not guarantee WAL truncation; safe to call repeatedly; not a durability boundary on its own. Routine maintenance only, not exposed on `EncryptedTursoDb`. `sce hooks post-commit` is the sole current caller: it runs `RepositoryAgentTraceDb::passive_checkpoint()` exactly once after post-commit Agent Trace persistence succeeds; a failing checkpoint is logged as a warning and never fails the hook or affects already-persisted data (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). +- `count_write_contention(body) -> (T, WriteContentionCounts)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports the write-contention `attempts`, `outer_retries`, and `exhaustions` it caused on the current thread. Test instrumentation only; not production telemetry and not shared across threads or processes. +- `record_write_contention_timeline(body) -> (T, Vec<(Instant, WriteContentionTimelineEvent)>)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and returns the `Instant`-stamped attempt start/end, backoff requested and backoff slept events of the write-contention retry on the current thread. Test-only measurement instrumentation used by the lock-contention suite; never logged in production. - `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query`/`query_values`/`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. - `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same structural shape (connection, runtime bridge, and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. - `EncryptedTursoDb` exposes the same public synchronous `execute()`, `query()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. @@ -35,10 +40,11 @@ `TursoDb` offers two generic multi-statement transaction primitives beyond the single-statement `execute()`/`query()`/`query_map()` wrappers. Both are -public on `TursoDb` only (not `EncryptedTursoDb`), and both route -retryability through the same `run_with_retry_sync` seam used everywhere -else in this module, with a local classification layer so a deterministic -failure is never retried like a transient one: +public on `TursoDb` only (not `EncryptedTursoDb`). Each attempt runs the +whole `BEGIN IMMEDIATE` → `COMMIT` unit and classifies every `turso::Error` +as retryable (`Busy`/`BusySnapshot`) or deterministic. On the Agent Trace DB +the unit runs under the write-contention retry above; on other databases it +runs under the generic `run_with_retry_sync` query retry: - `execute_transactional_insert_pair_if_absent(operation_name, retry_hint, exists_sql, exists_params, first_sql, first_params, second_sql, @@ -107,7 +113,21 @@ All three database areas (local DB, auth DB, Agent Trace DB) have lifecycle prov Migrations are deliberately outside the connection-open retry block. The generic constructors retry only local Turso open/connect; schema changes are not generically retried because migration SQL must not be replayed after partial execution. Service-specific resolvers may add bounded recovery around their own initialization contracts when they can prove safety; the repository-scoped Agent Trace resolver does this only for its one-file fresh schema and only repairs missing migration metadata after all required repository schema tables already exist. -`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` timeout, `25ms..100ms` backoff; default worst-case failure budget `<= 2_000ms`). `execute()`, `query()`, and `query_values()` convert caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic rendering by operator-facing services. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. +`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). `execute()`, `query()`, and `query_values()` convert caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic rendering by operator-facing services. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. + +Retry-policy budget versus wall-clock time: the generic query retry policy keeps its historical `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff configuration, and its earlier `<= 2_000ms` figure describes that configured retry/budget calculation, not a hard wall-clock operation timeout. `run_with_retry_sync` cannot interrupt a synchronous operation already in progress: it only compares an attempt's elapsed time with `timeout_ms` after the attempt returns, to label the error. Actual failure latency is the sum of each attempt's real duration plus backoff. For the Agent Trace write-contention set, each attempt can include up to `busy_timeout_ms` (default 1000 ms) of Turso busy waiting and at most one outer retry starts, so a write blocked by a long-held lock fails after about 2–2.1 s with the defaults (about 1–1.1 s under the earlier 500 ms / 1250 ms defaults). None of `query.timeout_ms`, the `<= 2_000ms` policy figure, or `contention_deadline_ms` is a hard wall-clock deadline. + +| Layer | Role | +| --- | --- | +| multiprocess WAL | cross-process correctness and locking | +| `busy_timeout_ms` | Turso-level `Busy` waiting inside one operation attempt (Agent Trace DB only) | +| outer write-contention retry | SCE retry after Turso's wait is exhausted; Agent Trace safe write units only, at most `2` attempts, full-jitter `0..=100ms` backoff | +| `contention_deadline_ms` | cutoff for starting another outer write-contention retry, not a hard operation timeout | +| generic query `RetryPolicy` (`query.timeout_ms`) | outer retry for every other operation, including Agent Trace reads; `timeout_ms` labels slow attempts and never interrupts them | + +Contention exhaustion: when the write-contention retry gives up after a typed `Busy`/`BusySnapshot`, the returned error keeps the prefix `Operation '' failed after attempt(s) under write contention` and then carries `db_name`, `operation`, `attempts`, `busy_timeout_ms`, `contention_deadline_ms` (worded as a retry-scheduling cutoff), `elapsed_ms`, and `cause=database busy (busy timeout exhausted)`, followed by Turso's last error (`database is locked`) and the `Try:` hint. The same code path emits one `tracing::warn!` event (target `sce`, `event_id = "sce.agent_trace_db.contention_exhausted"`) with those fields plus `last_error`. Deterministic failures and successful retries emit no event. Nothing is written to stdout. Production observability is the returned error: fail-open hook paths pass it to their `Logger` error events, which follow the configured log-file/stderr routing. The `tracing` event is a telemetry instrumentation point. Production runs with `NoopTelemetry` and installs no tracing subscriber, so the event reaches a sink only where one is installed. Hook fail-open behavior is unchanged. + +Measured behavior of this contract, its lock-contention test suite, and the observed failure modes live in [agent-trace-db-write-contention-evidence.md](agent-trace-db-write-contention-evidence.md). Existing databases created before migration metadata are upgraded by re-applying the current idempotent migration list and recording each migration ID. This lets later `sce setup` / lifecycle initialization runs apply migrations added after the database file already existed, including Agent Trace DB schema/index additions.