diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 186c4fdd3..8c3734dde 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -1422,6 +1422,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -1726,6 +1737,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hybrid-array" version = "0.4.13" @@ -2712,6 +2729,76 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-http" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" +dependencies = [ + "http", + "httpdate", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.4", + "thiserror", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -3736,17 +3823,24 @@ dependencies = [ "jsonschema", "keyring-core", "murmur3", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "owo-colors 4.3.0", + "proc-macro2", "quint-connect", "rand 0.8.6", "reqwest", "serde", "serde_json", "sha2 0.11.0", + "syn 2.0.118", "tempfile", "tokio", "toml", "tracing", + "tracing-opentelemetry", + "tracing-subscriber", "turso", "uuid", "windows-native-keyring-store", @@ -4467,6 +4561,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" +dependencies = [ + "js-sys", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b16e47124..630b5bae1 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,6 +25,9 @@ categories = ["command-line-utilities", "development-tools"] name = "sce" path = "src/main.rs" +[features] +telemetry-test-receiver = [] + [dependencies] anyhow = "1" chrono = "0.4" @@ -46,6 +49,11 @@ sha2 = "0.11" tokio = { version = "1", default-features = false, features = ["rt", "io-util", "sync", "time", "macros", "rt-multi-thread", "process", "fs"] } toml = "0.9" tracing = "0.1" +opentelemetry = { version = "=0.33.0", default-features = false, features = ["trace"] } +opentelemetry_sdk = { version = "=0.33.0", default-features = false, features = ["trace"] } +opentelemetry-otlp = { version = "=0.33.0", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client"] } +tracing-opentelemetry = { version = "=0.34.0", default-features = false } +tracing-subscriber = { version = "=0.3.23", default-features = false, features = ["registry", "std"] } uuid = { version = "1", features = ["v4", "v7"] } murmur3 = "0.5.2" @@ -66,6 +74,8 @@ sha2 = "0.11" [dev-dependencies] quint-connect = "0.1.2" tempfile = "3" +syn = { version = "2", features = ["full", "visit"] } +proc-macro2 = "1" [lints] workspace = true diff --git a/cli/src/app.rs b/cli/src/app.rs index ccbcec067..f8cf4adb4 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -5,9 +5,12 @@ use std::process::ExitCode; use crate::services; use services::app_support::{self, RunOutcome}; use services::error::CliError; -use services::observability::traits::{ - Logger as LoggerTrait, NoopTelemetry, Telemetry as TelemetryTrait, -}; +use services::observability::otel_policy::{CommandName, ErrorCategory}; +use services::observability::otel_runtime::lifecycle_markers::{mark, LifecycleMarker}; +use services::observability::otel_runtime::{select_runtime_telemetry, RuntimeTelemetry}; +use services::observability::trace_context::extract_remote_parent; +use services::observability::tracing_boundary::CommandSpan; +use services::observability::traits::{Logger as LoggerTrait, Telemetry as TelemetryTrait}; const REPEATED_COMMAND_DISPATCH_ERROR: &str = "Command lifecycle telemetry attempted to execute command dispatch more than once"; @@ -19,7 +22,7 @@ struct StartupContext { struct AppRuntime { logger: services::observability::Logger, - telemetry: NoopTelemetry, + telemetry: RuntimeTelemetry, fs: services::capabilities::StdFsOps, git: services::capabilities::ProcessGitOps, registry: services::command_registry::CommandRegistry, @@ -32,7 +35,7 @@ struct AppRuntime { pub struct AppContext< 'a, L: LoggerTrait = services::observability::Logger, - T: TelemetryTrait = NoopTelemetry, + T: TelemetryTrait = RuntimeTelemetry, F: services::capabilities::FsOps = services::capabilities::StdFsOps, G: services::capabilities::GitOps = services::capabilities::ProcessGitOps, > { @@ -46,7 +49,7 @@ pub struct AppContext< type ProductionAppContext<'a> = AppContext< 'a, services::observability::Logger, - NoopTelemetry, + RuntimeTelemetry, services::capabilities::StdFsOps, services::capabilities::ProcessGitOps, >; @@ -57,21 +60,16 @@ pub(crate) trait HasLogger { fn logger(&self) -> &Self::Logger; } -#[allow(dead_code)] pub(crate) trait HasTelemetry { type Telemetry: TelemetryTrait; fn telemetry(&self) -> &Self::Telemetry; } -#[allow(dead_code)] pub(crate) trait HasFs { type Fs: services::capabilities::FsOps; - - fn fs(&self) -> &Self::Fs; } -#[allow(dead_code)] pub(crate) trait HasGit { type Git: services::capabilities::GitOps; @@ -113,23 +111,12 @@ where HasLogger::logger(self) } - #[allow(dead_code)] - pub(crate) fn fs(&self) -> &F { - HasFs::fs(self) - } - - #[allow(dead_code)] - pub(crate) fn git(&self) -> &G { - HasGit::git(self) - } - fn telemetry(&self) -> &T { HasTelemetry::telemetry(self) } /// Returns a context for a command-scoped repository root while preserving /// the runtime logger, telemetry, and capability dependencies. - #[allow(dead_code)] pub(crate) fn with_repo_root(&self, repo_root: impl Into) -> Self { Self { logger: self.logger, @@ -139,15 +126,6 @@ where repo_root: Some(repo_root.into()), } } - - /// Returns the resolved repository root path when available. - /// - /// Lifecycle providers use this during setup to avoid re-resolving - /// the repository root independently. - #[allow(dead_code)] - pub fn repo_root(&self) -> Option<&Path> { - HasRepoRoot::repo_root(self) - } } impl HasLogger for AppContext<'_, L, T, F, G> @@ -186,10 +164,6 @@ where G: services::capabilities::GitOps, { type Fs = F; - - fn fs(&self) -> &Self::Fs { - self.fs - } } impl HasGit for AppContext<'_, L, T, F, G> @@ -265,18 +239,25 @@ where StdoutW: Write, StderrW: Write, { - app_support::render_run_outcome( - try_run_with_dependency_check(args, dependency_check, stderr).await, - stdout, - stderr, - ) + let (outcome, telemetry) = try_run_with_dependency_check(args, dependency_check, stderr).await; + let code = app_support::render_run_outcome(outcome, stdout, stderr); + let _ = stdout.flush(); + let _ = stderr.flush(); + mark(LifecycleMarker::ShutdownBegin); + telemetry.shutdown().await; + mark(LifecycleMarker::ShutdownEnd); + mark(LifecycleMarker::ProcessExitRequested); + code } async fn try_run_with_dependency_check( args: I, dependency_check: F, stderr: &mut StderrW, -) -> RunOutcome +) -> ( + RunOutcome, + RuntimeTelemetry, +) where I: IntoIterator, F: FnOnce() -> anyhow::Result<()>, @@ -286,24 +267,37 @@ where .and_then(|()| build_startup_context()) .and_then(initialize_runtime); + let args = args.into_iter().collect::>(); match result { - Ok(runtime) => { + Ok(mut runtime) => { + runtime.telemetry = select_runtime_telemetry(&args, &runtime.registry, &process_env); let startup_diagnostic = runtime.startup_diagnostic.clone(); let result = run_command_lifecycle(args, &runtime, stderr).await; - RunOutcome { - logger: Some(runtime.logger), - startup_diagnostic, - result, - } + mark(LifecycleMarker::CommandComplete); + ( + RunOutcome { + logger: Some(runtime.logger), + startup_diagnostic, + result, + }, + runtime.telemetry, + ) } - Err(error) => RunOutcome { - result: Err(error), - logger: None, - startup_diagnostic: None, - }, + Err(error) => ( + RunOutcome { + result: Err(error), + logger: None, + startup_diagnostic: None, + }, + RuntimeTelemetry::disabled(), + ), } } +fn process_env(key: &str) -> Option { + std::env::var(key).ok() +} + fn perform_dependency_check anyhow::Result<()>>( dependency_check: F, ) -> Result<(), CliError> { @@ -337,7 +331,7 @@ fn initialize_runtime(startup: StartupContext) -> Result { app_support::log_startup_configuration(&logger, &startup.observability_config); Ok(AppRuntime { logger, - telemetry: NoopTelemetry, + telemetry: RuntimeTelemetry::disabled(), fs: services::capabilities::StdFsOps, git: services::capabilities::ProcessGitOps, registry: services::command_registry::build_default_registry(), @@ -389,7 +383,22 @@ where ))); }; let command = parse_command_phase(command_args, registry, context)?; - app_support::execute_command_phase(&command, context, stderr).await + let span = CommandSpan::start( + CommandName::parse(command.name().as_ref()), + extract_remote_parent(&process_env), + ); + let result = span + .scope(app_support::execute_command_phase( + &command, context, stderr, + )) + .await; + span.finish( + result + .as_ref() + .err() + .map(|error| ErrorCategory::from(error.class())), + ); + result } }) .await @@ -415,12 +424,18 @@ where Ok(command) } +#[cfg(test)] +mod doctor_reconciliation_cli_tests; +#[cfg(test)] +mod telemetry_tests; + #[cfg(test)] mod tests { use std::future::{poll_fn, Future}; use std::sync::{Arc, Mutex}; use super::*; + use services::observability::traits::NoopTelemetry; #[derive(Default)] struct RecordingScope { diff --git a/cli/src/app/doctor_reconciliation_cli_tests.rs b/cli/src/app/doctor_reconciliation_cli_tests.rs new file mode 100644 index 000000000..ed1dfff00 --- /dev/null +++ b/cli/src/app/doctor_reconciliation_cli_tests.rs @@ -0,0 +1,596 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode}; + +use serde_json::Value; + +use super::run_with_dependency_check_and_streams; +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::default_paths::agent_trace_db_path_for_repository_at; +use crate::services::repository_identity::resolve::resolve_repository_identity; + +const ARGS_ENV: &str = "SCE_DOCTOR_RECONCILIATION_CLI_ARGS"; +const OUT_ENV: &str = "SCE_DOCTOR_RECONCILIATION_CLI_OUT"; +const CHILD_TEST: &str = "app::doctor_reconciliation_cli_tests::doctor_reconciliation_cli_child"; +const STATE_FILE: &str = "ref-maintenance.json"; +const LOCK_FILE: &str = "mutation-cursor.lock"; +const RECONCILIATION_DETAIL_PREFIX: &str = "Snapshot ref reconciliation"; + +const EXIT_SUCCESS: &str = "success"; +const EXIT_FAILURE: &str = "failure"; + +struct CliRun { + stdout: String, + stderr: String, +} + +impl CliRun { + fn json(&self) -> Value { + serde_json::from_str(&self.stdout) + .unwrap_or_else(|error| panic!("doctor stdout is not JSON ({error}):\n{}", self.stdout)) + } +} + +struct Sandbox { + dir: tempfile::TempDir, + repo: PathBuf, + repository_id: String, +} + +fn git(cwd: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git command"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +impl Sandbox { + fn new(remote: &str) -> Self { + let dir = tempfile::tempdir().expect("sandbox"); + let repo = dir.path().join("repo"); + std::fs::create_dir_all(&repo).expect("repo dir"); + git(&repo, &["init", "-q"]); + git(&repo, &["config", "user.email", "t@example.invalid"]); + git(&repo, &["config", "user.name", "T"]); + git(&repo, &["config", "commit.gpgsign", "false"]); + git(&repo, &["remote", "add", "origin", remote]); + std::fs::write(repo.join("tracked.txt"), "tracked\n").expect("tracked file"); + git(&repo, &["add", "tracked.txt"]); + git(&repo, &["commit", "-q", "-m", "initial"]); + let repository_id = resolve_repository_identity(&repo, None, "origin") + .expect("identity") + .identity + .repository_id; + Self { + dir, + repo, + repository_id, + } + } + + fn state_root(&self) -> PathBuf { + if cfg!(target_os = "macos") { + self.dir.path().join("Library").join("Application Support") + } else { + self.dir.path().join("state") + } + } + + fn db_path(&self) -> PathBuf { + agent_trace_db_path_for_repository_at(&self.state_root(), &self.repository_id) + .expect("db path") + } + + fn sce_dir(&self) -> PathBuf { + self.repo.join(".git").join("sce") + } + + fn state_file(&self) -> PathBuf { + self.sce_dir().join(STATE_FILE) + } + + async fn create_db(&self, repository_id: &str) { + let db = RepositoryAgentTraceDb::new_at(self.db_path()) + .await + .expect("create db"); + db.verify_or_initialize_repository_metadata(repository_id) + .await + .expect("initialize metadata"); + } + + fn orphan_pin(&self) -> String { + let tree = git(&self.repo, &["rev-parse", "HEAD^{tree}"]); + git( + &self.repo, + &[ + "update-ref", + &format!("refs/sce/mutation-cursor/main/{tree}"), + &tree, + ], + ); + tree + } + + fn refs(&self) -> String { + git( + &self.repo, + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/sce/", + ], + ) + } + + fn state_bytes(&self) -> Option> { + std::fs::read(self.state_file()).ok() + } + + fn state_json(&self) -> Value { + serde_json::from_slice(&self.state_bytes().expect("maintenance state was written")) + .expect("state JSON") + } + + fn run(&self, args: &[&str]) -> CliRun { + let out = self.dir.path().join(format!("out-{}", args.join("-"))); + std::fs::create_dir_all(&out).expect("out dir"); + let output = Command::new(std::env::current_exe().expect("current exe")) + .args(["--exact", CHILD_TEST, "--nocapture"]) + .current_dir(&self.repo) + .env(ARGS_ENV, args.join(" ")) + .env(OUT_ENV, &out) + .env("HOME", self.dir.path()) + .env("XDG_CONFIG_HOME", self.dir.path().join("config")) + .env("XDG_STATE_HOME", self.state_root()) + .env("XDG_CACHE_HOME", self.dir.path().join("cache")) + .env("NO_COLOR", "1") + .env_remove("SCE_CONFIG_FILE") + .output() + .expect("child runs"); + assert!( + output.status.success(), + "child harness failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let run = CliRun { + stdout: std::fs::read_to_string(out.join("stdout")).expect("stdout"), + stderr: std::fs::read_to_string(out.join("stderr")).expect("stderr"), + }; + let exit = std::fs::read_to_string(out.join("exit")).expect("exit"); + assert_eq!( + exit, + EXIT_SUCCESS, + "sce {} exit status:\n{}\n{}", + args.join(" "), + run.stdout, + run.stderr + ); + run + } +} + +fn non_reconciliation_fix_results(report: &Value, sandbox_root: &Path) -> Vec { + let root = sandbox_root.to_string_lossy().into_owned(); + report["fix_results"] + .as_array() + .expect("fix_results array") + .iter() + .filter(|entry| { + !entry["detail"] + .as_str() + .expect("detail") + .starts_with(RECONCILIATION_DETAIL_PREFIX) + }) + .map(|entry| { + let mut entry = entry.clone(); + let detail = entry["detail"] + .as_str() + .expect("detail") + .replace(&root, ""); + entry["detail"] = Value::String(detail); + entry + }) + .collect() +} + +fn reconciliation_fix_result(report: &Value) -> Value { + let matches: Vec = report["fix_results"] + .as_array() + .expect("fix_results array") + .iter() + .filter(|entry| { + entry["detail"] + .as_str() + .expect("detail") + .starts_with(RECONCILIATION_DETAIL_PREFIX) + }) + .cloned() + .collect(); + assert_eq!(matches.len(), 1, "reconciliation reports exactly once"); + matches.into_iter().next().expect("one entry") +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_reconciliation_cli_child() { + let (Some(args), Some(out)) = ( + std::env::var_os(ARGS_ENV), + std::env::var_os(OUT_ENV).map(PathBuf::from), + ) else { + return; + }; + let mut command_line = vec!["sce".to_string()]; + command_line.extend( + args.to_string_lossy() + .split(' ') + .filter(|part| !part.is_empty()) + .map(str::to_string), + ); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit_code = Box::pin(run_with_dependency_check_and_streams( + command_line, + || Ok(()), + &mut stdout, + &mut stderr, + )) + .await; + let exit = if exit_code == ExitCode::SUCCESS { + EXIT_SUCCESS + } else { + EXIT_FAILURE + }; + std::fs::write(out.join("exit"), exit).expect("exit"); + std::fs::write(out.join("stdout"), stdout).expect("stdout"); + std::fs::write(out.join("stderr"), stderr).expect("stderr"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_inspection_is_read_only_and_fix_reconciles_the_orphan_pin_once() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + let tree = sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + assert!(refs_before.contains(&tree)); + + let inspection = sandbox.run(&["doctor", "--format", "json"]); + let report = inspection.json(); + assert!( + report["ref_reconciliation_fix"].is_null(), + "plain doctor never reconciles: {}", + inspection.stderr + ); + assert_eq!(sandbox.refs(), refs_before); + assert_eq!(sandbox.state_bytes(), None); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "completed", "{}", fix.stdout); + assert_eq!(reconciliation["counts"]["deleted"], 1); + assert_eq!(reconciliation["counts"]["retained"], 0); + assert_eq!(reconciliation["counts"]["local_required"], 0); + assert!(reconciliation["failure"].is_null()); + assert!(reconciliation["state_warning"].is_null()); + assert_eq!(sandbox.refs(), ""); + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "fixed"); + assert_eq!(entry["category"], "mutation_scope_health"); + assert_eq!( + entry["detail"], + "Snapshot ref reconciliation completed: deleted 1, retained 0, locally required 0." + ); + + let state = sandbox.state_json(); + assert!(state["last_success"].is_number()); + assert_eq!(state["last_report"]["deleted"], 1); + assert!(state.get("consecutive_failures").is_none_or(|v| v == 0)); + + let again = sandbox.run(&["doctor", "--fix", "--format", "json"]).json(); + assert_eq!(again["ref_reconciliation_fix"]["counts"]["deleted"], 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_missing_database_without_creating_it_or_deleting_refs() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + let db_path = sandbox.db_path(); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "failed", "{}", fix.stdout); + assert_eq!(reconciliation["failure"]["kind"], "agent_trace_db_missing"); + assert!(reconciliation["counts"].is_null()); + assert_eq!(sandbox.refs(), refs_before); + assert!(!db_path.exists()); + let bootstrap: Vec = non_reconciliation_fix_results(&report, sandbox.dir.path()) + .iter() + .map(|entry| entry["detail"].as_str().expect("detail").to_string()) + .filter(|detail| detail.starts_with("Agent trace DB parent directory bootstrapped")) + .collect(); + assert_eq!( + bootstrap.len(), + 1, + "the parent directory comes from the separate global-state repair, not from reconciliation" + ); + for suffix in ["-wal", "-shm", "-tshm"] { + let mut sidecar = db_path.clone().into_os_string(); + sidecar.push(suffix); + assert!(!PathBuf::from(sidecar).exists()); + } + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "failed"); + assert!(entry["detail"] + .as_str() + .expect("detail") + .starts_with("Snapshot ref reconciliation failed (agent_trace_db_missing):")); + + let state = sandbox.state_json(); + assert_eq!(state["consecutive_failures"], 1); + assert!(state.get("last_success").is_none()); + assert!(report["fix_results"].is_array()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_repository_mismatch_without_touching_the_database_or_refs() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + let foreign = "some-other-repository"; + sandbox.create_db(foreign).await; + sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "failed", "{}", fix.stdout); + assert_eq!( + reconciliation["failure"]["kind"], + "agent_trace_db_repository_mismatch" + ); + assert!(reconciliation["failure"]["message"] + .as_str() + .expect("message") + .contains(foreign)); + assert_eq!(sandbox.refs(), refs_before); + assert_eq!(sandbox.state_json()["consecutive_failures"], 1); + + let (_, metadata) = + RepositoryAgentTraceDb::open_verified_existing_at(sandbox.db_path(), foreign) + .await + .expect("the foreign repository identity is untouched"); + assert_eq!(metadata.repository_id, foreign); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_busy_when_the_worktree_lock_is_held_and_keeps_other_repairs() { + let reference = Sandbox::new("https://example.invalid/org/repo-a.git"); + reference.create_db(&reference.repository_id).await; + reference.orphan_pin(); + let reference_report = reference + .run(&["doctor", "--fix", "--format", "json"]) + .json(); + + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + std::fs::create_dir_all(sandbox.sce_dir()).expect("sce dir"); + let held = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(sandbox.sce_dir().join(LOCK_FILE)) + .expect("lock file"); + held.try_lock().expect("hold the worktree lock externally"); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + drop(held); + + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "busy", "{}", fix.stdout); + assert!(reconciliation["counts"].is_null()); + assert!(reconciliation["failure"].is_null()); + assert_eq!(sandbox.refs(), refs_before); + assert_eq!(sandbox.state_bytes(), None); + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "skipped"); + assert!(entry["detail"] + .as_str() + .expect("detail") + .starts_with("Snapshot ref reconciliation skipped:")); + assert_eq!( + non_reconciliation_fix_results(&report, sandbox.dir.path()), + non_reconciliation_fix_results(&reference_report, reference.dir.path()), + "independent doctor repairs are preserved" + ); +} + +const FUTURE_SKEW_TOLERANCE_MS: i64 = 5 * 60 * 1000; +const STATE_TIMESTAMP_FIELDS: [&str; 4] = + ["anchor", "last_success", "last_attempt", "last_advised"]; + +fn now_ms() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_millis(), + ) + .expect("unix ms") +} + +impl Sandbox { + fn seed_state(&self, bytes: &[u8]) { + std::fs::create_dir_all(self.sce_dir()).expect("sce dir"); + std::fs::write(self.state_file(), bytes).expect("seed state"); + } + + fn make_state_unreadable(&self) { + std::fs::create_dir_all(self.state_file()).expect("directory in place of the state file"); + } +} + +fn assert_unreadable_state_outcome(fix: &CliRun, sandbox: &Sandbox, label: &str) { + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!( + reconciliation["outcome"], "completed_state_persist_failed", + "{label}: {}", + fix.stdout + ); + assert_eq!(reconciliation["counts"]["deleted"], 1, "{label}"); + assert_eq!(reconciliation["counts"]["retained"], 0, "{label}"); + assert_eq!(reconciliation["counts"]["local_required"], 0, "{label}"); + assert!(reconciliation["failure"].is_null(), "{label}"); + let warning = &reconciliation["state_warning"]; + assert_eq!(warning["kind"], "previous_state_unreadable", "{label}"); + assert!(warning["phase"].is_null(), "{label}"); + assert_eq!( + warning["message"], "Maintenance state could not be read; the update was not attempted.", + "{label}" + ); + assert!( + !warning["cause"].as_str().expect("cause").is_empty(), + "{label}" + ); + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "fixed", "{label}"); + let detail = entry["detail"].as_str().expect("detail"); + assert!( + detail.starts_with( + "Snapshot ref reconciliation completed: deleted 1, retained 0, locally required 0. Warning: Maintenance state could not be read; the update was not attempted. Cause: " + ), + "{label}: {detail}" + ); + assert!(!detail.contains("failed"), "{label}: {detail}"); + + assert_eq!(sandbox.refs(), "", "{label}: orphan pin must be deleted"); + assert!( + sandbox.state_file().is_dir(), + "{label}: the state path must stay a directory" + ); + assert_eq!( + std::fs::read_dir(sandbox.state_file()) + .expect("state directory") + .count(), + 0, + "{label}: the state directory must stay untouched" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + sandbox.orphan_pin(); + sandbox.make_state_unreadable(); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + + assert_unreadable_state_outcome(&fix, &sandbox, "unreadable"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_recovers_from_every_initial_maintenance_state_condition() { + let now = now_ms(); + let day_ms = 24 * 60 * 60 * 1000; + let year_ms = 365 * day_ms; + let failed_streak = serde_json::json!({ + "version": 1, + "anchor": now - 10 * day_ms, + "last_attempt": now - 60_000, + "last_attempt_outcome": "failed", + "consecutive_failures": 3, + "last_failure": {"kind": "agent_trace_db_missing", "message": "seeded failure"}, + }); + let expired_advisory = serde_json::json!({ + "version": 1, + "anchor": now - 90 * day_ms, + "last_success": now - 60 * day_ms, + "last_advised": now - 30 * day_ms, + }); + let invalid_timestamps = serde_json::json!({ + "version": 1, + "anchor": now + 10 * year_ms, + "last_success": now + 10 * year_ms, + "last_attempt": now + 10 * year_ms, + "last_advised": now + 10 * year_ms, + "last_attempt_outcome": "failed", + "consecutive_failures": 2, + }); + let cases: Vec<(&str, Vec)> = vec![ + ("failed_streak", failed_streak.to_string().into_bytes()), + ( + "expired_advisory", + expired_advisory.to_string().into_bytes(), + ), + ( + "invalid_timestamps", + invalid_timestamps.to_string().into_bytes(), + ), + ("corrupt", b"not json".to_vec()), + ("oversized", vec![b' '; 5000]), + ]; + + for (label, bytes) in cases { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + sandbox.orphan_pin(); + sandbox.seed_state(&bytes); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!( + reconciliation["outcome"], "completed", + "{label}: {}", + fix.stdout + ); + assert_eq!(reconciliation["counts"]["deleted"], 1, "{label}"); + assert!(reconciliation["failure"].is_null(), "{label}"); + assert!(reconciliation["state_warning"].is_null(), "{label}"); + assert_eq!(sandbox.refs(), "", "{label}: orphan pin must be reclaimed"); + + let after = now_ms(); + let state = sandbox.state_json(); + assert_eq!(state["version"], 1, "{label}"); + assert!(state["last_success"].is_number(), "{label}"); + assert_eq!(state["last_attempt_outcome"], "completed", "{label}"); + assert!( + state.get("consecutive_failures").is_none_or(|v| v == 0), + "{label}: failure streak must be cleared" + ); + assert!(state.get("last_failure").is_none(), "{label}"); + assert_eq!(state["last_report"]["deleted"], 1, "{label}"); + for field in STATE_TIMESTAMP_FIELDS { + if let Some(value) = state.get(field) { + let timestamp = value.as_i64().expect("timestamp"); + assert!( + timestamp <= after + FUTURE_SKEW_TOLERANCE_MS, + "{label}: {field} must be normalized, got {timestamp}" + ); + } + } + + let again = sandbox.run(&["doctor", "--fix", "--format", "json"]); + assert_eq!( + again.json()["ref_reconciliation_fix"]["outcome"], + "completed", + "{label}: the rewritten state is readable" + ); + } +} diff --git a/cli/src/app/telemetry_tests.rs b/cli/src/app/telemetry_tests.rs new file mode 100644 index 000000000..e21948ad1 --- /dev/null +++ b/cli/src/app/telemetry_tests.rs @@ -0,0 +1,559 @@ +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpListener}; +use std::path::Path; +use std::process::ExitCode; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use super::*; +use opentelemetry::trace::{SpanId, Status, TraceContextExt}; +use services::command_registry::build_default_registry; +use services::observability::otel_policy::OtelName; +use services::observability::otel_runtime::test_support::{attributes, Capture}; +use services::observability::otel_runtime::{ + provider_with_exporter, resolve_telemetry_mode, OtelTelemetry, RuntimeTelemetry, + ShutdownOutcome, TelemetryMode, SCE_TELEMETRY_ENV, +}; +use services::observability::tracing_boundary::test_spans::in_otel_span; +use services::observability::traits::NoopLogger; +use services::parse::command_runtime::parse_runtime_command; + +const CHILD_MARKER: &str = "SCE_TELEMETRY_TEST_CHILD"; +const CHILD_ARGS: &str = "SCE_TELEMETRY_TEST_ARGS"; +const CHILD_OUT: &str = "SCE_TELEMETRY_TEST_OUT"; +const HOOK_INVOCATIONS: [&[&str]; 3] = [ + &["sce", "hooks", "pre-commit"], + &["sce", "hooks", "mutation-scope"], + &["sce", "hooks", "codex"], +]; + +struct Receiver { + addr: SocketAddr, + requests: Arc, + connections: Arc, + stop: Arc, + thread: Option>, +} + +impl Receiver { + fn start(respond: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let connections = Arc::new(AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let held = Arc::new(Mutex::new(Vec::new())); + let thread = { + let (requests, connections, stop) = + (requests.clone(), connections.clone(), stop.clone()); + std::thread::spawn(move || { + while !stop.load(Ordering::SeqCst) { + let Ok((mut stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(10)); + continue; + }; + connections.fetch_add(1, Ordering::SeqCst); + if !respond { + held.lock().unwrap().push(stream); + continue; + } + let requests = requests.clone(); + std::thread::spawn(move || serve(&mut stream, &requests)); + } + }) + }; + Self { + addr, + requests, + connections, + stop, + thread: Some(thread), + } + } + + fn endpoint(&self) -> String { + format!("http://{}", self.addr) + } + + fn requests(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } + + fn connections(&self) -> usize { + self.connections.load(Ordering::SeqCst) + } + + fn wait_for_request(&self) -> bool { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if self.requests() > 0 { + return true; + } + std::thread::sleep(Duration::from_millis(20)); + } + false + } +} + +impl Drop for Receiver { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn serve(stream: &mut std::net::TcpStream, requests: &AtomicUsize) { + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut buffer = Vec::new(); + let mut chunk = [0u8; 4096]; + let header_end = loop { + let Ok(read) = stream.read(&mut chunk) else { + return; + }; + if read == 0 { + return; + } + buffer.extend_from_slice(&chunk[..read]); + if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + break position + 4; + } + }; + let head = String::from_utf8_lossy(&buffer[..header_end]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + while buffer.len() < header_end + length { + let Ok(read) = stream.read(&mut chunk) else { + break; + }; + if read == 0 { + break; + } + buffer.extend_from_slice(&chunk[..read]); + } + if head.starts_with("post /v1/traces") { + requests.fetch_add(1, Ordering::SeqCst); + } + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); +} + +fn closed_endpoint() -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + format!("http://{addr}") +} + +fn env_map(pairs: &[(&str, String)]) -> impl Fn(&str) -> Option { + let map: HashMap = pairs + .iter() + .map(|(key, value)| ((*key).to_string(), value.clone())) + .collect(); + move |key| map.get(key).cloned() +} + +fn standalone_env(endpoint: &str) -> impl Fn(&str) -> Option { + env_map(&[ + (SCE_TELEMETRY_ENV, "standalone".to_string()), + ("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint.to_string()), + ( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + format!("{endpoint}/v1/traces"), + ), + ("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc".to_string()), + ]) +} + +fn owned(args: &[&str]) -> Vec { + args.iter().map(|arg| (*arg).to_string()).collect() +} + +async fn emit_command_span(telemetry: &RuntimeTelemetry) { + telemetry + .with_default_subscriber(&mut || async { + in_otel_span(OtelName::Command, async { + tokio::task::yield_now().await; + }) + .await; + Ok(String::new()) + }) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_hook_export_gated_standalone_hook_invocations_export_nothing() { + let receiver = Receiver::start(true); + let env = standalone_env(&receiver.endpoint()); + let registry = build_default_registry(); + for hook in HOOK_INVOCATIONS { + let args = owned(hook); + let parsed = parse_runtime_command(args.iter().cloned(), ®istry, None).unwrap(); + assert!(parsed.is_hook_invocation(), "{hook:?}"); + let telemetry = select_runtime_telemetry(&args, ®istry, &env); + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_)), "{hook:?}"); + emit_command_span(&telemetry).await; + drop(telemetry); + } + let policy = owned(&["sce", "policy", "bash"]); + assert!( + parse_runtime_command(policy.iter().cloned(), ®istry, None) + .unwrap() + .is_hook_invocation() + ); + assert!(matches!( + select_runtime_telemetry(&policy, ®istry, &env), + RuntimeTelemetry::Noop(_) + )); + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!(receiver.connections(), 0); + assert_eq!(receiver.requests(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_hook_export_gated_unparseable_invocation_fails_closed() { + let receiver = Receiver::start(true); + let env = standalone_env(&receiver.endpoint()); + let telemetry = select_runtime_telemetry( + &owned(&["sce", "--definitely-not-a-flag"]), + &build_default_registry(), + &env, + ); + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); + assert_eq!(receiver.connections(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_hook_export_gated_non_hook_standalone_command_still_exports() { + let receiver = Receiver::start(true); + let env = standalone_env(&receiver.endpoint()); + let telemetry = + select_runtime_telemetry(&owned(&["sce", "version"]), &build_default_registry(), &env); + assert!(matches!(telemetry, RuntimeTelemetry::Otel(_))); + emit_command_span(&telemetry).await; + drop(telemetry); + assert!( + receiver.wait_for_request(), + "standalone trace was not exported" + ); +} + +#[test] +fn telemetry_hook_export_gated_resolution_never_inspects_invocation_unless_requested() { + let inspected = std::cell::Cell::new(false); + let disabled = env_map(&[( + "OTEL_EXPORTER_OTLP_ENDPOINT", + "http://127.0.0.1:1".to_string(), + )]); + assert_eq!( + resolve_telemetry_mode(&disabled, || { + inspected.set(true); + true + }), + TelemetryMode::Disabled + ); + assert!(!inspected.get()); + let standalone = env_map(&[(SCE_TELEMETRY_ENV, "standalone".to_string())]); + assert_eq!( + resolve_telemetry_mode(&standalone, || false), + TelemetryMode::Disabled + ); + assert_eq!( + resolve_telemetry_mode(&standalone, || true), + TelemetryMode::Standalone + ); +} + +#[test] +fn telemetry_disabled_baseline_resolution_reads_only_the_environment_switch() { + let keys = Mutex::new(Vec::new()); + let env = |key: &str| { + keys.lock().unwrap().push(key.to_string()); + None + }; + assert_eq!( + resolve_telemetry_mode(&env, || true), + TelemetryMode::Disabled + ); + assert_eq!(*keys.lock().unwrap(), vec![SCE_TELEMETRY_ENV.to_string()]); + for value in [ + "", + "Standalone", + "STANDALONE", + "on", + "true", + "1", + "managed", + "test-receiver", + ] { + let env = env_map(&[(SCE_TELEMETRY_ENV, value.to_string())]); + assert_eq!( + resolve_telemetry_mode(&env, || true), + TelemetryMode::Disabled, + "{value}" + ); + } +} + +fn run_child(args: &str, envs: &[(&str, String)], sandbox: &Path, label: &str) -> String { + let out = sandbox.join(format!("{label}.out")); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "app::telemetry_tests::isolated_command_output", + "--nocapture", + ]) + .current_dir(sandbox) + .env(CHILD_MARKER, "1") + .env(CHILD_ARGS, args) + .env(CHILD_OUT, &out) + .env("HOME", sandbox) + .env("XDG_CONFIG_HOME", sandbox.join("config")) + .env("XDG_STATE_HOME", sandbox.join("state")) + .env("XDG_CACHE_HOME", sandbox.join("cache")) + .env("NO_COLOR", "1") + .env_remove("SCE_CONFIG_FILE") + .env_remove(SCE_TELEMETRY_ENV); + for (key, value) in envs { + command.env(key, value); + } + let started = Instant::now(); + let output = command.output().unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + started.elapsed() < Duration::from_secs(20), + "child process did not exit promptly" + ); + std::fs::read_to_string(out).unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn isolated_command_output() { + if std::env::var_os(CHILD_MARKER).is_none() { + return; + } + let args = std::env::var(CHILD_ARGS).unwrap(); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let code = Box::pin(run_with_dependency_check_and_streams( + args.split_whitespace().map(String::from), + || Ok(()), + &mut stdout, + &mut stderr, + )) + .await; + let rendered = format!( + "{}\n---\n{}\n---\n{}", + if code == ExitCode::SUCCESS { + "success" + } else { + "failure" + }, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + std::fs::write(std::env::var(CHILD_OUT).unwrap(), rendered).unwrap(); +} + +#[test] +fn telemetry_disabled_baseline_ignores_otlp_environment_and_other_switch_values() { + let sandbox = tempfile::tempdir().unwrap(); + let receiver = Receiver::start(true); + let endpoint = receiver.endpoint(); + let baseline = run_child("sce version", &[], sandbox.path(), "baseline"); + assert!(baseline.starts_with("success\n---\nshared-context-engineering ")); + + let otlp_only = [ + ("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint.clone()), + ("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", endpoint.clone()), + ]; + assert_eq!( + run_child("sce version", &otlp_only, sandbox.path(), "otlp"), + baseline + ); + for (index, value) in ["off", "Standalone", "1"].into_iter().enumerate() { + let mut envs = otlp_only.to_vec(); + envs.push((SCE_TELEMETRY_ENV, value.to_string())); + assert_eq!( + run_child( + "sce version", + &envs, + sandbox.path(), + &format!("switch-{index}") + ), + baseline, + "{value}" + ); + } + assert_eq!(receiver.connections(), 0); + assert_eq!(receiver.requests(), 0); +} + +#[test] +fn telemetry_hook_export_gated_unreachable_receiver_preserves_hook_behavior() { + let sandbox = tempfile::tempdir().unwrap(); + let baseline = run_child("sce hooks pre-commit", &[], sandbox.path(), "hook-baseline"); + + let blackhole = Receiver::start(false); + let refused = closed_endpoint(); + let cases = [("blackhole", blackhole.endpoint()), ("refused", refused)]; + for (label, endpoint) in cases { + let envs = [ + (SCE_TELEMETRY_ENV, "standalone".to_string()), + ("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint), + ]; + assert_eq!( + run_child("sce hooks pre-commit", &envs, sandbox.path(), label), + baseline, + "{label}" + ); + } + assert_eq!(blackhole.connections(), 0); +} + +async fn lifecycle_spans( + args: &[&str], +) -> ( + Result, + Vec, +) { + let capture = Capture::default(); + let telemetry = OtelTelemetry::from_provider_with_budget( + provider_with_exporter(capture.clone()), + Duration::from_secs(5), + ); + let registry = build_default_registry(); + let mut stderr = Vec::new(); + let result = { + let context = AppContext::new( + &NoopLogger, + &telemetry, + &services::capabilities::StdFsOps, + &services::capabilities::ProcessGitOps, + None, + ); + Box::pin(run_command_lifecycle_with_context( + args.iter().map(|arg| (*arg).to_string()), + ®istry, + &context, + &mut stderr, + )) + .await + }; + assert_eq!(telemetry.shutdown().await, ShutdownOutcome::Completed); + assert_eq!(capture.shutdown_calls(), 1); + assert_eq!(capture.exports_after_shutdown(), 0); + (result, capture.spans()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_command_span_success_records_one_root_with_typed_attributes() { + let (result, spans) = lifecycle_spans(&["sce", "version"]).await; + assert!(result.is_ok()); + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.command"); + let ambient_parent = + services::observability::trace_context::extract_remote_parent(&process_env) + .map_or(SpanId::INVALID, |context| { + context.span().span_context().span_id() + }); + assert_eq!(span.parent_span_id, ambient_parent); + assert_ne!( + span.span_context.trace_id(), + opentelemetry::trace::TraceId::INVALID + ); + assert_eq!( + attributes(span), + vec![ + ("sce.command.name".to_string(), "version".to_string()), + ("sce.outcome".to_string(), "success".to_string()), + ] + ); + assert_eq!(span.status, Status::Unset); + assert!(span.events.is_empty()); + assert!(span.links.is_empty()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_command_span_failure_records_category_without_error_text() { + let (result, spans) = lifecycle_spans(&[ + "sce", + "config", + "validate", + "--config", + "/nonexistent-SECRETPATH-sce-config.json", + ]) + .await; + assert_eq!( + result.unwrap_err().class(), + services::error::FailureClass::Runtime + ); + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!( + attributes(span), + vec![ + ("sce.command.name".to_string(), "config".to_string()), + ("sce.error.category".to_string(), "runtime".to_string()), + ("sce.outcome".to_string(), "failure".to_string()), + ] + ); + assert_eq!(span.status, Status::error("")); + assert!(span.events.is_empty()); + let rendered = format!("{spans:?}"); + for leaked in [ + "SECRETPATH", + "nonexistent", + "No such file", + "stack", + "exception", + ] { + assert!(!rendered.contains(leaked), "{leaked}: {rendered}"); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_command_span_is_not_emitted_when_parsing_fails() { + let (result, spans) = lifecycle_spans(&["sce", "--invalid-boundary-option"]).await; + assert_eq!( + result.unwrap_err().class(), + services::error::FailureClass::Parse + ); + assert!(spans.is_empty(), "{spans:?}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn telemetry_command_span_dispatches_once_per_command_across_commands() { + for args in [&["sce", "help"][..], &["sce", "version"][..]] { + let (result, spans) = lifecycle_spans(args).await; + assert!(result.is_ok(), "{args:?}"); + assert_eq!( + spans + .iter() + .filter(|span| span.name == "sce.command") + .count(), + 1, + "{args:?}" + ); + assert_eq!(spans.len(), 1, "{args:?}"); + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 39cd9686f..313c38b66 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -2,7 +2,6 @@ mod app; mod cli_schema; mod command_surface; -#[allow(dead_code)] mod generated_migrations { include!(concat!(env!("OUT_DIR"), "/generated_migrations.rs")); } diff --git a/cli/src/services/agent_trace.rs b/cli/src/services/agent_trace.rs index 15c2bebf3..0120affdf 100644 --- a/cli/src/services/agent_trace.rs +++ b/cli/src/services/agent_trace.rs @@ -188,34 +188,22 @@ fn parse_commit_timestamp(commit_timestamp: &str) -> Result = OnceLock::new(); #[derive(Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) enum AgentTraceValidationError { - FileRead { path: String, message: String }, - InvalidJson { message: String }, SchemaValidation { errors: Vec }, } impl fmt::Display for AgentTraceValidationError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::FileRead { path, message } => { - write!(f, "Agent Trace JSON file could not be read at '{path}': {message}") - } - Self::InvalidJson { message } => { - write!(f, "Agent Trace JSON must be valid JSON: {message}") - } Self::SchemaValidation { errors } => write!( f, "Agent Trace JSON failed schema validation against embedded schema '{AGENT_TRACE_SCHEMA_PATH}': {}", @@ -227,7 +215,6 @@ impl fmt::Display for AgentTraceValidationError { impl Error for AgentTraceValidationError {} -#[allow(dead_code)] fn agent_trace_schema_validator() -> &'static Validator { AGENT_TRACE_SCHEMA_VALIDATOR.get_or_init(|| { let schema: Value = serde_json::from_str(AGENT_TRACE_SCHEMA_JSON) @@ -236,7 +223,6 @@ fn agent_trace_schema_validator() -> &'static Validator { }) } -#[allow(dead_code)] pub(crate) fn validate_agent_trace_value(value: &Value) -> Result<(), AgentTraceValidationError> { let mut errors = agent_trace_schema_validator() .iter_errors(value) @@ -252,26 +238,6 @@ pub(crate) fn validate_agent_trace_value(value: &Value) -> Result<(), AgentTrace Err(AgentTraceValidationError::SchemaValidation { errors }) } -#[allow(dead_code)] -pub(crate) fn validate_agent_trace_json(raw: &str) -> Result<(), AgentTraceValidationError> { - let value: Value = - serde_json::from_str(raw).map_err(|error| AgentTraceValidationError::InvalidJson { - message: error.to_string(), - })?; - - validate_agent_trace_value(&value) -} - -#[allow(dead_code)] -pub(crate) fn validate_agent_trace_file(path: &Path) -> Result<(), AgentTraceValidationError> { - let raw = - std::fs::read_to_string(path).map_err(|error| AgentTraceValidationError::FileRead { - path: path.display().to_string(), - message: error.to_string(), - })?; - - validate_agent_trace_json(&raw) -} /// Classification of a single hunk's origin relative to the AI candidate patch. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -305,7 +271,6 @@ pub struct Conversation { /// A related resource for a conversation entry. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] -#[allow(dead_code)] pub struct ConversationRelated { /// Free-form related resource type. #[serde(rename = "type")] @@ -371,40 +336,6 @@ pub struct AgentTrace { pub files: Vec, } -/// Classify a single `post_commit_patch` hunk against the corresponding -/// `intersection_patch` hunk (if any). -/// -/// Two hunks correspond when they share the same `old_start` value within the -/// same file. This is the slot-matching rule that aligns `intersection_patch` -/// hunks to `post_commit_patch` hunks for comparison. -/// -/// Returns: -/// - `HunkContributor::Ai` when the `intersection_patch` hunk exists and its -/// touched lines match the `post_commit_patch` hunk's touched lines exactly -/// (same count, same kind, same `line_number`, same content, in the same order). -/// - `HunkContributor::Mixed` when the `intersection_patch` hunk exists but its -/// touched lines differ from the `post_commit_patch` hunk's touched lines. -/// - `HunkContributor::Unknown` when no `intersection_patch` hunk with the same -/// `old_start` exists for this `post_commit_patch` hunk. -#[allow(dead_code)] -pub fn classify_hunk( - post_commit_hunk: &PatchHunk, - intersection_hunks: &[PatchHunk], -) -> HunkContributor { - let Some(intersection_hunk) = intersection_hunks - .iter() - .find(|h| h.old_start == post_commit_hunk.old_start) - else { - return HunkContributor::Unknown; - }; - - if hunks_match_exactly(post_commit_hunk, intersection_hunk) { - HunkContributor::Ai - } else { - HunkContributor::Mixed - } -} - type CoverageLineKey<'a> = (TouchedLineKind, u64, &'a str); fn coverage_line_key(line: &TouchedLine) -> CoverageLineKey<'_> { @@ -473,7 +404,6 @@ fn combined_model_id( } } -#[allow(dead_code)] pub(crate) fn patches_have_overlap( candidate_patch: &ParsedPatch, target_patch: &ParsedPatch, @@ -490,17 +420,6 @@ pub(crate) fn patch_has_touched_lines(patch: &ParsedPatch) -> bool { .any(|file| file.hunks.iter().any(|hunk| !hunk.lines.is_empty())) } -/// Check whether two hunks have identical touched lines in the same order. -fn hunks_match_exactly(left: &PatchHunk, right: &PatchHunk) -> bool { - if left.lines.len() != right.lines.len() { - return false; - } - left.lines.iter().zip(right.lines.iter()).all(|(ll, rl)| { - ll.kind == rl.kind && ll.line_number == rl.line_number && ll.content == rl.content - }) -} - -#[allow(dead_code)] pub(crate) fn range_content_hash(hunk: &PatchHunk) -> String { let mut input = Vec::new(); input.extend_from_slice(RANGE_CONTENT_HASH_INPUT_VERSION); @@ -650,25 +569,6 @@ fn build_trace_file( }) } -#[allow(dead_code)] -pub fn build_agent_trace( - constructed_patch: &ParsedPatch, - post_commit_patch: &ParsedPatch, - metadata: AgentTraceMetadataInput<'_>, -) -> Result { - let empty_mutation_ai_patch = ParsedPatch { files: Vec::new() }; - - build_agent_trace_from_evidence( - AgentTraceEvidence { - direct_patch: constructed_patch, - mutation_ai_patch: &empty_mutation_ai_patch, - }, - post_commit_patch, - metadata, - ) -} - -#[allow(dead_code)] pub fn build_agent_trace_from_evidence( evidence: AgentTraceEvidence<'_>, post_commit_patch: &ParsedPatch, diff --git a/cli/src/services/agent_trace/tests.rs b/cli/src/services/agent_trace/tests.rs index f6b1ef8e6..22ae65678 100644 --- a/cli/src/services/agent_trace/tests.rs +++ b/cli/src/services/agent_trace/tests.rs @@ -1,6 +1,6 @@ use super::{ - build_agent_trace, build_agent_trace_from_evidence, validate_agent_trace_value, - AgentTraceEvidence, AgentTraceMetadataInput, AgentTraceVcsType, AGENT_TRACE_VERSION, + build_agent_trace_from_evidence, validate_agent_trace_value, AgentTraceEvidence, + AgentTraceMetadataInput, AgentTraceVcsType, AGENT_TRACE_VERSION, }; use crate::services::{ agent_trace::agent_trace_conversation_url, @@ -8,6 +8,9 @@ use crate::services::{ }; use serde_json::Value; +const TEST_COMMIT_TIMESTAMP: &str = "2026-04-23T10:20:30Z"; +const TEST_COMMIT_REVISION: &str = "a0b1c2d3e4f5a6b7c8d9e0f11223344556677889"; + #[derive(Clone, Copy)] struct AgentTraceScenario { incremental: &'static [&'static str], @@ -15,9 +18,6 @@ struct AgentTraceScenario { golden: &'static str, } -const TEST_COMMIT_TIMESTAMP: &str = "2026-04-23T10:20:30Z"; -const TEST_COMMIT_REVISION: &str = "a0b1c2d3e4f5a6b7c8d9e0f11223344556677889"; - fn parse_fixtures(fixtures: &[&str]) -> Vec { fixtures .iter() @@ -60,8 +60,12 @@ fn assert_builds_expected_agent_trace(scenario: AgentTraceScenario) { parse_patch(scenario.post_commit, None).expect("fixture patch should parse"); let golden: Value = serde_json::from_str(scenario.golden).expect("golden json should load"); validate_agent_trace_value(&golden).expect("golden json should validate against schema"); - let actual = build_agent_trace( - &constructed_patch, + let empty_mutation_ai_patch = ParsedPatch { files: Vec::new() }; + let actual = build_agent_trace_from_evidence( + AgentTraceEvidence { + direct_patch: &constructed_patch, + mutation_ai_patch: &empty_mutation_ai_patch, + }, &post_commit_patch, AgentTraceMetadataInput { commit_timestamp: TEST_COMMIT_TIMESTAMP, diff --git a/cli/src/services/agent_trace_db/lifecycle.rs b/cli/src/services/agent_trace_db/lifecycle.rs index dd9f4fe86..b00acce28 100644 --- a/cli/src/services/agent_trace_db/lifecycle.rs +++ b/cli/src/services/agent_trace_db/lifecycle.rs @@ -8,22 +8,20 @@ use crate::services::db::{bootstrap_db_parent, collect_db_path_health, DbSpec}; use crate::services::default_paths::agent_trace_db_path_for_repository; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; use crate::services::repository_identity::resolve::{ resolve_repository_identity, RepositoryIdentitySource, }; -use super::repository::{RepositoryAgentTraceDb, RepositoryAgentTraceDbSpec}; +use super::repository::{ + ExistingRepositoryDbError, RepositoryAgentTraceDb, RepositoryAgentTraceDbSpec, +}; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct AgentTraceDbLifecycle; impl ServiceLifecycle for AgentTraceDbLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::AgentTraceDb - } - async fn diagnose(&self, ctx: &C) -> Vec { diagnose_agent_trace_db_health(ctx.repo_root()).await } @@ -161,11 +159,24 @@ pub async fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec { - if let Err(error) = db.ensure_schema_ready_for_hooks().await { - problems.push(HealthProblem { + problems.extend(inspect_existing_db_schema(&db_path, || {}).await); + problems +} + +async fn inspect_existing_db_schema( + db_path: &Path, + before_open: impl FnOnce(), +) -> Vec { + let mut problems = Vec::new(); + if db_path.exists() && !db_path.is_file() { + return problems; + } + + match RepositoryAgentTraceDb::open_existing_schema_ready_at(db_path, before_open).await { + Ok(_db) => {} + Err(ExistingRepositoryDbError::Missing { .. }) => {} + Err(ExistingRepositoryDbError::IncompatibleSchema(error)) => { + problems.push(HealthProblem { kind: HealthProblemKind::AgentTraceDbSchemaNotReady, category: HealthCategory::GlobalState, severity: HealthSeverity::Error, @@ -179,10 +190,13 @@ pub async fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec { - problems.push(HealthProblem { + } + Err(error) => { + let error = match error { + ExistingRepositoryDbError::Unreadable(source) => source.to_string(), + other => other.to_string(), + }; + problems.push(HealthProblem { kind: HealthProblemKind::AgentTraceDbConnectionFailed, category: HealthCategory::GlobalState, severity: HealthSeverity::Error, @@ -196,7 +210,6 @@ pub async fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec) -> Result Vec { + let mut names = std::fs::read_dir(dir) + .expect("read dir") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect::>(); + names.sort(); + names + } + + async fn schema_objects(path: &Path) -> Vec { + let db = RepositoryAgentTraceDb::open_existing_without_migrations_at(path) + .await + .expect("read-only open"); + let mut names = db + .query_map("SELECT name FROM sqlite_master ORDER BY name", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("schema query"); + names.sort(); + names + } + + async fn count_rows(path: &Path, table: &str) -> i64 { + let db = RepositoryAgentTraceDb::open_existing_without_migrations_at(path) + .await + .expect("read-only open"); + db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("count query")[0] + } + + #[tokio::test(flavor = "multi_thread")] + async fn missing_database_is_not_created_and_leaves_no_parent_directory() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir + .path() + .join("state") + .join("repos") + .join("id") + .join("agent-trace.db"); + + let problems = inspect_existing_db_schema(&path, || {}).await; + let again = inspect_existing_db_schema(&path, || {}).await; + + assert!(problems.is_empty()); + assert!(again.is_empty()); + assert!(!path.exists()); + assert!(dir_entries(dir.path()).is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn database_removed_immediately_before_open_is_not_recreated() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&path).await.expect("create"); + assert!(path.is_file()); + + let problems = inspect_existing_db_schema(&path, || { + for entry in std::fs::read_dir(dir.path()).expect("read dir") { + std::fs::remove_file(entry.expect("entry").path()).expect("remove"); + } + }) + .await; + + assert!(problems.is_empty()); + assert!(dir_entries(dir.path()).is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn valid_database_is_inspected_without_problems_or_logical_change() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + let db = RepositoryAgentTraceDb::new_at(&path).await.expect("create"); + db.verify_or_initialize_repository_metadata("repo") + .await + .expect("metadata"); + drop(db); + let objects_before = schema_objects(&path).await; + let migrations_before = count_rows(&path, "__sce_migrations").await; + let metadata_before = count_rows(&path, "repository_metadata").await; + + let problems = inspect_existing_db_schema(&path, || {}).await; + let first_files = dir_entries(dir.path()); + let repeat = inspect_existing_db_schema(&path, || {}).await; + + assert!(problems.is_empty()); + assert!(repeat.is_empty()); + assert_eq!(dir_entries(dir.path()), first_files); + assert_eq!(schema_objects(&path).await, objects_before); + assert_eq!( + count_rows(&path, "__sce_migrations").await, + migrations_before + ); + assert_eq!( + count_rows(&path, "repository_metadata").await, + metadata_before + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn incompatible_schema_is_diagnosed_without_initializing_or_migrating() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + let raw = RepositoryAgentTraceDb::open_without_migrations_at(&path) + .await + .expect("create empty"); + raw.execute("CREATE TABLE unrelated (id INTEGER)", ()) + .await + .expect("unrelated table"); + drop(raw); + let objects_before = schema_objects(&path).await; + + let problems = inspect_existing_db_schema(&path, || {}).await; + + assert_eq!(problems.len(), 1); + assert_eq!( + problems[0].kind, + HealthProblemKind::AgentTraceDbSchemaNotReady + ); + assert!(problems[0].summary.contains("is not ready")); + assert_eq!(schema_objects(&path).await, objects_before); + assert!(!objects_before.iter().any(|name| name == "__sce_migrations")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn non_database_file_is_unreadable_and_left_byte_identical() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + std::fs::write( + &path, + b"this is not a sqlite database at all, just some text bytes....", + ) + .expect("write"); + let before = std::fs::read(&path).expect("read"); + + let problems = inspect_existing_db_schema(&path, || {}).await; + + assert_eq!(problems.len(), 1); + assert_eq!( + problems[0].kind, + HealthProblemKind::AgentTraceDbConnectionFailed + ); + assert_eq!(std::fs::read(&path).expect("read"), before); + assert_eq!(dir_entries(dir.path()), vec!["agent-trace.db".to_string()]); + } +} diff --git a/cli/src/services/agent_trace_db/mod.rs b/cli/src/services/agent_trace_db/mod.rs index 8c42cc44a..9c69240ea 100644 --- a/cli/src/services/agent_trace_db/mod.rs +++ b/cli/src/services/agent_trace_db/mod.rs @@ -408,23 +408,6 @@ fn claude_model_state_observation_from_turso( }) } -#[allow(dead_code)] -async fn insert_message_with( - db: &TursoDb, - input: InsertMessageInsert, -) -> Result { - db.execute( - INSERT_MESSAGE_SQL, - ( - input.session_id, - input.message_id, - input.role.to_string(), - input.generated_at_unix_ms, - ), - ) - .await -} - async fn insert_messages_with( db: &TursoDb, inputs: Vec, @@ -453,21 +436,6 @@ async fn insert_messages_with( db.execute(&sql, params).await } -#[allow(dead_code)] -async fn insert_part_with(db: &TursoDb, input: InsertPartInsert) -> Result { - db.execute( - INSERT_PART_SQL, - ( - input.part_type.to_string(), - input.text, - input.message_id, - input.session_id, - input.generated_at_unix_ms, - ), - ) - .await -} - async fn insert_parts_with( db: &TursoDb, inputs: Vec, diff --git a/cli/src/services/agent_trace_db/repository.rs b/cli/src/services/agent_trace_db/repository.rs index 15de10f1e..037521477 100644 --- a/cli/src/services/agent_trace_db/repository.rs +++ b/cli/src/services/agent_trace_db/repository.rs @@ -10,11 +10,10 @@ use crate::{ use super::{ insert_agent_trace_with, insert_conversation_text_event_with, insert_diff_trace_with, - insert_message_with, insert_messages_with, insert_part_with, insert_parts_with, - insert_post_commit_patch_intersection_with, recent_diff_trace_patches_with, - upsert_claude_model_state_with, AgentTraceInsert, ClaudeModelStateObservation, DiffTraceInsert, - InsertMessageInsert, InsertPartInsert, PostCommitPatchIntersectionInsert, - RecentDiffTracePatches, + insert_messages_with, insert_parts_with, insert_post_commit_patch_intersection_with, + recent_diff_trace_patches_with, upsert_claude_model_state_with, AgentTraceInsert, + ClaudeModelStateObservation, DiffTraceInsert, InsertMessageInsert, InsertPartInsert, + PostCommitPatchIntersectionInsert, RecentDiffTracePatches, }; const REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE: &str = "Run 'sce setup'."; @@ -86,6 +85,53 @@ fn ensure_repository_id_matches(stored_repository_id: &str, repository_id: &str) Ok(()) } +#[derive(Debug)] +#[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" +)] +pub enum ExistingRepositoryDbError { + Missing { path: PathBuf }, + Unreadable(anyhow::Error), + IncompatibleSchema(anyhow::Error), + MissingMetadata, + RepositoryMismatch { stored: String, resolved: String }, +} + +impl std::fmt::Display for ExistingRepositoryDbError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExistingRepositoryDbError::Missing { path } => write!( + f, + "repository Agent Trace DB does not exist at '{}'. \ + {REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE}", + path.display() + ), + ExistingRepositoryDbError::Unreadable(source) => { + write!(f, "repository Agent Trace DB is unreadable: {source}") + } + ExistingRepositoryDbError::IncompatibleSchema(source) => { + write!( + f, + "repository Agent Trace DB schema is incompatible: {source}" + ) + } + ExistingRepositoryDbError::MissingMetadata => write!( + f, + "repository Agent Trace DB metadata is missing or has no valid source-instance ID. \ + {REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE}" + ), + ExistingRepositoryDbError::RepositoryMismatch { stored, resolved } => write!( + f, + "repository Agent Trace DB metadata mismatch: stored repository ID {stored} \ + does not match resolved repository ID {resolved}" + ), + } + } +} + +impl std::error::Error for ExistingRepositoryDbError {} + /// Repository-scoped Agent Trace database configuration. pub struct RepositoryAgentTraceDbSpec; @@ -123,6 +169,105 @@ impl RepositoryAgentTraceDb { TursoDb::::open_without_migrations_at(path).await } + #[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" + )] + pub async fn open_verified_existing_at( + path: impl AsRef, + repository_id: &str, + ) -> std::result::Result<(Self, RepositoryMetadata), ExistingRepositoryDbError> { + Self::open_verified_existing_with(path.as_ref(), repository_id, || {}, || {}).await + } + + #[cfg(test)] + pub(crate) async fn open_verified_existing_at_with_hooks( + path: &std::path::Path, + repository_id: &str, + before_open: impl FnOnce(), + after_open: impl FnOnce(), + ) -> std::result::Result<(Self, RepositoryMetadata), ExistingRepositoryDbError> { + Self::open_verified_existing_with(path, repository_id, before_open, after_open).await + } + + pub async fn open_existing_schema_ready_at( + path: &std::path::Path, + before_open: impl FnOnce(), + ) -> std::result::Result { + Self::open_existing_schema_ready_with(path, before_open, || {}).await + } + + async fn open_existing_schema_ready_with( + path: &std::path::Path, + before_open: impl FnOnce(), + after_open: impl FnOnce(), + ) -> std::result::Result { + before_open(); + let opened = + TursoDb::::open_existing_without_migrations_at(path).await; + after_open(); + let db = match opened { + Ok(db) => db, + Err(source) => { + return Err(match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + ExistingRepositoryDbError::Missing { + path: path.to_path_buf(), + } + } + Ok(_) | Err(_) => ExistingRepositoryDbError::Unreadable(source), + }); + } + }; + db.ensure_schema_ready_for_hooks() + .await + .map_err(ExistingRepositoryDbError::IncompatibleSchema)?; + Ok(db) + } + + async fn open_verified_existing_with( + path: &std::path::Path, + repository_id: &str, + before_open: impl FnOnce(), + after_open: impl FnOnce(), + ) -> std::result::Result<(Self, RepositoryMetadata), ExistingRepositoryDbError> { + let db = Self::open_existing_schema_ready_with(path, before_open, after_open).await?; + let metadata = db + .verify_existing_repository_metadata(repository_id) + .await?; + Ok((db, metadata)) + } + + #[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" + )] + pub async fn verify_existing_repository_metadata( + &self, + repository_id: &str, + ) -> std::result::Result { + let Some((stored_repository_id, source_instance_id)) = self + .select_repository_metadata_row() + .await + .map_err(ExistingRepositoryDbError::Unreadable)? + else { + return Err(ExistingRepositoryDbError::MissingMetadata); + }; + if stored_repository_id != repository_id { + return Err(ExistingRepositoryDbError::RepositoryMismatch { + stored: stored_repository_id, + resolved: repository_id.to_string(), + }); + } + if !is_valid_source_instance_id(&source_instance_id) { + return Err(ExistingRepositoryDbError::MissingMetadata); + } + Ok(RepositoryMetadata { + repository_id: stored_repository_id, + source_instance_id, + }) + } + /// Verify that the repository-scoped schema baseline already exists. pub async fn ensure_schema_ready_for_hooks(&self) -> Result<()> { self.ensure_schema_ready(REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE) @@ -286,25 +431,12 @@ impl RepositoryAgentTraceDb { recent_diff_trace_patches_with(self, cutoff_time_ms, end_time_ms).await } - /// Insert a message row, ignoring duplicate `(session_id, message_id)` - /// rows. - #[allow(dead_code)] - pub async fn insert_message(&self, input: InsertMessageInsert) -> Result { - insert_message_with(self, input).await - } - /// Insert message rows with one multi-row statement, ignoring duplicate /// `(session_id, message_id)` rows. pub async fn insert_messages(&self, inputs: Vec) -> Result { insert_messages_with(self, inputs).await } - /// Append a part row (no upsert; multiple rows per message allowed). - #[allow(dead_code)] - pub async fn insert_part(&self, input: InsertPartInsert) -> Result { - insert_part_with(self, input).await - } - /// Append part rows with one multi-row statement. pub async fn insert_parts(&self, inputs: Vec) -> Result { insert_parts_with(self, inputs).await diff --git a/cli/src/services/agent_trace_export/mod.rs b/cli/src/services/agent_trace_export/mod.rs index 5046ac895..64b1e77d1 100644 --- a/cli/src/services/agent_trace_export/mod.rs +++ b/cli/src/services/agent_trace_export/mod.rs @@ -76,6 +76,10 @@ pub struct AgentTracePartExportRow { } /// Owned, wire-compatible export row for the `diff_traces` capture stream. +#[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" +)] #[derive(Clone, Debug, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AgentTraceDiffTraceExportRow { @@ -116,6 +120,10 @@ WHERE id > ?1 ORDER BY id ASC LIMIT ?2"; +#[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" +)] const SELECT_DIFF_TRACES_AFTER_SQL: &str = "SELECT id, session_id, time_ms, patch, model_id, tool_name, tool_version, payload_type FROM diff_traces @@ -200,6 +208,10 @@ impl<'a> AgentTraceExportReader<'a> { /// Read `diff_traces` rows with `id > cursor`, ordered by `id ASC`, /// capped at `limit`. `patch` and `payload_type` are returned raw and /// unmodified: no patch parsing or normalization is performed. + #[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" + )] pub async fn read_diff_traces_after( &self, cursor: i64, @@ -283,6 +295,10 @@ fn part_export_row_from_turso(row: &turso::Row) -> Result Result { Ok(AgentTraceDiffTraceExportRow { source_row_id: row.get(0).context("failed to read diff_traces.id")?, diff --git a/cli/src/services/agent_trace_storage/mod.rs b/cli/src/services/agent_trace_storage/mod.rs index 62192f9f9..9d0a103c8 100644 --- a/cli/src/services/agent_trace_storage/mod.rs +++ b/cli/src/services/agent_trace_storage/mod.rs @@ -55,20 +55,6 @@ pub async fn resolve_agent_trace_storage( open_storage(repository_identity, db_path).await } -/// Resolution core against an explicit state root, so tests can exercise the -/// full path without touching the real user state directory. -pub async fn resolve_agent_trace_storage_at_state_root( - context: &AgentTraceStorageContext<'_>, - state_root: &Path, -) -> Result { - let repository_identity = resolve_identity(context)?; - let db_path = agent_trace_db_path_for_repository_at( - state_root, - &repository_identity.identity.repository_id, - )?; - open_storage(repository_identity, db_path).await -} - /// Resolves repository-scoped Agent Trace storage for high-frequency hook /// runtime callers using the canonical state root. /// @@ -222,3 +208,89 @@ async fn open_repository_db_for_hook_runtime( .await?; Ok((db, metadata)) } + +#[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" +)] +pub async fn resolve_existing_agent_trace_storage_for_maintenance( + context: &AgentTraceStorageContext<'_>, +) -> Result { + let repository_identity = resolve_identity(context)?; + let db_path = agent_trace_db_path_for_repository(&repository_identity.identity.repository_id)?; + open_existing_storage_for_maintenance(repository_identity, db_path).await +} + +#[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" +)] +async fn open_existing_storage_for_maintenance( + repository_identity: ResolvedRepositoryIdentity, + db_path: PathBuf, +) -> Result { + let repository_id = &repository_identity.identity.repository_id; + let (db, metadata) = + RepositoryAgentTraceDb::open_verified_existing_at(&db_path, repository_id).await?; + + Ok(ResolvedAgentTraceStorage { + repository_identity, + db_path, + db, + metadata, + }) +} + +#[cfg(test)] +pub(crate) async fn resolve_existing_agent_trace_storage_for_maintenance_at_state_root( + context: &AgentTraceStorageContext<'_>, + state_root: &Path, +) -> Result { + let repository_identity = resolve_identity(context)?; + let db_path = agent_trace_db_path_for_repository_at( + state_root, + &repository_identity.identity.repository_id, + )?; + open_existing_storage_for_maintenance(repository_identity, db_path).await +} + +#[cfg(test)] +mod tests { + use super::{ + resolve_existing_agent_trace_storage_for_maintenance_at_state_root, + AgentTraceStorageContext, + }; + use crate::services::agent_trace_db::repository::ExistingRepositoryDbError; + + #[tokio::test(flavor = "multi_thread")] + async fn agent_trace_storage_maintenance_open_of_missing_db_creates_nothing() { + let repository = tempfile::tempdir().expect("repository dir"); + let state_root = tempfile::tempdir().expect("state root"); + let context = AgentTraceStorageContext { + repository_root: repository.path(), + explicit_repository_id: Some("maintenance-open-missing"), + repository_remote: "origin", + }; + + let Err(error) = resolve_existing_agent_trace_storage_for_maintenance_at_state_root( + &context, + state_root.path(), + ) + .await + else { + panic!("a missing database must not open"); + }; + + assert!(matches!( + error.downcast_ref::(), + Some(ExistingRepositoryDbError::Missing { .. }) + )); + let entries: Vec<_> = std::fs::read_dir(state_root.path()) + .expect("state root listing") + .collect(); + assert!( + entries.is_empty(), + "state root must stay empty: {entries:?}" + ); + } +} diff --git a/cli/src/services/agent_trace_sync/control_plane.rs b/cli/src/services/agent_trace_sync/control_plane.rs index f87c7da03..a44857293 100644 --- a/cli/src/services/agent_trace_sync/control_plane.rs +++ b/cli/src/services/agent_trace_sync/control_plane.rs @@ -18,6 +18,7 @@ use crate::services::agent_trace_export::{ AgentTracePartExportRow, }; use crate::services::auth::{self, AuthError, TokenResponse}; +use crate::services::observability::tracing_boundary::spawn_in_current_scope; use crate::services::resilience::{run_with_retry, RetryPolicy}; use crate::services::token_storage::{self, StoredTokens, TokenStorageError}; @@ -392,6 +393,10 @@ impl AuthenticatedControlPlaneClient { self.post_batch(request).await } + #[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" + )] pub async fn ingest_diff_traces( &self, request: &AgentTraceIngestionBatchRequest, @@ -518,7 +523,7 @@ impl AuthenticatedControlPlaneClient { let workos_client_id = self.workos_client_id.clone(); let credential_store = Arc::clone(&self.credential_store); - tokio::spawn(async move { + spawn_in_current_scope(async move { let _refresh_guard = refresh_guard; let token = auth::renew_stored_token_from_refresh_token( &http, @@ -641,7 +646,10 @@ mod tests { use tokio::sync::{Notify, Semaphore}; - use super::{AuthenticatedControlPlaneClient, ControlPlaneError, CredentialStore}; + use super::{ + AgentTraceIngestionStateRequest, AgentTraceIngestionStateResponse, + AuthenticatedControlPlaneClient, ControlPlaneError, CredentialStore, + }; use crate::services::auth::TokenResponse; use crate::services::token_storage::StoredTokens; @@ -854,4 +862,121 @@ mod tests { REPLACEMENT_REFRESH_TOKEN ); } + + struct FixedStore; + + impl CredentialStore for FixedStore { + async fn load(&self) -> Result, ControlPlaneError> { + Ok(Some(StoredTokens { + access_token: "access".to_string(), + token_type: "Bearer".to_string(), + expires_in: 3600, + refresh_token: "refresh".to_string(), + scope: None, + stored_at_unix_seconds: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_secs(), + })) + } + + async fn save(&self, _token: &TokenResponse) -> Result { + unreachable!("fixed credentials never refresh") + } + } + + fn start_state_server(payload: &str) -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind state server"); + let base_url = format!("http://{}", listener.local_addr().expect("server addr")); + let payload = payload.to_string(); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; content_length]; + let _ = reader.read_exact(&mut body); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + base_url + } + + async fn request_state( + payload: &str, + ) -> Result { + let client = AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + start_state_server(payload), + "http://127.0.0.1:1", + "client", + FixedStore, + ); + client + .ingestion_state(&AgentTraceIngestionStateRequest { + repository_id: "repo".to_string(), + source_instance_id: "source".to_string(), + }) + .await + } + + #[tokio::test] + async fn state_response_decodes_required_diff_traces_cursor() { + let state = + request_state(r#"{"cursors":{"messages":1,"parts":2,"diffTraces":7,"agentTraces":4}}"#) + .await + .expect("valid state"); + + assert_eq!(state.cursors.diff_traces, 7); + } + + #[tokio::test] + async fn state_response_rejects_missing_diff_traces_cursor() { + let error = request_state(r#"{"cursors":{"messages":1,"parts":2,"agentTraces":4}}"#) + .await + .expect_err("missing diffTraces"); + + assert!( + matches!(&error, ControlPlaneError::InvalidResponse(_)), + "{error:?}" + ); + } + + #[tokio::test] + async fn state_response_rejects_out_of_range_diff_traces_cursor() { + let error = request_state( + r#"{"cursors":{"messages":1,"parts":2,"diffTraces":-1,"agentTraces":4}}"#, + ) + .await + .expect_err("negative diffTraces"); + + assert!( + matches!(&error, ControlPlaneError::InvalidResponse(reason) if reason.contains("cursors.diffTraces")), + "{error:?}" + ); + } } diff --git a/cli/src/services/auth_db/lifecycle.rs b/cli/src/services/auth_db/lifecycle.rs index ab8ca2f56..ffc82fd0b 100644 --- a/cli/src/services/auth_db/lifecycle.rs +++ b/cli/src/services/auth_db/lifecycle.rs @@ -5,7 +5,7 @@ use crate::services::db::{bootstrap_db_parent, collect_db_path_health, DbSpec}; use crate::services::default_paths::auth_db_path; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; use super::{AuthDb, AuthDbSpec}; @@ -14,10 +14,6 @@ use super::{AuthDb, AuthDbSpec}; pub struct AuthDbLifecycle; impl ServiceLifecycle for AuthDbLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::AuthDb - } - async fn diagnose(&self, _ctx: &C) -> Vec { diagnose_auth_db_health() } diff --git a/cli/src/services/capabilities.rs b/cli/src/services/capabilities.rs index 7d6ab0f0c..ed255b47c 100644 --- a/cli/src/services/capabilities.rs +++ b/cli/src/services/capabilities.rs @@ -1,51 +1,18 @@ -#![allow(dead_code)] - -use std::fs::{self, Metadata}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use anyhow::{anyhow, Context, Result}; -pub trait FsOps: Send + Sync { - fn read_file(&self, path: &Path) -> Result; - - fn write_file(&self, path: &Path, content: &str) -> Result<()>; - - fn metadata(&self, path: &Path) -> Result; - - fn exists(&self, path: &Path) -> bool; -} +pub trait FsOps: Send + Sync {} #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct StdFsOps; -impl FsOps for StdFsOps { - fn read_file(&self, path: &Path) -> Result { - fs::read_to_string(path) - .with_context(|| format!("Failed to read file '{}'", path.display())) - } - - fn write_file(&self, path: &Path, content: &str) -> Result<()> { - fs::write(path, content) - .with_context(|| format!("Failed to write file '{}'", path.display())) - } - - fn metadata(&self, path: &Path) -> Result { - fs::metadata(path).with_context(|| format!("Failed to inspect path '{}'", path.display())) - } - - fn exists(&self, path: &Path) -> bool { - path.exists() - } -} +impl FsOps for StdFsOps {} pub trait GitOps: Send + Sync { fn run_command(&self, repo: &Path, args: &[&str]) -> Result; - fn resolve_repository_root(&self, dir: &Path) -> Result; - - fn resolve_hooks_directory(&self, repo: &Path) -> Result; - fn is_available(&self) -> bool; } @@ -57,22 +24,6 @@ impl GitOps for ProcessGitOps { run_git_command(repo, args) } - fn resolve_repository_root(&self, dir: &Path) -> Result { - let output = run_git_command(dir, &["rev-parse", "--show-toplevel"])?; - Ok(PathBuf::from(output.trim())) - } - - fn resolve_hooks_directory(&self, repo: &Path) -> Result { - let output = run_git_command(repo, &["rev-parse", "--git-path", "hooks"])?; - let hooks_path = PathBuf::from(output.trim()); - - if hooks_path.is_absolute() { - Ok(hooks_path) - } else { - Ok(repo.join(hooks_path)) - } - } - fn is_available(&self) -> bool { Command::new("git") .arg("--version") diff --git a/cli/src/services/command_registry.rs b/cli/src/services/command_registry.rs index 0e6700023..66d1cae6a 100644 --- a/cli/src/services/command_registry.rs +++ b/cli/src/services/command_registry.rs @@ -54,13 +54,19 @@ impl RuntimeCommand { } } - #[allow(dead_code)] - pub async fn execute(&self, context: &C) -> Result - where - C: HasLogger + ContextWithRepoRoot + crate::app::HasGit + crate::app::HasFs, - { - let mut stderr = std::io::sink(); - self.execute_with_stderr(context, &mut stderr).await + pub fn is_hook_invocation(&self) -> bool { + match self { + Self::Hooks(_) | Self::Policy(_) => true, + Self::Help(_) + | Self::HelpText(_) + | Self::Auth(_) + | Self::Config(_) + | Self::Setup(_) + | Self::Doctor(_) + | Self::Version(_) + | Self::Completion(_) + | Self::Sync(_) => false, + } } pub async fn execute_with_stderr( diff --git a/cli/src/services/config/lifecycle.rs b/cli/src/services/config/lifecycle.rs index 541ea608f..b5178488a 100644 --- a/cli/src/services/config/lifecycle.rs +++ b/cli/src/services/config/lifecycle.rs @@ -6,7 +6,7 @@ use crate::app::HasRepoRoot; use crate::services::default_paths::{resolve_sce_default_locations, RepoPaths}; use crate::services::lifecycle::{ HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, - LifecycleProviderId, ServiceLifecycle, SetupOutcome, + ServiceLifecycle, SetupOutcome, }; use crate::services::setup::bootstrap_repo_local_config; @@ -16,10 +16,6 @@ use super::validate_config_file; pub struct ConfigLifecycle; impl ServiceLifecycle for ConfigLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::Config - } - async fn diagnose(&self, ctx: &C) -> Vec { let repository_root = match ctx.repo_root() { Some(path) => path.to_path_buf(), diff --git a/cli/src/services/config/policy.rs b/cli/src/services/config/policy.rs index 3a3302c2a..b64ac1904 100644 --- a/cli/src/services/config/policy.rs +++ b/cli/src/services/config/policy.rs @@ -33,7 +33,6 @@ pub(crate) struct BashPolicyConfig { } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) struct RuntimeBashPolicyPreset { pub(crate) id: String, pub(crate) argv_prefixes: Vec>, @@ -121,7 +120,6 @@ fn builtin_bash_policy_catalog() -> &'static BuiltinBashPolicyCatalog { }) } -#[allow(dead_code)] pub(crate) fn runtime_bash_policy_presets() -> Vec { builtin_bash_policy_catalog() .presets diff --git a/cli/src/services/config/types.rs b/cli/src/services/config/types.rs index 7de321d86..bed656c23 100644 --- a/cli/src/services/config/types.rs +++ b/cli/src/services/config/types.rs @@ -42,16 +42,6 @@ impl LogLevel { } } - pub(crate) fn parse_env(raw: &str, key: &str) -> anyhow::Result { - match raw { - "error" => Ok(Self::Error), - "warn" => Ok(Self::Warn), - "info" => Ok(Self::Info), - "debug" => Ok(Self::Debug), - _ => anyhow::bail!("Invalid {key} '{raw}'. Valid values: error, warn, info, debug."), - } - } - pub(crate) fn as_str(self) -> &'static str { match self { Self::Error => "error", @@ -88,14 +78,6 @@ impl LogFormat { } } - pub(crate) fn parse_env(raw: &str, key: &str) -> anyhow::Result { - match raw { - "text" => Ok(Self::Text), - "json" => Ok(Self::Json), - _ => anyhow::bail!("Invalid {key} '{raw}'. Valid values: text, json."), - } - } - pub(crate) fn as_str(self) -> &'static str { match self { Self::Text => "text", diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index f6b9d9833..1b382ec11 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -12,12 +12,14 @@ use std::{ use anyhow::{Context, Result}; use rand::SeedableRng; -use turso::Value as TursoValue; use crate::services::config::{AgentTraceDbRetryConfig, DatabaseRetryConfig}; use crate::services::lifecycle::{ HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, }; +use crate::services::observability::tracing_boundary::{ + emit_contention_event, DbName, OperationClass, CONTENTION_EXHAUSTED_CAUSE, +}; use crate::services::resilience::{run_with_retry_elapsed, RetryOperation, RetryPolicy}; const MIGRATIONS_TABLE_SQL: &str = "CREATE TABLE IF NOT EXISTS __sce_migrations ( @@ -163,6 +165,12 @@ fn sentence_case(value: &str) -> String { first.to_uppercase().collect::() + chars.as_str() } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OpenMode { + CreateIfMissing, + ExistingReadOnly, +} + fn ensure_db_parent_dir(db_name: &str, db_path: &Path) -> Result<()> { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent).with_context(|| { @@ -193,29 +201,24 @@ async fn run_embedded_migrations( Ok(()) } - async fn ensure_migrations_table(conn: &turso::Connection, db_name: &str) -> Result<()> { - async { - conn.execute(MIGRATIONS_TABLE_SQL, ()) - .await - .map_err(|e| anyhow::anyhow!("{db_name} migration metadata setup failed: {e}")) - } - .await?; + conn.execute(MIGRATIONS_TABLE_SQL, ()) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata setup failed: {e}"))?; Ok(()) } async fn is_migration_applied(conn: &turso::Connection, db_name: &str, id: &str) -> Result { - async { - let mut rows = conn.query(SELECT_MIGRATION_SQL, (id,)).await.map_err(|e| { - anyhow::anyhow!("{db_name} migration metadata query failed for {id}: {e}") - })?; + let mut rows = conn + .query(SELECT_MIGRATION_SQL, (id,)) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata query failed for {id}: {e}"))?; - rows.next().await.map(|row| row.is_some()).map_err(|e| { - anyhow::anyhow!("{db_name} migration metadata row fetch failed for {id}: {e}") - }) - } - .await + rows.next() + .await + .map(|row| row.is_some()) + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata row fetch failed for {id}: {e}")) } async fn apply_migration( @@ -224,23 +227,18 @@ async fn apply_migration( id: &str, sql: &str, ) -> Result<()> { - async { - // Migration files may contain multiple statements (the repository - // Agent Trace baseline is one multi-statement schema file), so batch - // execution is required; `execute` would stop after the first - // statement. - conn.execute_batch(sql) - .await - .map_err(|e| anyhow::anyhow!("{db_name} migration {id} failed: {e}"))?; - conn.execute(INSERT_MIGRATION_SQL, (id,)) - .await - .map_err(|e| { - anyhow::anyhow!("{db_name} migration metadata record failed for {id}: {e}") - })?; + // Migration files may contain multiple statements (the repository + // Agent Trace baseline is one multi-statement schema file), so batch + // execution is required; `execute` would stop after the first + // statement. + conn.execute_batch(sql) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration {id} failed: {e}"))?; + conn.execute(INSERT_MIGRATION_SQL, (id,)) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata record failed for {id}: {e}"))?; - Ok(()) - } - .await + Ok(()) } /// Body of [`TursoDb::execute_transactional_insert_pair_if_absent`], run @@ -298,7 +296,6 @@ async fn execute_insert_pair_if_absent_body( Ok(true) } -#[allow(dead_code)] pub struct TransactionStatement<'a> { sql: &'a str, params: turso::params::Params, @@ -306,7 +303,6 @@ pub struct TransactionStatement<'a> { } impl<'a> TransactionStatement<'a> { - #[allow(dead_code)] pub fn new(sql: &'a str, params: impl turso::params::IntoParams) -> Result { let params = turso::params::IntoParams::into_params(params) .map_err(|e| anyhow::anyhow!("parameter conversion failed: {sql}: {e}"))?; @@ -318,7 +314,6 @@ impl<'a> TransactionStatement<'a> { }) } - #[allow(dead_code)] pub fn expect_rows_affected(mut self, expected: u64) -> Self { self.expected_rows_affected = Some(expected); self @@ -470,13 +465,11 @@ fn classify_turso_error(db_name: &str, action: &str, error: &turso::Error) -> Wr } } -#[allow(dead_code)] enum CasBatchAttemptOutcome { Settled(bool), Deterministic(anyhow::Error), } -#[allow(dead_code)] fn cas_batch_failure_into_attempt_result( failure: WriteAttemptFailure, ) -> Result { @@ -486,7 +479,6 @@ fn cas_batch_failure_into_attempt_result( } } -#[allow(dead_code)] async fn execute_cas_batch_body( tx: &turso::transaction::Transaction<'_>, db_name: &str, @@ -760,9 +752,6 @@ async fn run_with_write_contention_retry_using { core: TursoConnectionCore, } -/// Fully fetched SQL query result for deterministic rendering outside the -/// async Turso row iterator lifetime. -#[derive(Clone, Debug, PartialEq)] -#[allow(dead_code)] -pub struct QueryRows { - pub columns: Vec, - pub rows: Vec>, -} - /// Generic encrypted Turso database adapter. /// /// Mirrors the structural seams of [`TursoDb`] while reserving encrypted local @@ -873,17 +848,22 @@ impl TursoDb { Self::open_without_migrations_at(db_path).await } - /// Open or create the database at an explicit path without running embedded - /// migrations. - /// - /// Parent directories are created automatically and the connection-open - /// retry policy is preserved. Runtime callers that use this path are - /// responsible for verifying schema readiness before query/write work. pub async fn open_without_migrations_at(db_path: impl AsRef) -> Result { + Self::open_at_path(db_path.as_ref(), OpenMode::CreateIfMissing).await + } + + pub async fn open_existing_without_migrations_at(db_path: impl AsRef) -> Result { + Self::open_at_path(db_path.as_ref(), OpenMode::ExistingReadOnly).await + } + + async fn open_at_path(db_path: &Path, mode: OpenMode) -> Result { let db_name = M::db_name(); - let db_path = db_path.as_ref().to_path_buf(); + let db_path = db_path.to_path_buf(); - ensure_db_parent_dir(db_name, &db_path)?; + if mode == OpenMode::CreateIfMissing { + ensure_db_parent_dir(db_name, &db_path)?; + } + let read_only = mode == OpenMode::ExistingReadOnly; let retry_policy = resolve_connection_open_retry_policy::(); let busy_timeout = resolve_busy_timeout::(); @@ -900,6 +880,7 @@ impl TursoDb { })?; let db = turso::Builder::new_local(path_str) .experimental_multiprocess_wal(true) + .read_only(read_only) .build() .await .map_err(|e| { @@ -984,96 +965,6 @@ impl TursoDb { .await } - /// Execute a SQL query that returns rows. - /// - /// # Arguments - /// * `sql` - SQL query, which may contain `?` placeholders. - /// * `params` - Parameter values implementing `IntoParams`. - /// - /// # Returns - /// A `turso::Rows` iterator over the result set. - #[allow(dead_code)] - pub async fn query( - &self, - sql: &str, - params: impl turso::params::IntoParams, - ) -> Result { - let params = turso::params::IntoParams::into_params(params).map_err(|e| { - anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) - })?; - let operation_name = format!("query {} database", M::db_name()); - - run_with_retry_elapsed( - resolve_query_retry_policy::(), - &operation_name, - QUERY_RETRY_HINT, - async |_| { - async { - self.core - .conn - .query(sql, params.clone()) - .await - .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) - } - .await - }, - ) - .await - } - - /// Execute a SQL query and synchronously fetch column names plus raw values. - #[allow(dead_code)] - pub async fn query_values( - &self, - sql: &str, - params: impl turso::params::IntoParams, - ) -> Result { - let params = turso::params::IntoParams::into_params(params).map_err(|e| { - anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) - })?; - let operation_name = format!("query and fetch {} database values", M::db_name()); - - run_with_retry_elapsed( - resolve_query_retry_policy::(), - &operation_name, - QUERY_RETRY_HINT, - async |_| { - async { - let mut rows = - self.core - .conn - .query(sql, params.clone()) - .await - .map_err(|e| { - anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name()) - })?; - let columns = rows.column_names(); - let column_count = rows.column_count(); - let mut fetched_rows = Vec::new(); - - while let Some(row) = rows.next().await.map_err(|e| { - anyhow::anyhow!("{} row fetch failed: {sql}: {e}", M::db_name()) - })? { - let mut values = Vec::with_capacity(column_count); - for column_index in 0..column_count { - values.push(row.get_value(column_index).map_err(|e| { - anyhow::anyhow!("{} value fetch failed: {sql}: {e}", M::db_name()) - })?); - } - fetched_rows.push(values); - } - - Ok(QueryRows { - columns, - rows: fetched_rows, - }) - } - .await - }, - ) - .await - } - /// Run an "insert row pair if absent" write transaction. /// /// If `exists_sql` (bound to `exists_params`) finds a matching row, no @@ -1210,7 +1101,6 @@ impl TursoDb { /// the whole transaction and no other operation on this adapter can run /// inside it. Dropping the future uncommitted schedules a rollback that /// runs on the connection's next use rather than immediately. - #[allow(dead_code)] pub async fn execute_transactional_cas_batch( &mut self, operation_name: &str, @@ -1477,43 +1367,6 @@ impl EncryptedTursoDb { .await } - /// Execute a SQL query that returns rows. - /// - /// # Arguments - /// * `sql` - SQL query, which may contain `?` placeholders. - /// * `params` - Parameter values implementing `IntoParams`. - /// - /// # Returns - /// A `turso::Rows` iterator over the result set. - #[allow(dead_code)] - pub async fn query( - &self, - sql: &str, - params: impl turso::params::IntoParams, - ) -> Result { - let params = turso::params::IntoParams::into_params(params).map_err(|e| { - anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) - })?; - let operation_name = format!("query encrypted {} database", M::db_name()); - - run_with_retry_elapsed( - resolve_query_retry_policy::(), - &operation_name, - QUERY_RETRY_HINT, - async |_| { - async { - self.core - .conn - .query(sql, params.clone()) - .await - .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) - } - .await - }, - ) - .await - } - /// Execute a SQL query and synchronously map all returned rows. pub async fn query_map( &self, @@ -1583,3 +1436,6 @@ impl EncryptedTursoDb { #[cfg(test)] mod transaction_cancellation_tests; + +#[cfg(test)] +mod tracing_boundary_tests; diff --git a/cli/src/services/db/tracing_boundary_tests.rs b/cli/src/services/db/tracing_boundary_tests.rs new file mode 100644 index 000000000..d187d7f47 --- /dev/null +++ b/cli/src/services/db/tracing_boundary_tests.rs @@ -0,0 +1,62 @@ +use std::time::Duration; + +use super::{contention_exhausted_error, WriteContentionPolicy}; +use crate::services::observability::tracing_boundary::test_capture::CapturingSubscriber; + +const SECRET: &str = "sk-live-SECRET-TOKEN-9f8e7d"; +const SENSITIVE_PATH: &str = "/home/victim/.local/state/sce/agent-trace.db"; + +fn policy(db_name: &'static str) -> WriteContentionPolicy { + WriteContentionPolicy { + db_name, + max_attempts: 3, + backoff_cap: Duration::from_millis(1), + busy_timeout: Duration::from_millis(5), + contention_deadline: Duration::from_millis(50), + } +} + +#[test] +fn tracing_boundary_a_contention_exhausted_event_omits_error_chain_and_classifies_strings() { + let last_error = + anyhow::anyhow!("open {SENSITIVE_PATH}: token {SECRET}").context(format!("outer {SECRET}")); + + let events = CapturingSubscriber::capture(|| { + let _ = contention_exhausted_error( + policy("repository Agent Trace DB"), + "execute repository Agent Trace DB database query", + 3, + Duration::from_millis(40), + &last_error, + "retry later", + ); + let _ = contention_exhausted_error( + policy(SECRET), + SENSITIVE_PATH, + 3, + Duration::from_millis(40), + &last_error, + "retry later", + ); + }); + + assert_eq!(events.len(), 2); + for event in &events { + let rendered = event.rendered(); + for needle in [SECRET, SENSITIVE_PATH, "outer"] { + assert!(!rendered.contains(needle), "leaked {needle}: {rendered}"); + } + assert!(event.field("last_error").is_none()); + assert_eq!( + event.field("event_id"), + Some("sce.agent_trace_db.contention_exhausted") + ); + } + assert_eq!( + events[0].field("db_name"), + Some("repository_agent_trace_db") + ); + assert_eq!(events[0].field("operation"), Some("db.execute_query")); + assert_eq!(events[1].field("db_name"), Some("unclassified")); + assert_eq!(events[1].field("operation"), Some("unclassified")); +} diff --git a/cli/src/services/default_paths.rs b/cli/src/services/default_paths.rs index 7155b947c..86ec44208 100644 --- a/cli/src/services/default_paths.rs +++ b/cli/src/services/default_paths.rs @@ -7,8 +7,11 @@ mod roots { use anyhow::{anyhow, Result}; - #[allow(dead_code)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] + #[allow( + dead_code, + reason = "constructed only under cfg(target_os) on non-Linux platforms" + )] pub(crate) enum PlatformFamily { Linux, Macos, @@ -56,11 +59,6 @@ mod roots { pub(crate) fn state_root(&self) -> &Path { &self.state_root } - - #[allow(dead_code)] - pub(crate) fn cache_root(&self) -> &Path { - &self.cache_root - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -76,15 +74,6 @@ mod roots { pub(crate) fn global_config_file(&self) -> PathBuf { self.roots.config_root().join("sce").join("config.json") } - - #[allow(dead_code)] - pub(crate) fn persisted_artifact_locations(&self) -> Vec { - vec![super::PersistedArtifactLocation { - id: super::PersistedArtifactId::GlobalConfig, - root_kind: super::PersistedArtifactRootKind::Config, - path: self.global_config_file(), - }] - } } pub(crate) fn resolve_state_data_root() -> Result { @@ -322,62 +311,26 @@ fn sce_state_dir() -> anyhow::Result { .join("sce")) } -#[allow(dead_code)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PersistedArtifactRootKind { - Config, - State, - Cache, -} - -#[allow(dead_code)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PersistedArtifactId { - GlobalConfig, -} - -#[allow(dead_code)] -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct PersistedArtifactLocation { - pub id: PersistedArtifactId, - pub root_kind: PersistedArtifactRootKind, - pub path: PathBuf, -} - pub(crate) mod repo_dir { pub const SCE: &str = ".sce"; pub const OPENCODE: &str = ".opencode"; pub const CLAUDE: &str = ".claude"; pub const PI: &str = ".pi"; pub const CODEX: &str = ".codex"; - pub const GIT: &str = ".git"; } pub(crate) mod repo_file { pub const SCE_CONFIG: &str = "config.json"; pub const OPENCODE_MANIFEST: &str = "opencode.json"; - pub const GIT_COMMIT_EDITMSG: &str = "COMMIT_EDITMSG"; -} - -pub(crate) mod hook_dir { - pub const HOOKS: &str = "hooks"; - pub const PRE_COMMIT: &str = "pre-commit"; - pub const COMMIT_MSG: &str = "commit-msg"; - pub const POST_COMMIT: &str = "post-commit"; } -#[allow(dead_code)] pub(crate) mod opencode_asset { - pub const OPENCODE_DIR: &str = "opencode"; pub const PLUGINS_DIR: &str = "plugins"; - pub const PLUGIN_FILE: &str = "sce-bash-policy.ts"; - pub const PLUGIN_MANIFEST_ENTRY: &str = "./plugins/sce-bash-policy.ts"; pub const LIB_DIR: &str = "lib"; pub const PRESET_CATALOG: &str = "bash-policy-presets.json"; pub const OPENCODE_AGENT_DIR: &str = "agent"; pub const OPENCODE_COMMAND_DIR: &str = "command"; pub const SKILLS_DIR: &str = "skills"; - pub const AGENTS_DIR: &str = "agents"; } pub(crate) mod claude_asset { @@ -415,7 +368,6 @@ pub(crate) mod context_file { pub const PATTERNS: &str = "patterns.md"; pub const CONTEXT_MAP: &str = "context-map.md"; pub const TMP_GITIGNORE: &str = ".gitignore"; - pub const SKILL_DEFINITION: &str = "SKILL.md"; } pub(crate) mod schema { @@ -428,7 +380,6 @@ pub(crate) struct RepoPaths { root: PathBuf, } -#[allow(dead_code)] impl RepoPaths { pub(crate) fn new(root: impl Into) -> Self { Self { root: root.into() } @@ -462,22 +413,6 @@ impl RepoPaths { self.root.join(repo_dir::CODEX) } - pub(crate) fn git_dir(&self) -> PathBuf { - self.root.join(repo_dir::GIT) - } - - pub(crate) fn git_hooks_dir(&self) -> PathBuf { - self.git_dir().join(hook_dir::HOOKS) - } - - pub(crate) fn git_hook_file(&self, hook_name: &str) -> PathBuf { - self.git_hooks_dir().join(hook_name) - } - - pub(crate) fn git_commit_editmsg(&self) -> PathBuf { - self.git_dir().join(repo_file::GIT_COMMIT_EDITMSG) - } - pub(crate) fn context_dir(&self) -> PathBuf { self.root.join(context_dir::CONTEXT_ROOT) } @@ -528,7 +463,6 @@ pub(crate) struct InstallTargetPaths { repo_root: PathBuf, } -#[allow(dead_code)] impl InstallTargetPaths { pub(crate) fn new(repo_root: impl Into) -> Self { Self { @@ -555,45 +489,9 @@ impl InstallTargetPaths { self.repo_root.clone() } - pub(crate) fn opencode_plugin_target(&self) -> PathBuf { - self.opencode_target_dir() - .join(opencode_asset::PLUGINS_DIR) - .join(opencode_asset::PLUGIN_FILE) - } - pub(crate) fn opencode_preset_catalog_target(&self) -> PathBuf { self.opencode_target_dir() .join(opencode_asset::LIB_DIR) .join(opencode_asset::PRESET_CATALOG) } - - pub(crate) fn skill_tile_relative_path(skill_name: &str) -> String { - format!( - "{}/{}/{}", - opencode_asset::SKILLS_DIR, - skill_name, - context_file::SKILL_DEFINITION - ) - } - - pub(crate) fn pre_commit_hook_path(&self) -> PathBuf { - self.repo_root - .join(repo_dir::GIT) - .join(hook_dir::HOOKS) - .join(hook_dir::PRE_COMMIT) - } - - pub(crate) fn commit_msg_hook_path(&self) -> PathBuf { - self.repo_root - .join(repo_dir::GIT) - .join(hook_dir::HOOKS) - .join(hook_dir::COMMIT_MSG) - } - - pub(crate) fn post_commit_hook_path(&self) -> PathBuf { - self.repo_root - .join(repo_dir::GIT) - .join(hook_dir::HOOKS) - .join(hook_dir::POST_COMMIT) - } } diff --git a/cli/src/services/doctor/inspect.rs b/cli/src/services/doctor/inspect.rs index f4437bafe..ef3de754b 100644 --- a/cli/src/services/doctor/inspect.rs +++ b/cli/src/services/doctor/inspect.rs @@ -50,7 +50,6 @@ pub(super) async fn build_report_with_lifecycle_problems( impl crate::services::capabilities::GitOps, impl Fn() -> anyhow::Result, impl Fn() -> anyhow::Result, - impl Fn(&Path) -> anyhow::Result<()>, impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, >, lifecycle_problems: Vec, @@ -78,7 +77,6 @@ async fn build_report_without_service_owned_problem_checks( impl crate::services::capabilities::GitOps, impl Fn() -> anyhow::Result, impl Fn() -> anyhow::Result, - impl Fn(&Path) -> anyhow::Result<()>, impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, >, mut problems: Vec, @@ -566,7 +564,6 @@ fn collect_global_state_locations( impl crate::services::capabilities::GitOps, impl Fn() -> anyhow::Result, impl Fn() -> anyhow::Result, - impl Fn(&Path) -> anyhow::Result<()>, impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, >, ) -> GlobalStateHealth { @@ -744,81 +741,6 @@ fn hook_managed_block_content_state( } } -#[allow(dead_code)] -fn inspect_repository_hooks( - repository_root: &Path, - git_available: bool, - bare_repository: bool, - detected_repository_root: Option<&Path>, - hooks_directory: Option<&Path>, - problems: &mut Vec, -) -> Vec { - if !git_available { - problems.push(DoctorProblem { - kind: ProblemKind::GitUnavailable, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from("Git is not available on this machine."), - remediation: String::from("Install an accessible 'git' binary and ensure it is on PATH before rerunning 'sce doctor'."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - return Vec::new(); - } - - if bare_repository { - problems.push(DoctorProblem { - kind: ProblemKind::BareRepository, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from( - "The current repository is bare and does not support local SCE hook rollout.", - ), - remediation: String::from("Run 'sce doctor' from a non-bare working tree clone to inspect repo-scoped SCE hook health."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - return Vec::new(); - } - - if detected_repository_root.is_none() { - problems.push(DoctorProblem { - kind: ProblemKind::NotInsideGitRepository, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from("The current directory is not inside a git repository."), - remediation: String::from("Run 'sce doctor' from inside the target repository working tree to inspect repo-scoped SCE hook health."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - return Vec::new(); - } - - if let Some(directory) = hooks_directory { - return collect_hook_health(directory, problems); - } - - let _ = repository_root; - problems.push(DoctorProblem { - kind: ProblemKind::UnableToResolveGitHooksDirectory, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from("Unable to resolve git hooks directory."), - remediation: String::from("Verify that git repository inspection succeeds and rerun 'sce doctor' inside a non-bare git repository."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - Vec::new() -} - /// Returns `true` when the doctor was able to check for integration targets /// and found none (neither configured in `.sce/config.json` nor detected /// via repo-root `.opencode/` / `.claude/` / `.pi/` directories). @@ -1109,235 +1031,6 @@ fn repair_merge_target_if_mismatched( ) } -#[allow(dead_code)] -fn collect_global_state_health( - repository_root: &Path, - problems: &mut Vec, - dependencies: &DoctorDependencies< - '_, - impl crate::services::capabilities::GitOps, - impl Fn() -> anyhow::Result, - impl Fn() -> anyhow::Result, - impl Fn(&Path) -> anyhow::Result<()>, - impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, - >, -) -> GlobalStateHealth { - let mut state_root_health = None; - let mut config_locations = Vec::new(); - - match (dependencies.resolve_state_root)() { - Ok(state_root) => { - state_root_health = Some(FileLocationHealth { - label: "State root", - state: if state_root.exists() { "present" } else { "expected" }, - path: state_root.clone(), - }); - } - Err(error) => problems.push(DoctorProblem { - kind: ProblemKind::UnableToResolveStateRoot, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!("Unable to resolve expected state root: {error}"), - remediation: String::from("Verify that the current platform exposes a writable SCE state directory before rerunning 'sce doctor'."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }), - } - - match (dependencies.resolve_global_config_path)() { - Ok(global_path) => { - if global_path.exists() { - if let Err(error) = (dependencies.validate_config_file)(&global_path) { - problems.push(DoctorProblem { - kind: ProblemKind::GlobalConfigValidationFailed, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!( - "Global config file '{}' failed validation: {error}", - global_path.display() - ), - remediation: format!( - "Repair or remove the invalid global config file at '{}' and rerun 'sce doctor'.", - global_path.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - } - } - config_locations.push(FileLocationHealth { - label: "Global config", - state: if global_path.exists() { "present" } else { "expected" }, - path: global_path, - }); - } - Err(error) => problems.push(DoctorProblem { - kind: ProblemKind::UnableToResolveGlobalConfigPath, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!("Unable to resolve expected global config path: {error}"), - remediation: String::from("Verify that the current platform exposes a writable SCE config directory before rerunning 'sce doctor'."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }), - } - - let local_path = RepoPaths::new(repository_root).sce_config_file(); - if local_path.exists() { - if let Err(error) = (dependencies.validate_config_file)(&local_path) { - problems.push(DoctorProblem { - kind: ProblemKind::LocalConfigValidationFailed, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!( - "Local config file '{}' failed validation: {error}", - local_path.display() - ), - remediation: format!( - "Repair or remove the invalid local config file at '{}' and rerun 'sce doctor'.", - local_path.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - } - } - config_locations.push(FileLocationHealth { - label: "Local config", - state: if local_path.exists() { - "present" - } else { - "expected" - }, - path: local_path, - }); - - GlobalStateHealth { - state_root: state_root_health, - config_locations, - } -} - -#[allow(dead_code)] -#[allow(clippy::too_many_lines)] -fn collect_hook_health(directory: &Path, problems: &mut Vec) -> Vec { - if !directory.exists() { - problems.push(DoctorProblem { - kind: ProblemKind::HooksDirectoryMissing, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!("Hooks directory '{}' does not exist.", directory.display()), - remediation: format!( - "Run 'sce doctor --fix' to install the canonical SCE-managed hooks into '{}', or run 'sce setup --hooks' directly.", - directory.display() - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } else if !directory.is_dir() { - problems.push(DoctorProblem { - kind: ProblemKind::HooksPathNotDirectory, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!("Hooks path '{}' is not a directory.", directory.display()), - remediation: format!( - "Replace '{}' with a writable hooks directory, then rerun 'sce doctor' or 'sce setup --hooks'.", - directory.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - } - - REQUIRED_HOOKS - .iter() - .map(|hook_name| { - let hook_path = directory.join(hook_name); - let metadata = fs::metadata(&hook_path).ok(); - let exists = metadata.is_some(); - let executable = metadata - .as_ref() - .is_some_and(|entry| entry.is_file() && is_executable(entry)); - let content_state = inspect_hook_content_state(hook_name, &hook_path, exists, problems); - - if !exists { - problems.push(DoctorProblem { - kind: ProblemKind::RequiredHookMissing, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!( - "Missing required hook '{}' at '{}'.", - hook_name, - hook_path.display() - ), - remediation: format!( - "Run 'sce doctor --fix' to install the canonical '{hook_name}' hook, or run 'sce setup --hooks' directly." - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } else if !executable { - problems.push(DoctorProblem { - kind: ProblemKind::HookNotExecutable, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!("Hook '{hook_name}' exists but is not executable."), - remediation: format!( - "Run 'sce doctor --fix' to restore the canonical executable hook, or run 'sce setup --hooks' / 'chmod +x {}' manually.", - hook_path.display() - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } - - if content_state == HookContentState::Stale { - problems.push(DoctorProblem { - kind: ProblemKind::HookContentStale, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!( - "Hook '{}' at '{}' differs from the canonical SCE-managed content.", - hook_name, - hook_path.display() - ), - remediation: format!( - "Run 'sce doctor --fix' to reinstall the canonical '{hook_name}' hook content, or run 'sce setup --hooks' directly." - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } - - HookFileHealth { - name: hook_name, - path: hook_path, - exists, - executable, - content_state, - } - }) - .collect() -} - fn inspect_opencode_integration_health( repository_root: &Path, integration_groups: &[IntegrationGroupHealth], @@ -2708,49 +2401,6 @@ fn path_is_file(path: &Path) -> bool { fs::metadata(path).is_ok_and(|metadata| metadata.is_file()) } -#[allow(dead_code)] -fn inspect_hook_content_state( - hook_name: &str, - hook_path: &Path, - exists: bool, - problems: &mut Vec, -) -> HookContentState { - if !exists { - return HookContentState::Missing; - } - - let Some(expected_hook) = - iter_required_hook_assets().find(|asset| asset.relative_path == hook_name) - else { - return HookContentState::Unknown; - }; - - match fs::read(hook_path) { - Ok(bytes) => hook_managed_block_content_state(hook_name, &bytes, expected_hook.bytes), - Err(error) => { - problems.push(DoctorProblem { - kind: ProblemKind::HookReadFailed, - category: ProblemCategory::FilesystemPermissions, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!( - "Unable to read hook '{}' at '{}': {error}", - hook_name, - hook_path.display() - ), - remediation: format!( - "Verify that '{}' is readable before rerunning 'sce doctor'.", - hook_path.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - HookContentState::Unknown - } - } -} - fn doctor_git_output(git: &impl GitOps, repository_root: &Path, args: &[&str]) -> Option { let output = git.run_command(repository_root, args).ok()?; let trimmed = output.trim(); diff --git a/cli/src/services/doctor/mod.rs b/cli/src/services/doctor/mod.rs index 450974ad8..b42906607 100644 --- a/cli/src/services/doctor/mod.rs +++ b/cli/src/services/doctor/mod.rs @@ -10,6 +10,10 @@ use crate::services::lifecycle::{ lifecycle_providers, FixOutcome, HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, LifecycleProvider, LifecycleProviderId, }; +use crate::services::mutation_trace::runtime::{ + inspect_reconciliation_recommendation, run_explicit_reconciliation, ReconciliationFix, + ReconciliationRecommendation, +}; use crate::services::output_format::OutputFormat; use crate::services::setup; @@ -50,17 +54,18 @@ pub struct DoctorRequest { pub format: DoctorFormat, } -struct DoctorDependencies<'a, G, S, C, V, P> { +struct DoctorDependencies<'a, G, S, C, P> { git: &'a G, resolve_state_root: &'a S, resolve_global_config_path: &'a C, - validate_config_file: &'a V, probe_codex_hook_policy: &'a P, } struct DoctorExecution { report: HookDoctorReport, fix_results: Vec, + ref_reconciliation: Option, + ref_reconciliation_fix: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -86,6 +91,8 @@ where &repository_root, &scoped_context, &mutation_scope_repair_seam::, + &run_explicit_reconciliation, + &inspect_reconciliation_recommendation, ) .await; render_report(request, &execution) @@ -100,6 +107,8 @@ async fn execute_doctor_with_context( &str, Option<&crate::services::observability::traits::NoopLogger>, ) -> anyhow::Result, + reconcile_seam: &impl std::ops::AsyncFn(&Path) -> ReconciliationFix, + recommendation_seam: &impl Fn(&Path) -> Option, ) -> DoctorExecution { execute_doctor_with_lifecycle_providers( request, @@ -111,10 +120,11 @@ async fn execute_doctor_with_context( resolve_global_config_path: &|| { Ok(resolve_sce_default_locations()?.global_config_file()) }, - validate_config_file: &crate::services::config::validate_config_file, probe_codex_hook_policy: &codex_hook_policy::probe_default, }, mutation_scope_seam, + reconcile_seam, + recommendation_seam, ) .await } @@ -128,7 +138,6 @@ async fn execute_doctor_with_lifecycle_providers( impl crate::services::capabilities::GitOps, impl Fn() -> Result, impl Fn() -> Result, - impl Fn(&Path) -> Result<()>, impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, >, mutation_scope_seam: &impl std::ops::AsyncFn( @@ -136,10 +145,9 @@ async fn execute_doctor_with_lifecycle_providers( &str, Option<&crate::services::observability::traits::NoopLogger>, ) -> anyhow::Result, + reconcile_seam: &impl std::ops::AsyncFn(&Path) -> ReconciliationFix, + recommendation_seam: &impl Fn(&Path) -> Option, ) -> DoctorExecution { - // Probed exactly once per doctor invocation, then reused for every - // Codex integration inspection below (initial report, `--fix`, and final - // report alike) instead of once per registration or once per report. let policy_readiness = (dependencies.probe_codex_hook_policy)(); let providers = lifecycle_providers(true); @@ -161,6 +169,8 @@ async fn execute_doctor_with_lifecycle_providers( return DoctorExecution { report: initial_report, fix_results: Vec::new(), + ref_reconciliation: recommendation_seam(repository_root), + ref_reconciliation_fix: None, }; } @@ -193,9 +203,53 @@ async fn execute_doctor_with_lifecycle_providers( &mutation_scope_repairs, )); + let reconciliation_fix = reconcile_seam(repository_root).await; + fix_results.push(reconciliation_fix_result(&reconciliation_fix)); + DoctorExecution { report: final_report, fix_results, + ref_reconciliation: None, + ref_reconciliation_fix: Some(reconciliation_fix), + } +} + +fn reconciliation_fix_result(fix: &ReconciliationFix) -> DoctorFixResultRecord { + let (outcome, detail) = match fix { + ReconciliationFix::Completed(counts) => ( + FixResult::Fixed, + format!( + "Snapshot ref reconciliation completed: deleted {}, retained {}, locally required {}.", + counts.deleted, counts.retained, counts.local_required + ), + ), + ReconciliationFix::CompletedStatePersistFailed { counts, warning } => ( + FixResult::Fixed, + format!( + "Snapshot ref reconciliation completed: deleted {}, retained {}, locally required {}. Warning: {warning}", + counts.deleted, counts.retained, counts.local_required + ), + ), + ReconciliationFix::Failed { + kind, + message, + state_warning, + } => { + let mut detail = format!("Snapshot ref reconciliation failed ({kind}): {message}"); + if let Some(warning) = state_warning { + detail = format!("{detail} Warning: {warning}"); + } + (FixResult::Failed, detail) + } + ReconciliationFix::Busy => ( + FixResult::Skipped, + "Snapshot ref reconciliation skipped: another mutation boundary or maintenance pass holds the worktree lock. Retry `sce doctor --fix`.".to_string(), + ), + }; + DoctorFixResultRecord { + category: ProblemCategory::MutationScopeHealth, + outcome, + detail, } } @@ -500,3 +554,133 @@ fn is_executable(metadata: &fs::Metadata) -> bool { fn is_executable(metadata: &fs::Metadata) -> bool { metadata.is_file() } + +#[cfg(test)] +mod reconciliation_fix_tests { + use super::{reconciliation_fix_result, FixResult, ReconciliationFix}; + use crate::services::mutation_trace::runtime::{ReconciliationCounts, StateWarning}; + + const COUNTS: ReconciliationCounts = ReconciliationCounts { + deleted: 3, + retained: 2, + local_required: 1, + }; + + fn not_applied() -> StateWarning { + StateWarning::NotApplied { + phase: "rename", + cause: "disk full".to_string(), + } + } + + fn durability_uncertain() -> StateWarning { + StateWarning::DurabilityUncertain { + phase: "sync_parent_directory", + cause: "io error".to_string(), + } + } + + #[test] + fn reconciliation_fix_results_render_expected_text() { + struct Case { + label: &'static str, + fix: ReconciliationFix, + outcome: FixResult, + required: Vec<&'static str>, + forbidden: Vec<&'static str>, + } + let cases = [ + Case { + label: "completed", + fix: ReconciliationFix::Completed(COUNTS), + outcome: FixResult::Fixed, + required: vec![ + "Snapshot ref reconciliation completed: deleted 3, retained 2, locally required 1.", + ], + forbidden: vec!["Warning", "failed"], + }, + Case { + label: "pre-rename persistence failure", + fix: ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: not_applied(), + }, + outcome: FixResult::Fixed, + required: vec![ + "deleted 3", + "Maintenance-state update was not applied.", + "disk full", + ], + forbidden: vec!["were not recorded", "failed"], + }, + Case { + label: "post-rename persistence failure", + fix: ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: durability_uncertain(), + }, + outcome: FixResult::Fixed, + required: vec![ + "deleted 3", + "Maintenance-state durability could not be confirmed.", + "io error", + ], + forbidden: vec!["not applied", "were not recorded", "failed"], + }, + Case { + label: "unreadable previous state", + fix: ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::PreviousStateUnreadable { + cause: "permission denied".to_string(), + }, + }, + outcome: FixResult::Fixed, + required: vec!["deleted 3", "could not be read", "permission denied"], + forbidden: vec!["were not recorded", "failed"], + }, + Case { + label: "failed pass", + fix: ReconciliationFix::Failed { + kind: "agent_trace_db_missing", + message: "db missing".to_string(), + state_warning: Some(not_applied()), + }, + outcome: FixResult::Failed, + required: vec![ + "agent_trace_db_missing", + "db missing", + "Maintenance-state update was not applied.", + ], + forbidden: vec!["completed"], + }, + Case { + label: "busy pass", + fix: ReconciliationFix::Busy, + outcome: FixResult::Skipped, + required: vec!["skipped"], + forbidden: vec!["completed", "deleted"], + }, + ]; + for case in cases { + let record = reconciliation_fix_result(&case.fix); + assert_eq!(record.outcome, case.outcome, "{}", case.label); + for fragment in case.required { + assert!( + record.detail.contains(fragment), + "{}: missing {fragment:?} in {:?}", + case.label, + record.detail + ); + } + for fragment in case.forbidden { + assert!( + !record.detail.contains(fragment), + "{}: unexpected {fragment:?} in {:?}", + case.label, + record.detail + ); + } + } + } +} diff --git a/cli/src/services/doctor/render.rs b/cli/src/services/doctor/render.rs index de83f0c53..587eea855 100644 --- a/cli/src/services/doctor/render.rs +++ b/cli/src/services/doctor/render.rs @@ -13,6 +13,9 @@ use super::types::{ PostCommitAutoSyncState, ProblemKind, ProblemSeverity, Readiness, }; use super::{DoctorExecution, DoctorFormat, DoctorMode, DoctorRequest, NAME}; +use crate::services::mutation_trace::runtime::{ + ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, StateWarning, +}; /// Guidance message rendered in the Integrations section when no integration /// targets are configured, detected, or both. @@ -48,9 +51,58 @@ fn format_execution(execution: &DoctorExecution) -> String { } } + if let Some(recommendation) = &execution.ref_reconciliation { + lines.push(format!("\n{}:", heading("Snapshot ref reconciliation"))); + lines.push(format!( + " [{}] {}", + value("recommended"), + value(&recommendation_text(recommendation)) + )); + } + lines.join("\n") } +fn recommendation_text(recommendation: &ReconciliationRecommendation) -> String { + let mut parts = vec!["Reconciliation is recommended. Run `sce doctor --fix`.".to_string()]; + if recommendation.last_attempt_failed { + parts.push(format!( + "The latest reconciliation attempt failed ({} consecutive failure(s)).", + recommendation.consecutive_failures + )); + } + if let Some(age) = recommendation.last_attempt_age_ms { + parts.push(format!( + "Last reconciliation attempt {} ago.", + format_age(age) + )); + } + if recommendation.state_unusable { + parts.push("Maintenance state is unreadable or invalid.".to_string()); + } + if recommendation.invalid_timestamps { + parts.push("Maintenance state has invalid timestamps.".to_string()); + } + if let Some(age) = recommendation.last_success_age_ms { + parts.push(format!("Last success {} ago.", format_age(age))); + } + if let Some(age) = recommendation.last_advised_age_ms { + parts.push(format!("Last reminder {} ago.", format_age(age))); + } + parts.join(" ") +} + +fn format_age(age_ms: i64) -> String { + let minutes = age_ms / 60_000; + if minutes < 60 { + format!("{minutes}m") + } else if minutes < 60 * 24 { + format!("{}h", minutes / 60) + } else { + format!("{}d", minutes / (60 * 24)) + } +} + fn format_report(report: &HookDoctorReport) -> String { format_report_with_color_policy(report, supports_color()) } @@ -887,6 +939,8 @@ fn render_report_json(execution: &DoctorExecution) -> Result { } else { Vec::new() }, + "ref_reconciliation": execution.ref_reconciliation.as_ref().map(ref_reconciliation_json), + "ref_reconciliation_fix": execution.ref_reconciliation_fix.as_ref().map(ref_reconciliation_fix_json), }); serde_json::to_string_pretty(&payload).context("failed to serialize doctor report to JSON") @@ -921,3 +975,252 @@ fn hook_content_state(state: HookContentState) -> &'static str { HookContentState::Unknown => "unknown", } } + +fn ref_reconciliation_json(recommendation: &ReconciliationRecommendation) -> serde_json::Value { + json!({ + "recommended": true, + "last_attempt_failed": recommendation.last_attempt_failed, + "consecutive_failures": recommendation.consecutive_failures, + "state_unusable": recommendation.state_unusable, + "invalid_timestamps": recommendation.invalid_timestamps, + "last_success_age_ms": recommendation.last_success_age_ms, + "last_attempt_age_ms": recommendation.last_attempt_age_ms, + "last_advised_age_ms": recommendation.last_advised_age_ms, + "text": recommendation_text(recommendation), + }) +} + +fn ref_reconciliation_fix_json(fix: &ReconciliationFix) -> serde_json::Value { + match fix { + ReconciliationFix::Completed(counts) => json!({ + "outcome": "completed", + "counts": reconciliation_counts_json(counts), + "failure": null, + "state_warning": null, + }), + ReconciliationFix::CompletedStatePersistFailed { counts, warning } => json!({ + "outcome": "completed_state_persist_failed", + "counts": reconciliation_counts_json(counts), + "failure": null, + "state_warning": state_warning_json(warning), + }), + ReconciliationFix::Failed { + kind, + message, + state_warning, + } => json!({ + "outcome": "failed", + "counts": null, + "failure": { "kind": kind, "message": message }, + "state_warning": state_warning.as_ref().map(state_warning_json), + }), + ReconciliationFix::Busy => json!({ + "outcome": "busy", + "counts": null, + "failure": null, + "state_warning": null, + }), + } +} + +fn reconciliation_counts_json(counts: &ReconciliationCounts) -> serde_json::Value { + json!({ + "deleted": counts.deleted, + "retained": counts.retained, + "local_required": counts.local_required, + }) +} + +fn state_warning_json(warning: &StateWarning) -> serde_json::Value { + json!({ + "kind": warning.kind(), + "phase": warning.phase(), + "message": warning.message(), + "cause": warning.cause(), + }) +} + +#[cfg(test)] +mod reconciliation_render_tests { + use serde_json::json; + + use super::{recommendation_text, ref_reconciliation_fix_json}; + use crate::services::mutation_trace::runtime::{ + ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, StateWarning, + }; + + const COUNTS: ReconciliationCounts = ReconciliationCounts { + deleted: 3, + retained: 2, + local_required: 1, + }; + + fn recommendation(last_attempt_age_ms: Option) -> ReconciliationRecommendation { + ReconciliationRecommendation { + invalid_timestamps: false, + state_unusable: false, + last_success_age_ms: Some(3_600_000 * 30), + last_attempt_age_ms, + last_advised_age_ms: Some(120_000), + last_attempt_failed: last_attempt_age_ms.is_some(), + consecutive_failures: 2, + } + } + + #[test] + fn recommendation_shows_last_attempt_age_for_failed_recent_attempt() { + let text = recommendation_text(&recommendation(Some(12 * 60_000))); + assert!(text.contains("Last reconciliation attempt 12m ago.")); + assert!( + text.contains("The latest reconciliation attempt failed (2 consecutive failure(s)).") + ); + assert!(text.contains("Last success 1d ago.")); + assert!(text.contains("Last reminder 2m ago.")); + } + + #[test] + fn recommendation_omits_last_attempt_age_when_no_attempt_exists() { + let text = recommendation_text(&recommendation(None)); + assert!(!text.contains("Last reconciliation attempt")); + assert!(text.contains("Reconciliation is recommended.")); + } + + #[test] + fn json_ref_reconciliation_fix_results_match_contract() { + let counts = json!({ "deleted": 3, "retained": 2, "local_required": 1 }); + let cases = [ + ( + "completed", + ReconciliationFix::Completed(COUNTS), + json!({ + "outcome": "completed", + "counts": counts, + "failure": null, + "state_warning": null, + }), + ), + ( + "failed without warning", + ReconciliationFix::Failed { + kind: "agent_trace_db_missing", + message: "db missing".to_string(), + state_warning: None, + }, + json!({ + "outcome": "failed", + "counts": null, + "failure": { "kind": "agent_trace_db_missing", "message": "db missing" }, + "state_warning": null, + }), + ), + ( + "failed with warning", + ReconciliationFix::Failed { + kind: "missing_required_pins", + message: "pins".to_string(), + state_warning: Some(StateWarning::NotApplied { + phase: "write_staging", + cause: "full".to_string(), + }), + }, + json!({ + "outcome": "failed", + "counts": null, + "failure": { "kind": "missing_required_pins", "message": "pins" }, + "state_warning": { + "kind": "not_applied", + "phase": "write_staging", + "message": "Maintenance-state update was not applied.", + "cause": "full", + }, + }), + ), + ( + "busy", + ReconciliationFix::Busy, + json!({ + "outcome": "busy", + "counts": null, + "failure": null, + "state_warning": null, + }), + ), + ]; + for (label, fix, expected) in cases { + assert_eq!(ref_reconciliation_fix_json(&fix), expected, "{label}"); + } + } + + #[test] + fn json_completed_state_persist_failed_results_match_contract() { + let counts = json!({ "deleted": 3, "retained": 2, "local_required": 1 }); + let cases = [ + ( + "completed + not applied", + ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::NotApplied { + phase: "rename", + cause: "denied".to_string(), + }, + }, + json!({ + "outcome": "completed_state_persist_failed", + "counts": counts, + "failure": null, + "state_warning": { + "kind": "not_applied", + "phase": "rename", + "message": "Maintenance-state update was not applied.", + "cause": "denied", + }, + }), + ), + ( + "completed + durability uncertain", + ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::DurabilityUncertain { + phase: "sync_parent_directory", + cause: "eio".to_string(), + }, + }, + json!({ + "outcome": "completed_state_persist_failed", + "counts": counts, + "failure": null, + "state_warning": { + "kind": "durability_uncertain", + "phase": "sync_parent_directory", + "message": "Maintenance-state durability could not be confirmed.", + "cause": "eio", + }, + }), + ), + ( + "completed + previous state unreadable", + ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::PreviousStateUnreadable { + cause: "eacces".to_string(), + }, + }, + json!({ + "outcome": "completed_state_persist_failed", + "counts": counts, + "failure": null, + "state_warning": { + "kind": "previous_state_unreadable", + "phase": null, + "message": + "Maintenance state could not be read; the update was not attempted.", + "cause": "eacces", + }, + }), + ), + ]; + for (label, fix, expected) in cases { + assert_eq!(ref_reconciliation_fix_json(&fix), expected, "{label}"); + } + } +} diff --git a/cli/src/services/doctor/types.rs b/cli/src/services/doctor/types.rs index 317b2e2f9..f03489545 100644 --- a/cli/src/services/doctor/types.rs +++ b/cli/src/services/doctor/types.rs @@ -179,31 +179,6 @@ impl IntegrationGroupHealth { pub(super) const fn display_label(&self) -> &'static str { self.key.display_label() } - - #[allow(dead_code)] - pub(super) fn display_node(&self) -> DoctorDisplayNode { - let children = self - .children - .iter() - .map(IntegrationChildHealth::display_node) - .collect::>(); - let status = children - .iter() - .fold(DoctorDisplayStatus::Pass, |status, child| { - status.worst(if child.status == DoctorDisplayStatus::Miss { - DoctorDisplayStatus::Fail - } else { - child.status - }) - }); - DoctorDisplayNode::branch_with_status( - DoctorDisplayNodeKind::Group, - self.display_label(), - status, - Vec::new(), - children, - ) - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -247,7 +222,6 @@ pub(super) enum IntegrationContentState { } #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -#[allow(dead_code)] pub(super) enum DoctorDisplayStatus { Pass, Warn, @@ -256,7 +230,6 @@ pub(super) enum DoctorDisplayStatus { } impl DoctorDisplayStatus { - #[allow(dead_code)] pub(super) const fn worst(self, other: Self) -> Self { match (self, other) { (Self::Fail, _) | (_, Self::Fail) => Self::Fail, @@ -268,7 +241,6 @@ impl DoctorDisplayStatus { } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(super) enum DoctorDisplayDetail { MissingPath(PathBuf), ContentMismatch { @@ -309,7 +281,6 @@ pub(super) enum DoctorDisplayDetail { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(super) enum DoctorDisplayNodeKind { Domain, Group, @@ -317,7 +288,6 @@ pub(super) enum DoctorDisplayNodeKind { } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(super) struct DoctorDisplayNode { pub(super) kind: DoctorDisplayNodeKind, pub(super) label: String, @@ -327,17 +297,6 @@ pub(super) struct DoctorDisplayNode { } impl DoctorDisplayNode { - #[allow(dead_code)] - pub(super) fn domain(label: impl Into, children: Vec) -> Self { - Self::branch(DoctorDisplayNodeKind::Domain, label, children) - } - - #[allow(dead_code)] - pub(super) fn group(label: impl Into, children: Vec) -> Self { - Self::branch(DoctorDisplayNodeKind::Group, label, children) - } - - #[allow(dead_code)] fn asset( label: impl Into, status: DoctorDisplayStatus, @@ -352,11 +311,6 @@ impl DoctorDisplayNode { } } - #[allow(dead_code)] - fn branch(kind: DoctorDisplayNodeKind, label: impl Into, children: Vec) -> Self { - Self::branch_with_status(kind, label, DoctorDisplayStatus::Pass, Vec::new(), children) - } - pub(super) fn branch_with_status( kind: DoctorDisplayNodeKind, label: impl Into, @@ -388,7 +342,6 @@ impl DoctorDisplayNode { } impl IntegrationChildHealth { - #[allow(dead_code)] pub(super) fn display_node(&self) -> DoctorDisplayNode { let (status, detail) = match &self.content_state { IntegrationContentState::Match => (DoctorDisplayStatus::Pass, None), diff --git a/cli/src/services/error.rs b/cli/src/services/error.rs index e97579828..a99303543 100644 --- a/cli/src/services/error.rs +++ b/cli/src/services/error.rs @@ -71,7 +71,6 @@ impl UserError { } } - #[allow(dead_code)] pub fn key(&self) -> &'static str { match self { Self::NotAuthenticated => "auth.not_authenticated", @@ -114,7 +113,6 @@ impl std::fmt::Display for UserError { /// observability and diagnostic rendering. #[derive(Debug)] pub enum CliError { - #[allow(dead_code)] User { error: UserError, source: Option, @@ -126,15 +124,6 @@ pub enum CliError { } impl CliError { - #[allow(dead_code)] - pub fn user(error: UserError) -> Self { - Self::User { - error, - source: None, - } - } - - #[allow(dead_code)] pub fn user_with_source(error: UserError, source: impl Into) -> Self { Self::User { error, @@ -206,7 +195,10 @@ mod tests { #[test] fn user_error_not_authenticated_classifies_as_runtime() { - let error = CliError::user(UserError::NotAuthenticated); + let error = CliError::User { + error: UserError::NotAuthenticated, + source: None, + }; assert_eq!(error.class(), FailureClass::Runtime); assert_eq!(error.code(), "SCE-ERR-RUNTIME"); @@ -216,7 +208,10 @@ mod tests { #[test] fn not_git_repository_has_stable_runtime_catalog_mapping() { - let error = CliError::user(UserError::NotGitRepository); + let error = CliError::User { + error: UserError::NotGitRepository, + source: None, + }; assert_eq!(error.class(), FailureClass::Runtime); assert_eq!(error.code(), "SCE-ERR-RUNTIME"); @@ -229,7 +224,10 @@ mod tests { #[test] fn unexpected_failure_has_stable_runtime_catalog_mapping() { - let error = CliError::user(UserError::UnexpectedFailure); + let error = CliError::User { + error: UserError::UnexpectedFailure, + source: None, + }; assert_eq!(error.class(), FailureClass::Runtime); assert_eq!(error.code(), "SCE-ERR-RUNTIME"); diff --git a/cli/src/services/hooks/claude_mutation_scope/events.rs b/cli/src/services/hooks/claude_mutation_scope/events.rs index 226b8cd2e..1191fa1ca 100644 --- a/cli/src/services/hooks/claude_mutation_scope/events.rs +++ b/cli/src/services/hooks/claude_mutation_scope/events.rs @@ -59,10 +59,6 @@ impl ClaudeToolIdentity { tool_use_id: self.tool_use_id.clone(), } } - - pub(crate) fn is_subagent(&self) -> bool { - self.agent_id.is_some() - } } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/cli/src/services/hooks/claude_mutation_scope/mod.rs b/cli/src/services/hooks/claude_mutation_scope/mod.rs index 75fecddcd..3a93ed92d 100644 --- a/cli/src/services/hooks/claude_mutation_scope/mod.rs +++ b/cli/src/services/hooks/claude_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; diff --git a/cli/src/services/hooks/claude_mutation_scope/state.rs b/cli/src/services/hooks/claude_mutation_scope/state.rs index 67cb07919..9cf2cfd8b 100644 --- a/cli/src/services/hooks/claude_mutation_scope/state.rs +++ b/cli/src/services/hooks/claude_mutation_scope/state.rs @@ -348,14 +348,6 @@ pub(crate) async fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> .await } -pub(crate) async fn mark_recovery_pending(git_dir: &Path) -> Result<()> { - with_locked_state(git_dir, |state| { - state.recovery_pending = true; - Ok(StateTransaction::Persist(())) - }) - .await -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ClearRecoveryOutcome { Cleared, diff --git a/cli/src/services/hooks/codex/apply_patch/normalize.rs b/cli/src/services/hooks/codex/apply_patch/normalize.rs index 20193640a..3a91288e8 100644 --- a/cli/src/services/hooks/codex/apply_patch/normalize.rs +++ b/cli/src/services/hooks/codex/apply_patch/normalize.rs @@ -29,7 +29,6 @@ fn normalize_error(message: impl Into) -> CodexPatchNormalizeError { } } -#[allow(dead_code)] pub(crate) fn normalize_codex_patch( patch: &CodexPatch, tool_use_id: &str, diff --git a/cli/src/services/hooks/codex/apply_patch/parser.rs b/cli/src/services/hooks/codex/apply_patch/parser.rs index 08e6d4221..46a924032 100644 --- a/cli/src/services/hooks/codex/apply_patch/parser.rs +++ b/cli/src/services/hooks/codex/apply_patch/parser.rs @@ -9,13 +9,11 @@ const END_OF_FILE_MARKER: &str = "*** End of File"; const CHANGE_CONTEXT_MARKER: &str = "@@"; const CHANGE_CONTEXT_MARKER_WITH_TEXT: &str = "@@ "; -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CodexPatch { pub(crate) operations: Vec, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum CodexFileOperation { Add { @@ -32,7 +30,6 @@ pub(crate) enum CodexFileOperation { }, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CodexHunk { pub(crate) context: Option, @@ -40,7 +37,6 @@ pub(crate) struct CodexHunk { pub(crate) is_end_of_file: bool, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum CodexHunkLine { Context(String), @@ -48,7 +44,6 @@ pub(crate) enum CodexHunkLine { Removed(String), } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CodexPatchParseError { pub(crate) message: String, @@ -68,7 +63,6 @@ fn error(message: impl Into) -> CodexPatchParseError { } } -#[allow(dead_code)] pub(crate) fn normalize_outer_apply_patch_input(raw: &str) -> Result { let trimmed = raw.trim(); let lines: Vec<&str> = trimmed.lines().collect(); @@ -106,7 +100,6 @@ fn has_canonical_boundaries(lines: &[&str]) -> bool { && lines.last().map(|line| line.trim()) == Some(END_PATCH_MARKER) } -#[allow(dead_code)] pub(crate) fn parse_codex_apply_patch(raw: &str) -> Result { let trimmed = raw.trim(); let lines: Vec<&str> = trimmed.lines().collect(); diff --git a/cli/src/services/hooks/codex/apply_patch/path.rs b/cli/src/services/hooks/codex/apply_patch/path.rs index 9b6402f32..410ed5816 100644 --- a/cli/src/services/hooks/codex/apply_patch/path.rs +++ b/cli/src/services/hooks/codex/apply_patch/path.rs @@ -32,17 +32,6 @@ pub(crate) fn resolve_codex_patch_paths( Ok(()) } -#[allow(dead_code)] -pub(crate) fn resolve_codex_patch_path( - repository_root: &Path, - event_cwd: &str, - codex_path: &str, -) -> Result { - let git_root = resolve_git_root(repository_root)?; - let event_cwd = resolve_event_cwd(&git_root, event_cwd)?; - resolve_path_from_cwd(&git_root, &event_cwd, codex_path) -} - fn resolve_git_root(repository_root: &Path) -> Result { let output = Command::new("git") .args(["rev-parse", "--show-toplevel"]) diff --git a/cli/src/services/hooks/codex/mod.rs b/cli/src/services/hooks/codex/mod.rs index 2e6181605..3f9d5b9a3 100644 --- a/cli/src/services/hooks/codex/mod.rs +++ b/cli/src/services/hooks/codex/mod.rs @@ -42,7 +42,6 @@ where } #[derive(Debug, Deserialize)] -#[allow(dead_code)] pub(crate) struct CodexHookEvent { pub(crate) hook_event_name: String, #[serde(default)] @@ -60,8 +59,6 @@ pub(crate) struct CodexHookEvent { #[serde(default)] pub(crate) tool_input: Option, #[serde(default)] - pub(crate) tool_response: Option, - #[serde(default)] pub(crate) prompt: Option, #[serde(default, deserialize_with = "deserialize_nullable_field")] pub(crate) last_assistant_message: NullableField, diff --git a/cli/src/services/hooks/codex_mutation_scope/events.rs b/cli/src/services/hooks/codex_mutation_scope/events.rs index c83864032..80dccd113 100644 --- a/cli/src/services/hooks/codex_mutation_scope/events.rs +++ b/cli/src/services/hooks/codex_mutation_scope/events.rs @@ -50,10 +50,6 @@ impl CodexToolIdentity { tool_use_id: self.tool_use_id.clone(), } } - - pub(crate) fn is_subagent(&self) -> bool { - self.agent_id.is_some() - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -104,12 +100,6 @@ pub(crate) enum ToolClassification { pub(super) const TRACKED_MUTATION_TOOL_NAMES: &[&str] = &["Bash", "apply_patch"]; pub(super) const DELEGATION_TOOL_NAMES: &[&str] = &["collaborationspawn_agent", "collaborationwait_agent"]; -pub(super) const MCP_TOOL_NAME_PREFIX: &str = "mcp__"; - -pub(crate) fn is_mcp_tool_name(tool_name: &str) -> bool { - tool_name.starts_with(MCP_TOOL_NAME_PREFIX) -} - pub(crate) fn classify_tool(tool_name: &str) -> ToolClassification { if TRACKED_MUTATION_TOOL_NAMES.contains(&tool_name) { ToolClassification::TrackedMutation diff --git a/cli/src/services/hooks/codex_mutation_scope/mod.rs b/cli/src/services/hooks/codex_mutation_scope/mod.rs index 003de8242..e29e38078 100644 --- a/cli/src/services/hooks/codex_mutation_scope/mod.rs +++ b/cli/src/services/hooks/codex_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; @@ -25,9 +23,8 @@ use serde_json::{json, Map, Value}; #[allow(unused_imports)] pub(crate) use events::{ classify_tool, codex_scope_close_event_id, codex_scope_start_event_id, format_codex_scope_id, - is_mcp_tool_name, parse_codex_hook_event, AttemptKey, CodexAgentIdentity, CodexHookEvent, - CodexSessionIdentity, CodexToolExecution, CodexToolIdentity, CodexTurnIdentity, - ToolClassification, + parse_codex_hook_event, AttemptKey, CodexAgentIdentity, CodexHookEvent, CodexSessionIdentity, + CodexToolExecution, CodexToolIdentity, CodexTurnIdentity, ToolClassification, }; #[allow(unused_imports)] use events::{ diff --git a/cli/src/services/hooks/codex_mutation_scope/state.rs b/cli/src/services/hooks/codex_mutation_scope/state.rs index 950b52024..bbd73b521 100644 --- a/cli/src/services/hooks/codex_mutation_scope/state.rs +++ b/cli/src/services/hooks/codex_mutation_scope/state.rs @@ -39,12 +39,6 @@ pub(crate) enum RecoveryState { }, } -impl RecoveryState { - pub(crate) fn is_clear(&self) -> bool { - matches!(self, RecoveryState::Clear) - } -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct AdapterAttempt { pub attempt_seq: u64, diff --git a/cli/src/services/hooks/lifecycle.rs b/cli/src/services/hooks/lifecycle.rs index e76744cf5..437d4c6e3 100644 --- a/cli/src/services/hooks/lifecycle.rs +++ b/cli/src/services/hooks/lifecycle.rs @@ -7,8 +7,8 @@ use anyhow::{anyhow, Context, Result}; use crate::app::HasRepoRoot; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, RequiredHookInstallStatus, - RequiredHooksInstallOutcome, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, RequiredHookInstallStatus, RequiredHooksInstallOutcome, + ServiceLifecycle, SetupOutcome, }; use crate::services::setup::{ hook_merge, install_required_git_hooks, iter_required_hook_assets, @@ -28,10 +28,6 @@ enum HookContentState { } impl ServiceLifecycle for HooksLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::Hooks - } - async fn diagnose(&self, ctx: &C) -> Vec { let repository_root = match ctx.repo_root() { Some(path) => path.to_path_buf(), diff --git a/cli/src/services/hooks/mod.rs b/cli/src/services/hooks/mod.rs index 14b114bd2..f0e0ffb73 100644 --- a/cli/src/services/hooks/mod.rs +++ b/cli/src/services/hooks/mod.rs @@ -5,7 +5,7 @@ mod commit_hooks; mod conversation_trace; mod diff_trace; -mod mutation_scope_lock; +pub(crate) mod mutation_scope_lock; mod runtime; diff --git a/cli/src/services/hooks/mutation_scope.rs b/cli/src/services/hooks/mutation_scope.rs index 211b496b0..67ce7dfc6 100644 --- a/cli/src/services/hooks/mutation_scope.rs +++ b/cli/src/services/hooks/mutation_scope.rs @@ -6,8 +6,9 @@ use serde_json::{json, Map, Value}; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::mutation_trace::runtime::{ - abandon_scope, arm_external_mutation_guard, coordinate, AbandonScopeError, AbandonScopeOutcome, - CoordinateError, CoordinateOutcome, GuardEvent, GuardRequest, RuntimeBoundary, StartProvenance, + abandon_scope, advise_after_completed_boundary, arm_external_mutation_guard, coordinate, + AbandonScopeError, AbandonScopeOutcome, AdvisoryReport, AdvisorySeverity, CoordinateError, + CoordinateOutcome, GuardEvent, GuardRequest, RuntimeBoundary, StartProvenance, }; use crate::services::mutation_trace::types::{ActorKind, EventId, ScopeId}; @@ -262,6 +263,7 @@ pub(crate) async fn run_mutation_scope_from_payload< stdin_payload, logger, super::open_agent_trace_db_for_hook_runtime, + &mut std::io::stderr(), ) .await } @@ -269,14 +271,17 @@ pub(crate) async fn run_mutation_scope_from_payload< async fn run_mutation_scope_from_payload_with< L: crate::services::observability::traits::Logger, O, + E, >( repository_root: &Path, stdin_payload: &str, logger: Option<&L>, open_db: O, + diagnostics: &mut E, ) -> Result where O: std::ops::AsyncFn(&Path, &'static str) -> Result + Copy, + E: Write, { let payload = parse_mutation_scope_payload(stdin_payload)?; @@ -284,6 +289,7 @@ where repository_root, payload, logger, + diagnostics, async |root, boundary| { coordinate(root, boundary, async || { open_db(root, MUTATION_SCOPE_DB_CONTEXT).await @@ -296,18 +302,28 @@ where }) .await }, + advise_after_completed_boundary, ) .await } -async fn drive_mutation_scope( +pub(crate) async fn drive_mutation_scope< + L: crate::services::observability::traits::Logger, + E: Write, + C, + A, + V, +>( repository_root: &Path, payload: MutationScopePayload, logger: Option<&L>, + diagnostics: &mut E, coordinate_boundary: C, abandon: A, + advise: V, ) -> Result where + V: FnOnce(&Path) -> AdvisoryReport, C: std::ops::AsyncFnOnce( &Path, &RuntimeBoundary, @@ -353,10 +369,69 @@ where } }; - classify_coordinate( - coordinate_boundary(repository_root, &boundary).await, - logger, - ) + let advisory_eligible = matches!( + boundary, + RuntimeBoundary::Close { .. } | RuntimeBoundary::Flush + ); + let result = coordinate_boundary(repository_root, &boundary).await; + let durably_completed = matches!( + result, + Ok(_) | Err(CoordinateError::MarkerClearAfterCommit { .. }) + ); + let classified = classify_coordinate(result, logger); + + if advisory_eligible && durably_completed { + report_advisory(logger, diagnostics, &advise(repository_root)); + } + + classified +} + +fn report_advisory( + logger: Option<&L>, + diagnostics: &mut E, + report: &AdvisoryReport, +) { + log_advisory_report(logger, report); + emit_advisory_diagnostic(diagnostics, report); +} + +fn emit_advisory_diagnostic(diagnostics: &mut E, report: &AdvisoryReport) { + let Some(diagnostic) = report.diagnostic.as_deref() else { + return; + }; + let _ = writeln!(diagnostics, "{diagnostic}").and_then(|()| diagnostics.flush()); +} + +fn log_advisory_report( + logger: Option<&L>, + report: &AdvisoryReport, +) { + let Some(log) = logger else { + return; + }; + let message = report + .recommendation + .unwrap_or("Reconciliation advisory check."); + let warning = report.warning.as_deref().unwrap_or(""); + let mut fields = vec![("outcome", report.outcome)]; + if !warning.is_empty() { + fields.push(("persistence_warning", warning)); + } + match report.severity { + AdvisorySeverity::Warn => log.warn( + "sce.hooks.mutation_scope.ref_reconciliation_advisory", + message, + &fields, + None, + ), + AdvisorySeverity::Debug => log.debug( + "sce.hooks.mutation_scope.ref_reconciliation_advisory", + message, + &fields, + None, + ), + } } fn classify_coordinate( @@ -634,3 +709,449 @@ fn write_guard_line(writer: &mut W, line: &str) -> std::io::Result<()> writeln!(writer, "{line}")?; writer.flush() } + +#[cfg(test)] +mod advisory_trigger_tests { + use std::cell::Cell; + use std::io::Write; + use std::path::Path; + use std::sync::Mutex; + + use anyhow::anyhow; + + use super::{ + drive_mutation_scope, AbandonScopeError, AbandonScopeOutcome, AdvisoryReport, + AdvisorySeverity, CoordinateError, CoordinateOutcome, MutationScopePayload, + RuntimeBoundary, + }; + use crate::services::error::CliError; + use crate::services::mutation_trace::protocol::CommitEvaluation; + use crate::services::mutation_trace::types::{ActorKind, ScopeId, TreeId, WorktreeId}; + use crate::services::observability::traits::Logger; + + const ADVISORY_EVENT_ID: &str = "sce.hooks.mutation_scope.ref_reconciliation_advisory"; + const RECOMMENDED: &str = "Reconciliation is recommended. Run sce doctor --fix."; + + #[derive(Debug, Eq, PartialEq)] + struct LogRecord { + level: &'static str, + event_id: String, + message: String, + fields: Vec<(String, String)>, + } + + #[derive(Default)] + struct RecordingLogger { + records: Mutex>, + } + + impl RecordingLogger { + fn record(&self, level: &'static str, event_id: &str, message: &str, f: &[(&str, &str)]) { + self.records.lock().unwrap().push(LogRecord { + level, + event_id: event_id.to_string(), + message: message.to_string(), + fields: f + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + }); + } + + fn take(&self) -> Vec { + std::mem::take(&mut self.records.lock().unwrap()) + } + } + + impl Logger for RecordingLogger { + fn info(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("info", event_id, message, fields); + } + fn debug(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("debug", event_id, message, fields); + } + fn warn(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("warn", event_id, message, fields); + } + fn error(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("error", event_id, message, fields); + } + fn log_cli_error(&self, _error: &CliError, _session_id: Option<&str>) {} + } + + struct FailingWriter; + + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("stderr closed")) + } + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::other("stderr closed")) + } + } + + fn outcome() -> CoordinateOutcome { + CoordinateOutcome { + worktree_id: WorktreeId("main".to_string()), + observed_tree: TreeId("tree".to_string()), + revision: 1, + evaluation: CommitEvaluation::default(), + mutation_event: None, + } + } + + fn report(outcome: &'static str, severity: AdvisorySeverity) -> AdvisoryReport { + AdvisoryReport { + outcome, + severity, + recommendation: None, + warning: None, + diagnostic: None, + } + } + + fn close() -> MutationScopePayload { + MutationScopePayload::Close { + scope_id: "scope".to_string(), + event_id: "event".to_string(), + actor_kind: ActorKind::Codex, + } + } + + fn start() -> MutationScopePayload { + MutationScopePayload::Start { + scope_id: "scope".to_string(), + event_id: "event".to_string(), + actor_kind: ActorKind::Codex, + provenance: None, + } + } + + fn advance() -> MutationScopePayload { + MutationScopePayload::Advance { + scope_id: "scope".to_string(), + event_id: "event".to_string(), + actor_kind: ActorKind::Codex, + } + } + + #[derive(Default)] + struct Calls { + coordinate: Cell, + abandon: Cell, + advise: Cell, + } + + async fn drive_with( + payload: MutationScopePayload, + coordinated: Result, + advised: AdvisoryReport, + logger: Option<&L>, + diagnostics: &mut E, + calls: &Calls, + ) -> anyhow::Result { + drive_mutation_scope( + Path::new("."), + payload, + logger, + diagnostics, + async |_root: &Path, _boundary: &RuntimeBoundary| { + calls.coordinate.set(calls.coordinate.get() + 1); + coordinated + }, + async |_root: &Path, + _scope: &ScopeId| + -> Result { + calls.abandon.set(calls.abandon.get() + 1); + Ok(AbandonScopeOutcome::Abandoned { + worktree_id: WorktreeId("main".to_string()), + scope: ScopeId("scope".to_string()), + revision: 1, + }) + }, + |_root: &Path| { + calls.advise.set(calls.advise.get() + 1); + advised + }, + ) + .await + } + + async fn drive( + payload: MutationScopePayload, + coordinated: Result, + calls: &Calls, + ) -> anyhow::Result { + drive_with( + payload, + coordinated, + report("no_action", AdvisorySeverity::Debug), + None::<&RecordingLogger>, + &mut Vec::new(), + calls, + ) + .await + } + + async fn deliver(advised: AdvisoryReport) -> (Vec, String) { + let logger = RecordingLogger::default(); + let mut sink = Vec::new(); + let result = drive_with( + close(), + Ok(outcome()), + advised, + Some(&logger), + &mut sink, + &Calls::default(), + ) + .await; + assert_eq!(result.unwrap(), ""); + (logger.take(), String::from_utf8(sink).unwrap()) + } + + fn field<'a>(record: &'a LogRecord, key: &str) -> Option<&'a str> { + record + .fields + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.as_str()) + } + + #[tokio::test] + async fn advisory_runs_once_after_completed_close_and_flush() { + let calls = Calls::default(); + assert_eq!(drive(close(), Ok(outcome()), &calls).await.unwrap(), ""); + assert_eq!(calls.advise.get(), 1); + assert_eq!( + drive(MutationScopePayload::Flush, Ok(outcome()), &calls) + .await + .unwrap(), + "" + ); + assert_eq!(calls.advise.get(), 2); + assert_eq!(calls.coordinate.get(), 2); + assert_eq!(calls.abandon.get(), 0); + } + + #[tokio::test] + async fn advisory_never_runs_for_start_or_advance() { + let calls = Calls::default(); + drive(start(), Ok(outcome()), &calls).await.unwrap(); + drive(advance(), Ok(outcome()), &calls).await.unwrap(); + assert_eq!(calls.advise.get(), 0); + } + + #[tokio::test] + async fn advisory_never_runs_after_failed_coordinate() { + let calls = Calls::default(); + let logger = RecordingLogger::default(); + let mut sink = Vec::new(); + let result = drive_with( + close(), + Err(CoordinateError::Other(anyhow!("boom"))), + AdvisoryReport { + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }, + Some(&logger), + &mut sink, + &calls, + ) + .await; + assert!(result.is_err()); + assert_eq!(calls.advise.get(), 0); + assert!(sink.is_empty()); + assert!(logger.take().is_empty()); + } + + #[tokio::test] + async fn advisory_never_runs_for_abandon() { + let calls = Calls::default(); + let payload = MutationScopePayload::Abandon { + scope_id: "scope".to_string(), + }; + drive(payload, Ok(outcome()), &calls).await.unwrap(); + assert_eq!(calls.advise.get(), 0); + assert_eq!(calls.abandon.get(), 1); + } + + #[tokio::test] + async fn quiet_outcomes_log_debug_and_emit_no_diagnostic() { + for name in ["anchored", "no_action", "busy"] { + let (records, stderr) = deliver(report(name, AdvisorySeverity::Debug)).await; + assert_eq!(stderr, "", "{name}"); + assert_eq!(records.len(), 1, "{name}"); + assert_eq!(records[0].level, "debug", "{name}"); + assert_eq!(records[0].event_id, ADVISORY_EVENT_ID); + assert_eq!(field(&records[0], "outcome"), Some(name)); + assert_eq!(field(&records[0], "persistence_warning"), None); + } + } + + #[tokio::test] + async fn advised_logs_warn_and_writes_recommendation_to_diagnostics() { + let diagnostic = format!("SCE: {RECOMMENDED}"); + let (records, stderr) = deliver(AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(diagnostic.clone()), + ..report("advised", AdvisorySeverity::Warn) + }) + .await; + assert_eq!(stderr, format!("{diagnostic}\n")); + assert_eq!(records.len(), 1); + assert_eq!(records[0].level, "warn"); + assert_eq!(records[0].event_id, ADVISORY_EVENT_ID); + assert_eq!(records[0].message, RECOMMENDED); + assert_eq!(field(&records[0], "outcome"), Some("advised")); + assert_eq!(field(&records[0], "persistence_warning"), None); + } + + #[tokio::test] + async fn durability_uncertain_keeps_persistence_warning_in_log_and_diagnostics() { + let diagnostic = format!( + "SCE: {RECOMMENDED}\nSCE: Advisory-state durability could not be confirmed: sync failed." + ); + let (records, stderr) = deliver(AdvisoryReport { + recommendation: Some(RECOMMENDED), + warning: Some("sync failed".to_string()), + diagnostic: Some(diagnostic.clone()), + ..report("advised_durability_uncertain", AdvisorySeverity::Warn) + }) + .await; + assert_eq!(stderr, format!("{diagnostic}\n")); + assert_eq!(records[0].level, "warn"); + assert_eq!( + field(&records[0], "outcome"), + Some("advised_durability_uncertain") + ); + assert_eq!( + field(&records[0], "persistence_warning"), + Some("sync failed") + ); + } + + #[tokio::test] + async fn state_failures_log_warn_and_write_diagnostic_without_claiming_advice() { + for (name, warning, diagnostic) in [ + ( + "state_write_failed", + Some("rename failed"), + "SCE: Reconciliation advisory state could not be written: rename failed. Run sce doctor to check reconciliation status.", + ), + ( + "state_unavailable", + None, + "SCE: Reconciliation advisory state is unavailable. Run sce doctor to check reconciliation status.", + ), + ] { + let (records, stderr) = deliver(AdvisoryReport { + warning: warning.map(str::to_string), + diagnostic: Some(diagnostic.to_string()), + ..report(name, AdvisorySeverity::Warn) + }) + .await; + assert_eq!(stderr, format!("{diagnostic}\n"), "{name}"); + assert!(!stderr.contains(RECOMMENDED), "{name}"); + assert_eq!(records[0].level, "warn", "{name}"); + assert_eq!(field(&records[0], "outcome"), Some(name)); + assert_eq!(field(&records[0], "persistence_warning"), warning); + } + } + + #[tokio::test] + async fn missing_logger_still_delivers_actionable_diagnostic() { + let mut sink = Vec::new(); + let result = drive_with( + close(), + Ok(outcome()), + AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }, + None::<&RecordingLogger>, + &mut sink, + &Calls::default(), + ) + .await; + assert_eq!(result.unwrap(), ""); + assert_eq!( + String::from_utf8(sink).unwrap(), + format!("SCE: {RECOMMENDED}\n") + ); + } + + #[tokio::test] + async fn diagnostic_write_failure_does_not_change_mutation_result() { + let calls = Calls::default(); + let advised = AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }; + let logger = RecordingLogger::default(); + let result = drive_with( + close(), + Ok(outcome()), + advised.clone(), + Some(&logger), + &mut FailingWriter, + &calls, + ) + .await; + assert_eq!(result.unwrap(), ""); + assert_eq!(logger.take().len(), 1); + + let marker_clear = CoordinateError::MarkerClearAfterCommit { + source: anyhow!("clear failed"), + committed: Box::new(outcome()), + }; + let result = drive_with( + MutationScopePayload::Flush, + Err(marker_clear), + advised, + None::<&RecordingLogger>, + &mut FailingWriter, + &calls, + ) + .await; + assert_eq!(result.unwrap(), ""); + } + + #[tokio::test] + async fn marker_clear_after_commit_logs_taint_warning_and_still_advises_once() { + let logger = RecordingLogger::default(); + let mut sink = Vec::new(); + let calls = Calls::default(); + let error = CoordinateError::MarkerClearAfterCommit { + source: anyhow!("clear failed"), + committed: Box::new(outcome()), + }; + let result = drive_with( + close(), + Err(error), + AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }, + Some(&logger), + &mut sink, + &calls, + ) + .await; + assert_eq!(result.unwrap(), ""); + let records = logger.take(); + assert_eq!(records.len(), 2); + assert_eq!( + records[0].event_id, + "sce.hooks.mutation_scope.marker_clear_after_durable_completion" + ); + assert_eq!(records[1].event_id, ADVISORY_EVENT_ID); + assert_eq!( + String::from_utf8(sink).unwrap(), + format!("SCE: {RECOMMENDED}\n") + ); + assert_eq!((calls.coordinate.get(), calls.advise.get()), (1, 1)); + } +} diff --git a/cli/src/services/hooks/mutation_scope_health.rs b/cli/src/services/hooks/mutation_scope_health.rs index 122628d98..6ea7e77cd 100644 --- a/cli/src/services/hooks/mutation_scope_health.rs +++ b/cli/src/services/hooks/mutation_scope_health.rs @@ -1,7 +1,6 @@ use crate::services::mutation_trace::types::ActorKind; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) enum MutationScopeHealthStatus { Healthy, Recovering, @@ -10,14 +9,12 @@ pub(crate) enum MutationScopeHealthStatus { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) enum Repairability { AutoFixable, ManualOnly, } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) struct MutationScopeAdapterHealth { pub(crate) adapter: ActorKind, pub(crate) status: MutationScopeHealthStatus, @@ -25,7 +22,6 @@ pub(crate) struct MutationScopeAdapterHealth { pub(crate) detail: Option, } -#[allow(dead_code)] impl MutationScopeAdapterHealth { pub(crate) fn new( adapter: ActorKind, diff --git a/cli/src/services/hooks/opencode_mutation_scope/mod.rs b/cli/src/services/hooks/opencode_mutation_scope/mod.rs index b7ef588b8..c5c5160e3 100644 --- a/cli/src/services/hooks/opencode_mutation_scope/mod.rs +++ b/cli/src/services/hooks/opencode_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; diff --git a/cli/src/services/hooks/opencode_mutation_scope/state.rs b/cli/src/services/hooks/opencode_mutation_scope/state.rs index 722bf91a0..e19403105 100644 --- a/cli/src/services/hooks/opencode_mutation_scope/state.rs +++ b/cli/src/services/hooks/opencode_mutation_scope/state.rs @@ -43,12 +43,6 @@ pub(crate) enum RecoveryState { }, } -impl RecoveryState { - pub(crate) fn is_clear(&self) -> bool { - matches!(self, RecoveryState::Clear) - } -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct AdapterAttempt { pub scope_id: String, @@ -366,22 +360,6 @@ pub(crate) async fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Pa .await } -pub(crate) async fn arm_recovery(git_dir: &Path) -> Result { - with_locked_state(git_dir, |state| { - let generation = match state.recovery { - RecoveryState::Pending { generation } => generation, - RecoveryState::Clear | RecoveryState::Flushing { .. } => { - let generation = state.next_recovery_generation; - state.next_recovery_generation += 1; - generation - } - }; - state.recovery = RecoveryState::Pending { generation }; - Ok(StateTransaction::Persist(generation)) - }) - .await -} - fn transition_to_pending_abandon_and_arm_flush( state: &mut AdapterState, scope_ids: &[String], @@ -451,24 +429,6 @@ pub(crate) async fn reprove_dead_owner_pending_start_and_begin_repair( .await } -pub(crate) async fn arm_and_begin_recovery_flush(git_dir: &Path) -> Result { - with_locked_state(git_dir, |state| { - let generation = match state.recovery { - RecoveryState::Pending { generation } | RecoveryState::Flushing { generation } => { - generation - } - RecoveryState::Clear => { - let generation = state.next_recovery_generation; - state.next_recovery_generation += 1; - generation - } - }; - state.recovery = RecoveryState::Flushing { generation }; - Ok(StateTransaction::Persist(generation)) - }) - .await -} - pub(crate) async fn complete_recovery_flush( git_dir: &Path, generation: u64, diff --git a/cli/src/services/hooks/pi_mutation_scope/mod.rs b/cli/src/services/hooks/pi_mutation_scope/mod.rs index aee856725..3ee6f8559 100644 --- a/cli/src/services/hooks/pi_mutation_scope/mod.rs +++ b/cli/src/services/hooks/pi_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; diff --git a/cli/src/services/hooks/pi_mutation_scope/state.rs b/cli/src/services/hooks/pi_mutation_scope/state.rs index db4d19fc2..05759bc45 100644 --- a/cli/src/services/hooks/pi_mutation_scope/state.rs +++ b/cli/src/services/hooks/pi_mutation_scope/state.rs @@ -43,12 +43,6 @@ pub(crate) enum RecoveryState { }, } -impl RecoveryState { - pub(crate) fn is_clear(&self) -> bool { - matches!(self, RecoveryState::Clear) - } -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct AdapterAttempt { pub attempt_seq: u64, diff --git a/cli/src/services/lifecycle.rs b/cli/src/services/lifecycle.rs index 027f5ecc9..a343f1b2c 100644 --- a/cli/src/services/lifecycle.rs +++ b/cli/src/services/lifecycle.rs @@ -132,9 +132,6 @@ pub struct SetupOutcome { } pub(crate) trait ServiceLifecycle: Send + Sync { - #[allow(dead_code)] - fn id(&self) -> LifecycleProviderId; - async fn diagnose(&self, _ctx: &C) -> Vec { Vec::new() } diff --git a/cli/src/services/local_db/lifecycle.rs b/cli/src/services/local_db/lifecycle.rs index 27893d4bc..329e661ef 100644 --- a/cli/src/services/local_db/lifecycle.rs +++ b/cli/src/services/local_db/lifecycle.rs @@ -5,7 +5,7 @@ use crate::services::db::{bootstrap_db_parent, collect_db_path_health, DbSpec}; use crate::services::default_paths::local_db_path; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; use super::{LocalDb, LocalDbSpec}; @@ -14,10 +14,6 @@ use super::{LocalDb, LocalDbSpec}; pub struct LocalDbLifecycle; impl ServiceLifecycle for LocalDbLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::LocalDb - } - async fn diagnose(&self, _ctx: &C) -> Vec { diagnose_local_db_health() } diff --git a/cli/src/services/mod.rs b/cli/src/services/mod.rs index 1f7690f8e..899ef6907 100644 --- a/cli/src/services/mod.rs +++ b/cli/src/services/mod.rs @@ -1,16 +1,12 @@ pub mod agent_trace; pub mod agent_trace_db; -#[allow(dead_code)] pub mod agent_trace_export; -#[allow(dead_code)] pub mod agent_trace_storage; -#[allow(dead_code)] pub mod agent_trace_sync; pub mod app_support; pub mod auth; pub mod auth_command; pub mod auth_db; -#[allow(dead_code)] pub mod bash_policy; pub mod capabilities; pub(crate) mod codex_hook_config; @@ -27,13 +23,11 @@ pub mod help; pub mod hooks; pub mod lifecycle; pub mod local_db; -#[allow(dead_code)] pub mod mutation_trace; pub mod observability; pub mod output_format; pub mod parse; pub mod patch; -#[allow(dead_code)] pub mod repository_identity; pub mod resilience; pub mod security; diff --git a/cli/src/services/mutation_trace/lineage.rs b/cli/src/services/mutation_trace/lineage.rs index 74e69527a..4f28203ed 100644 --- a/cli/src/services/mutation_trace/lineage.rs +++ b/cli/src/services/mutation_trace/lineage.rs @@ -72,10 +72,6 @@ impl MutationLineage { } } - pub fn reset_file(&mut self, path: &str, content: Option<&str>) { - self.files.insert(path.to_owned(), unknown_lines(content)); - } - pub fn reset_all(&mut self, files: &BTreeMap>) { self.files = files .iter() @@ -83,10 +79,6 @@ impl MutationLineage { .collect(); } - pub fn tracked_paths(&self) -> impl Iterator { - self.files.keys() - } - pub fn apply( &mut self, patch: &ParsedPatch, diff --git a/cli/src/services/mutation_trace/lineage/tests.rs b/cli/src/services/mutation_trace/lineage/tests.rs index d2f0f093a..4a49dd069 100644 --- a/cli/src/services/mutation_trace/lineage/tests.rs +++ b/cli/src/services/mutation_trace/lineage/tests.rs @@ -57,7 +57,6 @@ fn a_removed_line_loses_its_provenance_permanently() { lineage.provenance_at("f.rs", 2, "foo"), LineProvenance::Unknown ); - assert_eq!(lineage.tracked_paths().count(), 1); } #[test] @@ -161,31 +160,3 @@ fn duplicate_lines_do_not_let_provenance_jump_between_occurrences() { LineProvenance::Unknown ); } - -#[test] -fn a_deleted_file_drops_out_of_the_lineage() { - let mut lineage = baseline(&[("f.rs", Some("a\nb\n"))]); - lineage - .apply( - &diff("diff --git a/f.rs b/f.rs\ndeleted file mode 100644\n--- a/f.rs\n+++ /dev/null\n@@ -1,2 +0,0 @@\n-a\n-b\n"), - &TransitionOrigin::MutationNonAi, - ) - .expect("delete"); - assert_eq!(lineage.tracked_paths().count(), 0); -} - -#[test] -fn reset_file_returns_a_file_to_a_conservative_baseline() { - let mut lineage = baseline(&[("f.rs", Some("a\n"))]); - lineage - .apply( - &diff("diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n"), - &TransitionOrigin::MutationAi(scope("ai")), - ) - .expect("add"); - lineage.reset_file("f.rs", Some("a\nfoo\n")); - assert_eq!( - lineage.provenance_at("f.rs", 2, "foo"), - LineProvenance::Unknown - ); -} diff --git a/cli/src/services/mutation_trace/mbt/model.rs b/cli/src/services/mutation_trace/mbt/model.rs index c257e4088..4876c3451 100644 --- a/cli/src/services/mutation_trace/mbt/model.rs +++ b/cli/src/services/mutation_trace/mbt/model.rs @@ -443,6 +443,10 @@ pub struct ModelState { /// variables are still accepted, but only via an explicit [`IgnoredAny`] /// field — modeling a new Quint variable here as `IgnoredAny` is a /// deliberate "not semantic state" decision, not an oversight. +#[allow( + dead_code, + reason = "verification-only Quint variables are deserialized to satisfy deny_unknown_fields and never read" +)] #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct WireModelState { diff --git a/cli/src/services/mutation_trace/runtime/coordinator.rs b/cli/src/services/mutation_trace/runtime/coordinator.rs index ddfe688c4..1fa06e61d 100644 --- a/cli/src/services/mutation_trace/runtime/coordinator.rs +++ b/cli/src/services/mutation_trace/runtime/coordinator.rs @@ -1,4 +1,5 @@ use std::path::Path; +use std::time::Duration; use anyhow::Result; use uuid::Uuid; @@ -14,7 +15,7 @@ use crate::services::mutation_trace::types::{ }; use super::git_snapshot::GitSnapshotService; -use super::protected_worktree::{ProtectedWorktree, ProtectedWorktreeError}; +use super::protected_worktree::{ProtectedWorktree, ProtectedWorktreeError, WORKTREE_LOCK_TIMEOUT}; use super::worktree_lock::WorktreeLockLease; pub use super::protected_worktree::ExternalTaintOperation; @@ -50,10 +51,30 @@ pub enum RuntimeBoundary { #[derive(Debug)] pub struct CoordinateOutcome { + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub worktree_id: WorktreeId, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub observed_tree: TreeId, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub revision: u64, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub evaluation: protocol::CommitEvaluation, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub mutation_event: Option, } @@ -90,6 +111,10 @@ pub enum CoordinateError { /// next invocation conservatively recovers. MarkerClearAfterCommit { source: anyhow::Error, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] committed: Box, }, /// The caller-supplied Agent Trace DB provider returned `Err` after the @@ -181,6 +206,7 @@ where repository_root, boundary, open_db, + WORKTREE_LOCK_TIMEOUT, || {}, |_attempt| {}, |_attempt| Ok(()), @@ -192,6 +218,7 @@ pub(super) async fn coordinate_inner( repository_root: &Path, boundary: &RuntimeBoundary, open_db: P, + lock_timeout: Duration, on_lock_contention: F, after_load: L, after_recovery: R, @@ -202,9 +229,10 @@ where L: FnMut(u32), R: FnMut(u32) -> Result<()>, { - let protected = ProtectedWorktree::acquire_inner(repository_root, on_lock_contention) - .await - .map_err(protected_worktree_failure)?; + let protected = + ProtectedWorktree::acquire_with_timeout(repository_root, lock_timeout, on_lock_contention) + .await + .map_err(protected_worktree_failure)?; let outcome = coordinate_protected( repository_root, diff --git a/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs b/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs index 42f6b65fe..966cea930 100644 --- a/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs +++ b/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs @@ -40,6 +40,7 @@ pub(crate) enum GuardError { Wait(std::io::Error), Finish(CoordinateError), SupervisorWorker(tokio::task::JoinError), + #[allow(dead_code, reason = "constructed only under cfg(not(unix))")] UnsupportedPlatform, } @@ -104,21 +105,6 @@ where Err(GuardError::UnsupportedPlatform) } -#[cfg(not(unix))] -pub(crate) fn run_external_mutation_guard( - _repository_root: &Path, - _request: &GuardRequest, - _open_db: P, - _on_event: E, - _cancel_rx: mpsc::Receiver<()>, -) -> Result -where - P: std::ops::AsyncFnOnce() -> anyhow::Result, - E: FnMut(GuardEvent), -{ - Err(GuardError::UnsupportedPlatform) -} - #[cfg(unix)] mod unix_impl { use std::fs::File; @@ -824,30 +810,6 @@ mod unix_impl { .await } - pub(crate) async fn run_external_mutation_guard( - repository_root: &Path, - request: &GuardRequest, - open_db: P, - mut on_event: E, - cancel_rx: mpsc::Receiver<()>, - ) -> Result - where - P: std::ops::AsyncFnOnce() -> anyhow::Result, - E: FnMut(GuardEvent), - { - let guard = arm_external_mutation_guard( - repository_root, - open_db, - || { - on_event(GuardEvent::Armed); - Ok(()) - }, - cancel_rx, - ) - .await?; - guard.exec(request, on_event).await - } - #[cfg(test)] mod tests { use std::path::{Path, PathBuf}; @@ -857,11 +819,37 @@ mod unix_impl { use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use super::{run_external_mutation_guard, GuardEvent, GuardRequest}; + use super::{ + arm_external_mutation_guard, GuardError, GuardEvent, GuardOutcome, GuardRequest, + }; const WORKER_PROGRESS_LIMIT: Duration = Duration::from_millis(500); const READY_TIMEOUT: Duration = Duration::from_secs(30); + async fn arm_and_exec_guard( + repository_root: &Path, + request: &GuardRequest, + open_db: P, + mut on_event: E, + cancel_rx: mpsc::Receiver<()>, + ) -> Result + where + P: std::ops::AsyncFnOnce() -> anyhow::Result, + E: FnMut(GuardEvent), + { + let guard = arm_external_mutation_guard( + repository_root, + open_db, + || { + on_event(GuardEvent::Armed); + Ok(()) + }, + cancel_rx, + ) + .await?; + guard.exec(request, on_event).await + } + fn git(repo_root: &Path, args: &[&str]) { let output = Command::new("git") .args(args) @@ -972,7 +960,7 @@ mod unix_impl { let guard_task = tokio::spawn(async move { let mut ready_tx = Some(ready_tx); - run_external_mutation_guard( + arm_and_exec_guard( &repo_root, &guard_request("printf ready; sleep 2"), async || RepositoryAgentTraceDb::new_at(&db_path).await, @@ -1018,7 +1006,7 @@ mod unix_impl { let (_cancel_tx, cancel_rx) = mpsc::channel(); let mut events = Vec::new(); - let outcome = run_external_mutation_guard( + let outcome = arm_and_exec_guard( &repo_root, &guard_request("printf out; printf err >&2; exit 7"), async || RepositoryAgentTraceDb::new_at(&db_path).await, @@ -1042,6 +1030,4 @@ mod unix_impl { } #[cfg(unix)] -pub(crate) use unix_impl::{ - arm_external_mutation_guard, run_external_mutation_guard, ArmedExternalMutationGuard, -}; +pub(crate) use unix_impl::{arm_external_mutation_guard, ArmedExternalMutationGuard}; diff --git a/cli/src/services/mutation_trace/runtime/git_snapshot.rs b/cli/src/services/mutation_trace/runtime/git_snapshot.rs index 6d1271b33..0c64a8c9d 100644 --- a/cli/src/services/mutation_trace/runtime/git_snapshot.rs +++ b/cli/src/services/mutation_trace/runtime/git_snapshot.rs @@ -14,11 +14,6 @@ const SCE_RUNTIME_DIR: &str = "sce"; const TMP_INDEX_DIR: &str = "tmp"; const REF_NAMESPACE: &str = "refs/sce/mutation-cursor"; -/// `git for-each-ref` format for pin inventory: four `%00`-separated fields — -/// refname, target object name, target object type, and the symbolic-ref -/// target (empty for a direct ref). NUL-separated so no field can be split or -/// trimmed ambiguously; the trailing symref field is always present (possibly -/// empty), so every well-formed line has exactly four fields. const FOR_EACH_REF_PIN_FORMAT: &str = "--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)"; @@ -147,19 +142,6 @@ impl GitSnapshotService { Ok(String::from_utf8(output.stdout).ok()) } - /// Inventory every SCE snapshot pin owned by `worktree_id`. - /// - /// Runs `git for-each-ref` constrained to the single path prefix - /// `refs/sce/mutation-cursor//`, so a ref owned by any other - /// worktree or in an unrelated namespace is never returned. Each line is - /// validated against the shape `pin_tree` produces: a **direct** ref (never - /// a symbolic ref) whose target is a tree object and whose final path - /// component equals the target SHA. A symbolic ref anywhere in the - /// namespace is malformed state — it would let one worktree's pin resolve - /// through another worktree's ref — and is rejected rather than followed. A - /// `git for-each-ref` execution or exit failure is - /// [`PinInventoryError::Git`]; anything malformed inside the namespace is - /// [`PinInventoryError::MalformedRef`], matchable separately. pub async fn list_pins( &self, worktree_id: &WorktreeId, @@ -177,46 +159,16 @@ impl GitSnapshotService { .collect() } - /// Delete exactly `pins` in one atomic, no-dereference - /// `git update-ref --no-deref --stdin` transaction, each `delete` - /// conditioned on the tree SHA recorded in the [`PinnedRef`]. - /// - /// Two independent safety properties: - /// - /// - **Atomic** — `git update-ref --stdin` commits every command together - /// at end of input; if any command fails (including a failed old-value - /// check) the whole transaction aborts and no ref is changed. - /// - **No dereference** — `--no-deref` makes every `delete` operate on the - /// exact ref name given, never on a ref reached by resolving a symbolic - /// ref. Combined with a fail-closed re-check (below), a - /// direct-ref → symbolic-ref race between inventory and deletion can - /// never cause this call to touch the symref's target (for example a ref - /// owned by another worktree). - /// - /// Before issuing the transaction, each supplied ref is re-inventoried: it - /// must still exist, still be a direct ref to a tree, and still point at - /// the inventoried SHA. If any has changed — deleted, retargeted, or turned - /// into a symbolic ref — this returns `Err` and deletes nothing, preferring - /// failure over acting on unexpected namespace state. An empty slice is a - /// successful no-op. pub async fn delete_pins(&self, lease: WorktreeLockLease, pins: &[PinnedRef]) -> Result<()> { - self.delete_pins_inner(lease, pins, || {}).await + self.delete_pins_inner(lease, pins, || {}, || {}).await } - /// Body of [`delete_pins`] with a deterministic test seam that fires - /// **after** the fail-closed preflight re-inventory and **before** the - /// `git update-ref --no-deref --stdin` transaction is spawned. Production - /// calls it with a no-op hook; the inline atomicity test uses the hook to - /// mutate a ref *after* it has passed preflight, so the transaction is - /// actually issued and the per-`delete` expected-old-value check — not the - /// preflight — is what aborts the batch. This is the only proof that the - /// Git transaction itself is atomic; the preflight proves a different - /// property (unexpected ref state before the transaction is even attempted). async fn delete_pins_inner( &self, lease: WorktreeLockLease, pins: &[PinnedRef], after_preflight: impl FnOnce(), + on_worker_entered: impl FnOnce() + Send + 'static, ) -> Result<()> { if pins.is_empty() { return Ok(()); @@ -235,7 +187,7 @@ impl GitSnapshotService { stdin_payload.push('\n'); } - self.run_ref_mutation( + self.run_ref_mutation_inner( lease, vec![ "update-ref".to_string(), @@ -243,21 +195,13 @@ impl GitSnapshotService { "--stdin".to_string(), ], Some(stdin_payload.into_bytes()), + on_worker_entered, ) .await?; Ok(()) } - async fn run_ref_mutation( - &self, - lease: WorktreeLockLease, - args: Vec, - stdin: Option>, - ) -> Result { - self.run_ref_mutation_inner(lease, args, stdin, || {}).await - } - async fn run_ref_mutation_inner( &self, lease: WorktreeLockLease, @@ -277,14 +221,6 @@ impl GitSnapshotService { .map_err(|source| anyhow!("Git ref mutation worker failed: {source}"))? } - /// Fail closed unless every supplied pin is still exactly the direct ref - /// that was inventoried: present, a direct (non-symbolic) ref, targeting a - /// tree, and pointing at the recorded SHA. Re-inventoried in a single - /// `git for-each-ref` over the exact ref names, so no enumeration order is - /// relied on. This closes the common inventory→delete race cleanly; the - /// residual sub-transaction race is still contained by `--no-deref` plus - /// the per-`delete` old-value condition, which together cannot follow a - /// symbolic ref or mutate a ref the caller did not name. async fn assert_pins_are_unchanged_direct_refs(&self, pins: &[PinnedRef]) -> Result<()> { let mut args: Vec<&str> = vec!["for-each-ref", FOR_EACH_REF_PIN_FORMAT]; args.extend(pins.iter().map(|pin| pin.ref_name.as_str())); @@ -402,15 +338,12 @@ impl GitSnapshotService { } } -/// One SCE-owned snapshot pin: a ref under -/// `refs/sce/mutation-cursor//` and the tree object it protects. #[derive(Clone, Debug, Eq, PartialEq)] pub struct PinnedRef { pub ref_name: String, pub tree: TreeId, } -/// Why a worktree's pin inventory could not be produced. #[derive(Debug)] pub enum PinInventoryError { /// `git for-each-ref` itself failed to execute or exited non-zero. @@ -1019,6 +952,120 @@ mod tests { .is_empty()); } + #[tokio::test] + async fn cancelled_delete_keeps_worktree_lock_until_git_ref_deletion_completes() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + std::fs::write(repo_root.join("untracked.txt"), "untracked\n").expect("write file"); + + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let worktree_id = WorktreeId("main".to_string()); + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("snapshot service"); + let tree = snapshot.capture_tree().await.expect("capture tree"); + let pin_ref = format!("refs/sce/mutation-cursor/main/{}", tree.0); + + let pin_lock = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("pin lock"); + snapshot + .pin_tree(pin_lock.lease(), &worktree_id, &tree) + .await + .expect("pin orphan tree"); + drop(pin_lock); + let pins = snapshot.list_pins(&worktree_id).await.expect("list pins"); + assert_eq!( + pins, + vec![PinnedRef { + ref_name: pin_ref.clone(), + tree: tree.clone(), + }] + ); + + let lock_a = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock A"); + let lease = lock_a.lease(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + + let caller = tokio::spawn({ + let repo_root = repo_root.clone(); + let pins = pins.clone(); + async move { + let _lock_a = lock_a; + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("caller snapshot service"); + snapshot + .delete_pins_inner( + lease, + &pins, + || {}, + move || { + entered_tx.send(()).expect("signal worker entered"); + release_rx + .recv_timeout(LOCK_RELEASE_TIMEOUT) + .expect("release worker"); + }, + ) + .await + } + }); + + entered_rx.await.expect("ref deletion worker entered"); + assert_eq!( + snapshot.list_pins(&worktree_id).await.expect("list pins"), + pins, + "the parked worker must not have deleted the pin yet" + ); + + caller.abort(); + assert!(caller + .await + .expect_err("caller must be cancelled") + .is_cancelled()); + + assert!(matches!( + WorktreeLock::acquire_async(&git_dir, LOCK_CONTENDED_TIMEOUT).await, + Err(WorktreeLockError::TimedOut { .. }) + )); + assert_eq!( + snapshot + .list_pins(&worktree_id) + .await + .expect("list pins while the worker is still parked"), + pins + ); + + release_tx.send(()).expect("release worker"); + + let lock_b = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock B after the ref deletion completes"); + assert!(snapshot + .list_pins(&worktree_id) + .await + .expect("list pins after the released worker finished") + .is_empty()); + assert!(!git_dir.join(format!("{pin_ref}.lock")).exists()); + assert!(!git_dir.join("packed-refs.lock").exists()); + + snapshot + .pin_tree(lock_b.lease(), &worktree_id, &tree) + .await + .expect("a legitimate pin proceeds after the cancelled deletion"); + assert_eq!( + snapshot + .list_pins(&worktree_id) + .await + .expect("list pins after re-pin"), + pins + ); + } + #[tokio::test] async fn async_identity_resolution_distinguishes_main_and_linked_worktrees() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/cli/src/services/mutation_trace/runtime/maintenance_state.rs b/cli/src/services/mutation_trace/runtime/maintenance_state.rs new file mode 100644 index 000000000..e094a536b --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/maintenance_state.rs @@ -0,0 +1,740 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +pub(super) const RECONCILIATION_ADVISORY_AFTER_MS: i64 = 24 * 60 * 60 * 1000; +pub(super) const MAINTENANCE_STATE_MAX_BYTES: u64 = 4096; +pub(super) const FUTURE_SKEW_TOLERANCE_MS: i64 = 5 * 60 * 1000; +pub(super) const MAX_FAILURE_MESSAGE_CHARS: usize = 256; + +const MAINTENANCE_STATE_VERSION: u32 = 1; +const SCE_RUNTIME_DIR: &str = "sce"; +const MAINTENANCE_STATE_FILE: &str = "ref-maintenance.json"; +const STAGING_ATTEMPTS: u16 = 1000; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum AttemptOutcome { + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct StoredReport { + pub retained: usize, + pub deleted: usize, + pub local_required: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct StoredFailure { + pub kind: String, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct MaintenanceState { + pub version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub anchor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_success: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_attempt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_advised: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_attempt_outcome: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_report: Option, + #[serde(default)] + pub consecutive_failures: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_failure: Option, +} + +impl Default for MaintenanceState { + fn default() -> Self { + Self { + version: MAINTENANCE_STATE_VERSION, + anchor: None, + last_success: None, + last_attempt: None, + last_advised: None, + last_attempt_outcome: None, + last_report: None, + consecutive_failures: 0, + last_failure: None, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) enum StateRead { + Absent, + Unusable, + Valid(MaintenanceState), +} + +impl StateRead { + pub(super) fn into_state(self) -> MaintenanceState { + match self { + StateRead::Valid(state) => state, + StateRead::Absent | StateRead::Unusable => MaintenanceState::default(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct Evaluation { + pub recommended: bool, + pub advise: bool, + pub invalid_timestamps: bool, + pub reference_age_ms: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct Recommendation { + pub recommended: bool, + pub invalid_timestamps: bool, + pub last_success_age_ms: Option, + pub last_attempt_age_ms: Option, + pub last_advised_age_ms: Option, + pub last_attempt_outcome: Option, + pub consecutive_failures: u32, +} + +pub(super) fn system_now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|elapsed| i64::try_from(elapsed.as_millis()).ok()) + .unwrap_or(0) +} + +pub(super) fn state_path(git_dir: &Path) -> PathBuf { + git_dir.join(SCE_RUNTIME_DIR).join(MAINTENANCE_STATE_FILE) +} + +fn is_valid_timestamp(timestamp: i64, now: i64) -> bool { + timestamp <= now.saturating_add(FUTURE_SKEW_TOLERANCE_MS) +} + +fn age_ms(timestamp: i64, now: i64) -> i64 { + now.saturating_sub(timestamp).max(0) +} + +fn usable(timestamp: Option, now: i64) -> Option { + timestamp.filter(|value| is_valid_timestamp(*value, now)) +} + +pub(super) fn read_state(path: &Path) -> std::io::Result { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(StateRead::Absent); + } + Err(error) => return Err(error), + }; + + let mut bytes = Vec::new(); + file.take(MAINTENANCE_STATE_MAX_BYTES + 1) + .read_to_end(&mut bytes)?; + if u64::try_from(bytes.len()).map_or(true, |len| len > MAINTENANCE_STATE_MAX_BYTES) { + return Ok(StateRead::Unusable); + } + + Ok(match serde_json::from_slice::(&bytes) { + Ok(state) if state.version == MAINTENANCE_STATE_VERSION => StateRead::Valid(state), + Ok(_) | Err(_) => StateRead::Unusable, + }) +} + +fn failure_outstanding(state: &MaintenanceState) -> bool { + state.last_attempt_outcome == Some(AttemptOutcome::Failed) || state.consecutive_failures > 0 +} + +pub(super) fn evaluate(state: &MaintenanceState, now: i64) -> Evaluation { + let timestamps = [ + state.anchor, + state.last_success, + state.last_attempt, + state.last_advised, + ]; + let invalid_timestamps = timestamps + .iter() + .flatten() + .any(|timestamp| !is_valid_timestamp(*timestamp, now)); + + let reference = usable(state.last_success, now).or_else(|| usable(state.anchor, now)); + let reference_lost = + reference.is_none() && (state.last_success.is_some() || state.anchor.is_some()); + let reference_age_ms = reference.map(|timestamp| age_ms(timestamp, now)); + let due = reference_lost + || reference_age_ms.is_some_and(|age| age >= RECONCILIATION_ADVISORY_AFTER_MS); + let advised = usable(state.last_advised, now) + .filter(|timestamp| age_ms(*timestamp, now) < RECONCILIATION_ADVISORY_AFTER_MS); + let failure_outstanding = failure_outstanding(state); + let failure_reminded = advised.is_some_and(|advised_at| { + usable(state.last_attempt, now).is_none_or(|attempt| advised_at >= attempt) + }); + + Evaluation { + recommended: due || failure_outstanding || invalid_timestamps, + advise: (due && advised.is_none()) || (failure_outstanding && !failure_reminded), + invalid_timestamps, + reference_age_ms, + } +} + +pub(super) fn evaluate_recommendation(state: &MaintenanceState, now: i64) -> Recommendation { + let evaluation = evaluate(state, now); + let age_of = |timestamp: Option| usable(timestamp, now).map(|value| age_ms(value, now)); + + Recommendation { + recommended: evaluation.recommended, + invalid_timestamps: evaluation.invalid_timestamps, + last_success_age_ms: age_of(state.last_success), + last_attempt_age_ms: age_of(state.last_attempt), + last_advised_age_ms: age_of(state.last_advised), + last_attempt_outcome: state.last_attempt_outcome, + consecutive_failures: state.consecutive_failures, + } +} + +pub(super) fn normalize(state: &MaintenanceState, now: i64) -> MaintenanceState { + let mut normalized = state.clone(); + normalized.anchor = usable(state.anchor, now); + normalized.last_success = usable(state.last_success, now); + normalized.last_attempt = usable(state.last_attempt, now); + normalized.last_advised = usable(state.last_advised, now); + normalized +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum AdvisoryDecision { + Anchored, + NoAction, + Advised, +} + +pub(super) fn plan_advisory( + read: &StateRead, + now: i64, +) -> (AdvisoryDecision, Option) { + let StateRead::Valid(state) = read else { + let anchored = MaintenanceState { + anchor: Some(now), + ..MaintenanceState::default() + }; + return (AdvisoryDecision::Anchored, Some(anchored)); + }; + + let evaluation = evaluate(state, now); + let mut next = normalize(state, now); + let decision = if evaluation.advise { + next.last_advised = Some(now); + if next.anchor.is_none() && next.last_success.is_none() { + next.anchor = Some(now); + } + AdvisoryDecision::Advised + } else if next.anchor.is_none() && next.last_success.is_none() { + next.anchor = Some(now); + AdvisoryDecision::Anchored + } else { + AdvisoryDecision::NoAction + }; + + let changed = next != *state; + (decision, changed.then_some(next)) +} + +pub(super) fn record_success( + state: &MaintenanceState, + now: i64, + report: StoredReport, +) -> MaintenanceState { + let mut next = normalize(state, now); + next.last_attempt = Some(now); + next.last_success = Some(now); + next.last_attempt_outcome = Some(AttemptOutcome::Completed); + next.last_report = Some(report); + next.consecutive_failures = 0; + next.last_failure = None; + next +} + +pub(super) fn record_failure( + state: &MaintenanceState, + now: i64, + kind: &str, + message: &str, +) -> MaintenanceState { + let mut next = normalize(state, now); + next.last_attempt = Some(now); + next.last_attempt_outcome = Some(AttemptOutcome::Failed); + next.consecutive_failures = state.consecutive_failures.saturating_add(1); + next.last_failure = Some(StoredFailure { + kind: kind.to_string(), + message: message.chars().take(MAX_FAILURE_MESSAGE_CHARS).collect(), + }); + next +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum PersistPhase { + Serialize, + CreateStaging, + WriteStaging, + SyncStaging, + Rename, + OpenParentDirectory, + SyncParentDirectory, +} + +#[derive(Debug)] +pub(super) enum PersistFailure { + NotApplied { + phase: PersistPhase, + source: std::io::Error, + staging_cleanup: Option, + }, + DurabilityUncertain { + phase: PersistPhase, + source: std::io::Error, + }, +} + +impl std::fmt::Display for PersistFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PersistFailure::NotApplied { + phase, + source, + staging_cleanup, + } => { + write!( + f, + "maintenance state was not recorded ({phase:?}): {source}; previous state preserved" + )?; + if let Some(cleanup) = staging_cleanup { + write!(f, "; staging file cleanup failed: {cleanup}")?; + } + Ok(()) + } + PersistFailure::DurabilityUncertain { phase, source } => write!( + f, + "maintenance state was renamed into place but crash durability is unconfirmed ({phase:?}): {source}; the new state may be visible" + ), + } + } +} + +impl std::error::Error for PersistFailure {} + +pub(super) trait StateFilesystem { + fn create_staging(&self, directory: &Path) -> std::io::Result; + fn write_staging(&self, staging: &Path, bytes: &[u8]) -> std::io::Result<()>; + fn sync_staging(&self, staging: &Path) -> std::io::Result<()>; + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()>; + fn open_directory(&self, directory: &Path) -> std::io::Result; + fn sync_directory(&self, directory: &std::fs::File) -> std::io::Result<()>; + fn remove_staging(&self, staging: &Path) -> std::io::Result<()>; +} + +pub(super) struct RealStateFilesystem; + +impl StateFilesystem for RealStateFilesystem { + fn create_staging(&self, directory: &Path) -> std::io::Result { + create_staging_file(directory) + } + + fn write_staging(&self, staging: &Path, bytes: &[u8]) -> std::io::Result<()> { + let mut file = std::fs::OpenOptions::new().write(true).open(staging)?; + file.write_all(bytes) + } + + fn sync_staging(&self, staging: &Path) -> std::io::Result<()> { + std::fs::OpenOptions::new() + .write(true) + .open(staging)? + .sync_data() + } + + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + std::fs::rename(from, to) + } + + fn open_directory(&self, directory: &Path) -> std::io::Result { + std::fs::File::open(directory) + } + + fn sync_directory(&self, directory: &std::fs::File) -> std::io::Result<()> { + directory.sync_all() + } + + fn remove_staging(&self, staging: &Path) -> std::io::Result<()> { + std::fs::remove_file(staging) + } +} + +pub(super) fn write_state_atomically( + path: &Path, + state: &MaintenanceState, +) -> std::result::Result<(), PersistFailure> { + write_state_atomically_with(&RealStateFilesystem, path, state) +} + +fn not_applied( + phase: PersistPhase, + source: std::io::Error, + staging_cleanup: Option, +) -> PersistFailure { + PersistFailure::NotApplied { + phase, + source, + staging_cleanup, + } +} + +pub(super) fn write_state_atomically_with( + fs: &F, + path: &Path, + state: &MaintenanceState, +) -> std::result::Result<(), PersistFailure> { + let bytes = serde_json::to_vec(state) + .map_err(|error| not_applied(PersistPhase::Serialize, error.into(), None))?; + let directory = path.parent().ok_or_else(|| { + not_applied( + PersistPhase::CreateStaging, + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "maintenance state path has no parent directory", + ), + None, + ) + })?; + + let staging = fs + .create_staging(directory) + .map_err(|error| not_applied(PersistPhase::CreateStaging, error, None))?; + + let staged = fs + .write_staging(&staging, &bytes) + .map_err(|error| (PersistPhase::WriteStaging, error)) + .and_then(|()| { + fs.sync_staging(&staging) + .map_err(|error| (PersistPhase::SyncStaging, error)) + }) + .and_then(|()| { + fs.rename(&staging, path) + .map_err(|error| (PersistPhase::Rename, error)) + }); + if let Err((phase, source)) = staged { + return Err(not_applied( + phase, + source, + fs.remove_staging(&staging).err(), + )); + } + + let directory_handle = + fs.open_directory(directory) + .map_err(|source| PersistFailure::DurabilityUncertain { + phase: PersistPhase::OpenParentDirectory, + source, + })?; + fs.sync_directory(&directory_handle) + .map_err(|source| PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + source, + }) +} + +fn create_staging_file(directory: &Path) -> std::io::Result { + let epoch_nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(std::io::Error::other)? + .as_nanos(); + + for attempt in 0..STAGING_ATTEMPTS { + let candidate = directory.join(format!( + ".{MAINTENANCE_STATE_FILE}.staging-{epoch_nanos}-{}-{attempt}", + std::process::id() + )); + match std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&candidate) + { + Ok(_) => return Ok(candidate), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not allocate a unique maintenance state staging file", + )) +} + +pub(super) fn derive_git_dir(start: &Path) -> Option { + if ["GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE"] + .iter() + .any(|name| std::env::var_os(name).is_some()) + { + return None; + } + + for ancestor in start.ancestors() { + let dot_git = ancestor.join(".git"); + let Ok(metadata) = std::fs::symlink_metadata(&dot_git) else { + continue; + }; + + let git_dir = if metadata.is_dir() { + dot_git + } else if metadata.is_file() { + let contents = std::fs::read_to_string(&dot_git).ok()?; + let target = contents.lines().next()?.strip_prefix("gitdir:")?.trim(); + let target = Path::new(target); + if target.is_absolute() { + target.to_path_buf() + } else { + ancestor.join(target) + } + } else { + return None; + }; + + return git_dir + .is_dir() + .then(|| std::fs::canonicalize(&git_dir).ok()) + .flatten(); + } + + None +} + +#[cfg(test)] +pub(super) struct FaultInjectingFilesystem { + pub fail_at: Option, + pub fail_staging_cleanup: bool, +} + +#[cfg(test)] +impl FaultInjectingFilesystem { + pub(super) const fn failing_at(phase: PersistPhase) -> Self { + Self { + fail_at: Some(phase), + fail_staging_cleanup: false, + } + } + + fn check(&self, phase: PersistPhase) -> std::io::Result<()> { + if self.fail_at == Some(phase) { + return Err(std::io::Error::other(format!("injected {phase:?} failure"))); + } + Ok(()) + } +} + +#[cfg(test)] +impl StateFilesystem for FaultInjectingFilesystem { + fn create_staging(&self, directory: &Path) -> std::io::Result { + self.check(PersistPhase::CreateStaging)?; + RealStateFilesystem.create_staging(directory) + } + + fn write_staging(&self, staging: &Path, bytes: &[u8]) -> std::io::Result<()> { + self.check(PersistPhase::WriteStaging)?; + RealStateFilesystem.write_staging(staging, bytes) + } + + fn sync_staging(&self, staging: &Path) -> std::io::Result<()> { + self.check(PersistPhase::SyncStaging)?; + RealStateFilesystem.sync_staging(staging) + } + + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.check(PersistPhase::Rename)?; + RealStateFilesystem.rename(from, to) + } + + fn open_directory(&self, directory: &Path) -> std::io::Result { + self.check(PersistPhase::OpenParentDirectory)?; + RealStateFilesystem.open_directory(directory) + } + + fn sync_directory(&self, directory: &std::fs::File) -> std::io::Result<()> { + self.check(PersistPhase::SyncParentDirectory)?; + RealStateFilesystem.sync_directory(directory) + } + + fn remove_staging(&self, staging: &Path) -> std::io::Result<()> { + if self.fail_staging_cleanup { + return Err(std::io::Error::other("injected staging cleanup failure")); + } + RealStateFilesystem.remove_staging(staging) + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use super::{ + read_state, record_success, write_state_atomically, write_state_atomically_with, + FaultInjectingFilesystem, MaintenanceState, PersistFailure, PersistPhase, StateRead, + StoredReport, + }; + + const NOW: i64 = 1_000_000_000_000; + + fn state(marker: usize) -> MaintenanceState { + record_success( + &MaintenanceState::default(), + NOW, + StoredReport { + retained: marker, + deleted: 0, + local_required: 0, + }, + ) + } + + fn state_file(dir: &Path) -> PathBuf { + dir.join("ref-maintenance.json") + } + + fn stored_marker(path: &Path) -> Option { + match read_state(path).expect("read state") { + StateRead::Valid(state) => state.last_report.map(|report| report.retained), + StateRead::Absent => None, + StateRead::Unusable => panic!("state unusable"), + } + } + + fn staging_leftovers(dir: &Path) -> usize { + std::fs::read_dir(dir) + .expect("read dir") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains("staging")) + .count() + } + + fn assert_not_applied(phase: PersistPhase, previous_marker: Option) { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + if let Some(marker) = previous_marker { + write_state_atomically(&path, &state(marker)).expect("seed"); + } + let previous_bytes = std::fs::read(&path).ok(); + + let fs = FaultInjectingFilesystem::failing_at(phase); + let failure = + write_state_atomically_with(&fs, &path, &state(99)).expect_err("injected failure"); + let PersistFailure::NotApplied { + phase: reported, + staging_cleanup, + .. + } = failure + else { + panic!("expected NotApplied for {phase:?}, got {failure:?}"); + }; + assert_eq!(reported, phase); + assert!(staging_cleanup.is_none()); + assert_eq!(std::fs::read(&path).ok(), previous_bytes); + assert_eq!(path.exists(), previous_marker.is_some()); + assert_eq!(stored_marker(&path), previous_marker); + assert_eq!(staging_leftovers(dir.path()), 0); + + write_state_atomically(&path, &state(100)).expect("recovery write"); + assert_eq!(stored_marker(&path), Some(100)); + assert_eq!(staging_leftovers(dir.path()), 0); + } + + #[test] + fn maintenance_state_write_succeeds_end_to_end() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + write_state_atomically(&path, &state(1)).expect("first write"); + write_state_atomically(&path, &state(2)).expect("second write"); + assert_eq!(stored_marker(&path), Some(2)); + assert_eq!(staging_leftovers(dir.path()), 0); + } + + #[test] + fn maintenance_state_pre_rename_failures_preserve_previous_state() { + for (phase, previous_marker) in [ + (PersistPhase::CreateStaging, Some(1)), + (PersistPhase::WriteStaging, Some(1)), + (PersistPhase::SyncStaging, Some(1)), + (PersistPhase::Rename, Some(1)), + (PersistPhase::Rename, None), + ] { + assert_not_applied(phase, previous_marker); + } + } + + #[test] + fn maintenance_state_post_rename_failures_report_uncertain_durability() { + for phase in [ + PersistPhase::OpenParentDirectory, + PersistPhase::SyncParentDirectory, + ] { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + write_state_atomically(&path, &state(1)).expect("seed"); + + let fs = FaultInjectingFilesystem::failing_at(phase); + let failure = write_state_atomically_with(&fs, &path, &state(2)).expect_err("failure"); + let PersistFailure::DurabilityUncertain { + phase: reported, .. + } = failure + else { + panic!("expected DurabilityUncertain for {phase:?}, got {failure:?}"); + }; + assert_eq!(reported, phase); + let rendered = failure.to_string(); + assert!(rendered.contains("may be visible"), "{phase:?}: {rendered}"); + assert!( + !rendered.contains("previous state preserved"), + "{phase:?}: {rendered}" + ); + assert_eq!(stored_marker(&path), Some(2), "{phase:?}"); + assert_eq!(staging_leftovers(dir.path()), 0, "{phase:?}"); + + write_state_atomically(&path, &state(3)).expect("recovery write"); + assert_eq!(stored_marker(&path), Some(3), "{phase:?}"); + } + } + + #[test] + fn maintenance_state_recovers_after_interrupted_pre_rename_write() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + write_state_atomically(&path, &state(1)).expect("seed"); + + let fs = FaultInjectingFilesystem { + fail_at: Some(PersistPhase::Rename), + fail_staging_cleanup: true, + }; + let failure = write_state_atomically_with(&fs, &path, &state(2)).expect_err("failure"); + let PersistFailure::NotApplied { + staging_cleanup, .. + } = failure + else { + panic!("expected NotApplied"); + }; + assert!(staging_cleanup.is_some()); + assert_eq!(staging_leftovers(dir.path()), 1); + assert_eq!(stored_marker(&path), Some(1)); + + write_state_atomically(&path, &state(3)).expect("recovery write"); + assert_eq!(stored_marker(&path), Some(3)); + } +} diff --git a/cli/src/services/mutation_trace/runtime/mod.rs b/cli/src/services/mutation_trace/runtime/mod.rs index a7db7c910..4722da2da 100644 --- a/cli/src/services/mutation_trace/runtime/mod.rs +++ b/cli/src/services/mutation_trace/runtime/mod.rs @@ -2,8 +2,12 @@ mod coordinator; mod external_mutation_guard; mod external_taint; mod git_snapshot; +mod maintenance_state; mod mutation_attribution; mod protected_worktree; +mod ref_advisory; +mod ref_doctor; +mod ref_maintenance; mod ref_reconciliation; mod scope_runtime; mod worktree_lock; @@ -15,8 +19,8 @@ pub(crate) use coordinator::{ }; #[allow(unused_imports)] pub(crate) use external_mutation_guard::{ - arm_external_mutation_guard, run_external_mutation_guard, ArmedExternalMutationGuard, - GuardError, GuardEvent, GuardOutcome, GuardRequest, + arm_external_mutation_guard, ArmedExternalMutationGuard, GuardError, GuardEvent, GuardOutcome, + GuardRequest, }; #[allow(unused_imports)] pub(crate) use git_snapshot::{resolve_git_dir, resolve_worktree_id}; @@ -26,6 +30,11 @@ pub(crate) use mutation_attribution::{ BoundedMutationAttribution, MutationAttributionBarrier, MutationEventPageSource, TreeReadSource, MAX_MUTATION_ATTRIBUTION_EVENTS, }; +pub(crate) use ref_advisory::{advise_after_completed_boundary, AdvisoryReport, AdvisorySeverity}; +pub(crate) use ref_doctor::{ + inspect_reconciliation_recommendation, run_explicit_reconciliation, ReconciliationCounts, + ReconciliationFix, ReconciliationRecommendation, StateWarning, +}; #[allow(unused_imports)] pub(crate) use scope_runtime::{ abandon_scope, AbandonRecoveryReason, AbandonScopeError, AbandonScopeOutcome, diff --git a/cli/src/services/mutation_trace/runtime/protected_worktree.rs b/cli/src/services/mutation_trace/runtime/protected_worktree.rs index 89edc6486..eefd57665 100644 --- a/cli/src/services/mutation_trace/runtime/protected_worktree.rs +++ b/cli/src/services/mutation_trace/runtime/protected_worktree.rs @@ -106,7 +106,7 @@ impl ProtectedWorktree { Self::acquire_with_timeout(repository_root, WORKTREE_LOCK_TIMEOUT, on_lock_contention).await } - async fn acquire_with_timeout( + pub(super) async fn acquire_with_timeout( repository_root: &Path, lock_timeout: Duration, on_lock_contention: F, diff --git a/cli/src/services/mutation_trace/runtime/ref_advisory.rs b/cli/src/services/mutation_trace/runtime/ref_advisory.rs new file mode 100644 index 000000000..b70a368ab --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_advisory.rs @@ -0,0 +1,950 @@ +use std::path::Path; +use std::time::Duration; + +use super::maintenance_state::{ + derive_git_dir, plan_advisory, read_state, state_path, write_state_atomically, + AdvisoryDecision, MaintenanceState, PersistFailure, +}; +use super::worktree_lock::{acquire_inner, WorktreeLockError}; + +pub(super) const RECONCILIATION_RECOMMENDED_MESSAGE: &str = + "Reconciliation is recommended. Run sce doctor --fix."; +const DIAGNOSTIC_PREFIX: &str = "SCE: "; +const DURABILITY_UNCERTAIN_DIAGNOSTIC: &str = "Advisory-state durability could not be confirmed"; +const STATE_WRITE_FAILED_DIAGNOSTIC: &str = "Reconciliation advisory state could not be written"; +const STATE_DURABILITY_UNCONFIRMED_DIAGNOSTIC: &str = + "Reconciliation advisory-state durability could not be confirmed"; +const STATE_UNAVAILABLE_DIAGNOSTIC: &str = "Reconciliation advisory state is unavailable"; +const CHECK_DOCTOR_HINT: &str = "Run sce doctor to check reconciliation status."; + +#[derive(Debug)] +pub(super) enum AdvisoryOutcome { + Anchored, + NoAction, + Advised, + AdvisedDurabilityUncertain { warning: PersistFailure }, + Busy, + StateWriteFailed { warning: PersistFailure }, + StateUnavailable, +} + +impl AdvisoryOutcome { + pub(super) fn recommendation(&self) -> Option<&'static str> { + match self { + AdvisoryOutcome::Advised | AdvisoryOutcome::AdvisedDurabilityUncertain { .. } => { + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + } + AdvisoryOutcome::Anchored + | AdvisoryOutcome::NoAction + | AdvisoryOutcome::Busy + | AdvisoryOutcome::StateWriteFailed { .. } + | AdvisoryOutcome::StateUnavailable => None, + } + } + + pub(super) fn persistence_warning(&self) -> Option<&PersistFailure> { + match self { + AdvisoryOutcome::AdvisedDurabilityUncertain { warning } + | AdvisoryOutcome::StateWriteFailed { warning } => Some(warning), + AdvisoryOutcome::Anchored + | AdvisoryOutcome::NoAction + | AdvisoryOutcome::Advised + | AdvisoryOutcome::Busy + | AdvisoryOutcome::StateUnavailable => None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum AdvisorySeverity { + Debug, + Warn, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AdvisoryReport { + pub(crate) outcome: &'static str, + pub(crate) severity: AdvisorySeverity, + pub(crate) recommendation: Option<&'static str>, + pub(crate) warning: Option, + pub(crate) diagnostic: Option, +} + +impl AdvisoryOutcome { + pub(crate) fn report(&self) -> AdvisoryReport { + let (outcome, severity) = match self { + AdvisoryOutcome::Anchored => ("anchored", AdvisorySeverity::Debug), + AdvisoryOutcome::NoAction => ("no_action", AdvisorySeverity::Debug), + AdvisoryOutcome::Busy => ("busy", AdvisorySeverity::Debug), + AdvisoryOutcome::Advised => ("advised", AdvisorySeverity::Warn), + AdvisoryOutcome::AdvisedDurabilityUncertain { .. } => { + ("advised_durability_uncertain", AdvisorySeverity::Warn) + } + AdvisoryOutcome::StateWriteFailed { .. } => { + ("state_write_failed", AdvisorySeverity::Warn) + } + AdvisoryOutcome::StateUnavailable => ("state_unavailable", AdvisorySeverity::Warn), + }; + AdvisoryReport { + outcome, + severity, + recommendation: self.recommendation(), + warning: self.persistence_warning().map(ToString::to_string), + diagnostic: self.diagnostic(), + } + } + + fn diagnostic(&self) -> Option { + match self { + AdvisoryOutcome::Advised => Some(format!( + "{DIAGNOSTIC_PREFIX}{RECONCILIATION_RECOMMENDED_MESSAGE}" + )), + AdvisoryOutcome::AdvisedDurabilityUncertain { warning } => Some(format!( + "{DIAGNOSTIC_PREFIX}{RECONCILIATION_RECOMMENDED_MESSAGE}\n\ + {DIAGNOSTIC_PREFIX}{DURABILITY_UNCERTAIN_DIAGNOSTIC}: {warning}." + )), + AdvisoryOutcome::StateWriteFailed { + warning: warning @ PersistFailure::NotApplied { .. }, + } => Some(format!( + "{DIAGNOSTIC_PREFIX}{STATE_WRITE_FAILED_DIAGNOSTIC}: {warning}. {CHECK_DOCTOR_HINT}" + )), + AdvisoryOutcome::StateWriteFailed { + warning: warning @ PersistFailure::DurabilityUncertain { .. }, + } => Some(format!( + "{DIAGNOSTIC_PREFIX}{STATE_DURABILITY_UNCONFIRMED_DIAGNOSTIC}: {warning}. {CHECK_DOCTOR_HINT}" + )), + AdvisoryOutcome::StateUnavailable => Some(format!( + "{DIAGNOSTIC_PREFIX}{STATE_UNAVAILABLE_DIAGNOSTIC}. {CHECK_DOCTOR_HINT}" + )), + AdvisoryOutcome::Anchored | AdvisoryOutcome::NoAction | AdvisoryOutcome::Busy => None, + } + } +} + +fn system_unix_time_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|elapsed| i64::try_from(elapsed.as_millis()).ok()) + .unwrap_or(0) +} + +pub(crate) fn advise_after_completed_boundary(repository_root: &Path) -> AdvisoryReport { + advise_if_due(repository_root, system_unix_time_ms).report() +} + +fn outcome_after_write( + decision: AdvisoryDecision, + write: Result<(), PersistFailure>, +) -> AdvisoryOutcome { + match (decision, write) { + (AdvisoryDecision::Anchored, Ok(())) => AdvisoryOutcome::Anchored, + (AdvisoryDecision::NoAction, Ok(())) => AdvisoryOutcome::NoAction, + (AdvisoryDecision::Advised, Ok(())) => AdvisoryOutcome::Advised, + (AdvisoryDecision::Advised, Err(warning @ PersistFailure::DurabilityUncertain { .. })) => { + AdvisoryOutcome::AdvisedDurabilityUncertain { warning } + } + (_, Err(warning)) => AdvisoryOutcome::StateWriteFailed { warning }, + } +} + +pub(super) fn advise_if_due(repository_root: &Path, now: C) -> AdvisoryOutcome +where + C: Fn() -> i64, +{ + advise_if_due_with(repository_root, now, write_state_atomically) +} + +pub(super) fn advise_if_due_with( + repository_root: &Path, + now: C, + write_state: W, +) -> AdvisoryOutcome +where + C: Fn() -> i64, + W: Fn(&Path, &MaintenanceState) -> Result<(), PersistFailure>, +{ + let Some(git_dir) = derive_git_dir(repository_root) else { + return AdvisoryOutcome::StateUnavailable; + }; + let path = state_path(&git_dir); + + let Ok(unlocked) = read_state(&path) else { + return AdvisoryOutcome::StateUnavailable; + }; + let (decision, pending) = plan_advisory(&unlocked, now()); + if pending.is_none() && decision == AdvisoryDecision::NoAction { + return AdvisoryOutcome::NoAction; + } + + let lock = match acquire_inner(&git_dir, Duration::ZERO, || {}) { + Ok(lock) => lock, + Err(WorktreeLockError::TimedOut { .. }) => return AdvisoryOutcome::Busy, + Err(_) => return AdvisoryOutcome::StateUnavailable, + }; + + let Ok(locked) = read_state(&path) else { + return AdvisoryOutcome::StateUnavailable; + }; + let now_ms = now(); + let (decision, pending) = plan_advisory(&locked, now_ms); + + let result = match pending { + Some(next) => outcome_after_write(decision, write_state(&path, &next)), + None => outcome_after_write(decision, Ok(())), + }; + drop(lock); + result +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::process::Command; + + use super::super::maintenance_state::{ + evaluate, evaluate_recommendation, read_state, record_failure, record_success, state_path, + write_state_atomically, write_state_atomically_with, AttemptOutcome, + FaultInjectingFilesystem, MaintenanceState, PersistFailure, PersistPhase, StateRead, + StoredReport, FUTURE_SKEW_TOLERANCE_MS, RECONCILIATION_ADVISORY_AFTER_MS, + }; + use super::super::worktree_lock::acquire_inner; + use super::{ + advise_if_due_with, AdvisoryOutcome, AdvisorySeverity, RECONCILIATION_RECOMMENDED_MESSAGE, + }; + + const NOW: i64 = 1_000_000_000_000; + const RECOMMENDED: &str = RECONCILIATION_RECOMMENDED_MESSAGE; + + fn init_repo(root: &Path) { + std::fs::create_dir_all(root).expect("repo dir"); + let output = Command::new("git") + .args(["init", "-q"]) + .current_dir(root) + .output() + .expect("git init"); + assert!(output.status.success()); + } + + fn real_write(path: &Path, state: &MaintenanceState) -> Result<(), PersistFailure> { + write_state_atomically(path, state) + } + + fn advise(root: &Path, now: i64) -> AdvisoryOutcome { + advise_if_due_with(root, || now, real_write) + } + + fn stored(root: &Path) -> MaintenanceState { + match read_state(&state_path(&root.join(".git"))).expect("read state") { + StateRead::Valid(state) => state, + other => panic!("expected valid state, got {other:?}"), + } + } + + fn seed(root: &Path, state: &MaintenanceState) { + let path = state_path(&root.join(".git")); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + real_write(&path, state).expect("seed state"); + } + + #[test] + fn maintenance_report_maps_every_outcome_to_severity_and_diagnostic() { + let uncertain = || PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + source: std::io::Error::other("sync failed"), + }; + let not_applied = || PersistFailure::NotApplied { + phase: PersistPhase::Rename, + source: std::io::Error::other("rename failed"), + staging_cleanup: None, + }; + let recommended = format!("SCE: {RECOMMENDED}"); + let cases = [ + (AdvisoryOutcome::Anchored, "anchored", AdvisorySeverity::Debug, None), + (AdvisoryOutcome::NoAction, "no_action", AdvisorySeverity::Debug, None), + (AdvisoryOutcome::Busy, "busy", AdvisorySeverity::Debug, None), + ( + AdvisoryOutcome::Advised, + "advised", + AdvisorySeverity::Warn, + Some(recommended.clone()), + ), + ( + AdvisoryOutcome::StateUnavailable, + "state_unavailable", + AdvisorySeverity::Warn, + Some( + "SCE: Reconciliation advisory state is unavailable. Run sce doctor to check reconciliation status." + .to_string(), + ), + ), + ]; + for (outcome, name, severity, diagnostic) in cases { + let report = outcome.report(); + assert_eq!(report.outcome, name); + assert_eq!(report.severity, severity); + assert_eq!(report.diagnostic, diagnostic, "{name}"); + assert_eq!(report.warning, None, "{name}"); + } + + let report = AdvisoryOutcome::AdvisedDurabilityUncertain { + warning: uncertain(), + } + .report(); + assert_eq!(report.outcome, "advised_durability_uncertain"); + assert_eq!(report.severity, AdvisorySeverity::Warn); + assert_eq!( + report.recommendation, + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + let warning = report.warning.expect("persistence warning"); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with(&format!( + "{recommended}\nSCE: Advisory-state durability could not be confirmed: " + ))); + assert!(diagnostic.contains(&warning)); + + let report = AdvisoryOutcome::StateWriteFailed { + warning: not_applied(), + } + .report(); + assert_eq!(report.outcome, "state_write_failed"); + assert_eq!(report.severity, AdvisorySeverity::Warn); + assert_eq!(report.recommendation, None); + let warning = report.warning.expect("persistence warning"); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with("SCE: Reconciliation advisory state could not be written: ")); + assert!(diagnostic.contains(&warning)); + assert!(!diagnostic.contains(RECONCILIATION_RECOMMENDED_MESSAGE)); + + let report = AdvisoryOutcome::StateWriteFailed { + warning: uncertain(), + } + .report(); + assert_eq!(report.outcome, "state_write_failed"); + assert_eq!(report.severity, AdvisorySeverity::Warn); + assert_eq!(report.recommendation, None); + let warning = report.warning.expect("persistence warning"); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic + .starts_with("SCE: Reconciliation advisory-state durability could not be confirmed: ")); + assert!(diagnostic.ends_with("Run sce doctor to check reconciliation status.")); + assert!(diagnostic.contains(&warning)); + assert!(!diagnostic.contains("could not be written")); + assert!(!diagnostic.contains(RECONCILIATION_RECOMMENDED_MESSAGE)); + } + + #[test] + fn maintenance_advisory_anchors_once_then_leaves_state_untouched() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Anchored)); + let path = state_path(&dir.path().join(".git")); + let first = std::fs::read(&path).expect("state bytes"); + assert_eq!(stored(dir.path()).anchor, Some(NOW)); + + assert!(matches!( + advise(dir.path(), NOW + 1000), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), first); + } + + #[test] + fn maintenance_advisory_advises_once_per_window_and_persists_before_returning() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + + let due = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + assert!(matches!(advise(dir.path(), due), AdvisoryOutcome::Advised)); + assert_eq!(stored(dir.path()).last_advised, Some(due)); + assert!(matches!( + advise(dir.path(), due + 1000), + AdvisoryOutcome::NoAction + )); + assert!(matches!( + advise(dir.path(), due + RECONCILIATION_ADVISORY_AFTER_MS), + AdvisoryOutcome::Advised + )); + } + + #[test] + fn maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let invalid = MaintenanceState { + anchor: Some(NOW + FUTURE_SKEW_TOLERANCE_MS + 1), + last_success: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + seed(dir.path(), &invalid); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Advised)); + let normalized = stored(dir.path()); + assert_eq!(normalized.last_advised, Some(NOW)); + assert_eq!(normalized.anchor, Some(NOW)); + assert_eq!(normalized.last_success, None); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_advisory_reports_busy_and_leaves_invalid_state_byte_identical() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let invalid = MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + seed(dir.path(), &invalid); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + let held = acquire_inner(&dir.path().join(".git"), std::time::Duration::ZERO, || {}) + .expect("hold lock"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Busy)); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + drop(held); + + let recommendation = evaluate_recommendation(&invalid, NOW); + assert!(recommendation.recommended); + assert!(recommendation.invalid_timestamps); + } + + fn advise_with_fault(root: &Path, now: i64, phase: PersistPhase) -> AdvisoryOutcome { + advise_if_due_with( + root, + || now, + |path, state| { + write_state_atomically_with( + &FaultInjectingFilesystem::failing_at(phase), + path, + state, + ) + }, + ) + } + + fn failed_state(now: i64) -> MaintenanceState { + record_failure( + &MaintenanceState::default(), + now, + "agent_trace_db_unavailable", + "agent trace database is missing", + ) + } + + #[test] + fn maintenance_first_failed_explicit_pass_without_anchor_is_recommended() { + let state = failed_state(NOW); + assert_eq!(state.anchor, None); + assert_eq!(state.last_success, None); + + let recommendation = evaluate_recommendation(&state, NOW); + assert!(recommendation.recommended); + assert_eq!( + recommendation.last_attempt_outcome, + Some(AttemptOutcome::Failed) + ); + assert_eq!(recommendation.consecutive_failures, 1); + assert!(!evaluate_recommendation(&MaintenanceState::default(), NOW).recommended); + } + + #[test] + fn maintenance_failure_advisory_emits_once_and_doctor_still_recommends_repair() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &failed_state(NOW)); + + let first = advise(dir.path(), NOW + 1000); + assert!(matches!(first, AdvisoryOutcome::Advised)); + assert_eq!( + first.recommendation(), + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + let persisted = stored(dir.path()); + assert_eq!(persisted.last_advised, Some(NOW + 1000)); + assert_eq!(persisted.consecutive_failures, 1); + assert_eq!(persisted.last_attempt_outcome, Some(AttemptOutcome::Failed)); + assert_eq!(persisted.last_failure, failed_state(NOW).last_failure); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + let second = advise(dir.path(), NOW + 2000); + assert!(matches!(second, AdvisoryOutcome::NoAction)); + assert_eq!(second.recommendation(), None); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + + let recommendation = evaluate_recommendation(&stored(dir.path()), NOW + 2000); + assert!(recommendation.recommended); + assert!(recommendation.last_advised_age_ms.is_some()); + } + + #[test] + fn maintenance_failure_advisory_repeats_after_the_cadence_window() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &failed_state(NOW)); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Advised)); + let later = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + assert!(matches!( + advise(dir.path(), later), + AdvisoryOutcome::Advised + )); + assert_eq!(stored(dir.path()).consecutive_failures, 1); + } + + #[test] + fn maintenance_repeated_failed_passes_keep_recommendation_active() { + let first = failed_state(NOW); + let second = record_failure(&first, NOW + 10, "pin_inventory", "inventory failed"); + let third = record_failure(&second, NOW + 20, "pin_inventory", "inventory failed"); + assert_eq!(third.consecutive_failures, 3); + assert!(evaluate_recommendation(&first, NOW + 30).recommended); + assert!(evaluate_recommendation(&second, NOW + 30).recommended); + assert!(evaluate_recommendation(&third, NOW + 30).recommended); + } + + #[test] + fn maintenance_successful_pass_clears_failure_streak_and_recommendation() { + let failed = record_failure(&failed_state(NOW), NOW + 10, "pin_inventory", "failed"); + let healed = record_success( + &failed, + NOW + 20, + StoredReport { + retained: 1, + deleted: 0, + local_required: 0, + }, + ); + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.last_failure, None); + assert_eq!(healed.last_attempt_outcome, Some(AttemptOutcome::Completed)); + assert!(!evaluate_recommendation(&healed, NOW + 30).recommended); + } + + #[test] + fn maintenance_failure_after_recent_success_is_recommended_and_advised_despite_older_reminder() + { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let success = record_success( + &MaintenanceState::default(), + NOW, + StoredReport { + retained: 1, + deleted: 0, + local_required: 0, + }, + ); + let reminded = MaintenanceState { + last_advised: Some(NOW + 1000), + ..success + }; + assert!(!evaluate_recommendation(&reminded, NOW + 1500).recommended); + + let failed = record_failure(&reminded, NOW + 2000, "delete_transaction", "failed"); + assert!(evaluate_recommendation(&failed, NOW + 2500).recommended); + seed(dir.path(), &failed); + + assert!(matches!( + advise(dir.path(), NOW + 2500), + AdvisoryOutcome::Advised + )); + assert!(matches!( + advise(dir.path(), NOW + 3000), + AdvisoryOutcome::NoAction + )); + } + + #[test] + fn maintenance_advisory_rename_failure_preserves_state_and_next_call_can_advise() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + let due = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + let outcome = advise_with_fault(dir.path(), due, PersistPhase::Rename); + assert!(matches!( + outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::NotApplied { + phase: PersistPhase::Rename, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + assert!(outcome.persistence_warning().is_some()); + let diagnostic = outcome.report().diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with("SCE: Reconciliation advisory state could not be written: ")); + assert!(!diagnostic.contains("durability")); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + assert!(evaluate_recommendation(&stored(dir.path()), due).recommended); + assert!(matches!(advise(dir.path(), due), AdvisoryOutcome::Advised)); + } + + fn assert_durability_uncertain_advisory(phase: PersistPhase) { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + + let due = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + let outcome = advise_with_fault(dir.path(), due, phase); + assert!(!matches!(outcome, AdvisoryOutcome::Advised)); + let AdvisoryOutcome::AdvisedDurabilityUncertain { warning } = &outcome else { + panic!("expected AdvisedDurabilityUncertain for {phase:?}, got {outcome:?}"); + }; + assert!(matches!( + warning, + PersistFailure::DurabilityUncertain { phase: reported, .. } if *reported == phase + )); + assert_eq!( + outcome.recommendation(), + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + assert!(outcome.persistence_warning().is_some()); + let report = outcome.report(); + assert_eq!( + report.recommendation, + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with(&format!( + "SCE: {RECOMMENDED}\nSCE: Advisory-state durability could not be confirmed: " + ))); + assert!(!diagnostic.contains("could not be written")); + + let visible = stored(dir.path()); + assert_eq!(visible.last_advised, Some(due)); + + assert!(matches!( + advise(dir.path(), due + 1000), + AdvisoryOutcome::NoAction + )); + let recommendation = evaluate_recommendation(&visible, due + 1000); + assert!(recommendation.recommended); + assert!(recommendation.last_advised_age_ms.is_some()); + } + + #[test] + fn maintenance_advisory_parent_directory_open_failure_is_durability_uncertain() { + assert_durability_uncertain_advisory(PersistPhase::OpenParentDirectory); + } + + #[test] + fn maintenance_advisory_parent_directory_sync_failure_is_durability_uncertain() { + assert_durability_uncertain_advisory(PersistPhase::SyncParentDirectory); + } + + #[test] + fn maintenance_advisory_durability_uncertain_failure_state_stays_recommended() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &failed_state(NOW)); + + let outcome = advise_with_fault(dir.path(), NOW + 10, PersistPhase::SyncParentDirectory); + assert!(matches!( + outcome, + AdvisoryOutcome::AdvisedDurabilityUncertain { .. } + )); + let visible = stored(dir.path()); + assert_eq!(visible.last_advised, Some(NOW + 10)); + assert_eq!(visible.consecutive_failures, 1); + assert!(evaluate_recommendation(&visible, NOW + 20).recommended); + } + + #[test] + fn maintenance_advisory_accepts_valid_timestamp_offsets_without_rewrite() { + for ahead in [0, 60_000, FUTURE_SKEW_TOLERANCE_MS] { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let state = MaintenanceState { + last_success: Some(NOW + ahead), + ..MaintenanceState::default() + }; + seed(dir.path(), &state); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + assert!( + matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction), + "offset {ahead}" + ); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + assert!( + !evaluate_recommendation(&state, NOW).recommended, + "offset {ahead}" + ); + } + } + + #[test] + fn maintenance_advisory_clock_rollback_beyond_tolerance_normalizes_while_within_it_does_not() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + assert!(matches!( + advise(dir.path(), NOW - FUTURE_SKEW_TOLERANCE_MS), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + + let rolled_back = NOW - FUTURE_SKEW_TOLERANCE_MS - 1; + assert!(matches!( + advise(dir.path(), rolled_back), + AdvisoryOutcome::Advised + )); + let normalized = stored(dir.path()); + assert_within_skew_tolerance(&normalized, rolled_back); + assert_eq!(normalized.last_advised, Some(rolled_back)); + } + + #[test] + fn maintenance_advisory_reanchors_unusable_state_variants() { + let cases: [(&str, Vec); 3] = [ + ("wrong schema version", br#"{"version":9999}"#.to_vec()), + ("oversized state file", vec![b' '; 5000]), + ("corrupt JSON", b"not json".to_vec()), + ]; + for (label, bytes) in cases { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let path = state_path(&dir.path().join(".git")); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + std::fs::write(&path, &bytes).expect("unusable state"); + + assert!( + matches!(read_state(&path).expect("read"), StateRead::Unusable), + "{label}" + ); + let outcome = advise(dir.path(), NOW); + assert!( + matches!(outcome, AdvisoryOutcome::Anchored), + "{label}: {outcome:?}" + ); + assert_eq!(outcome.recommendation(), None, "{label}"); + let anchored = stored(dir.path()); + assert_eq!(anchored.anchor, Some(NOW), "{label}"); + assert!( + !evaluate_recommendation(&anchored, NOW).recommended, + "{label}" + ); + + let anchored_bytes = std::fs::read(&path).expect("state bytes"); + assert!( + matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction), + "{label}" + ); + assert_eq!( + std::fs::read(&path).expect("state bytes"), + anchored_bytes, + "{label}" + ); + } + } + + fn invalid_future_state() -> MaintenanceState { + MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + last_success: Some(NOW + FUTURE_SKEW_TOLERANCE_MS + 1), + ..MaintenanceState::default() + } + } + + fn assert_within_skew_tolerance(state: &MaintenanceState, now: i64) { + for timestamp in [ + state.anchor, + state.last_success, + state.last_attempt, + state.last_advised, + ] + .into_iter() + .flatten() + { + assert!(timestamp <= now + FUTURE_SKEW_TOLERANCE_MS); + } + } + + #[test] + fn maintenance_normalization_pre_rename_failure_keeps_previous_bytes_and_recovers_later() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &invalid_future_state()); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::Rename); + + assert!(matches!( + outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::NotApplied { + phase: PersistPhase::Rename, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + assert!(evaluate_recommendation(&stored(dir.path()), NOW).invalid_timestamps); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Advised)); + let normalized = stored(dir.path()); + assert_within_skew_tolerance(&normalized, NOW); + assert_eq!(normalized.last_advised, Some(NOW)); + } + + #[test] + fn maintenance_normalization_post_rename_failure_is_visible_but_durability_is_uncertain() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &invalid_future_state()); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::SyncParentDirectory); + + assert!(matches!( + &outcome, + AdvisoryOutcome::AdvisedDurabilityUncertain { + warning: PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + } + } + )); + assert!(outcome.persistence_warning().is_some()); + assert!(outcome.report().diagnostic.is_some()); + let visible = stored(dir.path()); + assert_within_skew_tolerance(&visible, NOW); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_advisory_anchor_write_failure_reports_state_write_failed_without_advice() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::Rename); + assert!(matches!(outcome, AdvisoryOutcome::StateWriteFailed { .. })); + assert_eq!(outcome.recommendation(), None); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Anchored)); + } + + #[test] + fn maintenance_anchor_post_rename_failure_is_state_write_failed_with_visible_anchor() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::SyncParentDirectory); + + assert!(matches!( + &outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + let report = outcome.report(); + assert!(report.warning.is_some()); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic + .starts_with("SCE: Reconciliation advisory-state durability could not be confirmed: ")); + assert!(!diagnostic.contains("could not be written")); + assert_eq!(stored(dir.path()).anchor, Some(NOW)); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_no_action_normalization_post_rename_failure_is_state_write_failed() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let invalid = MaintenanceState { + anchor: Some(NOW), + last_advised: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + seed(dir.path(), &invalid); + assert!(!evaluate(&invalid, NOW).advise); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::SyncParentDirectory); + + assert!(matches!( + &outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + let diagnostic = outcome.report().diagnostic.expect("diagnostic"); + assert!(diagnostic.contains("durability could not be confirmed")); + let visible = stored(dir.path()); + assert_eq!(visible.anchor, Some(NOW)); + assert_eq!(visible.last_advised, None); + assert_within_skew_tolerance(&visible, NOW); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_advisory_derives_linked_worktree_git_dir_from_subdirectory() { + let dir = tempfile::tempdir().expect("temp dir"); + let main = dir.path().join("main"); + init_repo(&main); + for args in [ + vec!["config", "user.email", "t@example.invalid"], + vec!["config", "user.name", "T"], + vec!["commit", "-q", "--allow-empty", "-m", "init"], + vec!["worktree", "add", "-q", "../linked"], + ] { + let output = Command::new("git") + .args(&args) + .current_dir(&main) + .output() + .expect("git"); + assert!(output.status.success(), "{args:?}"); + } + let linked = dir.path().join("linked"); + let sub = linked.join("nested"); + std::fs::create_dir_all(&sub).expect("subdir"); + + let expected = Command::new("git") + .args(["rev-parse", "--absolute-git-dir"]) + .current_dir(&sub) + .output() + .expect("rev-parse"); + let expected = std::fs::canonicalize(String::from_utf8_lossy(&expected.stdout).trim()) + .expect("canonical"); + + assert_eq!(super::derive_git_dir(&sub), Some(expected.clone())); + assert!(matches!(advise(&sub, NOW), AdvisoryOutcome::Anchored)); + assert!(state_path(&expected).is_file()); + } +} diff --git a/cli/src/services/mutation_trace/runtime/ref_doctor.rs b/cli/src/services/mutation_trace/runtime/ref_doctor.rs new file mode 100644 index 000000000..752e224f5 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_doctor.rs @@ -0,0 +1,405 @@ +use std::path::Path; + +use super::maintenance_state::{ + derive_git_dir, evaluate_recommendation, read_state, state_path, system_now_ms, AttemptOutcome, + MaintenanceState, PersistFailure, PersistPhase, StateRead, +}; +use super::ref_maintenance::{reconcile_explicit, ExplicitOutcome, StatePersistWarning}; +use super::ref_reconciliation::{ReconcileError, ReconciliationReport}; +use crate::services::agent_trace_db::repository::ExistingRepositoryDbError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationCounts { + pub(crate) deleted: usize, + pub(crate) retained: usize, + pub(crate) local_required: usize, +} + +const NOT_APPLIED_MESSAGE: &str = "Maintenance-state update was not applied."; +const DURABILITY_UNCERTAIN_MESSAGE: &str = "Maintenance-state durability could not be confirmed."; +const PREVIOUS_STATE_UNREADABLE_MESSAGE: &str = + "Maintenance state could not be read; the update was not attempted."; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum StateWarning { + NotApplied { phase: &'static str, cause: String }, + DurabilityUncertain { phase: &'static str, cause: String }, + PreviousStateUnreadable { cause: String }, +} + +impl StateWarning { + pub(crate) fn kind(&self) -> &'static str { + match self { + StateWarning::NotApplied { .. } => "not_applied", + StateWarning::DurabilityUncertain { .. } => "durability_uncertain", + StateWarning::PreviousStateUnreadable { .. } => "previous_state_unreadable", + } + } + + pub(crate) fn phase(&self) -> Option<&'static str> { + match self { + StateWarning::NotApplied { phase, .. } + | StateWarning::DurabilityUncertain { phase, .. } => Some(phase), + StateWarning::PreviousStateUnreadable { .. } => None, + } + } + + pub(crate) fn message(&self) -> &'static str { + match self { + StateWarning::NotApplied { .. } => NOT_APPLIED_MESSAGE, + StateWarning::DurabilityUncertain { .. } => DURABILITY_UNCERTAIN_MESSAGE, + StateWarning::PreviousStateUnreadable { .. } => PREVIOUS_STATE_UNREADABLE_MESSAGE, + } + } + + pub(crate) fn cause(&self) -> &str { + match self { + StateWarning::NotApplied { cause, .. } + | StateWarning::DurabilityUncertain { cause, .. } + | StateWarning::PreviousStateUnreadable { cause } => cause, + } + } +} + +impl std::fmt::Display for StateWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message())?; + match self.phase() { + Some(phase) => write!(f, " Cause ({phase}): {}", self.cause()), + None => write!(f, " Cause: {}", self.cause()), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ReconciliationFix { + Completed(ReconciliationCounts), + CompletedStatePersistFailed { + counts: ReconciliationCounts, + warning: StateWarning, + }, + Failed { + kind: &'static str, + message: String, + state_warning: Option, + }, + Busy, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationRecommendation { + pub(crate) invalid_timestamps: bool, + pub(crate) state_unusable: bool, + pub(crate) last_success_age_ms: Option, + pub(crate) last_attempt_age_ms: Option, + pub(crate) last_advised_age_ms: Option, + pub(crate) last_attempt_failed: bool, + pub(crate) consecutive_failures: u32, +} + +pub(crate) async fn run_explicit_reconciliation(repository_root: &Path) -> ReconciliationFix { + fix_from_outcome(reconcile_explicit(repository_root).await) +} + +pub(crate) fn inspect_reconciliation_recommendation( + repository_root: &Path, +) -> Option { + inspect_recommendation_at(repository_root, system_now_ms()) +} + +fn inspect_recommendation_at( + repository_root: &Path, + now: i64, +) -> Option { + let git_dir = derive_git_dir(repository_root)?; + let (state, state_unusable) = match read_state(&state_path(&git_dir)) { + Ok(StateRead::Valid(state)) => (state, false), + Ok(StateRead::Absent) => (MaintenanceState::default(), false), + Ok(StateRead::Unusable) | Err(_) => (MaintenanceState::default(), true), + }; + let recommendation = evaluate_recommendation(&state, now); + if !recommendation.recommended && !state_unusable { + return None; + } + Some(ReconciliationRecommendation { + invalid_timestamps: recommendation.invalid_timestamps, + state_unusable, + last_success_age_ms: recommendation.last_success_age_ms, + last_attempt_age_ms: recommendation.last_attempt_age_ms, + last_advised_age_ms: recommendation.last_advised_age_ms, + last_attempt_failed: recommendation.last_attempt_outcome == Some(AttemptOutcome::Failed), + consecutive_failures: recommendation.consecutive_failures, + }) +} + +fn fix_from_outcome(outcome: ExplicitOutcome) -> ReconciliationFix { + match outcome { + ExplicitOutcome::Completed(report) => ReconciliationFix::Completed(counts(report)), + ExplicitOutcome::CompletedStatePersistFailed { report, warning } => { + ReconciliationFix::CompletedStatePersistFailed { + counts: counts(report), + warning: state_warning_from(warning), + } + } + ExplicitOutcome::Failed { + error, + state_warning, + } => ReconciliationFix::Failed { + kind: failure_kind(&error), + message: error.to_string(), + state_warning: state_warning.map(state_warning_from), + }, + ExplicitOutcome::Skipped(_) => ReconciliationFix::Busy, + } +} + +fn state_warning_from(warning: StatePersistWarning) -> StateWarning { + match warning { + StatePersistWarning::PreviousStateUnreadable(error) => { + StateWarning::PreviousStateUnreadable { + cause: error.to_string(), + } + } + StatePersistWarning::WriteFailed(PersistFailure::NotApplied { phase, source, .. }) => { + StateWarning::NotApplied { + phase: phase_name(phase), + cause: source.to_string(), + } + } + StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { phase, source }) => { + StateWarning::DurabilityUncertain { + phase: phase_name(phase), + cause: source.to_string(), + } + } + } +} + +fn phase_name(phase: PersistPhase) -> &'static str { + match phase { + PersistPhase::Serialize => "serialize", + PersistPhase::CreateStaging => "create_staging", + PersistPhase::WriteStaging => "write_staging", + PersistPhase::SyncStaging => "sync_staging", + PersistPhase::Rename => "rename", + PersistPhase::OpenParentDirectory => "open_parent_directory", + PersistPhase::SyncParentDirectory => "sync_parent_directory", + } +} + +fn counts(report: ReconciliationReport) -> ReconciliationCounts { + ReconciliationCounts { + deleted: report.deleted, + retained: report.retained, + local_required: report.local_required, + } +} + +fn failure_kind(error: &ReconcileError) -> &'static str { + match error { + ReconcileError::GitDir(_) => "git_dir_unavailable", + ReconcileError::Lock(_) => "lock_unavailable", + ReconcileError::CheckoutIdentity(_) => "worktree_identity_unavailable", + ReconcileError::AgentTraceDbUnavailable(source) => { + match source.downcast_ref::() { + Some(ExistingRepositoryDbError::Missing { .. }) => "agent_trace_db_missing", + Some(ExistingRepositoryDbError::Unreadable(_)) => "agent_trace_db_unreadable", + Some(ExistingRepositoryDbError::IncompatibleSchema(_)) => { + "agent_trace_db_incompatible_schema" + } + Some(ExistingRepositoryDbError::MissingMetadata) => { + "agent_trace_db_missing_metadata" + } + Some(ExistingRepositoryDbError::RepositoryMismatch { .. }) => { + "agent_trace_db_repository_mismatch" + } + None => "agent_trace_db_unavailable", + } + } + ReconcileError::SnapshotService(_) => "snapshot_service_unavailable", + ReconcileError::PinInventory(_) => "pin_inventory_failed", + ReconcileError::MalformedPin { .. } => "malformed_ref", + ReconcileError::DurableRoots(_) => "durable_roots_unavailable", + ReconcileError::MissingRequiredPins { .. } => "missing_required_pins", + ReconcileError::DeleteTransaction(_) => "delete_transaction_failed", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::mutation_trace::runtime::maintenance_state::{ + record_failure, record_success, StoredReport, + }; + + fn write_state(git_dir: &Path, state: &MaintenanceState) { + let path = state_path(git_dir); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, serde_json::to_vec(state).unwrap()).unwrap(); + } + + fn repo() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join(".git")).unwrap(); + dir + } + + #[test] + fn recommendation_is_absent_for_fresh_and_recent_state() { + let repo = repo(); + assert_eq!(inspect_recommendation_at(repo.path(), 1_000), None); + + let state = record_success( + &MaintenanceState::default(), + 1_000, + StoredReport { + retained: 1, + deleted: 0, + local_required: 1, + }, + ); + write_state(&repo.path().join(".git"), &state); + assert_eq!(inspect_recommendation_at(repo.path(), 2_000), None); + } + + #[test] + fn recommendation_reports_failure_and_never_writes_state() { + let repo = repo(); + let git_dir = repo.path().join(".git"); + let state = record_failure(&MaintenanceState::default(), 1_000, "kind", "message"); + write_state(&git_dir, &state); + let before = std::fs::read(state_path(&git_dir)).unwrap(); + + let recommendation = inspect_recommendation_at(repo.path(), 2_000).unwrap(); + assert!(recommendation.last_attempt_failed); + assert_eq!(recommendation.consecutive_failures, 1); + assert!(!recommendation.state_unusable); + assert_eq!(std::fs::read(state_path(&git_dir)).unwrap(), before); + } + + #[test] + fn recommendation_flags_unusable_and_invalid_state_without_rewriting() { + let repo = repo(); + let git_dir = repo.path().join(".git"); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"not json").unwrap(); + + let unusable = inspect_recommendation_at(repo.path(), 2_000).unwrap(); + assert!(unusable.state_unusable); + assert_eq!(std::fs::read(&path).unwrap(), b"not json"); + + let state = MaintenanceState { + last_success: Some(i64::MAX / 2), + ..MaintenanceState::default() + }; + write_state(&git_dir, &state); + let before = std::fs::read(&path).unwrap(); + let invalid = inspect_recommendation_at(repo.path(), 2_000).unwrap(); + assert!(invalid.invalid_timestamps); + assert_eq!(std::fs::read(&path).unwrap(), before); + } + + fn report() -> ReconciliationReport { + ReconciliationReport { + local_required: 1, + retained: 2, + deleted: 3, + } + } + + fn io_error() -> std::io::Error { + std::io::Error::other("injected") + } + + #[test] + fn pre_rename_persist_failure_maps_to_not_applied() { + let fix = fix_from_outcome(ExplicitOutcome::CompletedStatePersistFailed { + report: report(), + warning: StatePersistWarning::WriteFailed(PersistFailure::NotApplied { + phase: PersistPhase::Rename, + source: io_error(), + staging_cleanup: None, + }), + }); + let ReconciliationFix::CompletedStatePersistFailed { counts, warning } = fix else { + panic!("expected persist-failed completion"); + }; + assert_eq!(counts.deleted, 3); + assert_eq!(warning.kind(), "not_applied"); + assert_eq!(warning.phase(), Some("rename")); + assert_eq!( + warning.message(), + "Maintenance-state update was not applied." + ); + } + + #[test] + fn post_rename_persist_failure_maps_to_durability_uncertain() { + let fix = fix_from_outcome(ExplicitOutcome::CompletedStatePersistFailed { + report: report(), + warning: StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + source: io_error(), + }), + }); + let ReconciliationFix::CompletedStatePersistFailed { warning, .. } = fix else { + panic!("expected persist-failed completion"); + }; + assert_eq!(warning.kind(), "durability_uncertain"); + assert_eq!(warning.phase(), Some("sync_parent_directory")); + assert_eq!( + warning.message(), + "Maintenance-state durability could not be confirmed." + ); + } + + #[test] + fn unreadable_previous_state_maps_to_its_own_warning_on_failure() { + let fix = fix_from_outcome(ExplicitOutcome::Failed { + error: ReconcileError::MissingRequiredPins { missing: vec![] }, + state_warning: Some(StatePersistWarning::PreviousStateUnreadable(io_error())), + }); + let ReconciliationFix::Failed { state_warning, .. } = fix else { + panic!("expected failure"); + }; + let warning = state_warning.expect("warning retained"); + assert_eq!(warning.kind(), "previous_state_unreadable"); + assert_eq!(warning.phase(), None); + assert!(!warning.message().contains("not recorded")); + } + + #[test] + fn outcomes_map_to_distinct_fix_results() { + let report = ReconciliationReport { + local_required: 1, + retained: 2, + deleted: 3, + }; + assert_eq!( + fix_from_outcome(ExplicitOutcome::Completed(report)), + ReconciliationFix::Completed(ReconciliationCounts { + deleted: 3, + retained: 2, + local_required: 1, + }) + ); + assert_eq!( + fix_from_outcome(ExplicitOutcome::Skipped( + super::super::ref_maintenance::SkipReason::Busy + )), + ReconciliationFix::Busy + ); + let failed = fix_from_outcome(ExplicitOutcome::Failed { + error: ReconcileError::MissingRequiredPins { missing: vec![] }, + state_warning: None, + }); + assert!(matches!( + failed, + ReconciliationFix::Failed { + kind: "missing_required_pins", + .. + } + )); + } +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance.rs new file mode 100644 index 000000000..0c5ce8d92 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance.rs @@ -0,0 +1,231 @@ +use std::path::Path; +use std::time::Duration; + +use anyhow::{Context, Result}; + +use super::git_snapshot::resolve_git_dir; +use super::maintenance_state::{ + read_state, record_failure, record_success, state_path, system_now_ms, write_state_atomically, + MaintenanceState, PersistFailure, StateRead, StoredReport, +}; +use super::ref_reconciliation::{ + reconcile_with_held_lock, ReconcileError, ReconcilePhase, ReconciliationOutcome, + ReconciliationReport, +}; +use super::worktree_lock::{acquire_inner_async, WorktreeLockError}; +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::agent_trace_storage::{ + resolve_existing_agent_trace_storage_for_maintenance, AgentTraceStorageContext, +}; +use crate::services::config; + +#[cfg(test)] +mod tests; + +const EXPLICIT_LOCK_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SkipReason { + Busy, +} + +#[derive(Debug)] +pub(super) enum StatePersistWarning { + PreviousStateUnreadable(std::io::Error), + WriteFailed(PersistFailure), +} + +impl std::fmt::Display for StatePersistWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StatePersistWarning::PreviousStateUnreadable(error) => { + write!(f, "maintenance state could not be read: {error}") + } + StatePersistWarning::WriteFailed(failure) => write!(f, "{failure}"), + } + } +} + +#[derive(Debug)] +pub(super) enum ExplicitOutcome { + Completed(ReconciliationReport), + CompletedStatePersistFailed { + report: ReconciliationReport, + warning: StatePersistWarning, + }, + Failed { + error: ReconcileError, + state_warning: Option, + }, + Skipped(SkipReason), +} + +pub(super) async fn reconcile_explicit(repository_root: &Path) -> ExplicitOutcome { + reconcile_explicit_with( + repository_root, + async || open_authoritative_db(repository_root).await, + system_now_ms, + write_state_atomically, + |_| {}, + EXPLICIT_LOCK_TIMEOUT, + ) + .await +} + +async fn open_authoritative_db(repository_root: &Path) -> Result { + let storage_config = config::resolve_agent_trace_storage_runtime_config(repository_root) + .context("Failed to resolve Agent Trace repository storage config.")?; + let context = AgentTraceStorageContext { + repository_root, + explicit_repository_id: storage_config.repository_id.as_deref(), + repository_remote: &storage_config.repository_remote, + }; + resolve_existing_agent_trace_storage_for_maintenance(&context) + .await + .map(|storage| storage.db) +} + +async fn reconcile_explicit_with( + repository_root: &Path, + open_db: P, + now: C, + write_state: W, + on_phase: H, + lock_timeout: Duration, +) -> ExplicitOutcome +where + P: std::ops::AsyncFnOnce() -> Result, + C: Fn() -> i64, + W: Fn(&Path, &MaintenanceState) -> std::result::Result<(), PersistFailure>, + H: FnMut(ReconcilePhase), +{ + let git_dir = match resolve_git_dir(repository_root).await { + Ok(git_dir) => git_dir, + Err(source) => { + return ExplicitOutcome::Failed { + error: ReconcileError::GitDir(source), + state_warning: None, + }; + } + }; + + let lock = match acquire_inner_async(&git_dir, lock_timeout, || {}).await { + Ok(lock) => lock, + Err(WorktreeLockError::TimedOut { .. }) => { + return ExplicitOutcome::Skipped(SkipReason::Busy); + } + Err(error) => { + return ExplicitOutcome::Failed { + error: ReconcileError::Lock(error), + state_warning: None, + }; + } + }; + + let result = reconcile_with_held_lock(repository_root, &lock, open_db, on_phase).await; + let path = state_path(&git_dir); + let now_ms = now(); + let previous = read_state(&path); + + let outcome = match result { + Ok(ReconciliationOutcome::Reconciled(report)) => { + let next = record_success( + &previous_state(&previous), + now_ms, + StoredReport { + retained: report.retained, + deleted: report.deleted, + local_required: report.local_required, + }, + ); + match persist(&previous, &write_state, &path, &next) { + Ok(()) => ExplicitOutcome::Completed(report), + Err(warning) => ExplicitOutcome::CompletedStatePersistFailed { report, warning }, + } + } + Err(error) => record_failed_pass(&previous, &write_state, &path, now_ms, error), + }; + + drop(lock); + outcome +} + +fn previous_state(previous: &std::io::Result) -> MaintenanceState { + match previous { + Ok(read) => read.clone().into_state(), + Err(_) => MaintenanceState::default(), + } +} + +fn persist( + previous: &std::io::Result, + write_state: &W, + path: &Path, + next: &MaintenanceState, +) -> std::result::Result<(), StatePersistWarning> +where + W: Fn(&Path, &MaintenanceState) -> std::result::Result<(), PersistFailure>, +{ + if let Err(error) = previous { + return Err(StatePersistWarning::PreviousStateUnreadable( + std::io::Error::new(error.kind(), error.to_string()), + )); + } + write_state(path, next).map_err(StatePersistWarning::WriteFailed) +} + +fn record_failed_pass( + previous: &std::io::Result, + write_state: &W, + path: &Path, + now_ms: i64, + error: ReconcileError, +) -> ExplicitOutcome +where + W: Fn(&Path, &MaintenanceState) -> std::result::Result<(), PersistFailure>, +{ + let next = record_failure( + &previous_state(previous), + now_ms, + failure_kind(&error), + &error.to_string(), + ); + let state_warning = persist(previous, write_state, path, &next).err(); + ExplicitOutcome::Failed { + error, + state_warning, + } +} + +fn failure_kind(error: &ReconcileError) -> &'static str { + match error { + ReconcileError::GitDir(_) => "git_dir", + ReconcileError::Lock(_) => "lock", + ReconcileError::CheckoutIdentity(_) => "checkout_identity", + ReconcileError::AgentTraceDbUnavailable(_) => "agent_trace_db_unavailable", + ReconcileError::SnapshotService(_) => "snapshot_service", + ReconcileError::PinInventory(_) => "pin_inventory", + ReconcileError::MalformedPin { .. } => "malformed_pin", + ReconcileError::DurableRoots(_) => "durable_roots", + ReconcileError::MissingRequiredPins { .. } => "missing_required_pins", + ReconcileError::DeleteTransaction(_) => "delete_transaction", + } +} + +#[cfg(test)] +async fn open_authoritative_db_at_state_root( + repository_root: &Path, + state_root: &Path, +) -> Result { + let context = AgentTraceStorageContext { + repository_root, + explicit_repository_id: None, + repository_remote: "origin", + }; + crate::services::agent_trace_storage::resolve_existing_agent_trace_storage_for_maintenance_at_state_root( + &context, + state_root, + ) + .await + .map(|storage| storage.db) +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs new file mode 100644 index 000000000..d19be1183 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs @@ -0,0 +1,940 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use super::super::git_snapshot::{resolve_git_dir, GitSnapshotService, PinnedRef}; +use super::super::maintenance_state::{ + evaluate_recommendation, read_state, state_path, write_state_atomically, + write_state_atomically_with, FaultInjectingFilesystem, MaintenanceState, PersistFailure, + PersistPhase, StateRead, +}; +use super::super::ref_reconciliation::{ + reconcile_with_held_lock, ReconcileError, ReconcilePhase, ReconciliationOutcome, + ReconciliationReport, +}; +use super::super::worktree_lock::acquire_inner_async; +use super::{ + open_authoritative_db_at_state_root, reconcile_explicit_with, ExplicitOutcome, SkipReason, + StatePersistWarning, +}; +use crate::services::agent_trace_db::repository::{ + ExistingRepositoryDbError, RepositoryAgentTraceDb, +}; +use crate::services::default_paths::agent_trace_db_path_for_repository_at; +use crate::services::mutation_trace::types::WorktreeId; +use crate::services::repository_identity::resolve::resolve_repository_identity; + +const NOW: i64 = 1_000_000_000_000; +const OTHER_REPOSITORY_ID: &str = "some-other-repository"; + +mod coordination; +mod cross_process; +mod fail_closed; +mod state_recovery; +mod support; +mod worktrees; + +struct Fixture { + dir: tempfile::TempDir, + root: PathBuf, + state_root: PathBuf, + repository_id: String, +} + +impl Fixture { + fn new(remote_url: &str) -> Self { + let dir = tempfile::tempdir().expect("temp dir"); + let root = dir.path().join("repo"); + init_repo(&root, remote_url); + let repository_id = resolve_repository_identity(&root, None, "origin") + .expect("identity") + .identity + .repository_id; + let state_root = dir.path().join("state"); + Self { + dir, + root, + state_root, + repository_id, + } + } + + fn db_path(&self) -> PathBuf { + agent_trace_db_path_for_repository_at(&self.state_root, &self.repository_id) + .expect("db path") + } + + async fn create_db(&self) { + create_db(&self.db_path(), &self.repository_id).await; + } + + async fn run(&self) -> (ExplicitOutcome, Vec) { + self.run_with(write_state_atomically).await + } + + async fn run_with(&self, write_state: W) -> (ExplicitOutcome, Vec) + where + W: Fn(&Path, &MaintenanceState) -> Result<(), PersistFailure>, + { + let mut phases = Vec::new(); + let outcome = reconcile_explicit_with( + &self.root, + async || open_authoritative_db_at_state_root(&self.root, &self.state_root).await, + || NOW, + write_state, + |phase| phases.push(phase), + Duration::from_secs(10), + ) + .await; + (outcome, phases) + } + + async fn pins(&self) -> Vec { + GitSnapshotService::new(&self.root) + .await + .expect("snapshot") + .list_pins(&WorktreeId("main".to_string())) + .await + .expect("pins") + } + + async fn stored_state(&self) -> StateRead { + let git_dir = resolve_git_dir(&self.root).await.expect("git dir"); + read_state(&state_path(&git_dir)).expect("state") + } +} + +fn git(cwd: &Path, args: &[&str]) { + let output = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git command"); + assert!(output.status.success(), "git {args:?} failed"); +} + +fn init_repo(root: &Path, remote_url: &str) { + std::fs::create_dir_all(root).expect("repo dir"); + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "t@example.invalid"]); + git(root, &["config", "user.name", "T"]); + git(root, &["config", "commit.gpgsign", "false"]); + git(root, &["remote", "add", "origin", remote_url]); + std::fs::write(root.join("tracked.txt"), "tracked\n").expect("write file"); + git(root, &["add", "tracked.txt"]); + git(root, &["commit", "-q", "-m", "initial"]); +} + +async fn create_db(path: &Path, repository_id: &str) { + let db = RepositoryAgentTraceDb::new_at(path) + .await + .expect("create db"); + db.verify_or_initialize_repository_metadata(repository_id) + .await + .expect("initialize metadata"); +} + +fn ref_inventory(root: &Path) -> Vec { + let output = Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname) %(objectname) %(symref)", + "refs/sce/", + ]) + .current_dir(root) + .output() + .expect("for-each-ref"); + assert!(output.status.success(), "for-each-ref failed"); + let mut lines: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect(); + lines.sort(); + lines +} + +async fn pin_orphan(root: &Path) -> String { + let snapshot = GitSnapshotService::new(root).await.expect("snapshot"); + let tree = snapshot.capture_tree().await.expect("tree"); + let git_dir = resolve_git_dir(root).await.expect("git dir"); + let lock = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("lock"); + snapshot + .pin_tree(lock.lease(), &WorktreeId("main".to_string()), &tree) + .await + .expect("pin"); + tree.0 +} + +async fn seed_durable_root(db_path: &Path, tree: &str) { + seed_durable_root_for(db_path, "main", tree).await; +} + +async fn seed_durable_root_for(db_path: &Path, worktree: &str, tree: &str) { + let db = RepositoryAgentTraceDb::open_without_migrations_at(db_path) + .await + .expect("writer"); + db.execute( + "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES (?1, ?2, x'0000000000000000', 0, 'healthy', 0)", + (worktree, tree), + ) + .await + .expect("seed durable root"); +} + +fn unavailable_cause(error: &ReconcileError) -> &ExistingRepositoryDbError { + let ReconcileError::AgentTraceDbUnavailable(source) = error else { + panic!("expected AgentTraceDbUnavailable, got {error:?}"); + }; + source + .downcast_ref::() + .expect("verification failure keeps its typed source") +} + +fn failing_write( + phase: PersistPhase, +) -> impl Fn(&Path, &MaintenanceState) -> Result<(), PersistFailure> { + move |path, state| { + write_state_atomically_with(&FaultInjectingFilesystem::failing_at(phase), path, state) + } +} + +fn staging_leftovers(dir: &Path) -> usize { + std::fs::read_dir(dir) + .expect("read dir") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains("staging")) + .count() +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_database_reconciles_orphan_pin_and_records_success() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + + let (outcome, phases) = fixture.run().await; + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 0, + deleted: 1, + } + ); + assert_eq!( + phases, + vec![ReconcilePhase::DbOpened, ReconcilePhase::PinsInventoried] + ); + assert!(fixture.pins().await.is_empty()); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.last_success, Some(NOW)); + assert_eq!(state.consecutive_failures, 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_database_of_another_repository_cannot_authorize_deletion() { + let repo_a = Fixture::new("https://example.invalid/org/repo-a.git"); + let repo_b = Fixture::new("https://example.invalid/org/repo-b.git"); + assert_ne!(repo_a.repository_id, repo_b.repository_id); + repo_b.create_db().await; + pin_orphan(&repo_a.root).await; + + let (outcome, phases) = repo_a.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + assert!(phases.is_empty()); + assert_eq!(repo_a.pins().await.len(), 1); + + create_db(&repo_a.db_path(), &repo_b.repository_id).await; + let (outcome, phases) = repo_a.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + let ExistingRepositoryDbError::RepositoryMismatch { stored, resolved } = + unavailable_cause(&error) + else { + panic!("expected repository mismatch"); + }; + assert_eq!(stored, &repo_b.repository_id); + assert_eq!(resolved, &repo_a.repository_id); + assert!(phases.is_empty()); + assert_eq!(repo_a.pins().await.len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_metadata_or_schema_fails_before_pin_inventory() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + RepositoryAgentTraceDb::new_at(fixture.db_path()) + .await + .expect("migrated db without metadata"); + let (outcome, phases) = fixture.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::MissingMetadata + )); + assert!(phases.is_empty()); + assert_eq!(fixture.pins().await.len(), 1); + + std::fs::remove_file(fixture.db_path()).expect("remove db"); + drop( + RepositoryAgentTraceDb::open_without_migrations_at(fixture.db_path()) + .await + .expect("unmigrated db"), + ); + let (outcome, phases) = fixture.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::IncompatibleSchema(_) + )); + assert!(phases.is_empty()); + assert_eq!(fixture.pins().await.len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_database_creates_nothing_and_deletes_nothing() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + let (outcome, phases) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + assert!(phases.is_empty()); + assert!(!fixture.state_root.exists()); + assert_eq!(fixture.pins().await.len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_first_failed_pass_without_anchor_is_recommended_and_advised_once() { + use super::super::ref_advisory::{advise_if_due_with, AdvisoryOutcome}; + + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.anchor, None); + assert_eq!(state.last_success, None); + let now = state.last_attempt.expect("attempt time"); + assert!(evaluate_recommendation(&state, now).recommended); + + let first = advise_if_due_with(&fixture.root, || now, write_state_atomically); + assert!(matches!(first, AdvisoryOutcome::Advised)); + let second = advise_if_due_with(&fixture.root, || now + 1000, write_state_atomically); + assert!(matches!(second, AdvisoryOutcome::NoAction)); + + let StateRead::Valid(after) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(after.consecutive_failures, 1); + assert_eq!(after.last_failure, state.last_failure); + assert!(evaluate_recommendation(&after, now + 1000).recommended); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_failure_streak_counts_but_lock_contention_never_does() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + for expected in 1..=2 { + let (outcome, _) = fixture.run().await; + assert!(matches!( + outcome, + ExplicitOutcome::Failed { + state_warning: None, + .. + } + )); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.consecutive_failures, expected); + assert_eq!(state.last_success, None); + } + + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let before = std::fs::read(state_path(&git_dir)).expect("state bytes"); + let held = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("hold lock"); + let mut phases = Vec::new(); + let outcome = reconcile_explicit_with( + &fixture.root, + async || -> anyhow::Result { unreachable!("db must not open") }, + || NOW, + |_, _| unreachable!("state must not be written on contention"), + |phase| phases.push(phase), + Duration::ZERO, + ) + .await; + drop(held); + + assert!(matches!( + outcome, + ExplicitOutcome::Skipped(SkipReason::Busy) + )); + assert!(phases.is_empty()); + assert_eq!( + std::fs::read(state_path(&git_dir)).expect("state bytes"), + before + ); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.consecutive_failures, 2); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_database_and_failed_persistence_keep_both_errors() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + let tree = pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture.run_with(failing_write(PersistPhase::Rename)).await; + + let ExplicitOutcome::Failed { + error, + state_warning, + } = outcome + else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + let Some(StatePersistWarning::WriteFailed(PersistFailure::NotApplied { phase, .. })) = + state_warning + else { + panic!("expected retained persistence warning, got {state_warning:?}"); + }; + assert_eq!(phase, PersistPhase::Rename); + let pins = fixture.pins().await; + assert_eq!(pins.len(), 1); + assert_eq!(pins[0].tree.0, tree); + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + assert_eq!( + staging_leftovers(state_path(&git_dir).parent().expect("parent")), + 0 + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_required_pin_and_failed_persistence_keep_both_errors() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + let snapshot = GitSnapshotService::new(&fixture.root) + .await + .expect("snapshot"); + let unpinned = snapshot.capture_tree().await.expect("tree"); + seed_durable_root(&fixture.db_path(), &unpinned.0).await; + + let (outcome, phases) = fixture + .run_with(failing_write(PersistPhase::WriteStaging)) + .await; + + let ExplicitOutcome::Failed { + error, + state_warning, + } = outcome + else { + panic!("expected failure, got {outcome:?}"); + }; + let ReconcileError::MissingRequiredPins { missing } = &error else { + panic!("expected missing required pins, got {error:?}"); + }; + assert_eq!(missing, &vec![unpinned]); + assert!(matches!( + state_warning, + Some(StatePersistWarning::WriteFailed( + PersistFailure::NotApplied { + phase: PersistPhase::WriteStaging, + .. + } + )) + )); + assert_eq!( + phases, + vec![ReconcilePhase::DbOpened, ReconcilePhase::PinsInventoried] + ); + assert!(fixture.pins().await.is_empty()); + assert!(matches!(fixture.stored_state().await, StateRead::Absent)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_completed_pass_with_unconfirmed_durability_keeps_deletion() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture + .run_with(failing_write(PersistPhase::SyncParentDirectory)) + .await; + + let ExplicitOutcome::CompletedStatePersistFailed { report, warning } = outcome else { + panic!("expected persist-failed completion, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(matches!( + warning, + StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + }) + )); + assert!(fixture.pins().await.is_empty()); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("new state is visible even though durability is unconfirmed"); + }; + assert_eq!(state.last_success, Some(NOW)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture + .run_with(failing_write(PersistPhase::CreateStaging)) + .await; + + let ExplicitOutcome::CompletedStatePersistFailed { report, warning } = outcome else { + panic!("expected persist-failed completion, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(warning.to_string().contains("previous state preserved")); + assert!(fixture.pins().await.is_empty()); + assert!(matches!(fixture.stored_state().await, StateRead::Absent)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_unreadable_previous_state_is_reported_without_overwriting() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let path = state_path(&git_dir); + std::fs::create_dir_all(&path).expect("directory in place of state file"); + + let pins_before = fixture.pins().await; + assert_eq!(pins_before.len(), 1); + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Failed { + error, + state_warning, + } = outcome + else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + assert!(matches!( + state_warning, + Some(StatePersistWarning::PreviousStateUnreadable(_)) + )); + assert!(path.is_dir()); + assert_eq!(std::fs::read_dir(&path).expect("read state dir").count(), 0); + assert_eq!(fixture.pins().await, pins_before); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_lock_held_implementation_never_reacquires_the_lock() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let lock = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("lock"); + + let outcome = tokio::time::timeout( + Duration::from_secs(30), + reconcile_with_held_lock( + &fixture.root, + &lock, + async || open_authoritative_db_at_state_root(&fixture.root, &fixture.state_root).await, + |_| {}, + ), + ) + .await + .expect("must not block on a second acquisition") + .expect("reconcile"); + drop(lock); + + let ReconciliationOutcome::Reconciled(report) = outcome; + assert_eq!(report.deleted, 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_never_creates_missing_database_or_directories() { + let dir = tempfile::tempdir().expect("temp dir"); + let missing = dir.path().join("nested").join("agent-trace.db"); + + let error = RepositoryAgentTraceDb::open_verified_existing_at(&missing, "repo") + .await + .err() + .expect("missing db fails"); + + assert!(matches!(error, ExistingRepositoryDbError::Missing { .. })); + assert!(!dir.path().join("nested").exists()); + + let missing_in_existing_dir = dir.path().join("agent-trace.db"); + let error = RepositoryAgentTraceDb::open_verified_existing_at(&missing_in_existing_dir, "repo") + .await + .err() + .expect("missing db fails"); + assert!(matches!(error, ExistingRepositoryDbError::Missing { .. })); + assert_eq!(std::fs::read_dir(dir.path()).expect("read dir").count(), 0); +} + +const DATABASE_SIDECAR_SUFFIXES: [&str; 3] = ["-wal", "-tshm", "-shm"]; + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +fn remove_database(path: &Path) { + std::fs::remove_file(path).ok(); + for suffix in DATABASE_SIDECAR_SUFFIXES { + std::fs::remove_file(with_suffix(path, suffix)).ok(); + } +} + +fn swap_in_database(source: &Path, target: &Path) { + remove_database(target); + for suffix in DATABASE_SIDECAR_SUFFIXES { + let sidecar = with_suffix(source, suffix); + if sidecar.exists() { + std::fs::rename(sidecar, with_suffix(target, suffix)).expect("move sidecar"); + } + } + std::fs::rename(source, target).expect("move database"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_rejects_database_removed_before_open() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + let existed_when_hook_ran = std::cell::Cell::new(false); + + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || { + existed_when_hook_ran.set(path.is_file()); + remove_database(&path); + }, + || {}, + ) + .await + .err() + .expect("removed db fails"); + + assert!(existed_when_hook_ran.get()); + assert!(matches!(error, ExistingRepositoryDbError::Missing { .. })); + assert!(!path.exists()); + assert_eq!(std::fs::read_dir(dir.path()).expect("read dir").count(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_rejects_database_replaced_before_open() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + let foreign = dir.path().join("foreign.db"); + create_db(&foreign, "someone-else").await; + + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&foreign, &path), + || {}, + ) + .await + .err() + .expect("replacement with another repository fails"); + assert!(matches!( + error, + ExistingRepositoryDbError::RepositoryMismatch { .. } + )); + + let empty = dir.path().join("empty.db"); + drop( + RepositoryAgentTraceDb::open_without_migrations_at(&empty) + .await + .expect("empty db"), + ); + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&empty, &path), + || {}, + ) + .await + .err() + .expect("replacement without schema fails"); + assert!(matches!( + error, + ExistingRepositoryDbError::IncompatibleSchema(_) + )); + + let unusable = dir.path().join("unusable.db"); + std::fs::write(&unusable, b"this is not a database").expect("garbage"); + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&unusable, &path), + || {}, + ) + .await + .err() + .expect("replacement with garbage fails"); + assert!(matches!( + error, + ExistingRepositoryDbError::IncompatibleSchema(_) | ExistingRepositoryDbError::Unreadable(_) + )); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_accepts_a_replacement_only_if_it_verifies_itself() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + let replacement = dir.path().join("replacement.db"); + create_db(&replacement, "repo").await; + + let (_, metadata) = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&replacement, &path), + || {}, + ) + .await + .expect("a replacement that passes every check is itself authoritative"); + assert_eq!(metadata.repository_id, "repo"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_never_recreates_a_database_removed_after_open() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + + let (db, metadata) = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || {}, + || remove_database(&path), + ) + .await + .expect("the opened handle is what gets verified"); + assert_eq!(metadata.repository_id, "repo"); + drop(db); + + assert!(!path.exists()); + assert_eq!(std::fs::read_dir(dir.path()).expect("read dir").count(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_is_read_only() { + const INSERT_WORKTREE: &str = "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES ('main', 'tree', x'0000000000000000', 0, 'healthy', 0)"; + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + + let writer = RepositoryAgentTraceDb::new_at(&path).await.expect("writer"); + writer.execute("BEGIN", ()).await.expect("begin"); + writer + .execute(INSERT_WORKTREE, ()) + .await + .expect("the statement is valid on a writable connection"); + writer.execute("ROLLBACK", ()).await.expect("rollback"); + drop(writer); + + let (db, _) = RepositoryAgentTraceDb::open_verified_existing_at(&path, "repo") + .await + .expect("verified open"); + let error = db + .execute(INSERT_WORKTREE, ()) + .await + .expect_err("verified connection must reject writes"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("database is readonly"), "{rendered}"); + drop(db); + + let inspector = RepositoryAgentTraceDb::new_at(&path) + .await + .expect("inspector"); + let rows = inspector + .query_map( + "SELECT worktree_id FROM mutation_trace_worktrees", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("rows"); + assert!(rows.is_empty()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_coexists_with_a_live_multiprocess_wal_writer() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + let writer = RepositoryAgentTraceDb::new_at(&path).await.expect("writer"); + writer + .verify_or_initialize_repository_metadata("repo") + .await + .expect("metadata"); + + let (reader, metadata) = RepositoryAgentTraceDb::open_verified_existing_at(&path, "repo") + .await + .expect("verified open alongside live writer"); + assert_eq!(metadata.repository_id, "repo"); + + writer + .execute( + "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES ('main', 'tree-after-open', x'0000000000000000', 0, 'healthy', 0)", + (), + ) + .await + .expect("writer still writes"); + let rows = reader + .query_map( + "SELECT cursor_tree FROM mutation_trace_worktrees", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("reader sees committed writer data"); + assert_eq!(rows, vec!["tree-after-open".to_string()]); + assert!(with_suffix(&path, "-tshm").exists()); +} + +async fn pin_distinct(root: &Path, content: &str) -> String { + std::fs::write(root.join("distinct.txt"), content).expect("write distinct file"); + pin_orphan(root).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_retains_every_repository_wide_durable_root_and_never_repairs_other_worktrees( +) { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + let local = pin_distinct(&fixture.root, "local\n").await; + let other = pin_distinct(&fixture.root, "other\n").await; + let orphan_one = pin_distinct(&fixture.root, "orphan-one\n").await; + let orphan_two = pin_distinct(&fixture.root, "orphan-two\n").await; + let unpinned_elsewhere = "e".repeat(40); + seed_durable_root(&fixture.db_path(), &local).await; + seed_durable_root_for(&fixture.db_path(), "worktrees/other", &other).await; + seed_durable_root_for(&fixture.db_path(), "worktrees/ghost", &unpinned_elsewhere).await; + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 1, + retained: 2, + deleted: 2, + } + ); + let mut remaining: Vec = fixture + .pins() + .await + .into_iter() + .map(|pin| pin.tree.0) + .collect(); + remaining.sort(); + let mut expected = vec![local, other]; + expected.sort(); + assert_eq!(remaining, expected); + assert!(!remaining.contains(&orphan_one) && !remaining.contains(&orphan_two)); + assert!(!remaining.contains(&unpinned_elsewhere)); + + let (second, _) = fixture.run().await; + let ExplicitOutcome::Completed(second) = second else { + panic!("expected idempotent completed pass"); + }; + assert_eq!((second.retained, second.deleted), (2, 0)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_malformed_namespace_ref_deletes_nothing_and_records_failure() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + git( + &fixture.root, + &[ + "symbolic-ref", + "refs/sce/mutation-cursor/main/not-a-tree", + "HEAD", + ], + ); + let inventory_before = ref_inventory(&fixture.root); + assert_eq!(inventory_before.len(), 2); + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + let ReconcileError::MalformedPin { ref_name, .. } = &error else { + panic!("expected MalformedPin, got {error:?}"); + }; + assert_eq!(ref_name, "refs/sce/mutation-cursor/main/not-a-tree"); + assert_eq!( + ref_inventory(&fixture.root), + inventory_before, + "no ref may be deleted, replaced or retargeted" + ); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.consecutive_failures, 1); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/coordination.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/coordination.rs new file mode 100644 index 000000000..e95d5c313 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/coordination.rs @@ -0,0 +1,741 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use tokio::sync::mpsc::unbounded_channel; + +use super::super::super::coordinator::coordinate_inner; +use super::super::super::external_taint::ExternalTaintMarker; +use super::super::super::git_snapshot::resolve_git_dir; +use super::super::super::maintenance_state::{ + state_path, write_state_atomically, MaintenanceState, PersistPhase, + RECONCILIATION_ADVISORY_AFTER_MS, +}; +use super::super::super::protected_worktree::WORKTREE_LOCK_TIMEOUT; +use super::super::super::ref_advisory::{advise_if_due, advise_if_due_with, AdvisoryOutcome}; +use super::super::super::ref_reconciliation::{ReconcilePhase, ReconciliationReport}; +use super::super::super::worktree_lock::acquire_inner; +use super::super::ExplicitOutcome; +use super::support::{ + await_signal, close_payload, durable_snapshot, hook_ok, main_worktree, no_advice, pinned_trees, + run_hook, sce_refs, spawn_parked_pass, start_payload, writer, WATCHDOG, +}; +use super::{failing_write, pin_orphan, Fixture, NOW}; +use crate::services::hooks::mutation_scope::MutationScopePayload; +use crate::services::mutation_trace::runtime::{advise_after_completed_boundary, RuntimeBoundary}; +use crate::services::mutation_trace::types::ScopeStatus; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; +const SCOPE: &str = "scope-1"; + +async fn git_dir_of(fixture: &Fixture) -> PathBuf { + resolve_git_dir(&fixture.root).await.expect("git dir") +} + +async fn started_fixture() -> Fixture { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + hook_ok(&fixture, start_payload(SCOPE, "start-1")).await; + fixture +} + +fn state_bytes(git_dir: &Path) -> Option> { + std::fs::read(state_path(git_dir)).ok() +} + +async fn assert_coordinate_blocked_until_pass_releases(phase: ReconcilePhase) { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let baseline = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(baseline.roots.len(), 1); + let cursor_tree = baseline.worktree.cursor_tree.0.clone(); + + super::support::write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + super::support::write_file(&fixture.root, "next.txt", "next\n"); + + let mut pass = spawn_parked_pass(&fixture, phase); + await_signal(&mut pass.parked).await; + let refs_while_parked = sce_refs(&fixture.root); + assert_eq!( + pinned_trees(&fixture.root, "main"), + [cursor_tree.clone(), orphan.clone()].into_iter().collect() + ); + + let (contended_tx, mut contended_rx) = unbounded_channel::<()>(); + let db_opened = Arc::new(AtomicBool::new(false)); + let refs_at_load = Arc::new(Mutex::new(Vec::>::new())); + let db_path = fixture.db_path(); + let root = fixture.root.clone(); + + let coordinate = coordinate_inner( + &root, + &RuntimeBoundary::Flush, + async || { + db_opened.store(true, Ordering::SeqCst); + writer(&db_path).await + }, + WATCHDOG, + move || { + let _ = contended_tx.send(()); + }, + |_attempt| { + refs_at_load + .lock() + .expect("refs at load") + .push(sce_refs(&root)); + }, + |_attempt| Ok(()), + ); + + let marker = ExternalTaintMarker::new(&git_dir); + let driver = async { + await_signal(&mut contended_rx).await; + assert!( + !db_opened.load(Ordering::SeqCst), + "the coordinator must not reach its protected section while the pass holds the lock" + ); + assert!( + !marker.exists().expect("marker"), + "the coordinator must not arm its marker before it owns the lock" + ); + assert_eq!(sce_refs(&fixture.root), refs_while_parked); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + baseline + ); + pass.finish().await + }; + + let (coordinated, outcome) = tokio::join!(coordinate, driver); + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 1, + retained: 1, + deleted: 1, + } + ); + let coordinated = coordinated.expect("the coordinator completes once the pass releases"); + let new_tree = coordinated.observed_tree.0.clone(); + assert_ne!(new_tree, cursor_tree); + assert_ne!(new_tree, orphan); + + assert_eq!( + pinned_trees(&fixture.root, "main"), + [cursor_tree, new_tree.clone()].into_iter().collect(), + "the pin created after the pass released the lock must survive; only the orphan is gone" + ); + assert!(!refs_while_parked + .iter() + .any(|line| line.contains(&new_tree))); + let seen_at_load = refs_at_load.lock().expect("refs at load").clone(); + assert_new_pin_preceded_the_durable_root(&fixture, &new_tree, &seen_at_load).await; + + drop( + acquire_inner(&git_dir, Duration::ZERO, || {}) + .expect("neither operation leaves the lock held"), + ); +} + +async fn assert_new_pin_preceded_the_durable_root( + fixture: &Fixture, + new_tree: &str, + refs_at_load: &[Vec], +) { + assert_eq!(refs_at_load.len(), 1); + assert!( + refs_at_load[0].iter().any(|line| line.contains(new_tree)), + "the new pin must already exist before the durable root commit" + ); + let after = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(after.worktree.cursor_tree.0, new_tree); + assert!(after.roots.iter().any(|root| root.0 == new_tree)); + for root in &after.roots { + assert!( + pinned_trees(&fixture.root, "main").contains(&root.0), + "every durable root stays reachable" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_db_open() { + assert_coordinate_blocked_until_pass_releases(ReconcilePhase::DbOpened).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_pin_inventory() { + assert_coordinate_blocked_until_pass_releases(ReconcilePhase::PinsInventoried).await; +} + +#[derive(Clone, Copy)] +enum Boundary { + Close, + Flush, +} + +impl Boundary { + fn payload(self) -> MutationScopePayload { + match self { + Boundary::Close => close_payload(SCOPE, "close-1"), + Boundary::Flush => MutationScopePayload::Flush, + } + } + + fn scope_after_success(self) -> ScopeStatus { + match self { + Boundary::Close => ScopeStatus::Closed, + Boundary::Flush => ScopeStatus::Active, + } + } +} + +async fn assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries( + boundary: Boundary, +) { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + super::support::write_file(&fixture.root, "work.txt", "work\n"); + let before = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + let refs_before = sce_refs(&fixture.root); + let pins_before = pinned_trees(&fixture.root, "main"); + assert_eq!(before.scope, Some(ScopeStatus::Active)); + + let mut pass = spawn_parked_pass(&fixture, ReconcilePhase::DbOpened); + await_signal(&mut pass.parked).await; + + let advisories = Arc::new(AtomicU32::new(0)); + let contended = Arc::new(AtomicU32::new(0)); + let counting = |advisories: &Arc| { + let advisories = Arc::clone(advisories); + move |root: &Path| { + advisories.fetch_add(1, Ordering::SeqCst); + no_advice(root) + } + }; + let failed = run_hook( + &fixture.root, + &fixture.db_path(), + boundary.payload(), + Duration::from_millis(300), + { + let contended = Arc::clone(&contended); + move || { + contended.fetch_add(1, Ordering::SeqCst); + } + }, + || {}, + counting(&advisories), + ) + .await + .expect_err("the lock cannot be acquired while the pass holds it"); + + let message = format!("{failed:#}"); + assert!( + message.contains("before durable completion") + && message.contains("Timed out after") + && message.contains("waiting for worktree lock"), + "unexpected error: {message}" + ); + assert_eq!(contended.load(Ordering::SeqCst), 1); + assert_eq!(advisories.load(Ordering::SeqCst), 0); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + before + ); + assert_eq!(sce_refs(&fixture.root), refs_before); + assert_eq!(pinned_trees(&fixture.root, "main"), pins_before); + assert!(!marker.exists().expect("marker")); + assert_eq!(state_bytes(&git_dir), None); + + let outcome = pass.finish().await; + assert!(matches!(outcome, ExplicitOutcome::Completed(_))); + let after_pass_state = state_bytes(&git_dir); + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + boundary.payload(), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + counting(&advisories), + ) + .await + .expect("the same boundary succeeds once the maintenance pass released the lock"); + assert_eq!(output, ""); + assert_eq!(advisories.load(Ordering::SeqCst), 1); + assert_eq!(state_bytes(&git_dir), after_pass_state); + + let after = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(after.scope, Some(boundary.scope_after_success())); + assert_eq!(after.worktree.revision, before.worktree.revision + 1); + assert_ne!(after.worktree.cursor_tree, before.worktree.cursor_tree); + assert!(pinned_trees(&fixture.root, "main").contains(&after.worktree.cursor_tree.0)); + assert!(after.roots.contains(&after.worktree.cursor_tree)); + assert!(!marker.exists().expect("marker")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries() +{ + assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary::Close) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_flush_lock_timeout_under_maintenance_commits_nothing_and_retries() { + assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary::Flush) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock() { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + super::support::write_file(&fixture.root, "work.txt", "work\n"); + let before = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + + let at_advisory = Arc::new(Mutex::new(None)); + let refs_around = Arc::new(Mutex::new(None)); + let db_path = fixture.db_path(); + let advise = { + let at_advisory = Arc::clone(&at_advisory); + let refs_around = Arc::clone(&refs_around); + let marker = marker.clone(); + let git_dir = git_dir.clone(); + move |root: &Path| { + drop( + acquire_inner(&git_dir, Duration::ZERO, || {}) + .expect("the coordinator lock is released before the advisory runs"), + ); + *at_advisory.lock().expect("snapshot slot") = Some(tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(durable_snapshot( + &db_path, + &main_worktree(), + SCOPE, + )) + })); + let marker_before = marker.exists().expect("marker"); + let refs_before = sce_refs(root); + let report = advise_after_completed_boundary(root); + *refs_around.lock().expect("refs slot") = Some((refs_before, sce_refs(root))); + assert_eq!( + marker_before, + marker.exists().expect("marker"), + "the advisory never creates, clears or modifies the marker" + ); + assert!(!marker_before); + report + } + }; + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload(SCOPE, "close-1"), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + advise, + ) + .await + .expect("hook succeeds"); + + assert_eq!(output, ""); + let committed = at_advisory + .lock() + .expect("snapshot slot") + .clone() + .expect("advisory callback ran"); + assert_eq!(committed.scope, Some(ScopeStatus::Closed)); + assert_eq!(committed.worktree.revision, before.worktree.revision + 1); + assert_ne!(committed.worktree.cursor_tree, before.worktree.cursor_tree); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + committed, + "the advisory must not change the committed result" + ); + let (refs_before, refs_after) = refs_around + .lock() + .expect("refs slot") + .clone() + .expect("refs recorded"); + assert_eq!(refs_before, refs_after, "the advisory never touches refs"); + assert!(state_bytes(&git_dir).is_some()); + assert!(!marker.exists().expect("marker")); + drop( + acquire_inner(&git_dir, Duration::ZERO, || {}) + .expect("no lock is held after the advisory returns"), + ); +} + +fn marker_fingerprint(path: &Path) -> (bool, Vec) { + let metadata = std::fs::symlink_metadata(path).expect("marker path exists"); + let mut children: Vec = if metadata.is_dir() { + std::fs::read_dir(path) + .expect("read marker dir") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect() + } else { + Vec::new() + }; + children.sort(); + (metadata.is_dir(), children) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate() +{ + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + let marker_path = git_dir.join("sce").join("mutation-cursor-tainted"); + let saved_path = git_dir.join("sce").join("mutation-cursor-tainted.saved"); + assert!(!marker.exists().expect("marker")); + + super::support::write_file(&fixture.root, "one.txt", "one\n"); + hook_ok( + &fixture, + super::support::start_payload("scope-2", "start-2"), + ) + .await; + let control = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(control.scope, Some(ScopeStatus::Active)); + assert!(!marker.exists().expect("marker")); + + super::support::write_file(&fixture.root, "two.txt", "two\n"); + let before_fault = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + let armed_len = Arc::new(Mutex::new(None)); + let fingerprints = Arc::new(Mutex::new(None)); + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload("scope-2", "close-2"), + WORKTREE_LOCK_TIMEOUT, + || {}, + { + let marker_path = marker_path.clone(); + let saved_path = saved_path.clone(); + let armed_len = Arc::clone(&armed_len); + move || { + *armed_len.lock().expect("len slot") = + Some(std::fs::metadata(&marker_path).expect("armed marker").len()); + std::fs::rename(&marker_path, &saved_path).expect("set armed marker aside"); + std::fs::create_dir(&marker_path).expect("block unlink with a directory"); + } + }, + { + let marker_path = marker_path.clone(); + let fingerprints = Arc::clone(&fingerprints); + move |root| { + let before = marker_fingerprint(&marker_path); + let refs_before = sce_refs(root); + let report = advise_after_completed_boundary(root); + let after = marker_fingerprint(&marker_path); + assert_eq!(refs_before, sce_refs(root)); + *fingerprints.lock().expect("fingerprint slot") = Some((before, after)); + report + } + }, + ) + .await + .expect("MarkerClearAfterCommit is a durable success for the hook"); + assert_eq!(output, ""); + + let (fingerprint_before, fingerprint_after) = fingerprints + .lock() + .expect("fingerprint slot") + .clone() + .expect("advisory ran after the failed marker clear"); + assert_eq!(fingerprint_before, fingerprint_after); + assert!(fingerprint_before.0, "the injected fault is still in place"); + + let after_fault = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!( + after_fault.worktree.revision, + before_fault.worktree.revision + 1 + ); + assert_eq!(after_fault.scope, Some(ScopeStatus::Active)); + + std::fs::remove_dir(&marker_path).expect("remove injected fault"); + std::fs::rename(&saved_path, &marker_path).expect("restore the armed marker"); + assert!(marker.exists().expect("marker")); + assert_eq!( + std::fs::metadata(&marker_path) + .expect("restored marker") + .len(), + armed_len + .lock() + .expect("len slot") + .expect("length recorded") + ); + + super::support::write_file(&fixture.root, "three.txt", "three\n"); + hook_ok(&fixture, close_payload("scope-2", "close-3")).await; + + let recovered = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!( + recovered.scope, + Some(ScopeStatus::Abandoned), + "inherited taint conservatively abandons the live scope" + ); + assert!(recovered.worktree.revision >= after_fault.worktree.revision + 2); + assert!(!recovered.worktree.tainted); + assert!(!recovered.worktree.needs_rebaseline); + assert!(pinned_trees(&fixture.root, "main").contains(&recovered.worktree.cursor_tree.0)); + assert!(!marker.exists().expect("marker")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_advisory_after_a_real_commit_is_busy_when_a_pass_wins_the_handoff() { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + super::support::write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + super::support::write_file(&fixture.root, "work.txt", "work\n"); + + let (handoff_tx, mut handoff_rx) = unbounded_channel::<()>(); + let (holding_tx, holding_rx) = std::sync::mpsc::channel::<()>(); + let (advisory_done_tx, mut advisory_done_rx) = unbounded_channel::<()>(); + + let orchestrator = { + let root = fixture.root.clone(); + let state_root = fixture.state_root.clone(); + tokio::spawn(async move { + await_signal(&mut handoff_rx).await; + let mut pass = spawn_parked_pass_for(&root, &state_root); + await_signal(&mut pass.parked).await; + holding_tx.send(()).expect("resume the advisory"); + await_signal(&mut advisory_done_rx).await; + pass.finish().await + }) + }; + + let observed = Arc::new(Mutex::new(None)); + let advise = { + let observed = Arc::clone(&observed); + let git_dir = git_dir.clone(); + move |root: &Path| { + handoff_tx.send(()).expect("announce the handoff point"); + holding_rx + .recv_timeout(WATCHDOG) + .expect("the pass takes the lock"); + let state_before = state_bytes(&git_dir); + let refs_before = sce_refs(root); + let report = advise_after_completed_boundary(root); + *observed.lock().expect("observed slot") = Some(( + state_before, + state_bytes(&git_dir), + refs_before, + sce_refs(root), + report.outcome, + )); + advisory_done_tx.send(()).expect("advisory finished"); + report + } + }; + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload(SCOPE, "close-1"), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + advise, + ) + .await + .expect("the committed boundary still reports success"); + assert_eq!(output, ""); + + let (state_before, state_after, refs_before, refs_after, outcome) = observed + .lock() + .expect("observed slot") + .clone() + .expect("advisory ran"); + assert_eq!(outcome, "busy"); + assert_eq!(state_before, state_after); + assert_eq!(refs_before, refs_after); + + let pass_outcome = orchestrator.await.expect("orchestrator joins"); + let ExplicitOutcome::Completed(report) = pass_outcome else { + panic!("expected completed pass, got {pass_outcome:?}"); + }; + assert_eq!(report.deleted, 1); + let committed = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(committed.scope, Some(ScopeStatus::Closed)); + let pins = pinned_trees(&fixture.root, "main"); + assert!(!pins.contains(&orphan)); + assert!(pins.contains(&committed.worktree.cursor_tree.0)); +} + +fn spawn_parked_pass_for(root: &Path, state_root: &Path) -> super::support::ParkedPass { + super::support::spawn_parked_pass_at(root, state_root, ReconcilePhase::DbOpened) +} + +async fn advisory_from_independent_task(root: &Path, now: i64) -> AdvisoryOutcome { + let root = root.to_path_buf(); + tokio::task::spawn_blocking(move || advise_if_due(&root, || now)) + .await + .expect("advisory task joins") +} + +async fn assert_independent_advisory_is_busy_while_parked(phase: ReconcilePhase) { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + let orphan = pin_orphan(&fixture.root).await; + let git_dir = git_dir_of(&fixture).await; + + let mut pass = spawn_parked_pass(&fixture, phase); + await_signal(&mut pass.parked).await; + let refs_while_parked = sce_refs(&fixture.root); + let state_while_parked = state_bytes(&git_dir); + + let advisory = + tokio::time::timeout(WATCHDOG, advisory_from_independent_task(&fixture.root, NOW)) + .await + .expect("the advisory must not wait for the pass"); + assert!(matches!(advisory, AdvisoryOutcome::Busy)); + assert_eq!(state_bytes(&git_dir), state_while_parked); + assert_eq!(sce_refs(&fixture.root), refs_while_parked); + assert!(pinned_trees(&fixture.root, "main").contains(&orphan)); + + let outcome = pass.finish().await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(pinned_trees(&fixture.root, "main").is_empty()); + + let recorded = state_bytes(&git_dir).expect("pass recorded state"); + assert!(matches!( + advisory_from_independent_task(&fixture.root, NOW).await, + AdvisoryOutcome::NoAction + )); + assert_eq!(state_bytes(&git_dir), Some(recorded)); + assert!(matches!( + advisory_from_independent_task(&fixture.root, NOW + RECONCILIATION_ADVISORY_AFTER_MS + 1) + .await, + AdvisoryOutcome::Advised + )); + let stored: MaintenanceState = + match super::super::super::maintenance_state::read_state(&state_path(&git_dir)) + .expect("state") + { + super::super::super::maintenance_state::StateRead::Valid(state) => state, + other => panic!("expected valid state, got {other:?}"), + }; + assert_eq!(stored.last_success, Some(NOW)); + assert_eq!( + stored.last_advised, + Some(NOW + RECONCILIATION_ADVISORY_AFTER_MS + 1) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_independent_advisory_task_is_busy_while_a_pass_is_parked_after_db_open() +{ + assert_independent_advisory_is_busy_while_parked(ReconcilePhase::DbOpened).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_independent_advisory_task_is_busy_while_a_pass_is_parked_after_pin_inventory( +) { + assert_independent_advisory_is_busy_while_parked(ReconcilePhase::PinsInventoried).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_advisory_failures_after_a_real_commit_never_change_the_hook_result() { + for (case, expected) in [ + ("write_failed", "state_write_failed"), + ("corrupt_state", "anchored"), + ("unavailable_state", "state_unavailable"), + ] { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + match case { + "corrupt_state" => std::fs::write(&path, b"not json").expect("corrupt state"), + "unavailable_state" => std::fs::create_dir(&path).expect("directory as state"), + _ => {} + } + super::support::write_file(&fixture.root, "work.txt", "work\n"); + + let observed = Arc::new(Mutex::new(None)); + let db_path = fixture.db_path(); + let advise = { + let observed = Arc::clone(&observed); + let marker = marker.clone(); + move |root: &Path| { + let committed = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(durable_snapshot( + &db_path, + &main_worktree(), + SCOPE, + )) + }); + let refs_before = sce_refs(root); + let marker_before = marker.exists().expect("marker"); + let outcome = if case == "write_failed" { + advise_if_due_with(root, || NOW, failing_write(PersistPhase::Rename)) + } else { + advise_if_due_with(root, || NOW, write_state_atomically) + }; + let report = outcome.report(); + *observed.lock().expect("observed slot") = Some(( + committed, + refs_before == sce_refs(root), + marker_before == marker.exists().expect("marker"), + report.outcome, + )); + report + } + }; + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload(SCOPE, "close-1"), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + advise, + ) + .await + .unwrap_or_else(|error| panic!("{case}: hook must succeed, got {error:#}")); + + assert_eq!(output, "", "{case}"); + let (committed, refs_unchanged, marker_unchanged, outcome) = observed + .lock() + .expect("observed slot") + .clone() + .expect("advisory ran"); + assert_eq!(outcome, expected, "{case}"); + assert!(refs_unchanged, "{case}"); + assert!(marker_unchanged, "{case}"); + assert_eq!(committed.scope, Some(ScopeStatus::Closed), "{case}"); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + committed, + "{case}" + ); + assert!(!marker.exists().expect("marker"), "{case}"); + } +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/cross_process.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/cross_process.rs new file mode 100644 index 000000000..58b5cb461 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/cross_process.rs @@ -0,0 +1,166 @@ +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; + +use super::super::super::maintenance_state::{ + read_state, state_path, write_state_atomically, MaintenanceState, StateRead, + RECONCILIATION_ADVISORY_AFTER_MS, +}; +use super::super::super::ref_advisory::{advise_if_due, advise_if_due_with}; +use super::support::WATCHDOG; +use super::{Fixture, NOW}; + +const ROLE_ENV: &str = "SCE_CROSS_PROCESS_ADVISORY_ROLE"; +const ROOT_ENV: &str = "SCE_CROSS_PROCESS_ADVISORY_ROOT"; +const NOW_ENV: &str = "SCE_CROSS_PROCESS_ADVISORY_NOW"; +const ROLE_PARK: &str = "park"; +const ROLE_PROBE: &str = "probe"; +const PARKED_LINE: &str = "PARKED"; +const OUTCOME_PREFIX: &str = "OUTCOME:"; +const CHILD_TEST: &str = "ref_reconciliation_cross_process_advisory_child"; + +fn child_test_path() -> String { + let module = module_path!(); + let (_, relative) = module + .split_once("::") + .expect("crate-qualified module path"); + format!("{relative}::{CHILD_TEST}") +} + +fn child_command(role: &str, root: &Path) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current exe")); + command + .args(["--exact", &child_test_path(), "--nocapture"]) + .env(ROLE_ENV, role) + .env(ROOT_ENV, root) + .env(NOW_ENV, NOW.to_string()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + command +} + +fn outcome_of(stdout: &str) -> String { + stdout + .lines() + .find_map(|line| line.strip_prefix(OUTCOME_PREFIX)) + .unwrap_or_else(|| panic!("child reported no outcome:\n{stdout}")) + .to_string() +} + +fn run_probe(root: &Path) -> String { + let output = child_command(ROLE_PROBE, root) + .stdin(Stdio::null()) + .output() + .expect("probe child runs"); + assert!( + output.status.success(), + "probe child failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + outcome_of(&String::from_utf8_lossy(&output.stdout)) +} + +#[test] +fn ref_reconciliation_cross_process_advisory_child() { + let Ok(role) = std::env::var(ROLE_ENV) else { + return; + }; + let root = PathBuf::from(std::env::var(ROOT_ENV).expect("root")); + let now: i64 = std::env::var(NOW_ENV) + .expect("now") + .parse() + .expect("numeric now"); + let outcome = match role.as_str() { + ROLE_PARK => advise_if_due_with( + &root, + || now, + |path, state| { + println!("{PARKED_LINE}"); + std::io::stdout().flush().expect("flush stdout"); + let mut release = String::new(); + std::io::stdin() + .read_line(&mut release) + .expect("wait for release"); + write_state_atomically(path, state) + }, + ), + ROLE_PROBE => advise_if_due(&root, || now), + other => panic!("unknown role {other}"), + }; + let name = format!("{outcome:?}"); + let name = name + .split([' ', '{']) + .next() + .unwrap_or_default() + .to_string(); + println!("{OUTCOME_PREFIX}{name}"); +} + +#[test] +fn ref_reconciliation_advisory_lock_serializes_independent_processes() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + let git_dir = fixture.root.join(".git"); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().expect("runtime dir")).expect("runtime dir"); + let due = MaintenanceState { + anchor: Some(NOW - RECONCILIATION_ADVISORY_AFTER_MS - 1), + ..MaintenanceState::default() + }; + write_state_atomically(&path, &due).expect("seed due state"); + let seeded = std::fs::read(&path).expect("seeded bytes"); + + let mut parked = child_command(ROLE_PARK, &fixture.root) + .stdin(Stdio::piped()) + .spawn() + .expect("park child spawns"); + let mut stdout = BufReader::new(parked.stdout.take().expect("park stdout")); + let (line_tx, line_rx) = mpsc::channel::(); + let reader = std::thread::spawn(move || { + let mut lines = Vec::new(); + let mut line = String::new(); + while stdout.read_line(&mut line).expect("read child stdout") > 0 { + let trimmed = line.trim_end().to_string(); + if trimmed == PARKED_LINE { + let _ = line_tx.send(trimmed.clone()); + } + lines.push(trimmed); + line.clear(); + } + lines + }); + assert_eq!( + line_rx.recv_timeout(WATCHDOG).expect("child parks"), + PARKED_LINE + ); + + assert_eq!(run_probe(&fixture.root), "Busy"); + assert_eq!( + std::fs::read(&path).expect("state while parked"), + seeded, + "a second process must not write while the first holds the lock" + ); + + let mut stdin = parked.stdin.take().expect("park stdin"); + stdin + .write_all(b"release\n") + .expect("release the parked child"); + drop(stdin); + let status = parked.wait().expect("park child exits"); + assert!(status.success()); + let lines = reader.join().expect("reader joins"); + assert_eq!( + outcome_of(&lines.join("\n")), + "Advised", + "the lock holder recorded the one advice" + ); + + assert_eq!(run_probe(&fixture.root), "NoAction"); + + let StateRead::Valid(stored) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_eq!(stored.last_advised, Some(NOW)); + assert_eq!(stored.anchor, due.anchor); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/fail_closed.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/fail_closed.rs new file mode 100644 index 000000000..d8183fb6c --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/fail_closed.rs @@ -0,0 +1,99 @@ +use std::time::Duration; + +use super::super::super::external_taint::ExternalTaintMarker; +use super::super::super::git_snapshot::resolve_git_dir; +use super::super::super::maintenance_state::{read_state, state_path, StateRead}; +use super::super::super::ref_reconciliation::{ReconcileError, ReconcilePhase}; +use super::super::reconcile_explicit_with; +use super::super::{ExplicitOutcome, SkipReason}; +use super::support::{await_signal, pinned_trees, spawn_parked_pass, write_file}; +use super::{pin_orphan, seed_durable_root, Fixture, NOW}; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_local_pin_fails_closed_while_other_orphans_exist() { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + write_file(&fixture.root, "required.txt", "required\n"); + let snapshot = super::super::super::git_snapshot::GitSnapshotService::new(&fixture.root) + .await + .expect("snapshot"); + let unpinned = snapshot.capture_tree().await.expect("tree"); + seed_durable_root(&fixture.db_path(), &unpinned.0).await; + let refs_before = pinned_trees(&fixture.root, "main"); + assert_eq!(refs_before, [orphan].into_iter().collect()); + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + let ReconcileError::MissingRequiredPins { missing } = error else { + panic!("expected missing required pins, got {error:?}"); + }; + assert_eq!(missing, vec![unpinned]); + assert_eq!( + pinned_trees(&fixture.root, "main"), + refs_before, + "the eligible orphan is not deleted when a required pin is missing" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_database_failure_never_arms_the_external_taint_marker() { + let fixture = Fixture::new(REMOTE); + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let marker = ExternalTaintMarker::new(&git_dir); + + let (outcome, _) = fixture.run().await; + + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + assert!(!marker.exists().expect("marker")); + assert_eq!(pinned_trees(&fixture.root, "main").len(), 1); + let StateRead::Valid(state) = read_state(&state_path(&git_dir)).expect("state") else { + panic!("expected recorded failure"); + }; + assert_eq!(state.consecutive_failures, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_second_explicit_pass_is_busy_and_records_exactly_one_pass() { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + + let mut first = spawn_parked_pass(&fixture, ReconcilePhase::PinsInventoried); + await_signal(&mut first.parked).await; + + let second = reconcile_explicit_with( + &fixture.root, + async || -> anyhow::Result<_> { unreachable!("a busy pass never opens the database") }, + || NOW, + |_, _| unreachable!("a busy pass never writes state"), + |_| {}, + Duration::ZERO, + ) + .await; + assert!(matches!(second, ExplicitOutcome::Skipped(SkipReason::Busy))); + assert!(matches!( + read_state(&state_path(&git_dir)).expect("state"), + StateRead::Absent + )); + + let outcome = first.finish().await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + let StateRead::Valid(state) = read_state(&state_path(&git_dir)).expect("state") else { + panic!("expected recorded success"); + }; + assert_eq!(state.last_success, Some(NOW)); + assert_eq!(state.consecutive_failures, 0); + assert!(pinned_trees(&fixture.root, "main").is_empty()); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/state_recovery.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/state_recovery.rs new file mode 100644 index 000000000..9a8e0fb6f --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/state_recovery.rs @@ -0,0 +1,413 @@ +use std::path::Path; +use std::time::Duration; + +use super::super::super::external_taint::ExternalTaintMarker; +use super::super::super::git_snapshot::resolve_git_dir; +use super::super::super::maintenance_state::{ + evaluate_recommendation, read_state, state_path, write_state_atomically, MaintenanceState, + PersistFailure, PersistPhase, StateRead, FUTURE_SKEW_TOLERANCE_MS, + RECONCILIATION_ADVISORY_AFTER_MS, +}; +use super::super::super::ref_advisory::{advise_if_due, AdvisoryOutcome}; +use super::super::super::worktree_lock::acquire_inner_async; +use super::super::{reconcile_explicit_with, ExplicitOutcome, SkipReason, StatePersistWarning}; +use super::support::pinned_trees; +use super::{ + failing_write, pin_orphan, remove_database, unavailable_cause, Fixture, NOW, + OTHER_REPOSITORY_ID, +}; +use crate::services::agent_trace_db::repository::{ + ExistingRepositoryDbError, RepositoryAgentTraceDb, +}; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; + +async fn seed_state(fixture: &Fixture, state: &MaintenanceState) -> std::path::PathBuf { + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + write_state_atomically(&path, state).expect("seed state"); + path +} + +fn invalid_state() -> MaintenanceState { + MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + last_success: Some(NOW + FUTURE_SKEW_TOLERANCE_MS + 1), + last_advised: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + } +} + +fn assert_within_tolerance(state: &MaintenanceState) { + for timestamp in [ + state.anchor, + state.last_success, + state.last_attempt, + state.last_advised, + ] + .into_iter() + .flatten() + { + assert!(timestamp <= NOW + FUTURE_SKEW_TOLERANCE_MS); + } +} + +async fn logical_dump(path: &Path) -> Vec { + let db = RepositoryAgentTraceDb::open_without_migrations_at(path) + .await + .expect("inspection open"); + let tables = db + .query_map( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("tables"); + let mut dump = Vec::new(); + for table in &tables { + let columns = db + .query_map( + &format!("SELECT name FROM pragma_table_info('{table}') ORDER BY cid"), + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("columns"); + let projection = columns + .iter() + .map(|column| format!("quote(\"{column}\")")) + .collect::>() + .join(" || '|' || "); + let rows = db + .query_map( + &format!("SELECT {projection} FROM \"{table}\" ORDER BY rowid"), + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("rows"); + dump.push(format!("{table}:{}", rows.join(";"))); + } + if tables.iter().any(|table| table == "__sce_migrations") { + let ids = db + .query_map( + "SELECT CAST(id AS TEXT) FROM __sce_migrations ORDER BY id", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("migrations"); + dump.push(format!("migrations:{}", ids.join(","))); + } + if tables.iter().any(|table| table == "repository_metadata") { + let ids = db + .query_map( + "SELECT repository_id FROM repository_metadata ORDER BY id", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("metadata"); + dump.push(format!("metadata:{}", ids.join(","))); + } + dump +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verification_failures_leave_the_logical_database_unchanged() { + for case in ["incompatible", "missing_metadata", "mismatch"] { + let fixture = Fixture::new(REMOTE); + match case { + "incompatible" => drop( + RepositoryAgentTraceDb::open_without_migrations_at(fixture.db_path()) + .await + .expect("unmigrated db"), + ), + "missing_metadata" => drop( + RepositoryAgentTraceDb::new_at(fixture.db_path()) + .await + .expect("migrated db"), + ), + _ => super::create_db(&fixture.db_path(), OTHER_REPOSITORY_ID).await, + } + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let before = logical_dump(&fixture.db_path()).await; + if case == "mismatch" { + assert!(before.iter().any(|line| line.contains(OTHER_REPOSITORY_ID))); + } + + let (outcome, phases) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("{case}: expected failure, got {outcome:?}"); + }; + match (case, unavailable_cause(&error)) { + ("incompatible", ExistingRepositoryDbError::IncompatibleSchema(_)) + | ("missing_metadata", ExistingRepositoryDbError::MissingMetadata) + | ("mismatch", ExistingRepositoryDbError::RepositoryMismatch { .. }) => {} + (_, other) => panic!("{case}: unexpected cause {other:?}"), + } + assert!(phases.is_empty(), "{case}"); + assert_eq!(logical_dump(&fixture.db_path()).await, before, "{case}"); + assert_eq!(pinned_trees(&fixture.root, "main").len(), 1, "{case}"); + assert!(!ExternalTaintMarker::new(&git_dir).exists().expect("marker")); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_busy_explicit_pass_leaves_invalid_state_byte_identical() { + let fixture = Fixture::new(REMOTE); + let path = seed_state(&fixture, &invalid_state()).await; + let before = std::fs::read(&path).expect("state bytes"); + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let held = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("hold lock"); + + let outcome = reconcile_explicit_with( + &fixture.root, + async || -> anyhow::Result { unreachable!("db must not open") }, + || NOW, + |_, _| unreachable!("state must not be written"), + |_| {}, + Duration::ZERO, + ) + .await; + drop(held); + + assert!(matches!( + outcome, + ExplicitOutcome::Skipped(SkipReason::Busy) + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + let StateRead::Valid(state) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert!(evaluate_recommendation(&state, NOW).invalid_timestamps); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_successful_pass_normalizes_invalid_state_and_a_later_read_does_not_rewrite( +) { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let near_future = NOW + FUTURE_SKEW_TOLERANCE_MS; + let seeded = MaintenanceState { + last_advised: Some(near_future), + ..invalid_state() + }; + let path = seed_state(&fixture, &seeded).await; + + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Completed(_))); + + let StateRead::Valid(normalized) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_within_tolerance(&normalized); + assert_eq!(normalized.last_success, Some(NOW)); + assert_eq!(normalized.last_advised, Some(near_future)); + let bytes = std::fs::read(&path).expect("state bytes"); + + assert!(matches!( + advise_if_due(&fixture.root, || NOW), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + assert!(!evaluate_recommendation(&normalized, NOW).invalid_timestamps); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_failed_state_write_during_normalization_has_two_distinct_legs() { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let path = seed_state(&fixture, &invalid_state()).await; + let before = std::fs::read(&path).expect("state bytes"); + + let (pre_rename, _) = fixture.run_with(failing_write(PersistPhase::Rename)).await; + let ExplicitOutcome::CompletedStatePersistFailed { report, warning } = pre_rename else { + panic!("expected persist-failed completion, got {pre_rename:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(matches!( + warning, + StatePersistWarning::WriteFailed(PersistFailure::NotApplied { + phase: PersistPhase::Rename, + .. + }) + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + + pin_orphan(&fixture.root).await; + let (post_rename, _) = fixture + .run_with(failing_write(PersistPhase::SyncParentDirectory)) + .await; + let ExplicitOutcome::CompletedStatePersistFailed { warning, .. } = post_rename else { + panic!("expected persist-failed completion, got {post_rename:?}"); + }; + assert!(matches!( + warning, + StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + }) + )); + let StateRead::Valid(visible) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_within_tolerance(&visible); + assert_eq!(visible.last_success, Some(NOW)); + + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!( + advise_if_due(&fixture.root, || NOW), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_explicit_success_resets_the_streak_and_failure_keeps_last_success() { + let fixture = Fixture::new(REMOTE); + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let path = state_path(&git_dir); + let stored = || match read_state(&path).expect("state") { + StateRead::Valid(state) => state, + other => panic!("expected valid state, got {other:?}"), + }; + + for expected in 1..=2 { + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + assert_eq!(stored().consecutive_failures, expected); + } + + fixture.create_db().await; + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Completed(_))); + let healed = stored(); + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.last_failure, None); + assert_eq!(healed.last_success, Some(NOW)); + + remove_database(&fixture.db_path()); + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + let failed = stored(); + assert_eq!(failed.consecutive_failures, 1); + assert_eq!(failed.last_success, Some(NOW)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_explicit_pass_runs_and_rewrites_valid_state_from_every_stored_condition( +) { + let aged = MaintenanceState { + anchor: Some(NOW - 3 * RECONCILIATION_ADVISORY_AFTER_MS), + last_advised: Some(NOW - 2 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + for case in ["aged", "invalid", "corrupt", "oversized"] { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let path = match case { + "aged" => seed_state(&fixture, &aged).await, + "invalid" => seed_state(&fixture, &invalid_state()).await, + _ => { + let path = seed_state(&fixture, &MaintenanceState::default()).await; + let junk = if case == "corrupt" { + b"not json".to_vec() + } else { + vec![b' '; 5000] + }; + std::fs::write(&path, junk).expect("unusable state"); + path + } + }; + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("{case}: expected completed pass, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1, "{case}"); + let StateRead::Valid(rewritten) = read_state(&path).expect("state") else { + panic!("{case}: state must be rewritten valid"); + }; + assert_eq!(rewritten.last_success, Some(NOW), "{case}"); + assert_within_tolerance(&rewritten); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_unnormalized_invalid_state_stays_eligible_until_a_successful_advisory() +{ + let fixture = Fixture::new(REMOTE); + let invalid = MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + let path = seed_state(&fixture, &invalid).await; + let before = std::fs::read(&path).expect("state bytes"); + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + + for step in 0..3 { + let now = NOW + step * RECONCILIATION_ADVISORY_AFTER_MS / 2; + let recommendation = evaluate_recommendation(&invalid, now); + assert!(recommendation.recommended); + assert!(recommendation.invalid_timestamps); + } + let held = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("hold lock"); + for step in 0..3 { + assert!(matches!( + advise_if_due(&fixture.root, || NOW + + step * RECONCILIATION_ADVISORY_AFTER_MS), + AdvisoryOutcome::Busy + )); + } + drop(held); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + + let later = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + assert!(matches!( + advise_if_due(&fixture.root, || later), + AdvisoryOutcome::Advised + )); + let StateRead::Valid(state) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_eq!(state.last_advised, Some(later)); + assert_eq!(state.anchor, Some(later)); + assert!(matches!( + advise_if_due(&fixture.root, || later), + AdvisoryOutcome::NoAction + )); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_invalid_last_advised_does_not_suppress_a_due_advisory() { + let fixture = Fixture::new(REMOTE); + let seeded = MaintenanceState { + last_success: Some(NOW - RECONCILIATION_ADVISORY_AFTER_MS - 1), + last_advised: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + let path = seed_state(&fixture, &seeded).await; + + assert!(matches!( + advise_if_due(&fixture.root, || NOW), + AdvisoryOutcome::Advised + )); + let StateRead::Valid(state) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_eq!(state.last_advised, Some(NOW)); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/support.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/support.rs new file mode 100644 index 000000000..0af626163 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/support.rs @@ -0,0 +1,263 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::mpsc; +use std::time::Duration; + +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; + +use super::super::super::coordinator::{coordinate_inner, CoordinateOutcome}; +use super::super::super::maintenance_state::write_state_atomically; +use super::super::super::protected_worktree::WORKTREE_LOCK_TIMEOUT; +use super::super::super::ref_reconciliation::ReconcilePhase; +use super::super::{open_authoritative_db_at_state_root, reconcile_explicit_with, ExplicitOutcome}; +use super::{Fixture, NOW}; +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::hooks::mutation_scope::{drive_mutation_scope, MutationScopePayload}; +use crate::services::mutation_trace::runtime::{ + AbandonScopeError, AbandonScopeOutcome, AdvisoryReport, RuntimeBoundary, +}; +use crate::services::mutation_trace::store::MutationTraceStore; +use crate::services::mutation_trace::types::{ + ActorKind, ScopeId, ScopeStatus, TreeId, WorktreeId, WorktreeState, +}; +use crate::services::observability::traits::NoopLogger; + +pub(super) const WATCHDOG: Duration = Duration::from_mins(1); + +pub(super) fn main_worktree() -> WorktreeId { + WorktreeId("main".to_string()) +} + +pub(super) fn start_payload(scope: &str, event: &str) -> MutationScopePayload { + MutationScopePayload::Start { + scope_id: scope.to_string(), + event_id: event.to_string(), + actor_kind: ActorKind::ClaudeCode, + provenance: None, + } +} + +pub(super) fn close_payload(scope: &str, event: &str) -> MutationScopePayload { + MutationScopePayload::Close { + scope_id: scope.to_string(), + event_id: event.to_string(), + actor_kind: ActorKind::ClaudeCode, + } +} + +pub(super) fn write_file(root: &Path, name: &str, content: &str) { + std::fs::write(root.join(name), content).expect("write working tree file"); +} + +pub(super) fn sce_refs(root: &Path) -> Vec { + let output = Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/sce/", + ]) + .current_dir(root) + .output() + .expect("for-each-ref"); + assert!(output.status.success(), "for-each-ref failed"); + let mut lines: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect(); + lines.sort(); + lines +} + +pub(super) fn pinned_trees(root: &Path, worktree: &str) -> BTreeSet { + let prefix = format!("refs/sce/mutation-cursor/{worktree}/"); + sce_refs(root) + .into_iter() + .filter_map(|line| { + let (name, tree) = line.split_once(' ')?; + name.strip_prefix(&prefix) + .filter(|rest| !rest.contains('/')) + .map(|_| tree.to_string()) + }) + .collect() +} + +pub(super) async fn writer(db_path: &Path) -> anyhow::Result { + RepositoryAgentTraceDb::open_without_migrations_at(db_path).await +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct DurableSnapshot { + pub worktree: WorktreeState, + pub scope: Option, + pub roots: BTreeSet, +} + +pub(super) async fn durable_snapshot( + db_path: &Path, + worktree: &WorktreeId, + scope: &str, +) -> DurableSnapshot { + let mut db = writer(db_path).await.expect("inspection db"); + let store = MutationTraceStore::new(&mut db); + let projection = store + .load_worktree(worktree, None, None) + .await + .expect("load worktree") + .expect("worktree row"); + let scope = store + .load_scope(&ScopeId(scope.to_string())) + .await + .expect("load scope") + .map(|state| state.status); + let roots = store.load_all_tree_roots().await.expect("roots"); + DurableSnapshot { + worktree: projection.worktree_state, + scope, + roots, + } +} + +pub(super) async fn run_hook( + root: &Path, + db_path: &Path, + payload: MutationScopePayload, + lock_timeout: Duration, + on_contention: C, + in_open_db: O, + advise: V, +) -> anyhow::Result +where + C: FnOnce() + Send + 'static, + O: FnOnce(), + V: FnOnce(&Path) -> AdvisoryReport, +{ + drive_mutation_scope( + root, + payload, + None::<&NoopLogger>, + &mut Vec::new(), + async |root: &Path, boundary: &RuntimeBoundary| { + coordinate_inner( + root, + boundary, + async || { + in_open_db(); + writer(db_path).await + }, + lock_timeout, + on_contention, + |_attempt| {}, + |_attempt| Ok(()), + ) + .await + }, + async |_root: &Path, _scope: &ScopeId| -> Result { + unreachable!("abandon is not driven by these tests") + }, + advise, + ) + .await +} + +pub(super) fn no_advice(_root: &Path) -> AdvisoryReport { + AdvisoryReport { + outcome: "no_action", + severity: crate::services::mutation_trace::runtime::AdvisorySeverity::Debug, + recommendation: None, + warning: None, + diagnostic: None, + } +} + +pub(super) async fn hook_ok(fixture: &Fixture, payload: MutationScopePayload) -> String { + run_hook( + &fixture.root, + &fixture.db_path(), + payload, + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + no_advice, + ) + .await + .expect("hook boundary completes") +} + +pub(super) async fn coordinate_flush(root: &Path, db_path: &Path) -> CoordinateOutcome { + coordinate_inner( + root, + &RuntimeBoundary::Flush, + async || writer(db_path).await, + WORKTREE_LOCK_TIMEOUT, + || {}, + |_attempt| {}, + |_attempt| Ok(()), + ) + .await + .expect("flush completes") +} + +pub(super) struct ParkedPass { + outcome: tokio::sync::oneshot::Receiver, + pub parked: UnboundedReceiver<()>, + release: mpsc::Sender<()>, +} + +impl ParkedPass { + pub(super) async fn finish(self) -> ExplicitOutcome { + self.release.send(()).expect("release the pass"); + tokio::time::timeout(WATCHDOG, self.outcome) + .await + .expect("pass watchdog") + .expect("pass thread reports its outcome") + } +} + +pub(super) fn spawn_parked_pass(fixture: &Fixture, target: ReconcilePhase) -> ParkedPass { + spawn_parked_pass_at(&fixture.root, &fixture.state_root, target) +} + +pub(super) fn spawn_parked_pass_at( + root: &Path, + state_root: &Path, + target: ReconcilePhase, +) -> ParkedPass { + let root: PathBuf = root.to_path_buf(); + let state_root: PathBuf = state_root.to_path_buf(); + let (parked_tx, parked) = unbounded_channel(); + let (release, release_rx) = mpsc::channel::<()>(); + let (outcome_tx, outcome) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("pass runtime"); + let result = runtime.block_on(reconcile_explicit_with( + &root, + async || open_authoritative_db_at_state_root(&root, &state_root).await, + || NOW, + write_state_atomically, + move |phase| { + if phase == target { + let _ = parked_tx.send(()); + let _ = release_rx.recv_timeout(WATCHDOG); + } + }, + Duration::from_secs(10), + )); + let _ = outcome_tx.send(result); + }); + ParkedPass { + outcome, + parked, + release, + } +} + +pub(super) async fn await_signal(receiver: &mut UnboundedReceiver<()>) { + tokio::time::timeout(WATCHDOG, receiver.recv()) + .await + .expect("watchdog expired while waiting for a synchronization signal") + .expect("signal sender dropped before signalling"); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/worktrees.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/worktrees.rs new file mode 100644 index 000000000..70064c806 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/worktrees.rs @@ -0,0 +1,263 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use super::super::super::git_snapshot::{resolve_worktree_id, GitSnapshotService}; +use super::super::super::maintenance_state::write_state_atomically; +use super::super::super::ref_reconciliation::{ReconcileError, ReconciliationReport}; +use super::super::{open_authoritative_db_at_state_root, reconcile_explicit_with, ExplicitOutcome}; +use super::support::{ + coordinate_flush, hook_ok, pinned_trees, sce_refs, start_payload, write_file, writer, +}; +use super::{git, pin_orphan, Fixture, NOW}; +use crate::services::mutation_trace::store::MutationTraceStore; +use crate::services::mutation_trace::types::WorktreeId; +use crate::services::repository_identity::resolve::resolve_repository_identity; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; + +struct Linked { + fixture: Fixture, + root: PathBuf, + id: String, +} + +impl Linked { + async fn new() -> Self { + let fixture = Fixture::new(REMOTE); + let root = fixture.dir.path().join("linked"); + git( + &fixture.root, + &[ + "worktree", + "add", + "-q", + "-b", + "linked-branch", + root.to_str().expect("utf-8 path"), + ], + ); + let main_identity = resolve_repository_identity(&fixture.root, None, "origin") + .expect("main identity") + .identity + .repository_id; + let linked_identity = resolve_repository_identity(&root, None, "origin") + .expect("linked identity") + .identity + .repository_id; + assert_eq!(main_identity, linked_identity); + assert_eq!( + resolve_worktree_id(&fixture.root).await.expect("main id"), + WorktreeId("main".to_string()) + ); + let id = resolve_worktree_id(&root).await.expect("linked id").0; + assert!(id.starts_with("worktrees/")); + fixture.create_db().await; + Self { fixture, root, id } + } + + async fn run(&self, root: &Path) -> ExplicitOutcome { + reconcile_explicit_with( + root, + async || open_authoritative_db_at_state_root(root, &self.fixture.state_root).await, + || NOW, + write_state_atomically, + |_| {}, + Duration::from_secs(10), + ) + .await + } + + async fn linked_hook( + &self, + payload: crate::services::hooks::mutation_scope::MutationScopePayload, + ) { + super::support::run_hook( + &self.root, + &self.fixture.db_path(), + payload, + super::super::super::protected_worktree::WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + super::support::no_advice, + ) + .await + .expect("linked hook completes"); + } + + fn linked_pins(&self) -> BTreeSet { + pinned_trees(&self.root, &self.id) + } + + fn main_pins(&self) -> BTreeSet { + pinned_trees(&self.fixture.root, "main") + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_historical_roots_of_a_real_coordinator_sequence_stay_protected() { + let linked = Linked::new().await; + let fixture = &linked.fixture; + hook_ok(fixture, start_payload("scope-1", "start-1")).await; + let db_path = fixture.db_path(); + + let mut trees = BTreeSet::new(); + let mut store_db = writer(&db_path).await.expect("db"); + let cursor = MutationTraceStore::new(&mut store_db) + .load_worktree(&WorktreeId("main".to_string()), None, None) + .await + .expect("load") + .expect("row") + .worktree_state + .cursor_tree + .0; + trees.insert(cursor); + for (name, content) in [("b.txt", "b\n"), ("c.txt", "c\n"), ("d.txt", "d\n")] { + write_file(&fixture.root, name, content); + let outcome = coordinate_flush(&fixture.root, &db_path).await; + trees.insert(outcome.observed_tree.0); + } + assert_eq!(trees.len(), 4); + assert_eq!(linked.main_pins(), trees); + + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + assert!(!trees.contains(&orphan)); + + let outcome = linked.run(&fixture.root).await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 4, + retained: 4, + deleted: 1, + } + ); + assert_eq!( + linked.main_pins(), + trees, + "historical A, B, C and D are all retained although only D is the cursor" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_pin_of_one_real_worktree_is_retained_while_another_needs_the_tree() { + let linked = Linked::new().await; + let fixture = &linked.fixture; + + write_file(&fixture.root, "shared.txt", "shared\n"); + let shared = pin_orphan(&fixture.root).await; + write_file(&linked.root, "shared.txt", "shared\n"); + linked + .linked_hook(start_payload("linked-scope", "linked-start")) + .await; + assert_eq!(linked.linked_pins(), [shared.clone()].into_iter().collect()); + + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + assert_eq!( + linked.main_pins(), + [shared.clone(), orphan.clone()].into_iter().collect() + ); + + let outcome = linked.run(&fixture.root).await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 1, + deleted: 1, + } + ); + assert_eq!( + linked.main_pins(), + [shared.clone()].into_iter().collect(), + "main's pin is the same tree the linked worktree durably requires" + ); + assert_eq!(linked.linked_pins(), [shared].into_iter().collect()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_pin_in_another_real_worktree_neither_fails_nor_is_repaired() { + let linked = Linked::new().await; + let fixture = &linked.fixture; + let db_path = fixture.db_path(); + + write_file(&fixture.root, "shared.txt", "shared\n"); + let shared = pin_orphan(&fixture.root).await; + write_file(&linked.root, "shared.txt", "shared\n"); + linked + .linked_hook(start_payload("linked-scope", "linked-start")) + .await; + write_file(&linked.root, "more.txt", "more\n"); + let flushed = coordinate_flush(&linked.root, &db_path).await; + let later = flushed.observed_tree.0; + assert_eq!( + linked.linked_pins(), + [shared.clone(), later.clone()].into_iter().collect() + ); + + let linked_ref = format!("refs/sce/mutation-cursor/{}/{shared}", linked.id); + git(&linked.root, &["update-ref", "-d", &linked_ref]); + assert_eq!(linked.linked_pins(), [later.clone()].into_iter().collect()); + + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + let refs_of_linked_before: Vec = sce_refs(&fixture.root) + .into_iter() + .filter(|line| line.contains(&linked.id)) + .collect(); + + let outcome = linked.run(&fixture.root).await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 1, + deleted: 1, + } + ); + assert!(!linked.main_pins().contains(&orphan)); + assert_eq!( + linked.main_pins(), + [shared.clone()].into_iter().collect(), + "the remaining protection for the other worktree's tree is retained" + ); + let refs_of_linked_after: Vec = sce_refs(&fixture.root) + .into_iter() + .filter(|line| line.contains(&linked.id)) + .collect(); + assert_eq!(refs_of_linked_before, refs_of_linked_after); + assert_eq!( + linked.linked_pins(), + [later].into_iter().collect(), + "reconciling main never recreates the other worktree's missing pin" + ); + + let degraded = linked.run(&linked.root).await; + let ExplicitOutcome::Failed { error, .. } = degraded else { + panic!("the degraded worktree fails closed in its own pass, got {degraded:?}"); + }; + let ReconcileError::MissingRequiredPins { missing } = error else { + panic!("expected missing required pins, got {error:?}"); + }; + assert_eq!(missing.len(), 1); + assert_eq!(missing[0].0, shared); + assert!(GitSnapshotService::new(&linked.root) + .await + .expect("snapshot") + .list_pins(&WorktreeId(linked.id.clone())) + .await + .expect("pins") + .iter() + .all(|pin| pin.tree.0 != shared)); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs b/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs index 0b7da0d9c..50c7f63f7 100644 --- a/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs +++ b/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs @@ -1,6 +1,5 @@ use std::collections::BTreeSet; use std::path::Path; -use std::time::Duration; use anyhow::Result; @@ -9,28 +8,9 @@ use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::mutation_trace::store::MutationTraceStore; use crate::services::mutation_trace::types::TreeId; -use super::git_snapshot::{resolve_git_dir, GitSnapshotService, PinInventoryError, PinnedRef}; -use super::worktree_lock::{acquire_inner_async, WorktreeLockError}; - -const RECONCILIATION_LOCK_TIMEOUT: Duration = Duration::from_secs(10); - -/// Outcome counts of one successful reconciliation pass. -/// -/// - `local_required` — the target worktree's own durable-root count -/// (`load_tree_roots(W).len()`), the left side of the local consistency -/// invariant. -/// - `retained` — `actual_W.len() - deleted`: pins left in place, whether -/// because the target worktree still needs their tree or because another -/// worktree in the repository durably does. -/// - `deleted` — pins actually removed (inventoried under `W`'s prefix, tree -/// absent from the repository-wide durable root set). -/// -/// `retained == local_required` is **not** an invariant — a pin retained only -/// because another worktree durably needs its tree counts toward `retained` -/// but not `local_required`. For `ReconciliationOutcome::Reconciled(report)` -/// the only relation that holds is `report.local_required <= report.retained`. -/// `ReconciliationOutcome::SkippedNoCheckoutIdentity` carries no report, so no -/// report invariant applies to it. +use super::git_snapshot::{GitSnapshotService, PinInventoryError, PinnedRef}; +use super::worktree_lock::{WorktreeLock, WorktreeLockError}; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ReconciliationReport { pub local_required: usize, @@ -38,50 +18,22 @@ pub struct ReconciliationReport { pub deleted: usize, } -/// Outcome of one reconciliation pass: a real pass that ran, carrying its -/// [`ReconciliationReport`], versus a skip because no current checkout identity -/// could be derived (an `Ok`, never an `Err`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ReconciliationOutcome { Reconciled(ReconciliationReport), - SkippedNoCheckoutIdentity, } -/// Why a reconciliation pass could not complete. One variant per fallible step, -/// no `Other` catch-all — mirroring `CoordinateError`'s convention. Every -/// non-`Ok` outcome leaves the SCE ref namespace in a consistent state: either -/// untouched, or (only on `Ok`) with exactly the stale refs gone. #[derive(Debug)] pub enum ReconcileError { - /// `resolve_git_dir` failed. GitDir(anyhow::Error), - /// The worktree's `WorktreeLock` could not be acquired (timeout or I/O). Lock(WorktreeLockError), - /// Git topology could not be resolved into a worktree identity. CheckoutIdentity(anyhow::Error), - /// The caller-supplied `open_db` provider returned `Err`. This is a - /// reconciliation maintenance error only: it never arms - /// `ExternalTaintMarker` and never becomes - /// `CoordinateError::AgentTraceDbUnavailable`, because no mutation boundary - /// is being coordinated. AgentTraceDbUnavailable(anyhow::Error), - /// `GitSnapshotService::new` failed. SnapshotService(anyhow::Error), - /// `git for-each-ref` itself failed to execute or exited non-zero - /// (`PinInventoryError::Git`). PinInventory(anyhow::Error), - /// A ref inside the SCE mutation-cursor namespace is not shaped like a - /// `pin_tree` output (`PinInventoryError::MalformedRef`). Reconciliation - /// deletes nothing. MalformedPin { ref_name: String, reason: String }, - /// `load_tree_roots` / `load_all_tree_roots` failed (DB query error, - /// migration `003` absent, ...). DurableRoots(anyhow::Error), - /// A durable root of the **target** worktree has no live pin — the local - /// consistency invariant is violated. Fail closed: nothing is deleted. MissingRequiredPins { missing: Vec }, - /// The atomic `delete_pins` transaction failed (including a ref that - /// changed since inventory). Nothing is deleted. DeleteTransaction(anyhow::Error), } @@ -115,40 +67,22 @@ impl std::fmt::Display for ReconcileError { impl std::error::Error for ReconcileError {} -/// Reconcile one worktree's SCE snapshot pins: remove orphan / unreferenced -/// pins while retaining every tree any current or historical durable -/// mutation-cursor state in the repository still references. -/// -/// Module-private to `runtime`, exactly like `coordinate` — never re-exported -/// outside mutation-trace `runtime`. It is a one-line delegation to -/// [`reconcile_worktree_inner`] with a no-op lock-contention closure. -pub async fn reconcile_worktree

( - repository_root: &Path, - open_db: P, -) -> std::result::Result -where - P: std::ops::AsyncFnOnce() -> Result, -{ - reconcile_worktree_inner(repository_root, open_db, || {}).await +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ReconcilePhase { + DbOpened, + PinsInventoried, } -pub(super) async fn reconcile_worktree_inner( +pub(super) async fn reconcile_with_held_lock( repository_root: &Path, + lock: &WorktreeLock, open_db: P, - on_lock_contention: F, + mut on_phase: H, ) -> std::result::Result where P: std::ops::AsyncFnOnce() -> Result, - F: FnOnce() + Send + 'static, + H: FnMut(ReconcilePhase), { - let git_dir = resolve_git_dir(repository_root) - .await - .map_err(ReconcileError::GitDir)?; - - let lock = acquire_inner_async(&git_dir, RECONCILIATION_LOCK_TIMEOUT, on_lock_contention) - .await - .map_err(ReconcileError::Lock)?; - let worktree_id = resolve_worktree_id(repository_root) .await .map_err(ReconcileError::CheckoutIdentity)?; @@ -156,13 +90,12 @@ where let mut db = open_db() .await .map_err(ReconcileError::AgentTraceDbUnavailable)?; + on_phase(ReconcilePhase::DbOpened); let snapshot = GitSnapshotService::new(repository_root) .await .map_err(ReconcileError::SnapshotService)?; - // Inventory the worktree's pins first, so every durable-root read that - // follows is compared against a fixed observation of the namespace. let actual = snapshot .list_pins(&worktree_id) .await @@ -172,13 +105,11 @@ where ReconcileError::MalformedPin { ref_name, reason } } })?; + on_phase(ReconcilePhase::PinsInventoried); let pinned_trees: BTreeSet = actual.iter().map(|pin| pin.tree.clone()).collect(); let store = MutationTraceStore::new(&mut db); - // Local consistency invariant (a strictly per-worktree check): every tree - // the target worktree's own durable evidence references must still have a - // live pin, or the pass fails closed and deletes nothing. let required_local = store .load_tree_roots(&worktree_id) .await @@ -190,10 +121,6 @@ where }); } - // Deletion safety invariant: an owned ref is removed only when its tree is - // outside the durable root set of **every** worktree in the repository — - // linked worktrees share one object database, so an A-owned ref may be the - // last SCE ref protecting a tree that only worktree B durably requires. let required_repository = store .load_all_tree_roots() .await diff --git a/cli/src/services/mutation_trace/runtime/scope_runtime.rs b/cli/src/services/mutation_trace/runtime/scope_runtime.rs index 59bc27cde..c8e220bb6 100644 --- a/cli/src/services/mutation_trace/runtime/scope_runtime.rs +++ b/cli/src/services/mutation_trace/runtime/scope_runtime.rs @@ -60,6 +60,10 @@ pub enum AbandonScopeError { }, MarkerClearAfterCompletion { source: anyhow::Error, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] completed: Box, }, Other(anyhow::Error), diff --git a/cli/src/services/mutation_trace/runtime/worktree_lock.rs b/cli/src/services/mutation_trace/runtime/worktree_lock.rs index 0610826da..cd7a404f3 100644 --- a/cli/src/services/mutation_trace/runtime/worktree_lock.rs +++ b/cli/src/services/mutation_trace/runtime/worktree_lock.rs @@ -15,7 +15,6 @@ const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(100); #[derive(Debug)] struct WorktreeLockInner { file: File, - path: PathBuf, unlock_on_drop: AtomicBool, } @@ -55,10 +54,6 @@ impl std::fmt::Display for WorktreeLockError { impl std::error::Error for WorktreeLockError {} impl WorktreeLock { - pub fn acquire(git_dir: &Path, timeout: Duration) -> Result { - acquire_inner(git_dir, timeout, || {}) - } - pub async fn acquire_async( git_dir: &Path, timeout: Duration, @@ -121,7 +116,6 @@ where return Ok(WorktreeLock { inner: Arc::new(WorktreeLockInner { file, - path: lock_path, unlock_on_drop: AtomicBool::new(true), }), }); diff --git a/cli/src/services/mutation_trace/store.rs b/cli/src/services/mutation_trace/store.rs index f200db063..8bdace440 100644 --- a/cli/src/services/mutation_trace/store.rs +++ b/cli/src/services/mutation_trace/store.rs @@ -194,6 +194,10 @@ pub fn encode_boundary_kind(kind: BoundaryKind) -> &'static str { } /// Decodes a [`BoundaryKind`] from `mutation_trace_events.boundary_kind`. +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] pub fn decode_boundary_kind(value: &str) -> Result { match value { "start" => Ok(BoundaryKind::Start), @@ -216,6 +220,10 @@ const SELECT_SCOPE_PROVENANCE_SQL: &str = "SELECT scope_id, session_id, model_id FROM mutation_trace_scope_provenance WHERE scope_id = ?1"; const SELECT_PROCESSED_EVENT_SQL: &str = "SELECT 1 FROM mutation_trace_processed_events WHERE scope_id = ?1 AND event_id = ?2"; +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] const SELECT_MUTATION_EVENT_SQL: &str = "SELECT before_tree, after_tree, tainted, failure_kind, attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id FROM mutation_trace_events WHERE worktree_id = ?1 AND revision = ?2"; @@ -236,21 +244,18 @@ const SELECT_LATEST_MUTATION_EVENT_REVISION_SQL: &str = "SELECT revision FROM mu WHERE worktree_id = ?1 ORDER BY revision DESC LIMIT 1"; +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] const SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL: &str = "SELECT scope_id FROM mutation_trace_event_active_scopes WHERE worktree_id = ?1 AND revision = ?2"; -/// One worktree's complete durable tree root set — its cursor tree plus the -/// `before_tree` / `after_tree` of every historical `mutation_trace_events` -/// row — as a single `UNION` statement so the whole set is read from one -/// database snapshot, never assembled from independent `SELECT`s. const SELECT_TREE_ROOTS_BY_WORKTREE_SQL: &str = "SELECT cursor_tree AS tree FROM mutation_trace_worktrees WHERE worktree_id = ?1 UNION SELECT before_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1 UNION SELECT after_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1"; -/// The same three `TreeId` columns unioned across **every** worktree in the -/// repository, in one statement / one snapshot — the reconciler's -/// repository-wide retention set. const SELECT_ALL_TREE_ROOTS_SQL: &str = "SELECT cursor_tree AS tree FROM mutation_trace_worktrees UNION SELECT before_tree AS tree FROM mutation_trace_events @@ -688,6 +693,10 @@ impl<'a> MutationTraceStore<'a> { /// revision)`, decoding its full `Attribution` and `Boundary`, or `None` /// when no such row exists. Never called from `load_worktree` or from /// any hook-boundary path. + #[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" + )] pub async fn load_mutation_event( &self, worktree: &WorktreeId, @@ -794,27 +803,6 @@ impl<'a> MutationTraceStore<'a> { Ok(rows.into_iter().next()) } - /// Reads `worktree`'s complete durable tree root set: its - /// `mutation_trace_worktrees.cursor_tree`, plus the `before_tree` and - /// `after_tree` of every `mutation_trace_events` row for `worktree`, - /// deduplicated. Returns an empty set (not an error) when `worktree` has - /// no durable row at all. - /// - /// Read-only, cold path — never called from `load_worktree` or any - /// hook-boundary path, exactly like [`MutationTraceStore::load_mutation_event`]. - /// It reads only the three `TreeId` columns above: never - /// `mutation_trace_scopes` / `mutation_trace_processed_events` / - /// `mutation_trace_event_active_scopes`, never another worktree's trees, - /// and never transient `AttemptState` / `external_taint`. - /// - /// The whole set is produced by **one** SQL statement (a `UNION` of the - /// three columns) through **one** `query_map` call, so a concurrent - /// mutation-cursor commit — which atomically moves `cursor_tree` from `T` - /// to `X` and inserts `MutationEvent { before_tree = T, after_tree = X }` - /// in the same transaction — cannot expose a torn root set that omits `T`: - /// the single statement observes either the pre-commit snapshot - /// (`cursor_tree` still contains `T`) or the post-commit snapshot - /// (`before_tree` contains `T`). pub async fn load_tree_roots(&self, worktree: &WorktreeId) -> Result> { let rows = self .db @@ -828,19 +816,6 @@ impl<'a> MutationTraceStore<'a> { Ok(rows.into_iter().collect()) } - /// Reads the repository-wide durable tree root set: the union of - /// `mutation_trace_worktrees.cursor_tree`, `mutation_trace_events.before_tree`, - /// and `mutation_trace_events.after_tree` across **every** worktree, - /// deduplicated. Returns an empty set (not an error) for a repository with - /// no mutation-cursor rows. - /// - /// This is the reconciler's retention set: linked worktrees share one Git - /// object database, so a ref owned by worktree `A` may be the last SCE ref - /// protecting a tree that only worktree `B` durably requires. Read-only, - /// cold path, and — like [`MutationTraceStore::load_tree_roots`] — one SQL - /// statement through one `query_map` call, so it cannot tear across a - /// concurrent atomic `cursor T -> X` + `event T -> X` commit on another - /// worktree. pub async fn load_all_tree_roots(&self) -> Result> { let rows = self .db @@ -985,6 +960,10 @@ impl<'a> MutationTraceStore<'a> { Ok(!rows.is_empty()) } + #[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" + )] async fn load_mutation_event_active_scopes( &self, worktree: &WorktreeId, @@ -1275,6 +1254,10 @@ fn mutation_event_page_row_from_turso(row: &turso::Row) -> Result, } +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] fn mutation_event_row_from_turso(row: &turso::Row) -> Result { let before_tree: String = row .get(0) @@ -1346,6 +1333,10 @@ fn reconstruct_attribution(kind: AttributionKind, scope_id: Option) -> R } } +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] fn reconstruct_boundary( kind: BoundaryKind, worktree: &WorktreeId, diff --git a/cli/src/services/mutation_trace/tests.rs b/cli/src/services/mutation_trace/tests.rs index 979a5c14a..912dfadfe 100644 --- a/cli/src/services/mutation_trace/tests.rs +++ b/cli/src/services/mutation_trace/tests.rs @@ -125,20 +125,15 @@ fn prepare_and_commit( } #[test] -fn scope_status_live_and_terminal_table() { +fn scope_status_live_table() { let cases = [ - (ScopeStatus::NeverSeen, false, false), - (ScopeStatus::Active, true, false), - (ScopeStatus::Closed, false, true), - (ScopeStatus::Abandoned, false, true), + (ScopeStatus::NeverSeen, false), + (ScopeStatus::Active, true), + (ScopeStatus::Closed, false), + (ScopeStatus::Abandoned, false), ]; - for (status, expected_live, expected_terminal) in cases { + for (status, expected_live) in cases { assert_eq!(is_live(status), expected_live, "is_live({status:?})"); - assert_eq!( - is_terminal(status), - expected_terminal, - "is_terminal({status:?})" - ); } } diff --git a/cli/src/services/mutation_trace/types.rs b/cli/src/services/mutation_trace/types.rs index 30a781c5c..7ec4584e1 100644 --- a/cli/src/services/mutation_trace/types.rs +++ b/cli/src/services/mutation_trace/types.rs @@ -152,12 +152,6 @@ pub fn is_live(status: ScopeStatus) -> bool { status == ScopeStatus::Active } -/// A scope has ended and can never be reactivated. Refines `isTerminal` -/// (`spec/mutation_cursor.qnt:169-170`). -pub fn is_terminal(status: ScopeStatus) -> bool { - matches!(status, ScopeStatus::Closed | ScopeStatus::Abandoned) -} - impl ScopeState { /// The worktree this scope belongs to. Refines `scopeWorktree` /// (`spec/mutation_cursor.qnt:151-157`). @@ -175,21 +169,10 @@ impl ScopeState { self.worktree_id.clone() } - /// The harness that owns this scope. Refines `scopeActor` - /// (`spec/mutation_cursor.qnt:159-165`). - pub fn scope_actor(&self) -> ActorKind { - self.actor_kind - } - /// Refines `isLive` applied to this scope's status. pub fn is_live(&self) -> bool { is_live(self.status) } - - /// Refines `isTerminal` applied to this scope's status. - pub fn is_terminal(&self) -> bool { - is_terminal(self.status) - } } /// The worktree a boundary applies to. Refines `boundaryWorktree` diff --git a/cli/src/services/observability.rs b/cli/src/services/observability.rs index 5e6adb641..a407abae2 100644 --- a/cli/src/services/observability.rs +++ b/cli/src/services/observability.rs @@ -1,27 +1,30 @@ use std::{ cmp::Ordering, - collections::BTreeMap, + collections::hash_map::DefaultHasher, fmt::Write as FmtWrite, fs::{self, OpenOptions}, + hash::{Hash, Hasher}, io::{self, ErrorKind, Write}, path::{Path, PathBuf}, - sync::{Arc, Mutex, OnceLock}, + sync::Mutex, time::SystemTime, }; +use crate::services::config::{self, LogFormat, LogLevel, ENV_LOG_DIR}; +use crate::services::error::CliError; +use crate::services::security::redact_sensitive_text; use anyhow::{bail, Context, Result}; use chrono::{Local, NaiveDate, Utc}; use serde_json::json; -use tracing::Level; - -use crate::services::config::{ - self, LogFormat, LogLevel, ENV_LOG_DIR, ENV_LOG_FORMAT, ENV_LOG_LEVEL, -}; -use crate::services::error::CliError; -use crate::services::security::redact_sensitive_text; +pub mod otel_policy; +pub mod otel_runtime; +pub mod trace_context; +pub mod tracing_boundary; pub mod traits; +use tracing_boundary::{emit_logger_event, EventId}; + pub const NAME: &str = "observability"; const LOG_FILE_PREFIX: &str = "sce"; const LOG_FILE_EXTENSION: &str = "log"; @@ -74,34 +77,6 @@ impl Logger { }) } - #[allow(dead_code)] - fn from_env_lookup(lookup: F) -> Result - where - F: Fn(&str) -> Option, - { - let mut config = ObservabilityConfig::default(); - - if let Some(raw) = lookup(ENV_LOG_LEVEL) { - config.level = LogLevel::parse_env(&raw, ENV_LOG_LEVEL)?; - } - - if let Some(raw) = lookup(ENV_LOG_FORMAT) { - config.format = LogFormat::parse_env(&raw, ENV_LOG_FORMAT)?; - } - - let mut log_dir = None; - if let Some(raw) = lookup(ENV_LOG_DIR) { - validate_log_dir(&raw)?; - log_dir = Some(PathBuf::from(raw)); - } - - Ok(Self { - config, - log_dir, - log_file_retention_limit: config::DEFAULT_LOG_FILE_RETENTION_LIMIT, - }) - } - pub fn info( &self, event_id: &str, @@ -132,7 +107,6 @@ impl Logger { self.log_forced(LogLevel::Warn, event_id, message, fields, session_id); } - #[cfg_attr(not(test), allow(dead_code))] pub fn error( &self, event_id: &str, @@ -152,9 +126,14 @@ impl Logger { .map(|(key, value)| (*key, value.as_str())) .collect(); - self.log( + if !self.enabled(LogLevel::Error) { + return; + } + + self.log_classified( LogLevel::Error, &event_id, + EventId::from_cli_error(error), &message, &field_refs, session_id, @@ -184,7 +163,26 @@ impl Logger { fields: &[(&str, &str)], session_id: Option<&str>, ) { - emit_tracing_event(level, event_id, message, fields); + self.log_classified( + level, + event_id, + EventId::classify(event_id), + message, + fields, + session_id, + ); + } + + fn log_classified( + &self, + level: LogLevel, + event_id: &str, + tracing_event_id: EventId, + message: &str, + fields: &[(&str, &str)], + session_id: Option<&str>, + ) { + emit_logger_event(level, tracing_event_id); let line = self.render_line(level, event_id, message, fields); let redacted_line = redact_sensitive_text(&line); @@ -370,23 +368,26 @@ where .with_context(|| format!("failed to create log directory '{}'", parent.display()))?; } - let lock = file_log_lock(path); - let _guard = lock.lock().map_err(|error| { - anyhow::anyhow!("failed to lock log file '{}': {error}", path.display()) - })?; + let primary_error = { + let _guard = log_lock_stripe(path).lock().map_err(|error| { + anyhow::anyhow!("failed to lock log file '{}': {error}", path.display()) + })?; - match persist_log_line(path, redacted_line) { - Ok(write_target) => { - run_log_retention_after_creation(path, write_target, cleanup); - Ok(()) + match persist_log_line(path, redacted_line) { + Ok(write_target) => { + run_log_retention_after_creation(path, write_target, cleanup); + return Ok(()); + } + Err(primary_error) => primary_error, } - Err(primary_error) => attempt_v2_log_fallback( - path, - redacted_line, - &primary_error, - |fallback_path, line| append_log_line_once_with_cleanup(fallback_path, line, cleanup), - ), - } + }; + + attempt_v2_log_fallback( + path, + redacted_line, + &primary_error, + |fallback_path, line| append_log_line_once_with_cleanup(fallback_path, line, cleanup), + ) } fn attempt_v2_log_fallback( @@ -417,8 +418,7 @@ where .with_context(|| format!("failed to create log directory '{}'", parent.display()))?; } - let lock = file_log_lock(path); - let _guard = lock.lock().map_err(|error| { + let _guard = log_lock_stripe(path).lock().map_err(|error| { anyhow::anyhow!("failed to lock log file '{}': {error}", path.display()) })?; @@ -604,17 +604,19 @@ fn collect_managed_log_files(log_dir: &Path) -> Result<(Vec, Vec Ok((managed_files, errors)) } -fn file_log_lock(path: &Path) -> Arc> { - static FILE_LOG_LOCKS: OnceLock>>>> = OnceLock::new(); - let locks = FILE_LOG_LOCKS.get_or_init(|| Mutex::new(BTreeMap::new())); - let mut locks = locks - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - Arc::clone( - locks - .entry(path.to_path_buf()) - .or_insert_with(|| Arc::new(Mutex::new(()))), - ) +const LOG_LOCK_STRIPES: usize = 64; + +static LOG_LOCKS: [Mutex<()>; LOG_LOCK_STRIPES] = [const { Mutex::new(()) }; LOG_LOCK_STRIPES]; + +fn log_lock_stripe_index(path: &Path) -> usize { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + usize::try_from(hasher.finish() % LOG_LOCK_STRIPES as u64) + .expect("log lock stripe index must fit usize") +} + +fn log_lock_stripe(path: &Path) -> &'static Mutex<()> { + &LOG_LOCKS[log_lock_stripe_index(path)] } #[cfg(unix)] @@ -633,74 +635,6 @@ fn emit_stderr_line(line: &str) { let _ = stderr.flush(); } -fn emit_tracing_event(level: LogLevel, event_id: &str, message: &str, fields: &[(&str, &str)]) { - emit_tracing_event_with_fields_json(level, event_id, message, || tracing_fields_json(fields)); -} - -fn tracing_event_enabled(level: LogLevel) -> bool { - match level { - LogLevel::Error => tracing::enabled!(target: "sce", Level::ERROR), - LogLevel::Warn => tracing::enabled!(target: "sce", Level::WARN), - LogLevel::Info => tracing::enabled!(target: "sce", Level::INFO), - LogLevel::Debug => tracing::enabled!(target: "sce", Level::DEBUG), - } -} - -fn tracing_fields_json(fields: &[(&str, &str)]) -> String { - let detail_fields = fields - .iter() - .map(|(key, value)| { - ( - (*key).to_string(), - serde_json::Value::String((*value).to_string()), - ) - }) - .collect::>(); - serde_json::Value::Object(detail_fields).to_string() -} - -fn emit_tracing_event_with_fields_json( - level: LogLevel, - event_id: &str, - message: &str, - fields_json: F, -) where - F: FnOnce() -> String, -{ - if !tracing_event_enabled(level) { - return; - } - - let fields_json = fields_json(); - - match level { - LogLevel::Error => tracing::error!( - target: "sce", - event_id = event_id, - event_message = message, - fields = %fields_json - ), - LogLevel::Warn => tracing::warn!( - target: "sce", - event_id = event_id, - event_message = message, - fields = %fields_json - ), - LogLevel::Info => tracing::info!( - target: "sce", - event_id = event_id, - event_message = message, - fields = %fields_json - ), - LogLevel::Debug => tracing::debug!( - target: "sce", - event_id = event_id, - event_message = message, - fields = %fields_json - ), - } -} - #[cfg(test)] mod tests { use super::*; @@ -737,7 +671,10 @@ mod tests { #[test] fn observability_fields_for_user_error_without_source_omit_error_source() { - let error = CliError::user(UserError::NotAuthenticated); + let error = CliError::User { + error: UserError::NotAuthenticated, + source: None, + }; let fields = cli_error_fields(&error); @@ -814,6 +751,388 @@ mod tests { assert!(!log_dir.exists()); } + const CHILD_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); + const CHILD_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(20); + const ENV_CHILD_PATH: &str = "SCE_LOG_LOCK_CHILD_PATH"; + const SAME_STRIPE_SENTINEL: &str = "SAME_STRIPE_FALLBACK_OK"; + const POISON_SENTINEL: &str = "POISONED_STRIPE_OK"; + + fn text_logger(log_dir: &Path) -> Logger { + Logger { + config: ObservabilityConfig { + level: LogLevel::Error, + format: LogFormat::Text, + log_to_file: true, + }, + log_dir: Some(log_dir.to_path_buf()), + log_file_retention_limit: 10, + } + } + + fn read_all_log_lines(log_dir: &Path) -> Vec { + let mut lines = Vec::new(); + for entry in fs::read_dir(log_dir).unwrap() { + let path = entry.unwrap().path(); + if path + .extension() + .is_some_and(|ext| ext == LOG_FILE_EXTENSION) + { + lines.extend(fs::read_to_string(path).unwrap().lines().map(String::from)); + } + } + lines + } + + fn run_bounded_child(test_name: &str, child_path: &Path) -> std::process::Output { + use std::io::Read; + use std::process::{Command, Stdio}; + + let sandbox = tempfile::tempdir().unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name, "--nocapture"]) + .current_dir(sandbox.path()) + .env(ENV_CHILD_PATH, child_path) + .env("HOME", sandbox.path()) + .env("XDG_CONFIG_HOME", sandbox.path().join("config")) + .env("XDG_STATE_HOME", sandbox.path().join("state")) + .env("XDG_CACHE_HOME", sandbox.path().join("cache")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let mut stdout_pipe = child.stdout.take().unwrap(); + let mut stderr_pipe = child.stderr.take().unwrap(); + let stdout_reader = std::thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = stdout_pipe.read_to_end(&mut buffer); + buffer + }); + let stderr_reader = std::thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = stderr_pipe.read_to_end(&mut buffer); + buffer + }); + + let deadline = std::time::Instant::now() + CHILD_DEADLINE; + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break Some(status); + } + if std::time::Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + break None; + } + std::thread::sleep(CHILD_POLL_INTERVAL); + }; + + let stdout = stdout_reader.join().unwrap(); + let stderr = stderr_reader.join().unwrap(); + let Some(status) = status else { + panic!( + "child '{test_name}' exceeded {CHILD_DEADLINE:?} and was killed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + }; + + std::process::Output { + status, + stdout, + stderr, + } + } + + fn assert_child_succeeded(output: &std::process::Output, sentinel: &str) { + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains(sentinel), + "child failed: {:?}\nstdout: {stdout}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + + fn blocked_primary_path(log_dir: &Path, stem: &str) -> PathBuf { + let primary = log_dir.join(format!("{stem}.log")); + fs::create_dir_all(&primary).unwrap(); + primary + } + + #[test] + fn concurrent_writers_to_one_path_produce_complete_unique_lines() { + const THREADS: usize = 8; + const LINES_PER_THREAD: usize = 50; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-concurrent.log"); + let padding = "x".repeat(512); + + std::thread::scope(|scope| { + for thread in 0..THREADS { + let path = &path; + let padding = &padding; + scope.spawn(move || { + for index in 0..LINES_PER_THREAD { + let line = format!("t{thread}-l{index}-{padding}-end"); + append_log_line_with_cleanup(path, &line, |_| Ok(())).unwrap(); + } + }); + } + }); + + let contents = fs::read_to_string(&path).unwrap(); + let mut seen = std::collections::BTreeSet::new(); + for line in contents.lines() { + assert!(line.ends_with("-end"), "interleaved line: {line}"); + assert!(seen.insert(line.to_string()), "duplicated line: {line}"); + } + assert_eq!(seen.len(), THREADS * LINES_PER_THREAD); + } + + #[test] + fn independent_logger_instances_serialize_writes_to_one_path() { + const INSTANCES: usize = 4; + const RECORDS_PER_INSTANCE: usize = 25; + + let dir = tempfile::tempdir().unwrap(); + let loggers: Vec = (0..INSTANCES).map(|_| text_logger(dir.path())).collect(); + + std::thread::scope(|scope| { + for (instance, logger) in loggers.iter().enumerate() { + scope.spawn(move || { + for index in 0..RECORDS_PER_INSTANCE { + let event_id = format!("sce.test.i{instance}.r{index}"); + logger.error(&event_id, "concurrent record", &[], Some("shared")); + } + }); + } + }); + + let lines = read_all_log_lines(dir.path()); + assert_eq!(lines.len(), INSTANCES * RECORDS_PER_INSTANCE); + for line in &lines { + assert!(line.starts_with("timestamp="), "malformed line: {line}"); + assert!( + line.ends_with("message=concurrent record"), + "malformed line: {line}" + ); + } + let unique: std::collections::BTreeSet<&String> = lines.iter().collect(); + assert_eq!(unique.len(), lines.len()); + } + + #[test] + fn stripe_selection_is_stable_bounded_and_independent_of_path_count() { + assert_eq!(LOG_LOCKS.len(), LOG_LOCK_STRIPES); + + let mut used = std::collections::BTreeSet::new(); + for index in 0..5000 { + let path = PathBuf::from(format!("/tmp/sce-distinct/sce-{index}.log")); + let stripe = log_lock_stripe_index(&path); + assert!(stripe < LOG_LOCK_STRIPES); + assert_eq!(stripe, log_lock_stripe_index(&path)); + assert!(std::ptr::eq( + log_lock_stripe(&path), + LOG_LOCKS.get(stripe).unwrap() + )); + used.insert(stripe); + } + + assert!(used.len() > 1); + assert_eq!(LOG_LOCKS.len(), LOG_LOCK_STRIPES); + } + + #[test] + fn fallback_on_a_different_stripe_writes_the_v2_file() { + let dir = tempfile::tempdir().unwrap(); + let primary = (0..1000) + .map(|index| dir.path().join(format!("sce-diff-{index}.log"))) + .find(|candidate| { + !std::ptr::eq( + log_lock_stripe(candidate), + log_lock_stripe(&v2_log_path(candidate)), + ) + }) + .unwrap(); + fs::create_dir_all(&primary).unwrap(); + + append_log_line_with_cleanup(&primary, "fallback-line", |_| Ok(())).unwrap(); + + assert_eq!( + fs::read_to_string(v2_log_path(&primary)).unwrap(), + "fallback-line\n" + ); + assert!(primary.is_dir()); + } + + #[test] + fn fallback_failure_reports_combined_primary_and_v2_error() { + let dir = tempfile::tempdir().unwrap(); + let primary = blocked_primary_path(dir.path(), "sce-both-fail"); + let fallback = v2_log_path(&primary); + fs::create_dir_all(&fallback).unwrap(); + + let error = append_log_line_with_cleanup(&primary, "line", |_| Ok(())) + .unwrap_err() + .to_string(); + + assert!( + error.starts_with(&format!( + "primary log file persistence failed for '{}': ", + primary.display() + )), + "{error}" + ); + assert!( + error.contains(&format!( + "; v2 fallback log file persistence failed for '{}': ", + fallback.display() + )), + "{error}" + ); + } + + #[test] + fn same_stripe_fallback_completes_in_bounded_child_process() { + let dir = tempfile::tempdir().unwrap(); + let primary = (0..10_000) + .map(|index| dir.path().join(format!("sce-collide-{index}.log"))) + .find(|candidate| { + std::ptr::eq( + log_lock_stripe(candidate), + log_lock_stripe(&v2_log_path(candidate)), + ) + }) + .expect("a primary path whose v2 path shares its stripe"); + fs::create_dir_all(&primary).unwrap(); + + let output = run_bounded_child( + "services::observability::tests::isolated_same_stripe_fallback", + &primary, + ); + + assert_child_succeeded(&output, SAME_STRIPE_SENTINEL); + assert_eq!( + fs::read_to_string(v2_log_path(&primary)).unwrap(), + "same-stripe-line\n" + ); + assert!(primary.is_dir()); + } + + #[test] + fn isolated_same_stripe_fallback() { + let Some(primary) = std::env::var_os(ENV_CHILD_PATH).map(PathBuf::from) else { + return; + }; + assert!(std::ptr::eq( + log_lock_stripe(&primary), + log_lock_stripe(&v2_log_path(&primary)) + )); + + append_log_line_with_cleanup(&primary, "same-stripe-line", |_| Ok(())).unwrap(); + + println!("{SAME_STRIPE_SENTINEL}"); + } + + #[test] + fn poisoned_stripe_fails_lock_without_fallback_in_child_process() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-poison.log"); + + let output = run_bounded_child( + "services::observability::tests::isolated_poisoned_stripe", + &path, + ); + + assert_child_succeeded(&output, POISON_SENTINEL); + assert!(!v2_log_path(&path).exists()); + } + + #[test] + fn isolated_poisoned_stripe() { + let Some(path) = std::env::var_os(ENV_CHILD_PATH).map(PathBuf::from) else { + return; + }; + + let poisoned = std::panic::catch_unwind(|| { + let _ = append_log_line_with_cleanup(&path, "first", |_| panic!("poison the stripe")); + }); + assert!(poisoned.is_err()); + + let error = append_log_line_with_cleanup(&path, "second", |_| Ok(())) + .unwrap_err() + .to_string(); + assert!( + error.starts_with(&format!("failed to lock log file '{}': ", path.display())), + "{error}" + ); + assert!(!v2_log_path(&path).exists()); + assert_eq!(fs::read_to_string(&path).unwrap(), "first\n"); + + println!("{POISON_SENTINEL}"); + } + + #[test] + fn retention_runs_once_for_concurrent_creation_of_one_path() { + const THREADS: usize = 8; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-retention.log"); + let cleanups = std::sync::atomic::AtomicUsize::new(0); + + std::thread::scope(|scope| { + for thread in 0..THREADS { + let path = &path; + let cleanups = &cleanups; + scope.spawn(move || { + append_log_line_with_cleanup(path, &format!("line-{thread}"), |log_dir| { + assert_eq!(log_dir, path.parent().unwrap()); + cleanups.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + .unwrap(); + }); + } + }); + + assert_eq!(cleanups.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!(fs::read_to_string(&path).unwrap().lines().count(), THREADS); + } + + #[cfg(unix)] + #[test] + fn created_log_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-perms.log"); + + append_log_line_with_cleanup(&path, "line", |_| Ok(())).unwrap(); + + let mode = fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + + #[test] + fn secrets_are_redacted_before_they_reach_the_log_file() { + let dir = tempfile::tempdir().unwrap(); + let logger = text_logger(dir.path()); + + logger.error( + "sce.test.redaction", + "request failed", + &[("password", "hunter2-super-secret")], + None, + ); + + let lines = read_all_log_lines(dir.path()); + assert_eq!(lines.len(), 1); + assert!(lines[0].contains("[REDACTED]")); + assert!(!lines[0].contains("hunter2-super-secret")); + } + #[test] fn observability_fields_for_internal_error_carry_surface_and_full_source_chain() { let source = anyhow::anyhow!("root cause").context("failed to do the thing"); diff --git a/cli/src/services/observability/otel_policy.rs b/cli/src/services/observability/otel_policy.rs new file mode 100644 index 000000000..0c0034c33 --- /dev/null +++ b/cli/src/services/observability/otel_policy.rs @@ -0,0 +1,742 @@ +use crate::services::error::FailureClass; +use crate::services::observability::tracing_boundary::OperationClass; +use crate::services::{ + auth_command, bash_policy, completion, config, doctor, help, hooks, setup, sync, version, +}; + +pub const OTEL_TARGET: &str = "sce::otel"; + +const KEY_COMMAND_NAME: &str = "sce.command.name"; +const KEY_OUTCOME: &str = "sce.outcome"; +const KEY_ERROR_CATEGORY: &str = "sce.error.category"; +const KEY_OPERATION_TYPE: &str = "sce.operation.type"; +const KEY_DURATION_MS: &str = "sce.duration_ms"; +const KEY_COUNT: &str = "sce.count"; +const KEY_ATTEMPTS: &str = "sce.attempts"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OtelName { + Command, + MutationScopeCoordinate, + WorktreeLock, + Reconciliation, + GitSnapshot, + DbOperation, + Sync, +} + +impl OtelName { + pub const ALL: [Self; 7] = [ + Self::Command, + Self::MutationScopeCoordinate, + Self::WorktreeLock, + Self::Reconciliation, + Self::GitSnapshot, + Self::DbOperation, + Self::Sync, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Self::Command => "sce.command", + Self::MutationScopeCoordinate => "sce.mutation_scope.coordinate", + Self::WorktreeLock => "sce.worktree.lock", + Self::Reconciliation => "sce.reconciliation", + Self::GitSnapshot => "sce.git.snapshot", + Self::DbOperation => "sce.db.operation", + Self::Sync => "sce.sync", + } + } + + pub fn parse(name: &str) -> Option { + Self::ALL + .into_iter() + .find(|candidate| candidate.as_str() == name) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CommandName { + Help, + Auth, + Config, + Setup, + Doctor, + Hooks, + Policy, + Version, + Completion, + Sync, +} + +impl CommandName { + pub const ALL: [Self; 10] = [ + Self::Help, + Self::Auth, + Self::Config, + Self::Setup, + Self::Doctor, + Self::Hooks, + Self::Policy, + Self::Version, + Self::Completion, + Self::Sync, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Self::Help => help::NAME, + Self::Auth => auth_command::NAME, + Self::Config => config::NAME, + Self::Setup => setup::NAME, + Self::Doctor => doctor::NAME, + Self::Hooks => hooks::NAME, + Self::Policy => bash_policy::NAME, + Self::Version => version::NAME, + Self::Completion => completion::NAME, + Self::Sync => sync::NAME, + } + } + + pub fn parse(value: &str) -> Option { + Self::ALL + .into_iter() + .find(|candidate| candidate.as_str() == value) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Outcome { + Success, + Failure, + Cancelled, + Timeout, +} + +impl Outcome { + pub const ALL: [Self; 4] = [Self::Success, Self::Failure, Self::Cancelled, Self::Timeout]; + + pub fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Failure => "failure", + Self::Cancelled => "cancelled", + Self::Timeout => "timeout", + } + } + + pub fn parse(value: &str) -> Option { + Self::ALL + .into_iter() + .find(|candidate| candidate.as_str() == value) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ErrorCategory { + Parse, + Validation, + Runtime, + Dependency, + Unclassified, +} + +impl ErrorCategory { + pub const ALL: [Self; 5] = [ + Self::Parse, + Self::Validation, + Self::Runtime, + Self::Dependency, + Self::Unclassified, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Self::Parse => "parse", + Self::Validation => "validation", + Self::Runtime => "runtime", + Self::Dependency => "dependency", + Self::Unclassified => "unclassified", + } + } + + pub fn parse(value: &str) -> Option { + Self::ALL + .into_iter() + .find(|candidate| candidate.as_str() == value) + } +} + +impl From for ErrorCategory { + fn from(class: FailureClass) -> Self { + match class { + FailureClass::Parse => Self::Parse, + FailureClass::Validation => Self::Validation, + FailureClass::Runtime => Self::Runtime, + FailureClass::Dependency => Self::Dependency, + } + } +} + +pub type OperationType = OperationClass; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OtelRawValue<'a> { + Number(u64), + Text(&'a str), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OtelValue { + Number(u64), + Static(&'static str), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OtelAttribute { + CommandName(CommandName), + Outcome(Outcome), + ErrorCategory(ErrorCategory), + OperationType(OperationType), + DurationMs(u64), + Count(u64), + Attempts(u64), +} + +impl OtelAttribute { + #[cfg_attr(not(test), allow(dead_code))] + pub const KEYS: [&'static str; 7] = [ + KEY_COMMAND_NAME, + KEY_OUTCOME, + KEY_ERROR_CATEGORY, + KEY_OPERATION_TYPE, + KEY_DURATION_MS, + KEY_COUNT, + KEY_ATTEMPTS, + ]; + + pub fn key(self) -> &'static str { + match self { + Self::CommandName(_) => KEY_COMMAND_NAME, + Self::Outcome(_) => KEY_OUTCOME, + Self::ErrorCategory(_) => KEY_ERROR_CATEGORY, + Self::OperationType(_) => KEY_OPERATION_TYPE, + Self::DurationMs(_) => KEY_DURATION_MS, + Self::Count(_) => KEY_COUNT, + Self::Attempts(_) => KEY_ATTEMPTS, + } + } + + pub fn value(self) -> OtelValue { + match self { + Self::CommandName(value) => OtelValue::Static(value.as_str()), + Self::Outcome(value) => OtelValue::Static(value.as_str()), + Self::ErrorCategory(value) => OtelValue::Static(value.as_str()), + Self::OperationType(value) => OtelValue::Static(value.as_str()), + Self::DurationMs(value) | Self::Count(value) | Self::Attempts(value) => { + OtelValue::Number(value) + } + } + } + + pub fn admit(key: &str, value: OtelRawValue<'_>) -> Option { + match (key, value) { + (KEY_COMMAND_NAME, OtelRawValue::Text(text)) => { + CommandName::parse(text).map(Self::CommandName) + } + (KEY_OUTCOME, OtelRawValue::Text(text)) => Outcome::parse(text).map(Self::Outcome), + (KEY_ERROR_CATEGORY, OtelRawValue::Text(text)) => { + ErrorCategory::parse(text).map(Self::ErrorCategory) + } + (KEY_OPERATION_TYPE, OtelRawValue::Text(text)) => { + OperationType::parse(text).map(Self::OperationType) + } + (KEY_DURATION_MS, OtelRawValue::Number(number)) => Some(Self::DurationMs(number)), + (KEY_COUNT, OtelRawValue::Number(number)) => Some(Self::Count(number)), + (KEY_ATTEMPTS, OtelRawValue::Number(number)) => Some(Self::Attempts(number)), + _ => None, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(not(test), allow(dead_code))] +pub struct OtelRecord { + pub name: OtelName, + pub attributes: Vec, +} + +#[cfg_attr(not(test), allow(dead_code))] +pub fn admit_otel_record( + target: &str, + name: &str, + attributes: &[(&str, OtelRawValue<'_>)], +) -> Option { + if target != OTEL_TARGET { + return None; + } + + let name = OtelName::parse(name)?; + let attributes = attributes + .iter() + .filter_map(|(key, value)| OtelAttribute::admit(key, *value)) + .collect(); + Some(OtelRecord { name, attributes }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::config::LogLevel; + use crate::services::observability::tracing_boundary::test_capture::CapturingSubscriber; + use crate::services::observability::tracing_boundary::{ + emit_contention_event, emit_logger_event, emit_retry_event, DbName, EventId, + SCE_TRACING_TARGET, + }; + + const SECRET: &str = "sk-live-SECRET-TOKEN-9f8e7d"; + const SENSITIVE_PATH: &str = "/home/victim/.ssh/id_ed25519"; + const WHITELIST_PASSING_SECRETS: [&str; 3] = ["hunter2", "password123", "secret_token_123"]; + + fn passes_old_character_whitelist(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-')) + } + + fn text_attributes() -> Vec { + let mut attributes = Vec::new(); + attributes.extend(CommandName::ALL.map(OtelAttribute::CommandName)); + attributes.extend(Outcome::ALL.map(OtelAttribute::Outcome)); + attributes.extend(ErrorCategory::ALL.map(OtelAttribute::ErrorCategory)); + attributes.extend(OperationType::ALL.map(OtelAttribute::OperationType)); + attributes + } + + fn text_of(attribute: OtelAttribute) -> &'static str { + match attribute.value() { + OtelValue::Static(text) => text, + OtelValue::Number(_) => panic!("expected text attribute"), + } + } + + #[test] + fn tracing_boundary_b_admits_only_otel_target_and_enumerated_names() { + let ok = admit_otel_record( + OTEL_TARGET, + "sce.command", + &[ + ("sce.outcome", OtelRawValue::Text("success")), + ("sce.duration_ms", OtelRawValue::Number(12)), + ], + ) + .expect("admitted"); + assert_eq!(ok.name, OtelName::Command); + assert_eq!( + ok.attributes, + vec![ + OtelAttribute::Outcome(Outcome::Success), + OtelAttribute::DurationMs(12) + ] + ); + + assert!(admit_otel_record(SCE_TRACING_TARGET, "sce.command", &[]).is_none()); + assert!(admit_otel_record("sce::services::resilience", "sce.command", &[]).is_none()); + assert!(admit_otel_record(OTEL_TARGET, "sce.arbitrary", &[]).is_none()); + assert!(admit_otel_record(OTEL_TARGET, SECRET, &[]).is_none()); + } + + #[test] + fn tracing_boundary_b_every_valid_enum_produces_expected_static_value() { + let commands = [ + (CommandName::Help, "help"), + (CommandName::Auth, "auth"), + (CommandName::Config, "config"), + (CommandName::Setup, "setup"), + (CommandName::Doctor, "doctor"), + (CommandName::Hooks, "hooks"), + (CommandName::Policy, "policy"), + (CommandName::Version, "version"), + (CommandName::Completion, "completion"), + (CommandName::Sync, "sync"), + ]; + assert_eq!(commands.len(), CommandName::ALL.len()); + for (variant, expected) in commands { + assert_eq!(variant.as_str(), expected); + assert_eq!(CommandName::parse(expected), Some(variant)); + assert_eq!( + OtelAttribute::CommandName(variant).value(), + OtelValue::Static(expected) + ); + } + + let outcomes = [ + (Outcome::Success, "success"), + (Outcome::Failure, "failure"), + (Outcome::Cancelled, "cancelled"), + (Outcome::Timeout, "timeout"), + ]; + assert_eq!(outcomes.len(), Outcome::ALL.len()); + for (variant, expected) in outcomes { + assert_eq!(variant.as_str(), expected); + assert_eq!(Outcome::parse(expected), Some(variant)); + assert_eq!( + OtelAttribute::Outcome(variant).value(), + OtelValue::Static(expected) + ); + } + + assert_eq!(OtelAttribute::DurationMs(7).value(), OtelValue::Number(7)); + assert_eq!(OtelAttribute::Count(8).value(), OtelValue::Number(8)); + assert_eq!(OtelAttribute::Attempts(9).value(), OtelValue::Number(9)); + } + + #[test] + fn tracing_boundary_b_every_valid_error_category_and_operation_type_produces_expected_static_value( + ) { + let categories = [ + (ErrorCategory::Parse, "parse"), + (ErrorCategory::Validation, "validation"), + (ErrorCategory::Runtime, "runtime"), + (ErrorCategory::Dependency, "dependency"), + (ErrorCategory::Unclassified, "unclassified"), + ]; + assert_eq!(categories.len(), ErrorCategory::ALL.len()); + for (variant, expected) in categories { + assert_eq!(variant.as_str(), expected); + assert_eq!(ErrorCategory::parse(expected), Some(variant)); + assert_eq!( + OtelAttribute::ErrorCategory(variant).value(), + OtelValue::Static(expected) + ); + } + + let operations = [ + (OperationType::AuthRefreshToken, "auth.refresh_token"), + ( + OperationType::AgentTraceSyncIngestionState, + "agent_trace_sync.ingestion_state", + ), + (OperationType::DbOpenConnection, "db.open_connection"), + ( + OperationType::DbOpenEncryptedConnection, + "db.open_encrypted_connection", + ), + (OperationType::DbExecuteQuery, "db.execute_query"), + ( + OperationType::DbExecuteEncryptedQuery, + "db.execute_encrypted_query", + ), + (OperationType::DbQueryRows, "db.query_rows"), + ( + OperationType::DbQueryEncryptedRows, + "db.query_encrypted_rows", + ), + (OperationType::DbCheckpointWal, "db.checkpoint_wal"), + (OperationType::Unclassified, "unclassified"), + ]; + assert_eq!(operations.len(), OperationType::ALL.len()); + for (variant, expected) in operations { + assert_eq!(variant.as_str(), expected); + assert_eq!(OperationType::parse(expected), Some(variant)); + assert_eq!( + OtelAttribute::OperationType(variant).value(), + OtelValue::Static(expected) + ); + } + + assert_eq!( + ErrorCategory::from(FailureClass::Parse), + ErrorCategory::Parse + ); + assert_eq!( + ErrorCategory::from(FailureClass::Validation), + ErrorCategory::Validation + ); + assert_eq!( + ErrorCategory::from(FailureClass::Runtime), + ErrorCategory::Runtime + ); + assert_eq!( + ErrorCategory::from(FailureClass::Dependency), + ErrorCategory::Dependency + ); + } + + #[test] + fn tracing_boundary_b_unknown_command_names_are_rejected() { + for candidate in [ + "hunter2", + "secret_token_123", + "password123", + "private_customer_name", + "Sync", + "SYNC", + " sync", + "sync ", + "sync\n", + "", + SECRET, + SENSITIVE_PATH, + ] { + assert_eq!(CommandName::parse(candidate), None, "{candidate:?}"); + assert_eq!( + OtelAttribute::admit("sce.command.name", OtelRawValue::Text(candidate)), + None, + "{candidate:?}" + ); + } + } + + #[test] + fn tracing_boundary_b_unknown_outcomes_cannot_be_exported() { + for candidate in [ + "hunter2", + "secret_token_123", + "password123", + "Success", + "success ", + "ok", + "", + SECRET, + SENSITIVE_PATH, + ] { + assert_eq!(Outcome::parse(candidate), None, "{candidate:?}"); + assert_eq!( + OtelAttribute::admit("sce.outcome", OtelRawValue::Text(candidate)), + None, + "{candidate:?}" + ); + } + } + + #[test] + fn tracing_boundary_b_unknown_error_categories_cannot_be_exported() { + for candidate in [ + "hunter2", + "secret_token_123", + "password123", + "SCE-ERR-RUNTIME", + "Runtime", + "boom", + "", + SECRET, + SENSITIVE_PATH, + ] { + assert_eq!(ErrorCategory::parse(candidate), None, "{candidate:?}"); + assert_eq!( + OtelAttribute::admit("sce.error.category", OtelRawValue::Text(candidate)), + None, + "{candidate:?}" + ); + } + } + + #[test] + fn tracing_boundary_b_unknown_operation_types_cannot_be_exported() { + for candidate in [ + "hunter2", + "secret_token_123", + "password123", + "private_customer_name", + "db.execute_query;drop", + "DB.EXECUTE_QUERY", + "rm -rf /", + "", + SECRET, + SENSITIVE_PATH, + ] { + assert_eq!(OperationType::parse(candidate), None, "{candidate:?}"); + assert_eq!( + OtelAttribute::admit("sce.operation.type", OtelRawValue::Text(candidate)), + None, + "{candidate:?}" + ); + } + } + + #[test] + fn tracing_boundary_b_secret_passing_old_character_whitelist_is_rejected() { + for secret in WHITELIST_PASSING_SECRETS { + assert!( + passes_old_character_whitelist(secret), + "{secret} must satisfy the legacy whitelist for this test to be meaningful" + ); + for key in &OtelAttribute::KEYS[..4] { + assert_eq!( + OtelAttribute::admit(key, OtelRawValue::Text(secret)), + None, + "{key}={secret}" + ); + } + let record = admit_otel_record( + OTEL_TARGET, + "sce.command", + &[ + ("sce.outcome", OtelRawValue::Text(secret)), + ("sce.command.name", OtelRawValue::Text(secret)), + ("sce.error.category", OtelRawValue::Text(secret)), + ("sce.operation.type", OtelRawValue::Text(secret)), + ], + ) + .expect("record admitted without attributes"); + assert!(record.attributes.is_empty()); + } + } + + #[test] + fn tracing_boundary_b_excludes_paths_credentials_exception_text_and_stack_traces() { + let record = admit_otel_record( + OTEL_TARGET, + "sce.db.operation", + &[ + ("exception.message", OtelRawValue::Text("boom")), + ("exception.stacktrace", OtelRawValue::Text("at main")), + ("exception.type", OtelRawValue::Text("io_error")), + ("path", OtelRawValue::Text(SENSITIVE_PATH)), + ( + "http.request.header.authorization", + OtelRawValue::Text(SECRET), + ), + ("authorization", OtelRawValue::Text(SECRET)), + ("password", OtelRawValue::Text("password123")), + ("sce.error.category", OtelRawValue::Text(SENSITIVE_PATH)), + ("sce.outcome", OtelRawValue::Text(SECRET)), + ("sce.count", OtelRawValue::Number(3)), + ("sce.operation.type", OtelRawValue::Text("db.execute_query")), + ], + ) + .expect("admitted"); + + assert_eq!( + record.attributes, + vec![ + OtelAttribute::Count(3), + OtelAttribute::OperationType(OperationType::DbExecuteQuery) + ] + ); + for key in OtelAttribute::KEYS { + assert!(!key.contains("exception")); + assert!(!key.contains("path")); + assert!(!key.contains("authorization")); + } + } + + #[test] + fn tracing_boundary_b_attribute_keys_and_values_cannot_be_mismatched() { + let attributes = text_attributes(); + for key in &OtelAttribute::KEYS[..4] { + for attribute in &attributes { + let text = text_of(*attribute); + match OtelAttribute::admit(key, OtelRawValue::Text(text)) { + Some(admitted) => { + assert_eq!(admitted.key(), *key, "{key} admitted {text}"); + assert_eq!(admitted.value(), OtelValue::Static(text)); + } + None => assert_ne!(attribute.key(), *key, "{key} must admit its own {text}"), + } + } + } + + assert_eq!( + OtelAttribute::admit("sce.outcome", OtelRawValue::Text("doctor")), + None + ); + assert_eq!( + OtelAttribute::admit("sce.command.name", OtelRawValue::Text("success")), + None + ); + assert_eq!( + OtelAttribute::admit("sce.error.category", OtelRawValue::Text("success")), + None + ); + assert_eq!( + OtelAttribute::admit("sce.operation.type", OtelRawValue::Text("parse")), + None + ); + + for key in &OtelAttribute::KEYS[..4] { + assert_eq!(OtelAttribute::admit(key, OtelRawValue::Number(1)), None); + } + for key in &OtelAttribute::KEYS[4..] { + assert_eq!(OtelAttribute::admit(key, OtelRawValue::Text("12")), None); + assert_eq!( + OtelAttribute::admit(key, OtelRawValue::Text("success")), + None + ); + assert!(OtelAttribute::admit(key, OtelRawValue::Number(12)).is_some()); + } + } + + #[test] + fn tracing_boundary_b_policy_names_and_keys_have_stable_shapes() { + for name in OtelName::ALL { + assert_eq!(OtelName::parse(name.as_str()), Some(name)); + } + let samples = [ + OtelAttribute::CommandName(CommandName::Sync), + OtelAttribute::Outcome(Outcome::Success), + OtelAttribute::ErrorCategory(ErrorCategory::Runtime), + OtelAttribute::OperationType(OperationType::DbQueryRows), + OtelAttribute::DurationMs(1), + OtelAttribute::Count(2), + OtelAttribute::Attempts(3), + ]; + let keys: Vec<_> = samples.iter().map(|sample| sample.key()).collect(); + assert_eq!(keys, OtelAttribute::KEYS.to_vec()); + for sample in samples { + let raw = match sample.value() { + OtelValue::Number(number) => OtelRawValue::Number(number), + OtelValue::Static(text) => OtelRawValue::Text(text), + }; + assert_eq!(OtelAttribute::admit(sample.key(), raw), Some(sample)); + } + } + + #[test] + fn tracing_boundary_b_sce_logger_events_cannot_enter_the_otel_export_surface() { + let events = CapturingSubscriber::capture(|| { + for level in [ + LogLevel::Error, + LogLevel::Warn, + LogLevel::Info, + LogLevel::Debug, + ] { + emit_logger_event(level, EventId::classify("sce.app.start")); + emit_logger_event(level, EventId::classify(SECRET)); + } + emit_retry_event(OperationClass::DbExecuteQuery, 1, 3, 1_000, 10); + emit_contention_event( + DbName::AuthDb, + OperationClass::DbOpenConnection, + 2, + 5, + 50, + 40, + ); + }); + + assert_eq!(events.len(), 10); + assert_ne!(OTEL_TARGET, SCE_TRACING_TARGET); + for event in &events { + assert_eq!(event.target, SCE_TRACING_TARGET); + let attributes: Vec<(&str, OtelRawValue<'_>)> = event + .fields + .iter() + .map(|(key, value)| (key.as_str(), OtelRawValue::Text(value.as_str()))) + .collect(); + for name in OtelName::ALL { + assert!( + admit_otel_record(&event.target, name.as_str(), &attributes).is_none(), + "{} admitted as {}", + event.rendered(), + name.as_str() + ); + } + let event_id = event.field("event_id").expect("event_id field"); + assert!(admit_otel_record(&event.target, event_id, &attributes).is_none()); + } + } +} diff --git a/cli/src/services/observability/otel_runtime.rs b/cli/src/services/observability/otel_runtime.rs new file mode 100644 index 000000000..ebaccac86 --- /dev/null +++ b/cli/src/services/observability/otel_runtime.rs @@ -0,0 +1,471 @@ +use std::future::Future; +use std::sync::mpsc::{sync_channel, Receiver, SendError}; +use std::time::Duration; + +use opentelemetry::trace::{Status, TracerProvider}; +use opentelemetry::{KeyValue, Value}; +use opentelemetry_otlp::{Protocol, RetryPolicy, WithExportConfig, WithHttpConfig}; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::trace::{ + BatchConfigBuilder, BatchSpanProcessor, SdkTracerProvider, SpanData, SpanEvents, SpanExporter, + SpanLinks, +}; +use opentelemetry_sdk::Resource; +use tracing_subscriber::filter::filter_fn; +use tracing_subscriber::Layer; + +use crate::services::command_registry::CommandRegistry; +use crate::services::error::CliError; +use crate::services::observability::otel_policy::{ + OtelAttribute, OtelName, OtelRawValue, OtelValue, OTEL_TARGET, +}; +use crate::services::observability::tracing_boundary::ScopedDispatch; +use crate::services::observability::traits::{NoopTelemetry, Telemetry}; +use crate::services::parse::command_runtime::parse_runtime_command; + +pub const SCE_TELEMETRY_ENV: &str = "SCE_TELEMETRY"; +pub const STANDALONE_MODE: &str = "standalone"; +const OTLP_ENDPOINT_ENV: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; +const OTLP_TRACES_ENDPOINT_ENV: &str = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"; +pub const SERVICE_NAME: &str = "sce"; +pub const FLUSH_TIMEOUT_ENV: &str = "SCE_TELEMETRY_FLUSH_TIMEOUT_MS"; +pub const DEFAULT_FLUSH_BUDGET: Duration = Duration::from_secs(1); +pub const MAX_FLUSH_BUDGET: Duration = Duration::from_secs(5); +const EXPORT_TIMEOUT: Duration = Duration::from_millis(750); +const SHUTDOWN_THREAD_NAME: &str = "sce-otel-shutdown"; +const INIT_THREAD_NAME: &str = "sce-otel-init"; + +#[cfg(feature = "telemetry-test-receiver")] +pub mod test_receiver; + +pub mod lifecycle_markers; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BatchSettings { + pub queue_size: usize, + pub batch_size: usize, + pub scheduled_delay: Duration, +} + +impl BatchSettings { + pub const PRODUCTION: Self = Self { + queue_size: 256, + batch_size: 64, + scheduled_delay: Duration::from_secs(2), + }; +} + +pub fn resolve_flush_budget(env: &impl Fn(&str) -> Option) -> Duration { + let max_millis = u64::try_from(MAX_FLUSH_BUDGET.as_millis()).unwrap_or(u64::MAX); + env(FLUSH_TIMEOUT_ENV) + .and_then(|value| value.trim().parse::().ok()) + .filter(|millis| (1..=max_millis).contains(millis)) + .map_or(DEFAULT_FLUSH_BUDGET, Duration::from_millis) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TelemetryMode { + Disabled, + Standalone, +} + +pub fn resolve_telemetry_mode( + env: &impl Fn(&str) -> Option, + export_permitted_for_invocation: impl FnOnce() -> bool, +) -> TelemetryMode { + if env(SCE_TELEMETRY_ENV).as_deref() != Some(STANDALONE_MODE) { + return TelemetryMode::Disabled; + } + if export_permitted_for_invocation() { + TelemetryMode::Standalone + } else { + TelemetryMode::Disabled + } +} + +#[derive(Debug)] +pub struct BoundaryBExporter { + inner: E, +} + +impl BoundaryBExporter { + pub fn new(inner: E) -> Self { + Self { inner } + } +} + +fn raw_attribute_value(value: &Value) -> Option> { + match value { + Value::I64(number) => Some(OtelRawValue::Number(u64::try_from(*number).ok()?)), + Value::String(text) => Some(OtelRawValue::Text(text.as_str())), + _ => None, + } +} + +fn canonical_number(text: &str) -> Option { + let number = text.parse::().ok()?; + (number.to_string() == text).then_some(number) +} + +fn admit_attribute(attribute: &KeyValue) -> Option { + let key = attribute.key.as_str(); + let raw = raw_attribute_value(&attribute.value)?; + let admitted = OtelAttribute::admit(key, raw).or_else(|| match raw { + OtelRawValue::Text(text) => { + OtelAttribute::admit(key, OtelRawValue::Number(canonical_number(text)?)) + } + OtelRawValue::Number(_) => None, + })?; + let value = match admitted.value() { + OtelValue::Number(number) => Value::I64(i64::try_from(number).unwrap_or(i64::MAX)), + OtelValue::Static(text) => Value::from(text), + }; + Some(KeyValue::new(admitted.key(), value)) +} + +fn admit_span(mut span: SpanData) -> Option { + OtelName::parse(span.name.as_ref())?; + span.attributes = span.attributes.iter().filter_map(admit_attribute).collect(); + span.dropped_attributes_count = 0; + span.events = SpanEvents::default(); + span.links = SpanLinks::default(); + if matches!(span.status, Status::Error { .. }) { + span.status = Status::error(""); + } + Some(span) +} + +impl SpanExporter for BoundaryBExporter { + fn export(&self, batch: Vec) -> impl Future + Send { + let admitted: Vec = batch.into_iter().filter_map(admit_span).collect(); + async move { + if admitted.is_empty() { + return Ok(()); + } + self.inner.export(admitted).await + } + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner.shutdown_with_timeout(timeout) + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn set_resource(&mut self, resource: &Resource) { + self.inner.set_resource(resource); + } +} + +pub fn scoped_dispatch(provider: &SdkTracerProvider) -> ScopedDispatch { + let layer = tracing_opentelemetry::layer() + .with_tracer(provider.tracer(SERVICE_NAME)) + .with_location(false) + .with_threads(false) + .with_tracked_inactivity(false) + .with_filter(filter_fn(|metadata| { + metadata.is_span() + && metadata.target() == OTEL_TARGET + && OtelName::parse(metadata.name()).is_some() + })); + ScopedDispatch::from_layer(layer) +} + +fn resource() -> Resource { + Resource::builder_empty() + .with_attributes([ + KeyValue::new("service.name", SERVICE_NAME), + KeyValue::new("service.version", env!("CARGO_PKG_VERSION")), + ]) + .build() +} + +pub fn provider_with_exporter(exporter: E) -> SdkTracerProvider { + provider_with_settings(exporter, BatchSettings::PRODUCTION) +} + +pub fn provider_with_settings( + exporter: E, + settings: BatchSettings, +) -> SdkTracerProvider { + let config = BatchConfigBuilder::default() + .with_max_queue_size(settings.queue_size) + .with_max_export_batch_size(settings.batch_size) + .with_scheduled_delay(settings.scheduled_delay) + .build(); + let processor = BatchSpanProcessor::builder(BoundaryBExporter::new(exporter)) + .with_batch_config(config) + .build(); + SdkTracerProvider::builder() + .with_resource(resource()) + .with_span_processor(processor) + .build() +} + +fn standalone_traces_endpoint(env: &impl Fn(&str) -> Option) -> Option { + let configured = |key| env(key).filter(|value| !value.is_empty()); + configured(OTLP_TRACES_ENDPOINT_ENV).or_else(|| { + configured(OTLP_ENDPOINT_ENV) + .map(|base| format!("{}/v1/traces", base.trim_end_matches('/'))) + }) +} + +fn http_client_builder() -> reqwest::blocking::ClientBuilder { + reqwest::blocking::Client::builder() + .timeout(EXPORT_TIMEOUT) + .connect_timeout(EXPORT_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) +} + +fn http_client() -> anyhow::Result { + http_client_builder() + .build() + .or_else(|_| { + http_client_builder() + .tls_certs_only(std::iter::empty()) + .build() + }) + .map_err(Into::into) +} + +fn build_provider(traces_endpoint: Option) -> anyhow::Result { + let builder = opentelemetry_otlp::SpanExporter::builder() + .with_http() + .with_http_client(http_client()?) + .with_protocol(Protocol::HttpBinary) + .with_retry_policy(RetryPolicy::disabled()) + .with_timeout(EXPORT_TIMEOUT); + let exporter = match traces_endpoint { + Some(endpoint) => builder.with_endpoint(endpoint), + None => builder, + } + .build()?; + Ok(provider_with_exporter(exporter)) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ShutdownOutcome { + Completed, + Incomplete, + Unavailable, +} + +struct Teardown { + dispatch: ScopedDispatch, + provider: SdkTracerProvider, +} + +impl Teardown { + /// Last-resort path when no shutdown thread can take ownership. Dropping + /// the final `SdkTracerProvider` reference runs the SDK's synchronous + /// shutdown, and `shutdown_with_timeout(Duration::ZERO)` is not guaranteed + /// to return immediately, so neither may run on the caller. Forgetting + /// skips both destructors: no SDK code runs, nothing is exported or + /// written, and the provider, dispatch and batch worker stay allocated + /// until the short-lived CLI process exits. Memory is not reclaimed before + /// then; the tradeoff is accepted because this path needs thread creation + /// or handoff to have failed. + fn abandon(self) { + std::mem::forget(self); + } +} + +struct ShutdownWorker { + handoff: Receiver, + done: tokio::sync::oneshot::Sender, + budget: Duration, +} + +impl ShutdownWorker { + fn run(self) { + let outcome = match self.handoff.recv() { + Ok(Teardown { dispatch, provider }) => { + let outcome = match provider.shutdown_with_timeout(self.budget) { + Ok(()) => ShutdownOutcome::Completed, + Err(_) => ShutdownOutcome::Incomplete, + }; + drop(dispatch); + drop(provider); + outcome + } + Err(_) => ShutdownOutcome::Unavailable, + }; + let _ = self.done.send(outcome); + } +} + +fn spawn_shutdown_thread(worker: ShutdownWorker) -> std::io::Result<()> { + std::thread::Builder::new() + .name(SHUTDOWN_THREAD_NAME.to_string()) + .spawn(move || worker.run()) + .map(drop) +} + +fn spawn_teardown( + teardown: Teardown, + budget: Duration, + spawn_worker: S, +) -> Option> +where + S: FnOnce(ShutdownWorker) -> std::io::Result<()>, +{ + let (done, done_receiver) = tokio::sync::oneshot::channel(); + let (handoff_sender, handoff) = sync_channel::(1); + let worker = ShutdownWorker { + handoff, + done, + budget, + }; + if spawn_worker(worker).is_err() { + teardown.abandon(); + return None; + } + if let Err(SendError(teardown)) = handoff_sender.send(teardown) { + teardown.abandon(); + return None; + } + Some(done_receiver) +} + +pub struct OtelTelemetry { + teardown: Option, + flush_budget: Duration, +} + +impl OtelTelemetry { + pub fn from_provider_with_budget(provider: SdkTracerProvider, flush_budget: Duration) -> Self { + Self { + teardown: Some(Teardown { + dispatch: scoped_dispatch(&provider), + provider, + }), + flush_budget, + } + } + + pub fn standalone(env: &impl Fn(&str) -> Option) -> anyhow::Result { + Self::with_endpoint(standalone_traces_endpoint(env), resolve_flush_budget(env)) + } + + pub fn with_endpoint( + traces_endpoint: Option, + flush_budget: Duration, + ) -> anyhow::Result { + let provider = std::thread::Builder::new() + .name(INIT_THREAD_NAME.to_string()) + .spawn(move || build_provider(traces_endpoint))? + .join() + .map_err(|_| anyhow::anyhow!("telemetry initialization panicked"))??; + Ok(Self::from_provider_with_budget(provider, flush_budget)) + } + + pub async fn shutdown(self) -> ShutdownOutcome { + self.shutdown_with(spawn_shutdown_thread).await + } + + async fn shutdown_with(mut self, spawn_worker: S) -> ShutdownOutcome + where + S: FnOnce(ShutdownWorker) -> std::io::Result<()>, + { + let Some(teardown) = self.teardown.take() else { + return ShutdownOutcome::Unavailable; + }; + let budget = self.flush_budget; + let Some(done) = spawn_teardown(teardown, budget, spawn_worker) else { + return ShutdownOutcome::Unavailable; + }; + match tokio::time::timeout(budget, done).await { + Ok(Ok(outcome)) => outcome, + Ok(Err(_)) | Err(_) => ShutdownOutcome::Incomplete, + } + } +} + +impl OtelTelemetry { + fn release_with(&mut self, spawn_worker: S) + where + S: FnOnce(ShutdownWorker) -> std::io::Result<()>, + { + if let Some(teardown) = self.teardown.take() { + drop(spawn_teardown(teardown, self.flush_budget, spawn_worker)); + } + } +} + +impl Drop for OtelTelemetry { + fn drop(&mut self) { + self.release_with(spawn_shutdown_thread); + } +} + +impl Telemetry for OtelTelemetry { + async fn with_default_subscriber(&self, action: &mut F) -> Result + where + F: FnMut() -> Fut, + Fut: Future>, + { + match &self.teardown { + Some(teardown) => teardown.dispatch.scope(action()).await, + None => action().await, + } + } +} + +pub enum RuntimeTelemetry { + Noop(NoopTelemetry), + Otel(OtelTelemetry), +} + +impl RuntimeTelemetry { + pub fn disabled() -> Self { + Self::Noop(NoopTelemetry) + } + + pub async fn shutdown(self) { + if let Self::Otel(telemetry) = self { + telemetry.shutdown().await; + } + } +} + +impl Telemetry for RuntimeTelemetry { + async fn with_default_subscriber(&self, action: &mut F) -> Result + where + F: FnMut() -> Fut, + Fut: Future>, + { + match self { + Self::Noop(telemetry) => telemetry.with_default_subscriber(action).await, + Self::Otel(telemetry) => telemetry.with_default_subscriber(action).await, + } + } +} + +pub fn select_runtime_telemetry( + args: &[String], + registry: &CommandRegistry, + env: &impl Fn(&str) -> Option, +) -> RuntimeTelemetry { + #[cfg(feature = "telemetry-test-receiver")] + if let Some(config) = test_receiver::resolve(env) { + return OtelTelemetry::with_endpoint(Some(config.traces_endpoint()), config.flush_budget) + .map_or_else(|_| RuntimeTelemetry::disabled(), RuntimeTelemetry::Otel); + } + let mode = resolve_telemetry_mode(env, || { + parse_runtime_command(args.iter().cloned(), registry, None) + .is_ok_and(|command| !command.is_hook_invocation()) + }); + match mode { + TelemetryMode::Disabled => RuntimeTelemetry::disabled(), + TelemetryMode::Standalone => OtelTelemetry::standalone(env) + .map_or_else(|_| RuntimeTelemetry::disabled(), RuntimeTelemetry::Otel), + } +} + +#[cfg(test)] +pub(crate) mod test_support; + +#[cfg(test)] +mod tests; diff --git a/cli/src/services/observability/otel_runtime/lifecycle_markers.rs b/cli/src/services/observability/otel_runtime/lifecycle_markers.rs new file mode 100644 index 000000000..6c1a4ed93 --- /dev/null +++ b/cli/src/services/observability/otel_runtime/lifecycle_markers.rs @@ -0,0 +1,47 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LifecycleMarker { + CommandComplete, + ShutdownBegin, + ShutdownEnd, + ProcessExitRequested, +} + +impl LifecycleMarker { + #[cfg(feature = "telemetry-test-receiver")] + fn as_str(self) -> &'static str { + match self { + Self::CommandComplete => "command_complete", + Self::ShutdownBegin => "shutdown_begin", + Self::ShutdownEnd => "shutdown_end", + Self::ProcessExitRequested => "process_exit_requested", + } + } +} + +#[cfg(feature = "telemetry-test-receiver")] +pub const LIFECYCLE_FILE_ENV: &str = "SCE_TELEMETRY_TEST_LIFECYCLE_FILE"; + +#[cfg(feature = "telemetry-test-receiver")] +pub fn mark(marker: LifecycleMarker) { + use std::io::Write; + use std::sync::OnceLock; + use std::time::Instant; + + static ORIGIN: OnceLock = OnceLock::new(); + + let Some(path) = std::env::var_os(LIFECYCLE_FILE_ENV) else { + return; + }; + let elapsed = ORIGIN.get_or_init(Instant::now).elapsed(); + let Ok(mut file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + else { + return; + }; + let _ = writeln!(file, "{} {}", marker.as_str(), elapsed.as_nanos()); +} + +#[cfg(not(feature = "telemetry-test-receiver"))] +pub fn mark(_marker: LifecycleMarker) {} diff --git a/cli/src/services/observability/otel_runtime/test_receiver.rs b/cli/src/services/observability/otel_runtime/test_receiver.rs new file mode 100644 index 000000000..befe89414 --- /dev/null +++ b/cli/src/services/observability/otel_runtime/test_receiver.rs @@ -0,0 +1,164 @@ +use std::net::SocketAddr; +use std::time::Duration; + +use super::resolve_flush_budget; + +pub const TEST_RECEIVER_MODE: &str = "test-receiver"; +pub const TEST_RECEIVER_ENDPOINT_ENV: &str = "SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT"; +const HEADER_OVERRIDE_ENVS: [&str; 2] = [ + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", +]; +const HTTP_SCHEME: &str = "http://"; +const TRACES_PATH: &str = "/v1/traces"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TestReceiverConfig { + pub endpoint: SocketAddr, + pub flush_budget: Duration, +} + +impl TestReceiverConfig { + pub fn traces_endpoint(&self) -> String { + format!("{HTTP_SCHEME}{}{TRACES_PATH}", self.endpoint) + } +} + +pub fn parse_loopback_endpoint(raw: &str) -> Option { + let authority = raw.strip_prefix(HTTP_SCHEME)?; + let authority = authority.strip_suffix('/').unwrap_or(authority); + let address = authority.parse::().ok()?; + (address.ip().is_loopback() && address.port() != 0).then_some(address) +} + +pub fn resolve(env: &impl Fn(&str) -> Option) -> Option { + if env(super::SCE_TELEMETRY_ENV).as_deref() != Some(TEST_RECEIVER_MODE) { + return None; + } + let endpoint = parse_loopback_endpoint(&env(TEST_RECEIVER_ENDPOINT_ENV)?)?; + if HEADER_OVERRIDE_ENVS + .iter() + .any(|key| env(key).is_some_and(|value| !value.is_empty())) + { + return None; + } + Some(TestReceiverConfig { + endpoint, + flush_budget: resolve_flush_budget(env), + }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + + use super::*; + + fn env_of(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let map: HashMap = pairs + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(); + move |key| map.get(key).cloned() + } + + #[test] + fn telemetry_test_receiver_accepts_only_plain_http_loopback_literals() { + for accepted in [ + "http://127.0.0.1:4318", + "http://127.0.0.1:4318/", + "http://127.200.3.4:1", + "http://[::1]:4318", + ] { + assert!(parse_loopback_endpoint(accepted).is_some(), "{accepted}"); + } + for rejected in [ + "http://localhost:4318", + "https://127.0.0.1:4318", + "HTTP://127.0.0.1:4318", + "http://10.0.0.1:4318", + "http://192.0.2.1:4318", + "http://0.0.0.0:4318", + "http://127.0.0.1", + "http://127.0.0.1:0", + "http://127.0.0.1:4318/v1/traces", + "http://127.0.0.1:4318?x=1", + "http://user@127.0.0.1:4318", + "http://[::ffff:127.0.0.1]:4318", + "http://[::]:4318", + "127.0.0.1:4318", + "", + ] { + assert!(parse_loopback_endpoint(rejected).is_none(), "{rejected}"); + } + } + + #[test] + fn telemetry_test_receiver_resolution_is_pure_and_carries_no_credential() { + let keys = Mutex::new(Vec::new()); + let base = env_of(&[ + ("SCE_TELEMETRY", "test-receiver"), + ( + "SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", + "http://127.0.0.1:4318", + ), + ("OTEL_EXPORTER_OTLP_ENDPOINT", "https://evil.example"), + ( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "https://evil.example/v1/traces", + ), + ("SCE_TELEMETRY_FLUSH_TIMEOUT_MS", "250"), + ]); + let recording = |key: &str| { + keys.lock().unwrap().push(key.to_string()); + base(key) + }; + let config = resolve(&recording).unwrap(); + assert_eq!(config.traces_endpoint(), "http://127.0.0.1:4318/v1/traces"); + assert_eq!(config.flush_budget, Duration::from_millis(250)); + let mut read = keys.lock().unwrap().clone(); + read.sort(); + read.dedup(); + assert_eq!( + read, + vec![ + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "SCE_TELEMETRY", + "SCE_TELEMETRY_FLUSH_TIMEOUT_MS", + "SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", + ] + ); + } + + #[test] + fn telemetry_test_receiver_resolution_requires_explicit_mode_and_valid_endpoint() { + let endpoint = ( + "SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", + "http://127.0.0.1:4318", + ); + assert!(resolve(&env_of(&[endpoint])).is_none()); + assert!(resolve(&env_of(&[("SCE_TELEMETRY", "standalone"), endpoint])).is_none()); + assert!(resolve(&env_of(&[("SCE_TELEMETRY", "test-receiver")])).is_none()); + assert!(resolve(&env_of(&[ + ("SCE_TELEMETRY", "test-receiver"), + ( + "SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", + "http://localhost:4318" + ), + ])) + .is_none()); + for header_key in HEADER_OVERRIDE_ENVS { + assert!( + resolve(&env_of(&[ + ("SCE_TELEMETRY", "test-receiver"), + endpoint, + (header_key, "authorization=Bearer x"), + ])) + .is_none(), + "{header_key}" + ); + } + } +} diff --git a/cli/src/services/observability/otel_runtime/test_support.rs b/cli/src/services/observability/otel_runtime/test_support.rs new file mode 100644 index 000000000..c49bb06c1 --- /dev/null +++ b/cli/src/services/observability/otel_runtime/test_support.rs @@ -0,0 +1,173 @@ +use std::future::{ready, Future}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::ThreadId; +use std::time::{Duration, Instant}; + +use opentelemetry::Value; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::trace::{SdkTracerProvider, SpanData, SpanExporter}; + +use super::{provider_with_exporter, scoped_dispatch}; +use crate::services::observability::tracing_boundary::ScopedDispatch; + +pub(crate) const BATCH_THREAD_NAME: &str = "OpenTelemetry.Traces.BatchProcessor"; +const STALL_CEILING: Duration = Duration::from_mins(1); + +pub(crate) type ExportThread = (Option, ThreadId); + +#[derive(Clone, Debug, Default)] +pub(crate) struct Stall { + state: Arc<(Mutex, Condvar)>, +} + +#[derive(Debug, Default)] +struct StallState { + entered: usize, + released: bool, +} + +impl Stall { + pub(crate) fn block(&self) { + let (lock, condvar) = &*self.state; + let mut state = lock.lock().unwrap(); + state.entered += 1; + condvar.notify_all(); + let deadline = Instant::now() + STALL_CEILING; + while !state.released { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + state = condvar.wait_timeout(state, remaining).unwrap().0; + } + } + + pub(crate) fn wait_entered(&self, timeout: Duration) -> bool { + let (lock, condvar) = &*self.state; + let deadline = Instant::now() + timeout; + let mut state = lock.lock().unwrap(); + while state.entered == 0 { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return false; + } + state = condvar.wait_timeout(state, remaining).unwrap().0; + } + true + } + + pub(crate) fn is_released(&self) -> bool { + self.state.0.lock().unwrap().released + } + + pub(crate) fn release(&self) { + let (lock, condvar) = &*self.state; + lock.lock().unwrap().released = true; + condvar.notify_all(); + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct Capture { + pub(crate) spans: Arc>>, + pub(crate) export_threads: Arc>>, + pub(crate) shutdown_calls: Arc, + pub(crate) exports_after_shutdown: Arc, + pub(crate) shutdown_threads: Arc>>>, + pub(crate) export_stall: Option, + pub(crate) shutdown_stall: Option, +} + +impl Capture { + pub(crate) fn spans(&self) -> Vec { + self.spans.lock().unwrap().clone() + } + + pub(crate) fn names(&self) -> Vec { + self.spans() + .iter() + .map(|span| span.name.to_string()) + .collect() + } + + pub(crate) fn shutdown_calls(&self) -> usize { + self.shutdown_calls.load(Ordering::SeqCst) + } + + pub(crate) fn exports_after_shutdown(&self) -> usize { + self.exports_after_shutdown.load(Ordering::SeqCst) + } +} + +impl SpanExporter for Capture { + fn export(&self, batch: Vec) -> impl Future + Send { + let current = std::thread::current(); + self.export_threads + .lock() + .unwrap() + .push((current.name().map(str::to_string), current.id())); + if self.shutdown_calls() > 0 { + self.exports_after_shutdown.fetch_add(1, Ordering::SeqCst); + } + self.spans.lock().unwrap().extend(batch); + if let Some(stall) = &self.export_stall { + stall.block(); + } + ready(Ok(())) + } + + fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult { + self.shutdown_threads + .lock() + .unwrap() + .push(std::thread::current().name().map(str::to_string)); + self.shutdown_calls.fetch_add(1, Ordering::SeqCst); + if let Some(stall) = &self.shutdown_stall { + stall.block(); + } + Ok(()) + } +} + +pub(crate) fn harness() -> (Capture, SdkTracerProvider, ScopedDispatch) { + let capture = Capture::default(); + let provider = provider_with_exporter(capture.clone()); + let dispatch = scoped_dispatch(&provider); + (capture, provider, dispatch) +} + +pub(crate) fn attributes(span: &SpanData) -> Vec<(String, String)> { + let mut attributes: Vec<(String, String)> = span + .attributes + .iter() + .map(|attribute| { + let value = match &attribute.value { + Value::I64(number) => number.to_string(), + Value::String(text) => text.to_string(), + other => format!("{other:?}"), + }; + (attribute.key.to_string(), value) + }) + .collect(); + attributes.sort(); + attributes +} + +pub(crate) fn span_named<'a>(spans: &'a [SpanData], name: &str) -> &'a SpanData { + spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("span {name} exported")) +} + +pub(crate) fn wait_until(timeout: Duration, mut condition: impl FnMut() -> bool) -> bool { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if condition() { + return true; + } + std::thread::sleep(Duration::from_millis(5)); + } + condition() +} diff --git a/cli/src/services/observability/otel_runtime/tests.rs b/cli/src/services/observability/otel_runtime/tests.rs new file mode 100644 index 000000000..1bb2fce8a --- /dev/null +++ b/cli/src/services/observability/otel_runtime/tests.rs @@ -0,0 +1,706 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Waker}; +use std::thread::ThreadId; +use std::time::{Duration, Instant}; + +use opentelemetry::trace::{SpanId, Status, TracerProvider}; + +use super::test_support::{ + attributes, harness, span_named, wait_until, Capture, Stall, BATCH_THREAD_NAME, +}; +use super::{ + provider_with_exporter, provider_with_settings, scoped_dispatch, BatchSettings, OtelTelemetry, + ShutdownOutcome, ShutdownWorker, SERVICE_NAME, +}; +use crate::services::hooks::mutation_scope_lock::{AdapterLockSpec, AdvisoryLockError}; +use crate::services::observability::otel_policy::OtelName; +use crate::services::observability::tracing_boundary::test_spans::{ + emit_raw_boundary_b_inputs, emit_target_matrix, in_otel_span, thread_default_is_none, +}; +use crate::services::observability::tracing_boundary::{spawn_in_current_scope, ScopedDispatch}; +use crate::services::observability::traits::Telemetry; + +const SECRET: &str = "sk-live-SECRET-TOKEN-9f8e7d"; +const STALL_PROBE_BUDGET: Duration = Duration::from_millis(300); +const SCHEDULING_SLACK: Duration = Duration::from_millis(700); +const LOCK_CONTENDER_BOUND: Duration = Duration::from_secs(2); +const EXPORT_START_TIMEOUT: Duration = Duration::from_secs(10); + +fn run_sync(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(future) +} + +fn stalled_telemetry( + export_stall: Option, + shutdown_stall: Option, + budget: Duration, +) -> (Capture, OtelTelemetry) { + let capture = Capture { + export_stall, + shutdown_stall, + ..Capture::default() + }; + let provider = provider_with_exporter(capture.clone()); + ( + capture, + OtelTelemetry::from_provider_with_budget(provider, budget), + ) +} + +async fn emit_command_scope(telemetry: &OtelTelemetry) { + telemetry + .with_default_subscriber(&mut || async { + in_otel_span(OtelName::Command, async { + tokio::task::yield_now().await; + }) + .await; + Ok(String::new()) + }) + .await + .unwrap(); +} +#[test] +fn tracing_boundary_b_exporter_layer_exports_only_allowlisted_spans_and_attributes() { + let (capture, provider, dispatch) = harness(); + run_sync(dispatch.scope(async { emit_raw_boundary_b_inputs(SECRET) })); + provider.force_flush().unwrap(); + + let spans = capture.spans(); + assert_eq!(capture.names(), vec!["sce.command"]); + let span = &spans[0]; + assert_eq!( + attributes(span), + vec![ + ("sce.duration_ms".to_string(), "5".to_string()), + ("sce.error.category".to_string(), "parse".to_string()), + ("sce.outcome".to_string(), "success".to_string()), + ] + ); + assert!(span.events.is_empty()); + assert!(span.links.is_empty()); + assert_eq!(span.status, Status::error("")); + let rendered = format!("{spans:?}"); + assert!(!rendered.contains(SECRET), "{rendered}"); + assert!(!rendered.contains("exception"), "{rendered}"); + assert!(!rendered.contains("password"), "{rendered}"); +} + +#[test] +fn tracing_boundary_b_exporter_wrapper_enforces_policy_without_layer_filter() { + let capture = Capture::default(); + let provider = provider_with_exporter(capture.clone()); + let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer(SERVICE_NAME)); + let dispatch = ScopedDispatch::from_layer(layer); + run_sync(dispatch.scope(async { emit_raw_boundary_b_inputs(SECRET) })); + provider.force_flush().unwrap(); + + let spans = capture.spans(); + assert!(!spans.is_empty()); + for span in &spans { + assert!(OtelName::parse(&span.name).is_some(), "{}", span.name); + assert!(span.events.is_empty()); + assert!(span.links.is_empty()); + assert!(matches!(span.status, Status::Unset | Status::Error { .. })); + if let Status::Error { description } = &span.status { + assert!(description.is_empty()); + } + for (key, value) in attributes(span) { + assert!( + crate::services::observability::otel_policy::OtelAttribute::KEYS + .contains(&key.as_str()), + "{key}" + ); + assert_ne!(value, SECRET); + } + } + let rendered = format!("{spans:?}"); + assert!(!rendered.contains(SECRET), "{rendered}"); + assert!(!rendered.contains("sce.arbitrary"), "{rendered}"); + assert!(!rendered.contains("otel.name"), "{rendered}"); +} + +#[test] +fn tracing_boundary_b_layer_filter_excludes_sce_target_spans() { + let (capture, provider, dispatch) = harness(); + run_sync(dispatch.scope(async { emit_raw_boundary_b_inputs(SECRET) })); + provider.force_flush().unwrap(); + assert_eq!(capture.spans().len(), 1); + assert_eq!( + attributes(&capture.spans()[0]) + .iter() + .filter(|(key, _)| key == "sce.outcome") + .count(), + 1 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn telemetry_async_scope_is_active_on_every_poll_with_nested_parents() { + let (capture, provider, dispatch) = harness(); + dispatch + .scope(async { + in_otel_span(OtelName::Command, async { + for _ in 0..3 { + tokio::task::yield_now().await; + } + in_otel_span(OtelName::Reconciliation, async { + tokio::task::yield_now().await; + in_otel_span(OtelName::GitSnapshot, async { + tokio::task::yield_now().await; + }) + .await; + }) + .await; + }) + .await; + }) + .await; + provider.force_flush().unwrap(); + + let spans = capture.spans(); + assert_eq!(spans.len(), 3); + let command = span_named(&spans, "sce.command"); + let reconciliation = span_named(&spans, "sce.reconciliation"); + let snapshot = span_named(&spans, "sce.git.snapshot"); + assert_eq!(command.parent_span_id, SpanId::INVALID); + assert_eq!( + reconciliation.parent_span_id, + command.span_context.span_id() + ); + assert_eq!( + snapshot.parent_span_id, + reconciliation.span_context.span_id() + ); + assert_eq!( + command.span_context.trace_id(), + snapshot.span_context.trace_id() + ); + assert!(thread_default_is_none()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn telemetry_async_scope_isolates_concurrent_commands() { + let (capture_a, provider_a, dispatch_a) = harness(); + let (capture_b, provider_b, dispatch_b) = harness(); + let mut tasks = Vec::new(); + for index in 0..16 { + let (dispatch, name) = if index % 2 == 0 { + (dispatch_a.clone(), OtelName::Command) + } else { + (dispatch_b.clone(), OtelName::Reconciliation) + }; + tasks.push(tokio::spawn(dispatch.scope(async move { + for _ in 0..4 { + in_otel_span(name, async { + tokio::task::yield_now().await; + }) + .await; + } + }))); + } + for task in tasks { + task.await.unwrap(); + } + provider_a.force_flush().unwrap(); + provider_b.force_flush().unwrap(); + + assert_eq!(capture_a.names(), vec!["sce.command"; 32]); + assert_eq!(capture_b.names(), vec!["sce.reconciliation"; 32]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn telemetry_async_scope_reaches_spawned_tasks_only_through_the_helper() { + let (capture, provider, dispatch) = harness(); + dispatch + .scope(async { + in_otel_span(OtelName::Command, async { + spawn_in_current_scope(async { + in_otel_span(OtelName::GitSnapshot, async { + tokio::task::yield_now().await; + }) + .await; + }) + .await + .unwrap(); + tokio::spawn(async { + in_otel_span(OtelName::DbOperation, async {}).await; + }) + .await + .unwrap(); + }) + .await; + }) + .await; + provider.force_flush().unwrap(); + + let spans = capture.spans(); + assert_eq!(spans.len(), 2, "{:?}", capture.names()); + let command = span_named(&spans, "sce.command"); + let snapshot = span_named(&spans, "sce.git.snapshot"); + assert_eq!(snapshot.parent_span_id, command.span_context.span_id()); + assert!(!capture.names().contains(&"sce.db.operation".to_string())); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn telemetry_async_scope_exports_from_dedicated_thread_without_global_subscriber() { + let (capture, provider, dispatch) = harness(); + let command_thread = std::thread::current().id(); + dispatch + .scope(async { + in_otel_span(OtelName::Command, async { + tokio::task::yield_now().await; + }) + .await; + }) + .await; + provider.force_flush().unwrap(); + + let threads = capture.export_threads.lock().unwrap().clone(); + assert!(!threads.is_empty()); + for (name, id) in threads { + assert_eq!(name.as_deref(), Some(BATCH_THREAD_NAME)); + assert_ne!(id, command_thread); + } + assert!(thread_default_is_none()); +} + +#[test] +fn tracing_boundary_b_production_subscriber_rejects_wrong_target() { + let (capture, provider, dispatch) = harness(); + run_sync(dispatch.scope(async { emit_target_matrix() })); + provider.force_flush().unwrap(); + + let spans = capture.spans(); + assert_eq!(capture.names(), vec!["sce.command"], "{spans:?}"); + let span = &spans[0]; + assert_eq!( + attributes(span), + vec![("sce.outcome".to_string(), "success".to_string())] + ); + assert!(span.events.is_empty()); + assert!(span.links.is_empty()); + let rendered = format!("{spans:?}"); + for rejected in ["failure", "timeout", "cancelled", "sce.not_enumerated"] { + assert!(!rendered.contains(rejected), "{rejected}: {rendered}"); + } + assert!(!rendered.contains("sce.app.start"), "{rendered}"); +} + +#[test] +fn tracing_boundary_b_exporter_wrapper_cannot_observe_the_tracing_target() { + let capture = Capture::default(); + let provider = provider_with_exporter(capture.clone()); + let layer = tracing_opentelemetry::layer() + .with_tracer(provider.tracer(SERVICE_NAME)) + .with_location(false) + .with_threads(false); + let dispatch = ScopedDispatch::from_layer(layer); + run_sync(dispatch.scope(async { emit_target_matrix() })); + provider.force_flush().unwrap(); + + let spans = capture.spans(); + let outcomes: Vec = spans + .iter() + .flat_map(attributes) + .filter(|(key, _)| key == "sce.outcome") + .map(|(_, value)| value) + .collect(); + assert!( + outcomes.contains(&"failure".to_string()) && outcomes.contains(&"timeout".to_string()), + "wrong-target spans reach the wrapper because SpanData carries no tracing target: {outcomes:?}" + ); + assert!(!spans.iter().any(|span| span.name == "sce.not_enumerated")); + for span in &spans { + assert!( + attributes(span).iter().all(|(key, _)| key != "target"), + "no target attribute is fabricated" + ); + } +} + +#[test] +fn telemetry_async_scope_survives_worker_thread_migration() { + type Handoff = Pin + Send>>; + + let (capture, provider, dispatch) = harness(); + let observed: Arc>> = Arc::default(); + + let future: Handoff = Box::pin(dispatch.scope({ + let observed = observed.clone(); + async move { + let record = |observed: &Mutex>| { + observed + .lock() + .unwrap() + .push((std::thread::current().id(), !thread_default_is_none())); + }; + in_otel_span(OtelName::Command, async { + in_otel_span(OtelName::Reconciliation, async { + record(&observed); + }) + .await; + let mut suspended = false; + std::future::poll_fn(|_| { + if suspended { + Poll::Ready(()) + } else { + suspended = true; + Poll::Pending + } + }) + .await; + in_otel_span(OtelName::GitSnapshot, async { + record(&observed); + }) + .await; + }) + .await; + } + })); + + let (handoff_sender, handoff_receiver) = mpsc::channel::(); + let thread_a = std::thread::Builder::new() + .name("migration-a".to_string()) + .spawn(move || { + let mut future = future; + let mut cx = Context::from_waker(Waker::noop()); + let first = future.as_mut().poll(&mut cx); + let id = std::thread::current().id(); + let clean = thread_default_is_none(); + handoff_sender.send(future).unwrap(); + (first.is_pending(), id, clean) + }) + .unwrap(); + let thread_b = std::thread::Builder::new() + .name("migration-b".to_string()) + .spawn(move || { + let mut future = handoff_receiver.recv().unwrap(); + let mut cx = Context::from_waker(Waker::noop()); + let second = future.as_mut().poll(&mut cx); + ( + second.is_ready(), + std::thread::current().id(), + thread_default_is_none(), + ) + }) + .unwrap(); + let (first_pending, id_a, clean_a) = thread_a.join().unwrap(); + let (second_ready, id_b, clean_b) = thread_b.join().unwrap(); + provider.force_flush().unwrap(); + + assert!(first_pending, "first poll must suspend on thread A"); + assert!(second_ready, "second poll must complete on thread B"); + assert_ne!(id_a, id_b); + assert!(clean_a && clean_b && thread_default_is_none()); + assert_eq!( + *observed.lock().unwrap(), + vec![(id_a, true), (id_b, true)], + "the subscriber was active on each polling thread" + ); + + let spans = capture.spans(); + assert_eq!(spans.len(), 3, "{:?}", capture.names()); + let command = span_named(&spans, "sce.command"); + let first_child = span_named(&spans, "sce.reconciliation"); + let second_child = span_named(&spans, "sce.git.snapshot"); + assert_eq!(command.parent_span_id, SpanId::INVALID); + assert_eq!(first_child.parent_span_id, command.span_context.span_id()); + assert_eq!(second_child.parent_span_id, command.span_context.span_id()); + for child in [first_child, second_child] { + assert_eq!( + child.span_context.trace_id(), + command.span_context.trace_id() + ); + } +} + +#[test] +fn telemetry_shutdown_exactly_once() { + run_sync(async { + let (capture, telemetry) = stalled_telemetry(None, None, Duration::from_secs(5)); + emit_command_scope(&telemetry).await; + assert_eq!(capture.shutdown_calls(), 0); + + let outcome = telemetry.shutdown().await; + assert_eq!(outcome, ShutdownOutcome::Completed); + assert_eq!(capture.shutdown_calls(), 1); + assert_eq!(capture.names(), vec!["sce.command"]); + assert_eq!(capture.exports_after_shutdown(), 0); + let threads = capture.shutdown_threads.lock().unwrap().clone(); + assert_eq!(threads, vec![Some(BATCH_THREAD_NAME.to_string())]); + }); +} + +#[test] +fn telemetry_shutdown_exactly_once_when_clones_outlive_the_handle() { + run_sync(async { + let capture = Capture::default(); + let provider = provider_with_exporter(capture.clone()); + let retained_provider = provider.clone(); + let retained_dispatch = scoped_dispatch(&provider); + let telemetry = OtelTelemetry::from_provider_with_budget(provider, Duration::from_secs(5)); + emit_command_scope(&telemetry).await; + + assert_eq!(telemetry.shutdown().await, ShutdownOutcome::Completed); + drop(retained_dispatch); + drop(retained_provider); + assert_eq!(capture.shutdown_calls(), 1); + assert_eq!(capture.exports_after_shutdown(), 0); + assert_eq!(capture.names(), vec!["sce.command"]); + }); +} + +#[test] +fn telemetry_shutdown_exactly_once_when_dropped_without_explicit_shutdown() { + let stall = Stall::default(); + let (capture, telemetry) = stalled_telemetry(None, Some(stall.clone()), Duration::from_secs(5)); + run_sync(emit_command_scope(&telemetry)); + + let started = Instant::now(); + drop(telemetry); + assert!( + started.elapsed() < SCHEDULING_SLACK, + "dropping the handle must not run the blocking teardown on the caller" + ); + assert!(stall.wait_entered(EXPORT_START_TIMEOUT)); + assert_eq!(capture.shutdown_calls(), 1); + stall.release(); + assert!(wait_until(EXPORT_START_TIMEOUT, || capture.names() + == vec!["sce.command"])); + std::thread::sleep(Duration::from_millis(100)); + assert_eq!(capture.shutdown_calls(), 1); + assert_eq!(capture.exports_after_shutdown(), 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn telemetry_shutdown_stalled_exporter_does_not_hold_tokio_worker() { + let export_stall = Stall::default(); + let shutdown_stall = Stall::default(); + let (capture, telemetry) = stalled_telemetry( + Some(export_stall.clone()), + Some(shutdown_stall.clone()), + STALL_PROBE_BUDGET, + ); + emit_command_scope(&telemetry).await; + + let ticks = Arc::new(AtomicUsize::new(0)); + let max_gap_micros = Arc::new(AtomicUsize::new(0)); + let ticker = tokio::spawn({ + let ticks = ticks.clone(); + let max_gap_micros = max_gap_micros.clone(); + async move { + let mut last = Instant::now(); + loop { + tokio::time::sleep(Duration::from_millis(10)).await; + let now = Instant::now(); + let gap = + usize::try_from(now.duration_since(last).as_micros()).unwrap_or(usize::MAX); + max_gap_micros.fetch_max(gap, Ordering::SeqCst); + last = now; + ticks.fetch_add(1, Ordering::SeqCst); + } + } + }); + let started = Instant::now(); + let outcome = tokio::spawn(async move { telemetry.shutdown().await }) + .await + .unwrap(); + let elapsed = started.elapsed(); + ticker.abort(); + + assert_eq!(outcome, ShutdownOutcome::Incomplete); + assert!(elapsed >= STALL_PROBE_BUDGET, "{elapsed:?}"); + assert!( + elapsed < STALL_PROBE_BUDGET + SCHEDULING_SLACK, + "{elapsed:?}" + ); + assert!( + ticks.load(Ordering::SeqCst) >= 10, + "the only Tokio worker kept polling other tasks while the exporter was stalled" + ); + let max_gap = Duration::from_micros(max_gap_micros.load(Ordering::SeqCst) as u64); + assert!( + max_gap < STALL_PROBE_BUDGET / 2, + "the only Tokio worker was starved for {max_gap:?} while the exporter was stalled" + ); + assert!(export_stall.wait_entered(EXPORT_START_TIMEOUT)); + let threads = capture.shutdown_threads.lock().unwrap().clone(); + assert!(threads + .iter() + .flatten() + .all(|name| !name.starts_with("tokio"))); + export_stall.release(); + shutdown_stall.release(); +} + +#[test] +fn telemetry_exporter_cannot_block_locks() { + let dir = tempfile::tempdir().unwrap(); + let lock_dir = dir.path().to_path_buf(); + let spec = AdapterLockSpec::state("telemetry-exporter-isolation.lock"); + + let locked_operation = |lock_dir: std::path::PathBuf| async move { + let held = spec.acquire_async(&lock_dir).await.unwrap(); + let contended = spec + .acquire_with_timeout_async(&lock_dir, Duration::from_millis(150)) + .await; + assert!(matches!(contended, Err(AdvisoryLockError::TimedOut { .. }))); + std::fs::write(lock_dir.join("state.txt"), "committed").unwrap(); + drop(held); + let started = Instant::now(); + let second_holder = spec + .acquire_with_timeout_async(&lock_dir, LOCK_CONTENDER_BOUND) + .await + .unwrap(); + let waited = started.elapsed(); + drop(second_holder); + let state_path = lock_dir.join("state.txt"); + let value = spec + .run_locked_blocking(&lock_dir, move || { + Ok(std::fs::read_to_string(state_path).ok()) + }) + .await + .unwrap(); + (waited, value) + }; + + let baseline_dir = tempfile::tempdir().unwrap(); + let baseline = run_sync(async { + let (waited, _) = locked_operation(baseline_dir.path().to_path_buf()).await; + let state = std::fs::read_to_string(baseline_dir.path().join("state.txt")).unwrap(); + (waited < LOCK_CONTENDER_BOUND, state) + }); + + let export_stall = Stall::default(); + let capture = Capture { + export_stall: Some(export_stall.clone()), + shutdown_stall: Some(export_stall.clone()), + ..Capture::default() + }; + let provider = provider_with_settings( + capture.clone(), + BatchSettings { + queue_size: 8, + batch_size: 1, + scheduled_delay: Duration::from_millis(10), + }, + ); + let telemetry = OtelTelemetry::from_provider_with_budget(provider, STALL_PROBE_BUDGET); + let enabled = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() + .block_on(async { + let result = telemetry + .with_default_subscriber(&mut || { + let lock_dir = lock_dir.clone(); + let export_stall = export_stall.clone(); + async move { + in_otel_span(OtelName::Command, async { + in_otel_span(OtelName::WorktreeLock, async {}).await; + assert!( + export_stall.wait_entered(EXPORT_START_TIMEOUT), + "background export must have started and be stalled" + ); + let (waited, _) = locked_operation(lock_dir.clone()).await; + assert!(!export_stall.is_released()); + let state = + std::fs::read_to_string(lock_dir.join("state.txt")).unwrap(); + Ok(format!("{}|{state}", waited < LOCK_CONTENDER_BOUND)) + }) + .await + } + }) + .await + .unwrap(); + let outcome = telemetry.shutdown().await; + (result, outcome) + }); + + assert_eq!(baseline, (true, "committed".to_string())); + assert_eq!(enabled.0, "true|committed"); + assert_eq!(enabled.1, ShutdownOutcome::Incomplete); + export_stall.release(); +} + +fn failing_spawn(_: ShutdownWorker) -> std::io::Result<()> { + Err(std::io::Error::other("injected shutdown thread failure")) +} + +fn assert_exporter_shutdown_never_entered(capture: &Capture, stall: &Stall) { + std::thread::sleep(Duration::from_millis(100)); + assert_eq!(capture.shutdown_calls(), 0); + assert!(!stall.wait_entered(Duration::from_millis(50))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn telemetry_shutdown_thread_spawn_failure_is_nonblocking() { + let stall = Stall::default(); + let (capture, telemetry) = stalled_telemetry(None, Some(stall.clone()), Duration::from_secs(5)); + emit_command_scope(&telemetry).await; + + let ticks = Arc::new(AtomicUsize::new(0)); + let ticker = tokio::spawn({ + let ticks = ticks.clone(); + async move { + loop { + tokio::time::sleep(Duration::from_millis(10)).await; + ticks.fetch_add(1, Ordering::SeqCst); + } + } + }); + let started = Instant::now(); + let outcome = tokio::spawn(telemetry.shutdown_with(failing_spawn)) + .await + .unwrap(); + let elapsed = started.elapsed(); + tokio::time::sleep(Duration::from_millis(100)).await; + ticker.abort(); + + assert_eq!(outcome, ShutdownOutcome::Unavailable); + assert!(elapsed < SCHEDULING_SLACK, "{elapsed:?}"); + assert!(ticks.load(Ordering::SeqCst) >= 5); + assert_exporter_shutdown_never_entered(&capture, &stall); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn telemetry_shutdown_handoff_failure_is_nonblocking() { + let stall = Stall::default(); + let (capture, telemetry) = stalled_telemetry(None, Some(stall.clone()), Duration::from_secs(5)); + emit_command_scope(&telemetry).await; + + let started = Instant::now(); + let outcome = tokio::spawn(telemetry.shutdown_with(|worker| { + drop(worker); + Ok(()) + })) + .await + .unwrap(); + + assert_eq!(outcome, ShutdownOutcome::Unavailable); + assert!(started.elapsed() < SCHEDULING_SLACK); + assert_exporter_shutdown_never_entered(&capture, &stall); +} + +#[test] +fn telemetry_drop_without_shutdown_spawn_failure_is_nonblocking() { + let stall = Stall::default(); + let (capture, mut telemetry) = + stalled_telemetry(None, Some(stall.clone()), Duration::from_secs(5)); + run_sync(emit_command_scope(&telemetry)); + + let started = Instant::now(); + telemetry.release_with(failing_spawn); + drop(telemetry); + assert!(started.elapsed() < SCHEDULING_SLACK); + assert_exporter_shutdown_never_entered(&capture, &stall); +} diff --git a/cli/src/services/observability/trace_context.rs b/cli/src/services/observability/trace_context.rs new file mode 100644 index 000000000..6e1fc6c76 --- /dev/null +++ b/cli/src/services/observability/trace_context.rs @@ -0,0 +1,23 @@ +use std::collections::HashMap; + +use opentelemetry::propagation::TextMapPropagator; +use opentelemetry::trace::TraceContextExt; +use opentelemetry::Context; +use opentelemetry_sdk::propagation::TraceContextPropagator; + +pub const TRACEPARENT_ENV: &str = "TRACEPARENT"; +pub const TRACESTATE_ENV: &str = "TRACESTATE"; + +pub fn extract_remote_parent(env: &impl Fn(&str) -> Option) -> Option { + let mut carrier = HashMap::new(); + carrier.insert("traceparent".to_string(), env(TRACEPARENT_ENV)?); + if let Some(state) = env(TRACESTATE_ENV) { + carrier.insert("tracestate".to_string(), state); + } + let context = TraceContextPropagator::new().extract(&carrier); + let span_context = context.span().span_context().clone(); + (span_context.is_valid() && span_context.is_remote()).then_some(context) +} + +#[cfg(test)] +mod tests; diff --git a/cli/src/services/observability/trace_context/tests.rs b/cli/src/services/observability/trace_context/tests.rs new file mode 100644 index 000000000..99110309b --- /dev/null +++ b/cli/src/services/observability/trace_context/tests.rs @@ -0,0 +1,131 @@ +use std::time::Duration; + +use opentelemetry::trace::{SpanId, TraceContextExt, TraceFlags, TraceId}; +use opentelemetry_sdk::trace::SpanData; + +use super::{extract_remote_parent, TRACEPARENT_ENV, TRACESTATE_ENV}; +use crate::services::observability::otel_policy::CommandName; +use crate::services::observability::otel_runtime::test_support::Capture; +use crate::services::observability::otel_runtime::{ + provider_with_exporter, OtelTelemetry, ShutdownOutcome, +}; +use crate::services::observability::tracing_boundary::CommandSpan; +use crate::services::observability::traits::Telemetry; + +const TRACE_ID: &str = "4bf92f3577b34da6a3ce929d0e0e4736"; +const PARENT_ID: &str = "00f067aa0ba902b7"; + +fn env_of(traceparent: Option<&str>, tracestate: Option<&str>) -> impl Fn(&str) -> Option { + let traceparent = traceparent.map(str::to_string); + let tracestate = tracestate.map(str::to_string); + move |key| match key { + TRACEPARENT_ENV => traceparent.clone(), + TRACESTATE_ENV => tracestate.clone(), + _ => None, + } +} + +async fn command_span_with(traceparent: Option<&str>, tracestate: Option<&str>) -> Vec { + let capture = Capture::default(); + let telemetry = OtelTelemetry::from_provider_with_budget( + provider_with_exporter(capture.clone()), + Duration::from_secs(5), + ); + let env = env_of(traceparent, tracestate); + telemetry + .with_default_subscriber(&mut || async { + let span = + CommandSpan::start(CommandName::parse("version"), extract_remote_parent(&env)); + span.scope(async { + tokio::task::yield_now().await; + }) + .await; + span.finish(None); + Ok(String::new()) + }) + .await + .unwrap(); + assert_eq!(telemetry.shutdown().await, ShutdownOutcome::Completed); + capture.spans() +} + +fn sampled_header() -> String { + format!("00-{TRACE_ID}-{PARENT_ID}-01") +} + +#[test] +fn trace_context_extracts_sampled_remote_parent() { + let env = env_of(Some(&sampled_header()), None); + let context = extract_remote_parent(&env).expect("valid context"); + let span = context.span(); + let span_context = span.span_context(); + assert_eq!(span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(span_context.span_id().to_string(), PARENT_ID); + assert!(span_context.is_sampled()); + assert!(span_context.is_remote()); +} + +#[test] +fn trace_context_extracts_unsampled_remote_parent() { + let header = format!("00-{TRACE_ID}-{PARENT_ID}-00"); + let context = extract_remote_parent(&env_of(Some(&header), None)).expect("valid context"); + assert_eq!( + context.span().span_context().trace_flags(), + TraceFlags::default() + ); +} + +#[test] +fn trace_context_rejects_absent_and_malformed_values() { + assert!(extract_remote_parent(&env_of(None, Some("vendor=value"))).is_none()); + let zero_trace = format!("00-{}-{PARENT_ID}-01", "0".repeat(32)); + let zero_span = format!("00-{TRACE_ID}-{}-01", "0".repeat(16)); + let upper = format!("00-{}-{PARENT_ID}-01", TRACE_ID.to_uppercase()); + for malformed in [ + "", + "garbage", + "00-short-short-01", + "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7", + zero_trace.as_str(), + zero_span.as_str(), + upper.as_str(), + ] { + assert!( + extract_remote_parent(&env_of(Some(malformed), None)).is_none(), + "{malformed}" + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn trace_context_valid_parent_sets_trace_id_parent_span_and_sampled_flag() { + let spans = command_span_with(Some(&sampled_header()), Some("vendor=value")).await; + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!(span.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(span.parent_span_id.to_string(), PARENT_ID); + assert_ne!(span.span_context.span_id().to_string(), PARENT_ID); + assert!(span.span_context.is_sampled()); + assert_eq!(span.span_context.trace_state().header(), "vendor=value"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn trace_context_unsampled_parent_is_honored() { + let header = format!("00-{TRACE_ID}-{PARENT_ID}-00"); + let spans = command_span_with(Some(&header), None).await; + assert!(spans.is_empty(), "{spans:?}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn trace_context_malformed_or_absent_yields_independent_root() { + for traceparent in [None, Some("garbage"), Some("00-short-short-01")] { + let spans = command_span_with(traceparent, Some("vendor=value")).await; + assert_eq!(spans.len(), 1, "{traceparent:?}"); + let span = &spans[0]; + assert_eq!(span.parent_span_id, SpanId::INVALID, "{traceparent:?}"); + assert_ne!(span.span_context.trace_id(), TraceId::INVALID); + assert_ne!(span.span_context.trace_id().to_string(), TRACE_ID); + assert!(span.span_context.trace_state().header().is_empty()); + } +} diff --git a/cli/src/services/observability/tracing_boundary.rs b/cli/src/services/observability/tracing_boundary.rs new file mode 100644 index 000000000..3466f7b15 --- /dev/null +++ b/cli/src/services/observability/tracing_boundary.rs @@ -0,0 +1,884 @@ +use std::future::Future; + +use opentelemetry::Context; +use tracing::instrument::{Instrument, WithSubscriber}; +use tracing::Dispatch; +use tracing::Level; +use tracing_opentelemetry::OpenTelemetrySpanExt; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::{Layer, Registry}; + +use crate::services::config::LogLevel; +use crate::services::error::{CliError, FailureClass}; +use crate::services::observability::otel_policy::{ + CommandName, ErrorCategory, Outcome, OTEL_TARGET, +}; + +#[cfg(test)] +mod audit; + +pub const SCE_TRACING_TARGET: &str = "sce"; +pub const CONTENTION_EXHAUSTED_CAUSE: &str = "database busy (busy timeout exhausted)"; +pub const UNCLASSIFIED_EVENT_ID: &str = "sce.unclassified"; +pub const UNCLASSIFIED_LABEL: &str = "unclassified"; +pub const RESILIENCE_RETRY_EVENT_ID: &str = "sce.resilience.retry"; +pub const CONTENTION_EXHAUSTED_EVENT_ID: &str = "sce.agent_trace_db.contention_exhausted"; + +pub const TRACING_EVENT_IDS: &[&str] = &[ + "sce.agent_trace_db.contention_exhausted", + "sce.agent_trace_db.passive_checkpoint_failed", + "sce.app.start", + "sce.command.completed", + "sce.command.dispatch_end", + "sce.command.dispatch_start", + "sce.command.parsed", + "sce.command.raw_args", + "sce.config.file_discovered", + "sce.config.invalid_config", + "sce.error.dependency", + "sce.error.parse", + "sce.error.runtime", + "sce.error.validation", + "sce.hooks.claude_model_state.agent_trace_db_open_failed", + "sce.hooks.claude_model_state.agent_trace_db_write_failed", + "sce.hooks.claude_model_state.error", + "sce.hooks.claude_mutation_scope.model_state_unavailable", + "sce.hooks.claude_mutation_scope.pre_tool_use_fail_closed", + "sce.hooks.codex.apply_patch.normalize_failed", + "sce.hooks.codex.apply_patch.parse_failed", + "sce.hooks.codex.apply_patch.path_resolution_failed", + "sce.hooks.codex.error", + "sce.hooks.codex_mutation_scope.pre_tool_use_fail_closed", + "sce.hooks.commit_msg.ai_overlap_error", + "sce.hooks.conversation_trace.agent_trace_db_batch_failed", + "sce.hooks.conversation_trace.agent_trace_db_open_failed", + "sce.hooks.conversation_trace.error", + "sce.hooks.conversation_trace.payload_skipped", + "sce.hooks.diff_trace.agent_trace_db_open_failed", + "sce.hooks.diff_trace.agent_trace_db_time_invalid", + "sce.hooks.diff_trace.agent_trace_db_write_failed", + "sce.hooks.diff_trace.error", + "sce.hooks.mutation_scope.marker_clear_after_durable_completion", + "sce.hooks.mutation_scope.ref_reconciliation_advisory", + "sce.hooks.opencode_mutation_scope.start_fail_closed", + "sce.hooks.pi_mutation_scope.start_fail_closed", + "sce.resilience.retry", + "sce.unclassified", +]; + +pub fn classify_event_id(event_id: &str) -> &'static str { + match TRACING_EVENT_IDS.binary_search(&event_id) { + Ok(index) => TRACING_EVENT_IDS[index], + Err(_) => UNCLASSIFIED_EVENT_ID, + } +} + +pub fn cli_error_tracing_event_id(error: &CliError) -> &'static str { + match error.class() { + FailureClass::Parse => "sce.error.parse", + FailureClass::Validation => "sce.error.validation", + FailureClass::Runtime => "sce.error.runtime", + FailureClass::Dependency => "sce.error.dependency", + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct EventId(&'static str); + +impl EventId { + pub const RESILIENCE_RETRY: Self = Self(RESILIENCE_RETRY_EVENT_ID); + pub const CONTENTION_EXHAUSTED: Self = Self(CONTENTION_EXHAUSTED_EVENT_ID); + + pub fn classify(event_id: &str) -> Self { + Self(classify_event_id(event_id)) + } + + pub fn from_cli_error(error: &CliError) -> Self { + Self(cli_error_tracing_event_id(error)) + } + + pub fn as_str(self) -> &'static str { + self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperationClass { + AuthRefreshToken, + AgentTraceSyncIngestionState, + DbOpenConnection, + DbOpenEncryptedConnection, + DbExecuteQuery, + DbExecuteEncryptedQuery, + DbQueryRows, + DbQueryEncryptedRows, + DbCheckpointWal, + Unclassified, +} + +impl OperationClass { + pub const ALL: [Self; 10] = [ + Self::AuthRefreshToken, + Self::AgentTraceSyncIngestionState, + Self::DbOpenConnection, + Self::DbOpenEncryptedConnection, + Self::DbExecuteQuery, + Self::DbExecuteEncryptedQuery, + Self::DbQueryRows, + Self::DbQueryEncryptedRows, + Self::DbCheckpointWal, + Self::Unclassified, + ]; + + pub fn parse(value: &str) -> Option { + Self::ALL + .into_iter() + .find(|candidate| candidate.as_str() == value) + } + + pub fn classify(operation: &str) -> Self { + match operation { + "auth.refresh_token" => return Self::AuthRefreshToken, + "agent_trace_sync.ingestion_state" => return Self::AgentTraceSyncIngestionState, + _ => {} + } + + let framed = |prefix: &str, suffix: &str| { + operation.len() >= prefix.len() + suffix.len() + && operation.starts_with(prefix) + && operation.ends_with(suffix) + }; + + if framed("open encrypted ", " database connection") { + Self::DbOpenEncryptedConnection + } else if framed("open ", " database connection") { + Self::DbOpenConnection + } else if framed("execute encrypted ", " database query") { + Self::DbExecuteEncryptedQuery + } else if framed("execute ", " database query") { + Self::DbExecuteQuery + } else if framed("query and fetch encrypted ", " database rows") { + Self::DbQueryEncryptedRows + } else if framed("query and fetch ", " database rows") { + Self::DbQueryRows + } else if framed("checkpoint ", " database WAL") { + Self::DbCheckpointWal + } else { + Self::Unclassified + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::AuthRefreshToken => "auth.refresh_token", + Self::AgentTraceSyncIngestionState => "agent_trace_sync.ingestion_state", + Self::DbOpenConnection => "db.open_connection", + Self::DbOpenEncryptedConnection => "db.open_encrypted_connection", + Self::DbExecuteQuery => "db.execute_query", + Self::DbExecuteEncryptedQuery => "db.execute_encrypted_query", + Self::DbQueryRows => "db.query_rows", + Self::DbQueryEncryptedRows => "db.query_encrypted_rows", + Self::DbCheckpointWal => "db.checkpoint_wal", + Self::Unclassified => UNCLASSIFIED_LABEL, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DbName { + LocalDb, + AuthDb, + RepositoryAgentTraceDb, + Unclassified, +} + +impl DbName { + pub fn classify(db_name: &str) -> Self { + match db_name { + "local DB" => Self::LocalDb, + "auth DB" => Self::AuthDb, + "repository Agent Trace DB" => Self::RepositoryAgentTraceDb, + _ => Self::Unclassified, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::LocalDb => "local_db", + Self::AuthDb => "auth_db", + Self::RepositoryAgentTraceDb => "repository_agent_trace_db", + Self::Unclassified => UNCLASSIFIED_LABEL, + } + } +} + +pub fn emit_logger_event(level: LogLevel, event_id: EventId) { + let enabled = match level { + LogLevel::Error => tracing::enabled!(target: SCE_TRACING_TARGET, Level::ERROR), + LogLevel::Warn => tracing::enabled!(target: SCE_TRACING_TARGET, Level::WARN), + LogLevel::Info => tracing::enabled!(target: SCE_TRACING_TARGET, Level::INFO), + LogLevel::Debug => tracing::enabled!(target: SCE_TRACING_TARGET, Level::DEBUG), + }; + if !enabled { + return; + } + + let event_id = event_id.as_str(); + let log_level = level.as_str(); + match level { + LogLevel::Error => tracing::error!( + target: SCE_TRACING_TARGET, + event_id, + log_level, + "sce log event" + ), + LogLevel::Warn => tracing::warn!( + target: SCE_TRACING_TARGET, + event_id, + log_level, + "sce log event" + ), + LogLevel::Info => tracing::info!( + target: SCE_TRACING_TARGET, + event_id, + log_level, + "sce log event" + ), + LogLevel::Debug => tracing::debug!( + target: SCE_TRACING_TARGET, + event_id, + log_level, + "sce log event" + ), + } +} + +pub fn emit_retry_event( + operation: OperationClass, + attempt: u32, + max_attempts: u32, + timeout_ms: u64, + backoff_ms: u64, +) { + let event_id = EventId::RESILIENCE_RETRY.as_str(); + let operation = operation.as_str(); + tracing::warn!( + target: SCE_TRACING_TARGET, + event_id, + operation, + attempt, + max_attempts, + timeout_ms, + backoff_ms, + "Retrying operation after transient failure" + ); +} + +pub fn emit_contention_event( + database: DbName, + operation: OperationClass, + attempts: u32, + busy_timeout_ms: u64, + contention_deadline_ms: u64, + elapsed_ms: u64, +) { + let event_id = EventId::CONTENTION_EXHAUSTED.as_str(); + let db_name = database.as_str(); + let operation = operation.as_str(); + let cause = CONTENTION_EXHAUSTED_CAUSE; + tracing::warn!( + target: SCE_TRACING_TARGET, + event_id, + db_name, + operation, + attempts, + busy_timeout_ms, + contention_deadline_ms, + elapsed_ms, + cause, + "Agent Trace DB write contention retries exhausted" + ); +} + +#[derive(Clone)] +pub struct ScopedDispatch(Dispatch); + +impl ScopedDispatch { + pub fn from_layer(layer: L) -> Self + where + L: Layer + Send + Sync + 'static, + { + Self(Dispatch::new(Registry::default().with(layer))) + } + + pub fn scope(&self, future: F) -> impl Future { + future.with_subscriber(self.0.clone()) + } +} + +#[derive(Clone)] +pub struct CommandSpan(tracing::Span); + +impl CommandSpan { + pub fn start(command: Option, remote_parent: Option) -> Self { + let span = tracing::span!( + target: OTEL_TARGET, + Level::INFO, + "sce.command", + "sce.command.name" = tracing::field::Empty, + "sce.outcome" = tracing::field::Empty, + "sce.error.category" = tracing::field::Empty, + "otel.status_code" = tracing::field::Empty + ); + if let Some(command) = command { + span.record("sce.command.name", command.as_str()); + } + if let Some(parent) = remote_parent { + let _ = span.set_parent(parent); + } + Self(span) + } + + pub fn scope(&self, future: F) -> impl Future { + future.instrument(self.0.clone()) + } + + pub fn finish(&self, failure: Option) { + match failure { + None => { + self.0.record("sce.outcome", Outcome::Success.as_str()); + } + Some(category) => { + self.0.record("sce.outcome", Outcome::Failure.as_str()); + self.0.record("sce.error.category", category.as_str()); + self.0.record("otel.status_code", "error"); + } + } + } +} + +pub fn spawn_in_current_scope(future: F) -> tokio::task::JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + tokio::spawn(future.with_current_subscriber().in_current_span()) +} + +#[cfg(test)] +pub(crate) mod test_capture { + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }; + + use tracing::{ + field::{Field, Visit}, + span, Event, Metadata, Subscriber, + }; + + #[derive(Clone, Debug)] + pub struct CapturedEvent { + pub target: String, + pub fields: Vec<(String, String)>, + } + + impl CapturedEvent { + pub fn field(&self, name: &str) -> Option<&str> { + self.fields + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + } + + pub fn rendered(&self) -> String { + let mut rendered = self.target.clone(); + for (key, value) in &self.fields { + rendered.push(' '); + rendered.push_str(key); + rendered.push('='); + rendered.push_str(value); + } + rendered + } + } + + #[derive(Default)] + struct FieldCollector(Vec<(String, String)>); + + impl Visit for FieldCollector { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.push((field.name().to_string(), value.to_string())); + } + + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0 + .push((field.name().to_string(), format!("{value:?}"))); + } + } + + #[derive(Clone, Default)] + pub struct CapturingSubscriber { + events: Arc>>, + next_span_id: Arc, + } + + impl CapturingSubscriber { + pub fn events(&self) -> Vec { + self.events.lock().expect("capture lock").clone() + } + + pub fn capture(run: impl FnOnce()) -> Vec { + let subscriber = Self::default(); + let handle = subscriber.clone(); + tracing::subscriber::with_default(subscriber, run); + handle.events() + } + } + + impl Subscriber for CapturingSubscriber { + fn enabled(&self, _metadata: &Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _attributes: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(self.next_span_id.fetch_add(1, Ordering::SeqCst) + 1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + + fn event(&self, event: &Event<'_>) { + let mut collector = FieldCollector::default(); + event.record(&mut collector); + self.events + .lock() + .expect("capture lock") + .push(CapturedEvent { + target: event.metadata().target().to_string(), + fields: collector.0, + }); + } + + fn enter(&self, _span: &span::Id) {} + + fn exit(&self, _span: &span::Id) {} + } +} + +#[cfg(test)] +pub(crate) mod test_spans { + use std::future::Future; + + use tracing::instrument::Instrument; + use tracing::Level; + + use crate::services::config::LogLevel; + use crate::services::observability::otel_policy::{OtelName, OTEL_TARGET}; + + pub fn in_otel_span(name: OtelName, future: F) -> impl Future { + let span = match name { + OtelName::Command => tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.command"), + OtelName::MutationScopeCoordinate => { + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.mutation_scope.coordinate") + } + OtelName::WorktreeLock => { + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.worktree.lock") + } + OtelName::Reconciliation => { + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.reconciliation") + } + OtelName::GitSnapshot => { + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.git.snapshot") + } + OtelName::DbOperation => { + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.db.operation") + } + OtelName::Sync => tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.sync"), + }; + future.instrument(span) + } + + pub fn emit_raw_boundary_b_inputs(secret: &'static str) { + let allowed = tracing::span!( + target: OTEL_TARGET, + Level::INFO, + "sce.command", + "sce.outcome" = "success", + "sce.duration_ms" = 5u64, + "sce.command.name" = secret, + "sce.error.category" = "parse", + "sce.arbitrary_key" = secret, + password = secret, + "otel.status_code" = "error", + "otel.status_description" = secret + ); + { + let _entered = allowed.enter(); + tracing::event!(target: OTEL_TARGET, Level::ERROR, exception = secret, "exception message"); + } + drop(allowed); + drop(tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.arbitrary")); + drop(tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.command", "otel.name" = secret)); + drop(tracing::span!(target: "sce", Level::INFO, "sce.command", "sce.outcome" = "success")); + drop(tracing::span!(target: "sce::services::other", Level::INFO, "sce.command")); + } + + pub fn emit_target_matrix() { + drop( + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.command", "sce.outcome" = "success"), + ); + drop(tracing::span!(target: "sce", Level::INFO, "sce.command", "sce.outcome" = "failure")); + drop( + tracing::span!(target: "sce::services::other", Level::INFO, "sce.command", "sce.outcome" = "timeout"), + ); + drop( + tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.not_enumerated", "sce.outcome" = "cancelled"), + ); + tracing::event!(target: "sce", Level::INFO, event_id = "sce.command", "sce.command"); + tracing::event!(target: OTEL_TARGET, Level::INFO, event_id = "sce.command", "sce.command"); + super::emit_logger_event(LogLevel::Info, super::EventId::classify("sce.app.start")); + } + + pub fn thread_default_is_none() -> bool { + tracing::dispatcher::get_default(|dispatch| { + dispatch.is::() + }) + } +} + +#[cfg(test)] +mod tests { + use super::test_capture::CapturingSubscriber; + use super::*; + use crate::services::config::{LogFormat, LogLevel}; + use crate::services::error::UserError; + use crate::services::observability::{Logger, ObservabilityConfig}; + + const SECRET: &str = "sk-live-SECRET-TOKEN-9f8e7d"; + const SENSITIVE_PATH: &str = "/home/victim/.ssh/id_ed25519"; + + fn memory_logger() -> Logger { + Logger { + config: ObservabilityConfig { + level: LogLevel::Debug, + format: LogFormat::Text, + log_to_file: false, + }, + log_dir: None, + log_file_retention_limit: 1, + } + } + + fn assert_clean(events: &[super::test_capture::CapturedEvent], forbidden: &[&str]) { + assert!(!events.is_empty(), "expected captured events"); + for event in events { + let rendered = event.rendered(); + for needle in forbidden { + assert!( + !rendered.contains(needle), + "captured event leaked {needle:?}: {rendered}" + ); + } + for banned in ["event_message", "error_source", "last_error"] { + assert!(event.field(banned).is_none(), "banned field {banned}"); + } + assert!(event.field("fields").is_none(), "banned field fields"); + assert!(event.field("error").is_none(), "banned field error"); + } + } + + #[test] + fn tracing_boundary_a_registry_is_sorted_unique_and_well_shaped() { + for pair in TRACING_EVENT_IDS.windows(2) { + assert!( + pair[0] < pair[1], + "{} must sort before {}", + pair[0], + pair[1] + ); + } + for id in TRACING_EVENT_IDS { + assert!( + (1..=64).contains(&id.len()) + && id + .bytes() + .all(|b| matches!(b, b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.')), + "event id {id:?} violates shape" + ); + } + assert!(TRACING_EVENT_IDS.contains(&UNCLASSIFIED_EVENT_ID)); + assert!(TRACING_EVENT_IDS.contains(&RESILIENCE_RETRY_EVENT_ID)); + assert!(TRACING_EVENT_IDS.contains(&CONTENTION_EXHAUSTED_EVENT_ID)); + } + + #[test] + fn tracing_boundary_a_unknown_and_dynamic_event_ids_become_unclassified() { + assert_eq!(classify_event_id("sce.app.start"), "sce.app.start"); + assert_eq!(classify_event_id("sce.test.error"), UNCLASSIFIED_EVENT_ID); + assert_eq!(classify_event_id(SECRET), UNCLASSIFIED_EVENT_ID); + assert_eq!(classify_event_id(SENSITIVE_PATH), UNCLASSIFIED_EVENT_ID); + assert_eq!( + classify_event_id("sce.error.SCE-ERR-RUNTIME"), + UNCLASSIFIED_EVENT_ID + ); + } + + #[test] + fn tracing_boundary_a_logger_events_carry_only_classified_values() { + let logger = memory_logger(); + let secret_message = format!("token={SECRET} path={SENSITIVE_PATH}"); + let secret_fields = [ + ("operation", SECRET), + ("db_name", SENSITIVE_PATH), + ("detail", "password=hunter2"), + ]; + + let events = CapturingSubscriber::capture(|| { + logger.info(SECRET, &secret_message, &secret_fields, None); + logger.warn(SENSITIVE_PATH, &secret_message, &secret_fields, None); + logger.error("sce.unknown.event", &secret_message, &secret_fields, None); + logger.debug("sce.app.start", &secret_message, &secret_fields, None); + }); + + assert_eq!(events.len(), 4); + assert_clean( + &events, + &[SECRET, SENSITIVE_PATH, "hunter2", "password", "token="], + ); + let ids: Vec<_> = events.iter().map(|e| e.field("event_id")).collect(); + assert_eq!( + ids, + vec![ + Some(UNCLASSIFIED_EVENT_ID), + Some(UNCLASSIFIED_EVENT_ID), + Some(UNCLASSIFIED_EVENT_ID), + Some("sce.app.start"), + ] + ); + } + + #[test] + fn tracing_boundary_a_cli_errors_map_event_id_from_typed_class_without_source_chain() { + let logger = memory_logger(); + let error = CliError::user_with_source( + UserError::NotAuthenticated, + anyhow::anyhow!("open {SENSITIVE_PATH}: token {SECRET}") + .context(format!("outer {SECRET}")), + ); + + let events = CapturingSubscriber::capture(|| logger.log_cli_error(&error, None)); + + assert_eq!(events.len(), 1); + assert_clean(&events, &[SECRET, SENSITIVE_PATH, "outer"]); + assert_eq!(events[0].field("event_id"), Some("sce.error.runtime")); + } + + #[test] + fn tracing_boundary_a_local_log_output_keeps_raw_event_ids() { + let logger = memory_logger(); + let line = logger.render_line(LogLevel::Info, "sce.test.raw", "message", &[("k", "v")]); + assert!(line.contains("event_id=sce.test.raw")); + assert!(line.contains("message=message")); + assert!(line.contains("k=v")); + } + + #[test] + fn tracing_boundary_a_operation_and_db_name_classify_to_enums() { + assert_eq!( + OperationClass::classify("open local DB database connection"), + OperationClass::DbOpenConnection + ); + assert_eq!( + OperationClass::classify("open encrypted auth DB database connection"), + OperationClass::DbOpenEncryptedConnection + ); + assert_eq!( + OperationClass::classify("auth.refresh_token"), + OperationClass::AuthRefreshToken + ); + assert_eq!( + OperationClass::classify(&format!("rm -rf {SENSITIVE_PATH}")), + OperationClass::Unclassified + ); + assert_eq!(OperationClass::classify(""), OperationClass::Unclassified); + assert_eq!(DbName::classify("auth DB"), DbName::AuthDb); + assert_eq!(DbName::classify(SECRET), DbName::Unclassified); + assert_eq!( + OperationClass::classify(SECRET).as_str(), + UNCLASSIFIED_LABEL + ); + } + + #[test] + fn tracing_boundary_a_emission_functions_accept_only_closed_types() { + let _: fn(LogLevel, EventId) = emit_logger_event; + let _: fn(OperationClass, u32, u32, u64, u64) = emit_retry_event; + let _: fn(DbName, OperationClass, u32, u64, u64, u64) = emit_contention_event; + } + + #[test] + fn tracing_boundary_a_event_ids_only_come_from_the_classified_registry() { + assert_eq!(EventId::classify("sce.app.start").as_str(), "sce.app.start"); + assert_eq!(EventId::classify(SECRET).as_str(), UNCLASSIFIED_EVENT_ID); + assert_eq!( + EventId::classify(SENSITIVE_PATH).as_str(), + UNCLASSIFIED_EVENT_ID + ); + assert_eq!( + EventId::classify("sce.unknown.event").as_str(), + UNCLASSIFIED_EVENT_ID + ); + assert_eq!( + EventId::RESILIENCE_RETRY.as_str(), + RESILIENCE_RETRY_EVENT_ID + ); + assert_eq!( + EventId::CONTENTION_EXHAUSTED.as_str(), + CONTENTION_EXHAUSTED_EVENT_ID + ); + let error = CliError::user_with_source(UserError::NotAuthenticated, anyhow::anyhow!("x")); + assert_eq!( + EventId::from_cli_error(&error).as_str(), + "sce.error.runtime" + ); + } + + #[test] + fn tracing_boundary_a_logger_emitter_records_exact_target_and_closed_fields() { + let events = CapturingSubscriber::capture(|| { + for level in [ + LogLevel::Error, + LogLevel::Warn, + LogLevel::Info, + LogLevel::Debug, + ] { + emit_logger_event(level, EventId::classify("sce.app.start")); + } + }); + + assert_eq!(events.len(), 4); + for (event, level) in events.iter().zip(["error", "warn", "info", "debug"]) { + assert_eq!(event.target, SCE_TRACING_TARGET); + let names: Vec<_> = event.fields.iter().map(|(key, _)| key.as_str()).collect(); + assert_eq!(names, vec!["message", "event_id", "log_level"]); + assert_eq!(event.field("event_id"), Some("sce.app.start")); + assert_eq!(event.field("log_level"), Some(level)); + assert_eq!(event.field("message"), Some("sce log event")); + } + } + + #[test] + fn tracing_boundary_a_retry_and_contention_emitters_record_exact_closed_fields() { + let events = CapturingSubscriber::capture(|| { + emit_retry_event(OperationClass::DbQueryRows, 2, 5, 1_500, 40); + emit_contention_event( + DbName::RepositoryAgentTraceDb, + OperationClass::DbCheckpointWal, + 3, + 5_000, + 30_000, + 12_345, + ); + }); + + assert_eq!(events.len(), 2); + let retry = &events[0]; + assert_eq!(retry.target, SCE_TRACING_TARGET); + let names: Vec<_> = retry.fields.iter().map(|(key, _)| key.as_str()).collect(); + assert_eq!( + names, + vec![ + "message", + "event_id", + "operation", + "attempt", + "max_attempts", + "timeout_ms", + "backoff_ms" + ] + ); + assert_eq!(retry.field("event_id"), Some("sce.resilience.retry")); + assert_eq!(retry.field("operation"), Some("db.query_rows")); + assert_eq!(retry.field("attempt"), Some("2")); + assert_eq!(retry.field("max_attempts"), Some("5")); + assert_eq!(retry.field("timeout_ms"), Some("1500")); + assert_eq!(retry.field("backoff_ms"), Some("40")); + assert_eq!( + retry.field("message"), + Some("Retrying operation after transient failure") + ); + + let contention = &events[1]; + assert_eq!(contention.target, SCE_TRACING_TARGET); + let names: Vec<_> = contention + .fields + .iter() + .map(|(key, _)| key.as_str()) + .collect(); + assert_eq!( + names, + vec![ + "message", + "event_id", + "db_name", + "operation", + "attempts", + "busy_timeout_ms", + "contention_deadline_ms", + "elapsed_ms", + "cause" + ] + ); + assert_eq!( + contention.field("event_id"), + Some("sce.agent_trace_db.contention_exhausted") + ); + assert_eq!( + contention.field("db_name"), + Some("repository_agent_trace_db") + ); + assert_eq!(contention.field("operation"), Some("db.checkpoint_wal")); + assert_eq!(contention.field("attempts"), Some("3")); + assert_eq!(contention.field("busy_timeout_ms"), Some("5000")); + assert_eq!(contention.field("contention_deadline_ms"), Some("30000")); + assert_eq!(contention.field("elapsed_ms"), Some("12345")); + assert_eq!( + contention.field("cause"), + Some("database busy (busy timeout exhausted)") + ); + } + + #[test] + fn tracing_boundary_a_emitters_only_export_values_from_closed_sets() { + let events = CapturingSubscriber::capture(|| { + for operation in OperationClass::ALL { + emit_retry_event(operation, 1, 2, 3, 4); + for database in [ + DbName::LocalDb, + DbName::AuthDb, + DbName::RepositoryAgentTraceDb, + DbName::Unclassified, + ] { + emit_contention_event(database, operation, 1, 2, 3, 4); + } + } + }); + + let operations: Vec<_> = OperationClass::ALL.iter().map(|o| o.as_str()).collect(); + let databases = [ + "local_db", + "auth_db", + "repository_agent_trace_db", + UNCLASSIFIED_LABEL, + ]; + assert_eq!(events.len(), OperationClass::ALL.len() * 5); + for event in &events { + assert!(operations.contains(&event.field("operation").expect("operation"))); + if let Some(database) = event.field("db_name") { + assert!(databases.contains(&database)); + } + } + } +} diff --git a/cli/src/services/observability/tracing_boundary/audit.rs b/cli/src/services/observability/tracing_boundary/audit.rs new file mode 100644 index 000000000..8d65be40a --- /dev/null +++ b/cli/src/services/observability/tracing_boundary/audit.rs @@ -0,0 +1,814 @@ +use std::collections::HashMap; +use std::{fs, path::Path}; + +use crate::services::observability::otel_policy::OtelName; +use proc_macro2::{TokenStream, TokenTree}; +use syn::visit::{self, Visit}; +use syn::{ + Attribute, Expr, FnArg, ImplItemFn, ItemExternCrate, ItemFn, ItemMod, ItemUse, Macro, Meta, + Pat, Stmt, TraitItemFn, Type, UseTree, +}; + +const TRUSTED_RELATIVE_PATH: &str = "services/observability/tracing_boundary.rs"; +const TRACING_ROOT: &str = "tracing"; +const EVENT_MACROS: [&str; 6] = ["error", "warn", "info", "debug", "trace", "event"]; +const BOUNDARY_EVENT_MACROS: [&str; 4] = ["error", "warn", "info", "debug"]; +const BOUNDARY_ALLOWED_USES: [&str; 4] = [ + "tracing::Level", + "tracing::Dispatch", + "tracing::instrument::Instrument", + "tracing::instrument::WithSubscriber", +]; +const TARGET_CONSTANT: &str = "SCE_TRACING_TARGET"; +const SPAN_TARGET_TOKENS: &str = "target:OTEL_TARGET"; +const SPAN_LEVEL_TOKENS: &str = "Level::INFO"; +const SPAN_EMPTY_VALUE_TOKENS: &str = "tracing::field::Empty"; +const SPAN_FIELDS: [&str; 4] = [ + "sce.command.name", + "sce.outcome", + "sce.error.category", + "otel.status_code", +]; +const APPROVED_CONSTANTS: [&str; 1] = ["CONTENTION_EXHAUSTED_CAUSE"]; +const NUMERIC_TYPES: [&str; 2] = ["u32", "u64"]; +const CLOSED_TYPES: [&str; 4] = ["LogLevel", "EventId", "OperationClass", "DbName"]; +const ALLOWED_FIELDS: [&str; 13] = [ + "event_id", + "log_level", + "operation", + "attempt", + "max_attempts", + "timeout_ms", + "backoff_ms", + "db_name", + "attempts", + "busy_timeout_ms", + "contention_deadline_ms", + "elapsed_ms", + "cause", +]; +const REQUIRED_EMITTERS: [&str; 3] = [ + "emit_logger_event", + "emit_retry_event", + "emit_contention_event", +]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Zone { + Trusted, + Untrusted, +} + +#[derive(Debug, Default)] +struct Report { + violations: Vec, + emitting_fns: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Binding { + Numeric, + ClosedValue, + ClosedString, + Unsafe, +} + +impl Binding { + fn is_exportable(self) -> bool { + matches!(self, Self::Numeric | Self::ClosedString) + } +} + +struct FnScope { + name: String, + bindings: HashMap, +} + +struct Audit { + zone: Zone, + report: Report, + scopes: Vec, +} + +fn audit_source(source: &str, zone: Zone) -> Result { + let file = syn::parse_file(source)?; + let mut audit = Audit { + zone, + report: Report::default(), + scopes: Vec::new(), + }; + audit.visit_file(&file); + audit.report.emitting_fns.sort(); + audit.report.emitting_fns.dedup(); + Ok(audit.report) +} + +fn is_cfg_test(attrs: &[Attribute]) -> bool { + attrs.iter().any(|attr| match &attr.meta { + Meta::List(list) => list.path.is_ident("cfg") && list.tokens.to_string() == "test", + _ => false, + }) +} + +fn type_name(ty: &Type) -> Option { + match ty { + Type::Path(path) => path + .path + .segments + .last() + .map(|segment| segment.ident.to_string()), + _ => None, + } +} + +fn single_ident(expr: &Expr) -> Option { + match expr { + Expr::Path(path) if path.qself.is_none() => path.path.get_ident().map(ToString::to_string), + _ => None, + } +} + +fn classify_param(ty: &Type) -> Binding { + match type_name(ty) { + Some(name) if NUMERIC_TYPES.contains(&name.as_str()) => Binding::Numeric, + Some(name) if CLOSED_TYPES.contains(&name.as_str()) => Binding::ClosedValue, + _ => Binding::Unsafe, + } +} + +fn classify_initializer(expr: &Expr, bindings: &HashMap) -> Binding { + match expr { + Expr::MethodCall(call) if call.method == "as_str" && call.args.is_empty() => { + let closed_receiver = match &*call.receiver { + Expr::Path(path) if path.qself.is_none() => { + let segments: Vec<_> = path.path.segments.iter().collect(); + match segments.as_slice() { + [only] => { + bindings.get(&only.ident.to_string()) == Some(&Binding::ClosedValue) + } + [owner, _] => CLOSED_TYPES.contains(&owner.ident.to_string().as_str()), + _ => false, + } + } + _ => false, + }; + if closed_receiver { + Binding::ClosedString + } else { + Binding::Unsafe + } + } + other => match single_ident(other) { + Some(name) if APPROVED_CONSTANTS.contains(&name.as_str()) => Binding::ClosedString, + _ => Binding::Unsafe, + }, + } +} + +fn build_scope(name: &str, inputs: Vec<&FnArg>, block: Option<&syn::Block>) -> FnScope { + let mut bindings = HashMap::new(); + for input in inputs { + if let FnArg::Typed(typed) = input { + if let Pat::Ident(ident) = &*typed.pat { + bindings.insert(ident.ident.to_string(), classify_param(&typed.ty)); + } + } + } + if let Some(block) = block { + for statement in &block.stmts { + if let Stmt::Local(local) = statement { + if let (Pat::Ident(ident), Some(init)) = (&local.pat, &local.init) { + let binding = classify_initializer(&init.expr, &bindings); + bindings.insert(ident.ident.to_string(), binding); + } + } + } + } + FnScope { + name: name.to_string(), + bindings, + } +} + +fn flatten_use(tree: &UseTree, prefix: &str, out: &mut Vec) { + match tree { + UseTree::Path(path) => { + let next = format!("{prefix}{}::", path.ident); + flatten_use(&path.tree, &next, out); + } + UseTree::Name(name) => out.push(format!("{prefix}{}", name.ident)), + UseTree::Rename(rename) => { + out.push(format!("{prefix}{} as {}", rename.ident, rename.rename)); + } + UseTree::Glob(_) => out.push(format!("{prefix}*")), + UseTree::Group(group) => { + for item in &group.items { + flatten_use(item, prefix, out); + } + } + } +} + +fn mentions_tracing_path(tokens: &TokenStream) -> bool { + let trees: Vec = tokens.clone().into_iter().collect(); + trees.iter().enumerate().any(|(index, tree)| match tree { + TokenTree::Ident(ident) if *ident == TRACING_ROOT => matches!( + trees.get(index + 1), + Some(TokenTree::Punct(punct)) if punct.as_char() == ':' + ), + TokenTree::Group(group) => mentions_tracing_path(&group.stream()), + _ => false, + }) +} + +fn split_arguments(tokens: &TokenStream) -> Vec> { + let mut segments = vec![Vec::new()]; + for tree in tokens.clone() { + match &tree { + TokenTree::Punct(punct) if punct.as_char() == ',' => segments.push(Vec::new()), + _ => segments + .last_mut() + .expect("segments is never empty") + .push(tree), + } + } + if segments.last().is_some_and(Vec::is_empty) { + segments.pop(); + } + segments +} + +impl Audit { + fn context(&self) -> String { + self.scopes + .last() + .map_or_else(|| "".to_string(), |scope| scope.name.clone()) + } + + fn violation(&mut self, message: &str) { + let entry = format!("{}: {message}", self.context()); + self.report.violations.push(entry); + } + + fn with_scope(&mut self, scope: FnScope, walk: impl FnOnce(&mut Self)) { + self.scopes.push(scope); + walk(self); + self.scopes.pop(); + } + + fn check_use_leaf(&mut self, leaf: &str) { + let rooted = leaf == TRACING_ROOT + || leaf.starts_with("tracing::") + || leaf.starts_with("tracing as "); + if !rooted { + return; + } + let allowed = self.zone == Zone::Trusted && BOUNDARY_ALLOWED_USES.contains(&leaf); + if !allowed { + self.violation(&format!("import `{leaf}` is not permitted here")); + } + } + + fn audit_boundary_macro(&mut self, segments: &[String], last: &str, tokens: &TokenStream) { + let rooted = segments.first().is_some_and(|first| first == TRACING_ROOT); + if !rooted && !EVENT_MACROS.contains(&last) { + if mentions_tracing_path(tokens) { + self.violation(&format!("macro `{last}!` smuggles a `tracing` path")); + } + return; + } + if !(rooted && segments.len() == 2) { + self.violation(&format!( + "`{}!` must be invoked as `tracing::{last}!`", + segments.join("::") + )); + return; + } + match last { + "enabled" => {} + "span" => self.check_boundary_span(tokens), + name if BOUNDARY_EVENT_MACROS.contains(&name) => { + self.check_boundary_event(name, tokens); + } + _ => self.violation(&format!("`tracing::{last}!` is not permitted")), + } + } + + fn check_boundary_span(&mut self, tokens: &TokenStream) { + if self.scopes.last().is_none() { + self.violation("`span!` outside a function"); + return; + } + let segments = split_arguments(tokens); + let render = + |trees: &[TokenTree]| trees.iter().map(ToString::to_string).collect::(); + let name_is_enumerated = segments.get(2).is_some_and(|name| { + syn::parse2::(name.iter().cloned().collect()) + .is_ok_and(|literal| OtelName::parse(&literal.value()).is_some()) + }); + let head_is_typed = segments.len() >= 3 + && render(&segments[0]) == SPAN_TARGET_TOKENS + && render(&segments[1]) == SPAN_LEVEL_TOKENS + && name_is_enumerated; + if !head_is_typed { + self.violation("`tracing::span!` must use the `OTEL_TARGET`, `Level::INFO` and an enumerated span name"); + return; + } + for field in &segments[3..] { + let rendered = render(field); + let allowed = rendered + .strip_suffix(SPAN_EMPTY_VALUE_TOKENS) + .and_then(|key| key.strip_suffix('=')) + .map(|key| key.trim_matches('"')) + .is_some_and(|key| SPAN_FIELDS.contains(&key)); + if !allowed { + self.violation(&format!("`tracing::span!` field `{rendered}` must be an allowlisted key recorded as `Empty`")); + } + } + } + + fn check_boundary_event(&mut self, macro_name: &str, tokens: &TokenStream) { + if let Some(scope) = self.scopes.last() { + let name = scope.name.clone(); + self.report.emitting_fns.push(name); + } else { + self.violation(&format!("`{macro_name}!` outside a function")); + return; + } + + let segments = split_arguments(tokens); + let Some((message, fields)) = segments.split_last() else { + self.violation(&format!("`{macro_name}!` has no message")); + return; + }; + + let static_message = match message.as_slice() { + [tree @ TokenTree::Literal(_)] => { + syn::parse2::(TokenStream::from(tree.clone())).is_ok_and(|literal| { + let value = literal.value(); + !value.contains('{') && !value.contains('}') + }) + } + _ => false, + }; + if !static_message { + self.violation(&format!( + "`{macro_name}!` message must be one static string literal without interpolation" + )); + } + + for field in fields { + self.check_boundary_field(macro_name, field); + } + } + + fn check_boundary_field(&mut self, macro_name: &str, field: &[TokenTree]) { + match field { + [TokenTree::Ident(key), TokenTree::Punct(colon), TokenTree::Ident(value)] + if *key == "target" && colon.as_char() == ':' => + { + if *value != TARGET_CONSTANT { + self.violation(&format!( + "`{macro_name}!` target must be `{TARGET_CONSTANT}`" + )); + } + } + [TokenTree::Ident(name)] => { + let name = name.to_string(); + self.check_field_value(macro_name, &name, &name); + } + [TokenTree::Ident(name), TokenTree::Punct(equals), TokenTree::Ident(value)] + if equals.as_char() == '=' => + { + self.check_field_value(macro_name, &name.to_string(), &value.to_string()); + } + _ => self.violation(&format!( + "`{macro_name}!` field must be a plain `name` or `name = binding`" + )), + } + } + + fn check_field_value(&mut self, macro_name: &str, name: &str, value: &str) { + if !ALLOWED_FIELDS.contains(&name) { + self.violation(&format!("`{macro_name}!` field `{name}` is not classified")); + return; + } + if APPROVED_CONSTANTS.contains(&value) { + return; + } + let binding = if value == name { + self.scopes + .last() + .and_then(|scope| scope.bindings.get(value)) + .copied() + } else { + None + }; + if !binding.is_some_and(Binding::is_exportable) { + self.violation(&format!( + "`{macro_name}!` field `{name}` value `{value}` is not a closed-enum string or numeric parameter bound to the same name" + )); + } + } +} + +impl<'ast> Visit<'ast> for Audit { + fn visit_item_mod(&mut self, node: &'ast ItemMod) { + if self.zone == Zone::Trusted && is_cfg_test(&node.attrs) { + return; + } + visit::visit_item_mod(self, node); + } + + fn visit_item_fn(&mut self, node: &'ast ItemFn) { + if self.zone == Zone::Trusted && is_cfg_test(&node.attrs) { + return; + } + let scope = build_scope( + &node.sig.ident.to_string(), + node.sig.inputs.iter().collect(), + Some(&node.block), + ); + self.with_scope(scope, |audit| visit::visit_item_fn(audit, node)); + } + + fn visit_impl_item_fn(&mut self, node: &'ast ImplItemFn) { + let scope = build_scope( + &node.sig.ident.to_string(), + node.sig.inputs.iter().collect(), + Some(&node.block), + ); + self.with_scope(scope, |audit| visit::visit_impl_item_fn(audit, node)); + } + + fn visit_trait_item_fn(&mut self, node: &'ast TraitItemFn) { + let scope = build_scope( + &node.sig.ident.to_string(), + node.sig.inputs.iter().collect(), + node.default.as_ref(), + ); + self.with_scope(scope, |audit| visit::visit_trait_item_fn(audit, node)); + } + + fn visit_item_use(&mut self, node: &'ast ItemUse) { + if self.zone == Zone::Trusted && is_cfg_test(&node.attrs) { + return; + } + let mut leaves = Vec::new(); + flatten_use(&node.tree, "", &mut leaves); + for leaf in leaves { + self.check_use_leaf(&leaf); + } + } + + fn visit_item_extern_crate(&mut self, node: &'ast ItemExternCrate) { + if node.ident == TRACING_ROOT { + self.violation("`extern crate tracing` is not permitted"); + } + } + + fn visit_path(&mut self, node: &'ast syn::Path) { + if self.zone == Zone::Untrusted + && node + .segments + .first() + .is_some_and(|segment| segment.ident == TRACING_ROOT) + { + self.violation("direct `tracing` path outside the typed boundary"); + } + visit::visit_path(self, node); + } + + fn visit_macro(&mut self, node: &'ast Macro) { + let segments: Vec = node + .path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect(); + let Some(last) = segments.last().cloned() else { + return; + }; + let rooted = segments.first().is_some_and(|first| first == TRACING_ROOT); + let event_named = EVENT_MACROS.contains(&last.as_str()); + + match self.zone { + Zone::Untrusted => { + if rooted || event_named { + self.violation(&format!( + "`{}!` emits or reaches tracing outside the typed boundary", + segments.join("::") + )); + } else if mentions_tracing_path(&node.tokens) { + self.violation(&format!( + "macro `{last}!` references `tracing` outside the typed boundary" + )); + } + } + Zone::Trusted => self.audit_boundary_macro(&segments, &last, &node.tokens), + } + } +} + +fn relative_source_files(src: &Path) -> Vec<(String, std::path::PathBuf)> { + fn walk(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).expect("read src dir") { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + walk(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } + } + + let mut files = Vec::new(); + walk(src, &mut files); + files.sort(); + files + .into_iter() + .map(|path| { + let relative = path + .strip_prefix(src) + .expect("relative path") + .to_string_lossy() + .replace('\\', "/"); + (relative, path) + }) + .collect() +} + +fn violations(source: &str, zone: Zone) -> Vec { + audit_source(source, zone) + .expect("fixture parses") + .violations +} + +#[test] +fn tracing_boundary_a_every_tracing_macro_site_records_only_classified_fields() { + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let files = relative_source_files(&src); + assert!( + files + .iter() + .any(|(relative, _)| relative == TRUSTED_RELATIVE_PATH), + "trusted boundary file must exist" + ); + + let mut failures = Vec::new(); + let mut trusted_report = None; + for (relative, path) in &files { + let source = fs::read_to_string(path).expect("read source"); + let zone = if relative == TRUSTED_RELATIVE_PATH { + Zone::Trusted + } else { + Zone::Untrusted + }; + let report = audit_source(&source, zone) + .unwrap_or_else(|error| panic!("{relative} must parse: {error}")); + for violation in &report.violations { + failures.push(format!("{relative}: {violation}")); + } + if zone == Zone::Trusted { + trusted_report = Some(report); + } + } + + assert!( + failures.is_empty(), + "tracing boundary bypasses:\n{}", + failures.join("\n") + ); + let trusted = trusted_report.expect("trusted boundary audited"); + for emitter in REQUIRED_EMITTERS { + assert!( + trusted.emitting_fns.iter().any(|name| name == emitter), + "audit must observe emitter {emitter}" + ); + } +} + +#[test] +fn tracing_boundary_a_audit_rejects_interpolated_message_capture() { + let source = r#" + fn leak(secret: &str) { + tracing::warn!("token={secret}"); + } + "#; + assert!(!violations(source, Zone::Untrusted).is_empty()); + let trusted = violations(source, Zone::Trusted); + assert!( + trusted.iter().any(|v| v.contains("static string literal")), + "{trusted:?}" + ); +} + +#[test] +fn tracing_boundary_a_audit_rejects_untrusted_field_value() { + let source = r#" + fn leak(untrusted_value: &str) { + tracing::warn!( + operation = untrusted_value, + "retry" + ); + } + "#; + assert!(!violations(source, Zone::Untrusted).is_empty()); + let trusted = violations(source, Zone::Trusted); + assert!( + trusted.iter().any(|v| v.contains("untrusted_value")), + "{trusted:?}" + ); +} + +#[test] +fn tracing_boundary_a_audit_rejects_same_named_binding_of_unclassified_type() { + let source = r#" + fn leak(operation: &str) { + tracing::warn!(operation, "retry"); + } + fn leak_string(operation: &str) { + let operation = operation.to_string(); + tracing::warn!(operation, "retry"); + } + fn leak_as_str(raw: &str) { + let operation = raw.as_str(); + tracing::warn!(operation, "retry"); + } + "#; + assert_eq!(violations(source, Zone::Trusted).len(), 3); +} + +#[test] +fn tracing_boundary_a_audit_rejects_display_capture_and_unclassified_field_names() { + let source = r#" + fn leak(secret: &str) { + tracing::error!( + password = %secret, + "authentication failed" + ); + tracing::error!(secret = ?secret, "authentication failed"); + tracing::error!(error = secret, "authentication failed"); + } + "#; + assert!(!violations(source, Zone::Untrusted).is_empty()); + assert_eq!(violations(source, Zone::Trusted).len(), 3); +} + +#[test] +fn tracing_boundary_a_audit_rejects_imported_macro_aliases() { + let unqualified = r#" + use tracing::warn; + + fn leak(secret: &str) { + warn!("secret={secret}"); + } + "#; + let untrusted = violations(unqualified, Zone::Untrusted); + assert!( + untrusted.iter().any(|v| v.contains("import")), + "{untrusted:?}" + ); + assert!( + untrusted.iter().any(|v| v.contains("warn!")), + "{untrusted:?}" + ); + let trusted = violations(unqualified, Zone::Trusted); + assert!(trusted.iter().any(|v| v.contains("import")), "{trusted:?}"); + assert!( + trusted.iter().any(|v| v.contains("must be invoked")), + "{trusted:?}" + ); + + for source in [ + "use tracing::warn as w; fn f() { w!(\"x\"); }", + "use tracing::*; fn f() { warn!(\"x\"); }", + "use tracing::{warn, info}; fn f() { info!(\"x\"); }", + "use ::tracing::warn; fn f() { warn!(\"x\"); }", + "use tracing as t; fn f() { t::warn!(\"x\"); }", + "use tracing; fn f() { tracing::warn!(\"x\"); }", + "extern crate tracing as t; fn f() { t::warn!(\"x\"); }", + "fn f() { ::tracing::warn!(\"x\"); }", + "fn f() { tracing::event!(tracing::Level::WARN, \"x\"); }", + "fn f() { tracing::trace!(\"x\"); }", + "fn f() { let _ = tracing::Level::WARN; }", + "#[tracing::instrument] fn f() {}", + "macro_rules! emit { () => { tracing::warn!(\"x\") }; }", + "fn f() { wrap!(tracing::warn!(\"x\")); }", + ] { + assert!( + !violations(source, Zone::Untrusted).is_empty(), + "untrusted zone must reject {source}" + ); + } + + for source in [ + "use tracing::warn as w; fn f() { w!(\"x\"); }", + "use tracing::*;", + "use tracing::{Level, warn};", + "fn f() { tracing::event!(tracing::Level::WARN, \"x\"); }", + "fn f() { tracing::trace!(\"x\"); }", + "extern crate tracing as t;", + "macro_rules! emit { () => { tracing::warn!(\"x\") }; }", + "fn f() { wrap!(tracing::warn!(\"x\")); }", + ] { + assert!( + !violations(source, Zone::Trusted).is_empty(), + "trusted zone must reject {source}" + ); + } +} + +#[test] +fn tracing_boundary_a_audit_ignores_comments_and_string_literals() { + let source = r##" + //! tracing::warn!("doc comment {secret}"); + /// tracing::warn!("doc {secret}"); + fn quiet() { + // tracing::warn!("line comment {secret}"); + /* tracing::warn!("block comment {secret}"); */ + let plain = "tracing::warn!(\"token={secret}\")"; + let raw = r#"tracing::warn!("token={secret}"); use tracing::warn;"#; + let _ = (plain, raw); + } + "##; + assert!(violations(source, Zone::Untrusted).is_empty()); + assert!(violations(source, Zone::Trusted).is_empty()); +} + +#[test] +fn tracing_boundary_a_audit_accepts_conforming_boundary_emitter() { + let source = r#" + use tracing::Level; + + pub fn emit_example(operation: OperationClass, attempt: u32, level: LogLevel) { + let operation = operation.as_str(); + let log_level = level.as_str(); + let cause = CONTENTION_EXHAUSTED_CAUSE; + tracing::warn!( + target: SCE_TRACING_TARGET, + operation, + attempt, + log_level, + cause, + "Static message" + ); + let _ = tracing::enabled!(target: SCE_TRACING_TARGET, Level::WARN); + } + + #[cfg(test)] + mod tests { + fn capture() { + tracing::subscriber::with_default(1, || tracing::warn!("{x}")); + } + } + "#; + let report = audit_source(source, Zone::Trusted).expect("parses"); + assert!(report.violations.is_empty(), "{:?}", report.violations); + assert_eq!(report.emitting_fns, vec!["emit_example".to_string()]); +} + +#[test] +fn tracing_boundary_a_audit_rejects_non_constant_target_and_missing_static_message() { + let source = r#" + pub fn emit(operation: OperationClass) { + let operation = operation.as_str(); + tracing::warn!(target: "other", operation, "message"); + tracing::warn!(target: SCE_TRACING_TARGET, operation, concat!("a", "b")); + tracing::warn!(target: SCE_TRACING_TARGET, operation, "a {}", operation); + tracing::warn!(target: SCE_TRACING_TARGET, operation); + } + const FN_FREE: () = { tracing::warn!("x"); }; + "#; + assert!(violations(source, Zone::Trusted).len() >= 5); +} + +#[test] +fn tracing_boundary_a_audit_accepts_only_typed_otel_span_in_the_boundary() { + let conforming = r#" + pub fn start() { + let _ = tracing::span!( + target: OTEL_TARGET, + Level::INFO, + "sce.command", + "sce.outcome" = tracing::field::Empty, + "otel.status_code" = tracing::field::Empty + ); + } + "#; + assert!(violations(conforming, Zone::Trusted).is_empty()); + assert!(!violations(conforming, Zone::Untrusted).is_empty()); + + for rejected in [ + r#"pub fn s(v: &str) { let _ = tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.command", "sce.outcome" = v); }"#, + r#"pub fn s() { let _ = tracing::span!(target: "sce", Level::INFO, "sce.command"); }"#, + r#"pub fn s() { let _ = tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.arbitrary"); }"#, + r#"pub fn s() { let _ = tracing::span!(target: OTEL_TARGET, Level::INFO, "sce.command", "password" = tracing::field::Empty); }"#, + r#"pub fn s() { let _ = tracing::span!(target: OTEL_TARGET, Level::DEBUG, "sce.command"); }"#, + ] { + assert!( + !violations(rejected, Zone::Trusted).is_empty(), + "{rejected}" + ); + } +} diff --git a/cli/src/services/observability/traits.rs b/cli/src/services/observability/traits.rs index 330854885..a0d71d32c 100644 --- a/cli/src/services/observability/traits.rs +++ b/cli/src/services/observability/traits.rs @@ -46,7 +46,6 @@ pub trait Telemetry: Send + Sync { } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -#[allow(dead_code)] pub struct NoopLogger; impl Logger for NoopLogger { diff --git a/cli/src/services/output_format.rs b/cli/src/services/output_format.rs index 2e980c6f3..d403befcf 100644 --- a/cli/src/services/output_format.rs +++ b/cli/src/services/output_format.rs @@ -1,4 +1,3 @@ -use anyhow::{bail, Result}; use clap::ValueEnum; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] @@ -7,16 +6,3 @@ pub enum OutputFormat { Text, Json, } - -impl OutputFormat { - #[allow(dead_code)] - pub fn parse(raw: &str, help_command: &str) -> Result { - match raw { - "text" => Ok(Self::Text), - "json" => Ok(Self::Json), - _ => bail!( - "Invalid --format value '{raw}'. Valid values: text, json. Run '{help_command}' to see valid usage." - ), - } - } -} diff --git a/cli/src/services/patch.rs b/cli/src/services/patch.rs index 9e68c31ba..60572dda6 100644 --- a/cli/src/services/patch.rs +++ b/cli/src/services/patch.rs @@ -95,7 +95,6 @@ pub enum TouchedLineKind { Removed, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub struct ParseError { pub message: String, @@ -115,7 +114,6 @@ impl std::error::Error for ParseError {} /// payload could not be reconstructed into a valid `ParsedPatch`. Common /// causes include malformed JSON syntax, missing required fields, or type /// mismatches in the serialized structure. -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub struct PatchLoadError { pub message: String, @@ -147,24 +145,6 @@ pub fn load_patch_from_json(input: &str) -> Result }) } -/// Load a `ParsedPatch` from JSON bytes previously produced by serializing -/// a `ParsedPatch`. -/// -/// This is the bytes-oriented counterpart to [`load_patch_from_json`], -/// convenient when the caller has raw bytes (for example, from a database -/// BLOB column or a file read) rather than a UTF-8 string. -/// -/// # Errors -/// -/// Returns `PatchLoadError` with an actionable message when the input is -/// not valid JSON or does not match the expected `ParsedPatch` structure. -#[allow(dead_code)] -pub fn load_patch_from_json_bytes(input: &[u8]) -> Result { - serde_json::from_slice(input).map_err(|e| PatchLoadError { - message: format!("invalid patch JSON: {e}"), - }) -} - /// Compute the touched-line intersection of two patches. /// /// Returns a `ParsedPatch` containing only the touched lines from @@ -196,7 +176,6 @@ pub fn load_patch_from_json_bytes(input: &[u8]) -> Result(full_path: &[&'a str], suffix_candidate: &[&'a s /// /// let combined = combine_patches(&[patch_a, patch_b]); /// ``` -#[allow(dead_code)] pub fn combine_patches(patches: &[ParsedPatch]) -> ParsedPatch { use std::collections::HashMap; @@ -514,7 +492,6 @@ pub fn combine_patches(patches: &[ParsedPatch]) -> ParsedPatch { /// /// Returns `ParseError` with an actionable message when the input is malformed, /// such as an invalid hunk header or a `---`/`+++` line that cannot be parsed. -#[allow(dead_code)] pub fn parse_patch(input: &str, session_id: Option<&str>) -> Result { let mut files: Vec = Vec::new(); let mut current_file: Option = None; diff --git a/cli/src/services/repository_identity/mod.rs b/cli/src/services/repository_identity/mod.rs index 405d0b875..c795169d7 100644 --- a/cli/src/services/repository_identity/mod.rs +++ b/cli/src/services/repository_identity/mod.rs @@ -26,15 +26,6 @@ pub struct RepositoryIdentity { pub repository_id: String, } -impl RepositoryIdentity { - /// Human-readable on-disk directory segment (`-`) for this - /// identity. Convenience wrapper over [`repository_dir_segment`]; the - /// authoritative identity is still [`RepositoryIdentity::repository_id`]. - pub fn dir_segment(&self) -> String { - repository_dir_segment(&self.canonical_identity) - } -} - /// Canonicalization failure. Variants carry no input fragments so /// credential-bearing URLs never leak into error output. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -101,55 +92,6 @@ pub fn derive_repository_id(canonical_identity: &str) -> String { hex_encode(&hasher.finalize()) } -/// Length in hex chars of the disambiguating short hash in a directory segment. -const DIR_SEGMENT_SHORT_HASH_LEN: usize = 4; - -/// Derives the human-readable on-disk directory segment for a canonical -/// identity as `-`, where `slug` is the lowercased canonical -/// identity with every run of non-alphanumeric characters collapsed to a -/// single `-` and leading/trailing `-` trimmed, and `short` is the first four -/// hex chars of `SHA256(canonical_identity)` computed with **no** domain -/// prefix. The short hash is deliberately distinct from [`derive_repository_id`], -/// which keeps its `sce-repository-id-v1\0` domain separation. This is a pure -/// display/layout helper: the authoritative identity remains the repository ID. -pub fn repository_dir_segment(canonical_identity: &str) -> String { - let slug = slugify(canonical_identity); - let short = derive_short_hash(canonical_identity); - if slug.is_empty() { - short - } else { - format!("{slug}-{short}") - } -} - -/// Lowercases and collapses every run of non-alphanumeric characters to a -/// single `-`, trimming leading and trailing `-`. -fn slugify(input: &str) -> String { - let mut slug = String::with_capacity(input.len()); - let mut pending_dash = false; - for ch in input.chars() { - if ch.is_ascii_alphanumeric() { - if pending_dash && !slug.is_empty() { - slug.push('-'); - } - pending_dash = false; - slug.push(ch.to_ascii_lowercase()); - } else { - pending_dash = true; - } - } - slug -} - -/// First [`DIR_SEGMENT_SHORT_HASH_LEN`] hex chars of the un-prefixed -/// `SHA256(canonical_identity)`. -fn derive_short_hash(canonical_identity: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(canonical_identity.as_bytes()); - let full = hex_encode(&hasher.finalize()); - full[..DIR_SEGMENT_SHORT_HASH_LEN].to_string() -} - /// Canonicalizes a Git remote URL to the scheme-neutral form /// `host[:port]/path` with credentials stripped, hostname lowercased, /// default ports removed, and query/fragment/trailing-slash/trailing-`.git` @@ -483,26 +425,6 @@ mod tests { ); } - fn un_prefixed_sha256_prefix(input: &str, len: usize) -> String { - let mut hasher = Sha256::new(); - hasher.update(input.as_bytes()); - hex_encode(&hasher.finalize())[..len].to_string() - } - - #[test] - fn dir_segment_matches_slug_and_un_prefixed_short_hash() { - // Slug: lowercased, non-alphanumeric runs (`.`,`/`) collapsed to `-`, - // trimmed. Short: first 4 hex of the un-prefixed SHA-256, deliberately - // distinct from the domain-prefixed repository ID. - let canonical = "github.com/crocoder-dev/shared-context-engineering"; - let short = un_prefixed_sha256_prefix(canonical, 4); - assert_eq!( - repository_dir_segment(canonical), - format!("github-com-crocoder-dev-shared-context-engineering-{short}") - ); - assert_ne!(short, derive_repository_id(canonical)[..4]); - } - #[test] fn missing_path_variants_error() { assert_eq!( diff --git a/cli/src/services/resilience.rs b/cli/src/services/resilience.rs index 7bbeb9175..0abcd9acb 100644 --- a/cli/src/services/resilience.rs +++ b/cli/src/services/resilience.rs @@ -1,9 +1,10 @@ use std::future::Future; -use std::thread; use std::time::{Duration, Instant}; use anyhow::{anyhow, ensure, Result}; +use crate::services::observability::tracing_boundary::{emit_retry_event, OperationClass}; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RetryPolicy { pub max_attempts: u32, @@ -74,15 +75,12 @@ where } let backoff = policy.backoff_for_attempt(attempt + 1); - tracing::warn!( - event_id = "sce.resilience.retry", - operation = operation_name, + emit_retry_event( + OperationClass::classify(operation_name), attempt, - max_attempts = policy.max_attempts, - timeout_ms = policy.timeout_ms, - backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), - error = %last_error, - "Retrying operation after transient failure" + policy.max_attempts, + policy.timeout_ms, + u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), ); tokio::time::sleep(backoff).await; } @@ -98,78 +96,6 @@ where )) } -#[allow(dead_code)] -pub fn run_with_retry_sync( - policy: RetryPolicy, - operation_name: &str, - retry_hint: &str, - mut operation: Op, -) -> Result -where - Op: FnMut(u32) -> Result, -{ - ensure!( - policy.max_attempts > 0, - "Retry policy requires max_attempts >= 1" - ); - ensure!( - policy.timeout_ms > 0, - "Retry policy requires timeout_ms >= 1" - ); - ensure!( - policy.max_backoff_ms >= policy.initial_backoff_ms, - "Retry policy requires max_backoff_ms >= initial_backoff_ms" - ); - - let mut last_error = String::new(); - - for attempt in 1..=policy.max_attempts { - let started_at = Instant::now(); - let outcome = operation(attempt); - - match outcome { - Ok(value) => return Ok(value), - Err(error) => { - last_error = if started_at.elapsed() >= policy.timeout() { - format!( - "attempt {attempt} exceeded {}ms and failed: {error}", - policy.timeout_ms - ) - } else { - error.to_string() - }; - } - } - - if attempt == policy.max_attempts { - break; - } - - let backoff = policy.backoff_for_attempt(attempt + 1); - tracing::warn!( - event_id = "sce.resilience.retry", - operation = operation_name, - attempt, - max_attempts = policy.max_attempts, - timeout_ms = policy.timeout_ms, - backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), - error = %last_error, - "Retrying operation after transient failure" - ); - thread::sleep(backoff); - } - - Err(anyhow!( - "Operation '{operation_name}' failed after {} attempt(s) (timeout={}ms, backoff={}..{}ms). Last error: {}. Try: {}", - policy.max_attempts, - policy.timeout_ms, - policy.initial_backoff_ms, - policy.max_backoff_ms, - last_error, - retry_hint - )) -} - /// One retryable attempt. Closures returning a future implement it directly; /// operations that must lend an exclusive borrow (such as a transaction on a /// single connection) implement it on a struct, because an `FnMut` closure @@ -235,15 +161,12 @@ where } let backoff = policy.backoff_for_attempt(attempt + 1); - tracing::warn!( - event_id = "sce.resilience.retry", - operation = operation_name, + emit_retry_event( + OperationClass::classify(operation_name), attempt, - max_attempts = policy.max_attempts, - timeout_ms = policy.timeout_ms, - backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), - error = %last_error, - "Retrying operation after transient failure" + policy.max_attempts, + policy.timeout_ms, + u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), ); tokio::time::sleep(backoff).await; } @@ -258,3 +181,51 @@ where retry_hint )) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::observability::tracing_boundary::test_capture::CapturingSubscriber; + + const SECRET: &str = "sk-live-SECRET-TOKEN-9f8e7d"; + const SENSITIVE_PATH: &str = "/home/victim/.config/sce/auth.json"; + + #[test] + fn tracing_boundary_a_retry_event_omits_error_text_and_classifies_operation() { + let policy = RetryPolicy { + max_attempts: 2, + timeout_ms: 1_000, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + + let events = CapturingSubscriber::capture(|| { + runtime.block_on(async { + let _ = run_with_retry(policy, SENSITIVE_PATH, "retry later", |_| async { + Err::<(), _>(anyhow!("open {SENSITIVE_PATH} with {SECRET}")) + }) + .await; + let _ = run_with_retry(policy, "auth.refresh_token", "retry later", |_| async { + Err::<(), _>(anyhow!("denied {SECRET}")) + }) + .await; + }); + }); + + assert_eq!(events.len(), 2); + for event in &events { + let rendered = event.rendered(); + for needle in [SECRET, SENSITIVE_PATH] { + assert!(!rendered.contains(needle), "leaked {needle}: {rendered}"); + } + assert!(event.field("error").is_none()); + assert_eq!(event.field("event_id"), Some("sce.resilience.retry")); + } + assert_eq!(events[0].field("operation"), Some("unclassified")); + assert_eq!(events[1].field("operation"), Some("auth.refresh_token")); + } +} diff --git a/cli/src/services/setup/mod.rs b/cli/src/services/setup/mod.rs index 7e55db104..0c8e9ace1 100644 --- a/cli/src/services/setup/mod.rs +++ b/cli/src/services/setup/mod.rs @@ -97,14 +97,6 @@ pub struct EmbeddedAsset { pub sha256: [u8; 32], } -#[allow(dead_code)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum RequiredHookAsset { - PreCommit, - CommitMsg, - PostCommit, -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct OptionalWorkflow { pub id: &'static str, @@ -121,19 +113,6 @@ pub fn iter_required_hook_assets() -> std::slice::Iter<'static, EmbeddedAsset> { HOOK_EMBEDDED_ASSETS.iter() } -#[allow(dead_code)] -pub fn get_required_hook_asset(hook: RequiredHookAsset) -> Option<&'static EmbeddedAsset> { - let hook_name = match hook { - RequiredHookAsset::PreCommit => default_paths::hook_dir::PRE_COMMIT, - RequiredHookAsset::CommitMsg => default_paths::hook_dir::COMMIT_MSG, - RequiredHookAsset::PostCommit => default_paths::hook_dir::POST_COMMIT, - }; - - HOOK_EMBEDDED_ASSETS - .iter() - .find(|asset| asset.relative_path == hook_name) -} - fn embedded_assets_for_concrete_target(target: SetupTarget) -> &'static [EmbeddedAsset] { match target { SetupTarget::OpenCode => OPENCODE_EMBEDDED_ASSETS, @@ -952,19 +931,6 @@ fn setup_prompt_target_label(target: SetupPromptTarget) -> String { prompt::setup_prompt_target_label(target) } -#[allow(dead_code)] -fn setup_prompt_target_label_with_color_policy( - target: SetupPromptTarget, - color_enabled: bool, -) -> String { - prompt::setup_prompt_target_label_with_color_policy(target, color_enabled) -} - -#[allow(dead_code)] -fn setup_prompt_title_with_color_policy(color_enabled: bool) -> String { - prompt::setup_prompt_title_with_color_policy(color_enabled) -} - mod prompt; pub fn resolve_setup_dispatch

( diff --git a/cli/src/services/setup/prompt.rs b/cli/src/services/setup/prompt.rs index cf92217b4..2e69e192a 100644 --- a/cli/src/services/setup/prompt.rs +++ b/cli/src/services/setup/prompt.rs @@ -1,9 +1,7 @@ use anyhow::{bail, Result}; use inquire::{Confirm, InquireError, MultiSelect, Select}; -use crate::services::style::{ - prompt_label, prompt_label_with_color_policy, prompt_value_with_color_policy, -}; +use crate::services::style::{prompt_label, prompt_value_with_color_policy}; use super::{OptionalWorkflow, SetupDispatch, SetupMode, SetupPromptTarget, SetupTarget}; @@ -182,8 +180,3 @@ pub(super) fn setup_prompt_target_label_with_color_policy( prompt_value_with_color_policy(label, color_enabled) } - -#[allow(dead_code)] -pub(super) fn setup_prompt_title_with_color_policy(color_enabled: bool) -> String { - prompt_label_with_color_policy("Select setup target", color_enabled) -} diff --git a/cli/src/services/style.rs b/cli/src/services/style.rs index 7c1f2e8e3..329f22ffb 100644 --- a/cli/src/services/style.rs +++ b/cli/src/services/style.rs @@ -259,9 +259,7 @@ fn is_help_placeholder(token: &str) -> bool { /// Gradient color endpoints for the ASCII art banner. /// Right side: cyan (0, 255, 255), Left side: magenta (255, 0, 255). -#[allow(dead_code)] const GRADIENT_COLOR_RIGHT: (u8, u8, u8) = (0, 255, 255); -#[allow(dead_code)] const GRADIENT_COLOR_LEFT: (u8, u8, u8) = (255, 0, 255); /// Applies a right-to-left color gradient to ASCII art banner lines. @@ -270,7 +268,6 @@ const GRADIENT_COLOR_LEFT: (u8, u8, u8) = (255, 0, 255); /// column position using a cyan-to-magenta gradient (cyan on the right, /// magenta on the left). Spaces are left unstyled. /// When color is disabled, returns the plain ASCII art unchanged. -#[allow(dead_code)] #[must_use] pub fn banner_with_gradient(lines: &[&str]) -> String { banner_with_gradient_with_color_policy(lines, supports_color()) @@ -278,7 +275,7 @@ pub fn banner_with_gradient(lines: &[&str]) -> String { /// Internal variant of [`banner_with_gradient`] that accepts an explicit /// color policy flag for testability. -#[allow(dead_code, clippy::cast_precision_loss)] +#[allow(clippy::cast_precision_loss)] #[must_use] pub(crate) fn banner_with_gradient_with_color_policy( lines: &[&str], @@ -325,7 +322,7 @@ pub(crate) fn banner_with_gradient_with_color_policy( /// /// Returns `a + (b - a) * t`, clamped to the `u8` range by construction /// since `a`, `b` are `u8` and `t` is in `[0, 1]`. -#[allow(dead_code, clippy::cast_possible_truncation, clippy::cast_sign_loss)] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] fn lerp_u8(a: u8, b: u8, t: f64) -> u8 { (f64::from(a) + (f64::from(b) - f64::from(a)) * t).round() as u8 } diff --git a/cli/src/services/sync/command.rs b/cli/src/services/sync/command.rs index ce97b1f5b..77adb11fb 100644 --- a/cli/src/services/sync/command.rs +++ b/cli/src/services/sync/command.rs @@ -43,15 +43,6 @@ fn classify_sync_error(err: TraceSyncError) -> CliError { } impl SyncCommand { - #[allow(dead_code)] - pub async fn execute(&self, context: &C) -> Result - where - C: ContextWithRepoRoot, - { - let mut stderr = std::io::sink(); - self.execute_with_stderr(context, &mut stderr).await - } - pub async fn execute_with_stderr( &self, context: &C, diff --git a/cli/src/services/sync/mod.rs b/cli/src/services/sync/mod.rs index fac259368..62046d2a4 100644 --- a/cli/src/services/sync/mod.rs +++ b/cli/src/services/sync/mod.rs @@ -1,6 +1,5 @@ //! Top-level `sce sync` command and Agent Trace synchronization service. -#[allow(dead_code)] pub mod auto_sync; pub mod command; pub mod progress; diff --git a/cli/src/services/sync/render_sync.rs b/cli/src/services/sync/render_sync.rs index 2bdcd8d0e..987eb4aca 100644 --- a/cli/src/services/sync/render_sync.rs +++ b/cli/src/services/sync/render_sync.rs @@ -56,3 +56,55 @@ fn stream_json(stream: &StreamSyncReport) -> serde_json::Value { "batches": stream.batches, }) } + +#[cfg(test)] +mod tests { + use serde_json::{json, Value}; + + use super::render; + use crate::services::output_format::OutputFormat; + use crate::services::sync::sync::{AgentTraceSyncReport, StreamSyncReport, StreamSyncReports}; + + const DIFF_TRACES_CURSOR: i64 = 42; + + fn stream(uploaded: usize) -> StreamSyncReport { + StreamSyncReport { + uploaded, + initial_cursor: 0, + final_cursor: i64::try_from(uploaded).expect("small count"), + batches: usize::from(uploaded > 0), + } + } + + #[test] + fn json_report_keeps_zero_upload_diff_traces_stream() { + let report = AgentTraceSyncReport { + repository_id: "repo".to_string(), + source_instance_id: "source".to_string(), + streams: StreamSyncReports { + messages: stream(3), + parts: stream(0), + diff_traces: StreamSyncReport { + uploaded: 0, + initial_cursor: DIFF_TRACES_CURSOR, + final_cursor: DIFF_TRACES_CURSOR, + batches: 0, + }, + agent_traces: stream(0), + }, + }; + + let rendered = render(&report, OutputFormat::Json).expect("render json"); + let rendered: Value = serde_json::from_str(&rendered).expect("valid json"); + + assert_eq!( + rendered["streams"]["diffTraces"], + json!({ + "uploaded": 0, + "initialCursor": DIFF_TRACES_CURSOR, + "finalCursor": DIFF_TRACES_CURSOR, + "batches": 0, + }) + ); + } +} diff --git a/cli/src/services/sync/sync.rs b/cli/src/services/sync/sync.rs index f3abde4ae..a97e82681 100644 --- a/cli/src/services/sync/sync.rs +++ b/cli/src/services/sync/sync.rs @@ -18,14 +18,14 @@ use crate::services::agent_trace_storage::{resolve_agent_trace_storage, AgentTra use crate::services::agent_trace_sync::control_plane::{ AgentTraceCursors, AgentTraceIngestionBatchRequest, AgentTraceIngestionBatchResponse, AgentTraceIngestionStateRequest, AuthenticatedControlPlaneClient, ControlPlaneError, - IngestionStream, + CredentialStore, IngestionStream, }; use crate::services::agent_trace_sync::{ sync_stream, AgentTraceExportRow, BatchAttemptOutcome, StreamSyncError, }; use crate::services::auth; use crate::services::config; -use crate::services::sync::progress::{NoopProgressReporter, ProgressReporter}; +use crate::services::sync::progress::ProgressReporter; #[derive(Clone, Debug, PartialEq, Eq)] pub struct AgentTraceSyncReport { @@ -127,7 +127,6 @@ impl TraceSyncError { /// (`ControlPlane`) or from a stream's batch/refresh path (`Stream`). /// `Runtime` never carries a `ControlPlaneError` and is never an /// authentication failure. - #[allow(dead_code)] pub fn is_authentication_failure(&self) -> bool { match self { Self::Runtime(_) => false, @@ -148,24 +147,6 @@ impl TraceSyncError { } } -#[allow(dead_code)] -pub async fn run_current_sync(repo_root: &Path) -> Result { - let mut progress = NoopProgressReporter; - run_current_sync_with_progress(repo_root, &mut progress).await -} - -/// Production entry point with an injectable progress sink. -pub async fn run_current_sync_with_progress( - repo_root: &Path, - progress: &mut S, -) -> Result -where - S: ProgressReporter, -{ - let clock = SystemSyncProgressClock; - run_current_sync_with_progress_and_clock(repo_root, progress, &clock).await -} - /// Production sync entry point with injectable progress sink and clock. pub async fn run_current_sync_with_progress_and_clock( repo_root: &Path, @@ -203,31 +184,24 @@ where let storage = resolve_agent_trace_storage(&context) .await .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; - let result = async { - let storage = &storage; - - let auth_config = config::resolve_auth_runtime_config(repo_root) - .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; - let client = AuthenticatedControlPlaneClient::new( - reqwest::Client::new(), - auth_config.control_plane_base_url.value.unwrap_or_default(), - auth::WORKOS_DEFAULT_BASE_URL, - auth_config.workos_client_id.value.unwrap_or_default(), - ); - - let result = run_sync_against_without_progress( - &storage.metadata.repository_id, - &storage.metadata.source_instance_id, - &storage.db, - &client, - progress, - ) - .await; - client.wait_for_pending_refresh().await; - result - } + let auth_config = config::resolve_auth_runtime_config(repo_root) + .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; + let client = AuthenticatedControlPlaneClient::new( + reqwest::Client::new(), + auth_config.control_plane_base_url.value.unwrap_or_default(), + auth::WORKOS_DEFAULT_BASE_URL, + auth_config.workos_client_id.value.unwrap_or_default(), + ); + + let result = run_sync_against_without_progress( + &storage.metadata.repository_id, + &storage.metadata.source_instance_id, + &storage.db, + &client, + progress, + ) .await; - drop(storage); + client.wait_for_pending_refresh().await; result } @@ -246,15 +220,16 @@ where run_sync_async(repository_id, source_instance_id, &reader, client, progress).await } -async fn run_sync_async<'a, S>( +async fn run_sync_async<'a, S, K>( repository_id: &'a str, source_instance_id: &'a str, reader: &'a AgentTraceExportReader<'a>, - client: &'a AuthenticatedControlPlaneClient, + client: &'a AuthenticatedControlPlaneClient, progress: &'a mut S, ) -> Result where S: ProgressReporter + 'a, + K: CredentialStore, { let state_request = AgentTraceIngestionStateRequest { repository_id: repository_id.to_string(), @@ -283,7 +258,7 @@ where state.cursors.messages, "messages", async |cursor, limit| reader.read_messages_after(cursor, limit).await, - async |request| async move { client.ingest_messages(&request).await }.await, + async |request| client.ingest_messages(&request).await, Rc::clone(&progress), &terminated, ), @@ -295,7 +270,7 @@ where state.cursors.parts, "parts", async |cursor, limit| reader.read_parts_after(cursor, limit).await, - async |request| async move { client.ingest_parts(&request).await }.await, + async |request| client.ingest_parts(&request).await, Rc::clone(&progress), &terminated, ), @@ -307,7 +282,7 @@ where state.cursors.agent_traces, "agent_traces", async |cursor, limit| reader.read_agent_traces_after(cursor, limit).await, - async |request| async move { client.ingest_agent_traces(&request).await }.await, + async |request| client.ingest_agent_traces(&request).await, Rc::clone(&progress), &terminated, ), @@ -408,8 +383,8 @@ impl JoinSlot { } #[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn sync_one_stream<'a, T, ReadFn, IngestFn, S>( - client: &'a AuthenticatedControlPlaneClient, +async fn sync_one_stream<'a, T, ReadFn, IngestFn, S, K>( + client: &'a AuthenticatedControlPlaneClient, repository_id: &'a str, source_instance_id: &'a str, stream: IngestionStream, @@ -423,6 +398,7 @@ async fn sync_one_stream<'a, T, ReadFn, IngestFn, S>( where T: AgentTraceExportRow + Clone + 'a, S: ProgressReporter + 'a, + K: CredentialStore, ReadFn: std::ops::AsyncFnMut(i64, usize) -> anyhow::Result> + 'a, IngestFn: std::ops::AsyncFnMut( AgentTraceIngestionBatchRequest, @@ -792,3 +768,187 @@ mod tests { assert_eq!(sibling_ingests.get(), 0); } } + +#[cfg(test)] +mod contract_tests { + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::TcpListener; + use std::sync::{Arc, Mutex}; + use std::thread; + use std::time::{SystemTime, UNIX_EPOCH}; + + use serde_json::{json, Value}; + + use super::run_sync_async; + use crate::services::agent_trace_db::transaction_tests::{message, open, part}; + use crate::services::agent_trace_db::{AgentTraceInsert, DiffTraceInsert}; + use crate::services::agent_trace_export::AgentTraceExportReader; + use crate::services::agent_trace_sync::control_plane::{ + AuthenticatedControlPlaneClient, ControlPlaneError, CredentialStore, + }; + use crate::services::auth::TokenResponse; + use crate::services::sync::progress::NoopProgressReporter; + use crate::services::token_storage::StoredTokens; + + const DIFF_TRACES_CURSOR: i64 = 7; + + struct FixedCredentialStore; + + impl CredentialStore for FixedCredentialStore { + async fn load(&self) -> Result, ControlPlaneError> { + Ok(Some(StoredTokens { + access_token: "access".to_string(), + token_type: "Bearer".to_string(), + expires_in: 3600, + refresh_token: "refresh".to_string(), + scope: None, + stored_at_unix_seconds: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_secs(), + })) + } + + async fn save(&self, _token: &TokenResponse) -> Result { + unreachable!("fixed credentials never refresh") + } + } + + type CapturedRequests = Arc>>; + + fn start_control_plane() -> (String, CapturedRequests) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind control plane"); + let base_url = format!("http://{}", listener.local_addr().expect("server addr")); + let requests = Arc::new(Mutex::new(Vec::new())); + let thread_requests = Arc::clone(&requests); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); + let mut request_line = String::new(); + let _ = reader.read_line(&mut request_line); + let path = request_line + .split_whitespace() + .nth(1) + .unwrap_or_default() + .to_string(); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; content_length]; + let _ = reader.read_exact(&mut body); + let body: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + + let payload = if path.ends_with("/state") { + json!({"cursors": { + "messages": 0, + "parts": 0, + "diffTraces": DIFF_TRACES_CURSOR, + "agentTraces": 0, + }}) + } else { + let rows = body["rows"].as_array().cloned().unwrap_or_default(); + let cursor = rows + .last() + .and_then(|row| row["sourceRowId"].as_i64()) + .unwrap_or(0); + json!({"accepted": rows.len(), "cursor": cursor}) + }; + thread_requests + .lock() + .expect("requests lock") + .push((path, body)); + let payload = payload.to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + (base_url, requests) + } + + #[tokio::test] + async fn sync_uploads_only_three_streams_and_reports_zero_upload_diff_traces() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut db = open(&dir.path().join("agent-trace.db")).await; + db.insert_conversation_text_event(message("m1"), part("m1")) + .await + .expect("insert message and part"); + db.insert_agent_trace(AgentTraceInsert { + commit_id: "commit", + commit_time_ms: 1, + trace_json: "{}", + agent_trace_id: "trace", + url: "https://example.test/trace", + remote_url: "https://example.test/repo.git", + }) + .await + .expect("insert agent trace"); + db.insert_diff_trace(DiffTraceInsert { + time_ms: 1, + session_id: "session", + patch: "patch", + model_id: None, + tool_name: "tool", + tool_version: None, + payload_type: "patch", + }) + .await + .expect("insert diff trace"); + + let (base_url, requests) = start_control_plane(); + let client = AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + base_url, + "http://127.0.0.1:1", + "client", + FixedCredentialStore, + ); + let reader = AgentTraceExportReader::new(&db); + let mut reporter = NoopProgressReporter; + + let report = run_sync_async("repo", "source", &reader, &client, &mut reporter) + .await + .expect("sync converges"); + + let requests = requests.lock().expect("requests lock").clone(); + let mut streams: Vec<&str> = requests + .iter() + .filter(|(path, _)| path.ends_with("/batch")) + .map(|(_, body)| body["stream"].as_str().expect("stream field")) + .collect(); + streams.sort_unstable(); + assert_eq!(streams, ["agent_traces", "messages", "parts"]); + + assert_eq!(report.streams.messages.uploaded, 1); + assert_eq!(report.streams.parts.uploaded, 1); + assert_eq!(report.streams.agent_traces.uploaded, 1); + assert_eq!(report.streams.diff_traces.uploaded, 0); + assert_eq!(report.streams.diff_traces.batches, 0); + assert_eq!( + report.streams.diff_traces.initial_cursor, + DIFF_TRACES_CURSOR + ); + assert_eq!(report.streams.diff_traces.final_cursor, DIFF_TRACES_CURSOR); + } +} diff --git a/cli/tests/support/mod.rs b/cli/tests/support/mod.rs new file mode 100644 index 000000000..1d3cb8c52 --- /dev/null +++ b/cli/tests/support/mod.rs @@ -0,0 +1,291 @@ +#![allow(dead_code)] + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub const SCE: &str = env!("CARGO_BIN_EXE_sce"); +pub const HARD_TIMEOUT: Duration = Duration::from_secs(5); +pub const NON_ROUTABLE_ENDPOINT: &str = "http://192.0.2.1:4318"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Observed { + pub stdout: String, + pub stderr: String, + pub code: Option, +} + +pub struct Run { + pub observed: Observed, + pub elapsed: Duration, +} + +pub struct Sandbox { + dir: tempfile::TempDir, +} + +impl Sandbox { + pub fn new() -> Self { + Self { + dir: tempfile::tempdir().unwrap(), + } + } + + pub fn path(&self) -> &Path { + self.dir.path() + } + + pub fn marker_file(&self, label: &str) -> PathBuf { + self.dir.path().join(format!("{label}.lifecycle")) + } + + pub fn run(&self, args: &[&str], envs: &[(&str, String)]) -> Run { + self.try_run(args, envs) + .unwrap_or_else(|dump| panic!("{dump}")) + } + + pub fn try_run(&self, args: &[&str], envs: &[(&str, String)]) -> Result { + let mut command = Command::new(SCE); + command + .args(args) + .current_dir(self.path()) + .env_clear() + .env("PATH", std::env::var_os("PATH").unwrap_or_default()) + .env("HOME", self.path()) + .env("XDG_CONFIG_HOME", self.path().join("config")) + .env("XDG_STATE_HOME", self.path().join("state")) + .env("XDG_CACHE_HOME", self.path().join("cache")) + .env("NO_COLOR", "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (key, value) in envs { + command.env(key, value); + } + let started = Instant::now(); + let mut child = command.spawn().unwrap(); + let stdout = drain(child.stdout.take().unwrap()); + let stderr = drain(child.stderr.take().unwrap()); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if started.elapsed() > HARD_TIMEOUT { + let dump = thread_dump(child.id()); + let _ = child.kill(); + let _ = child.wait(); + return Err(format!( + "sce {args:?} hung past the {HARD_TIMEOUT:?} hard timeout\n{dump}" + )); + } + std::thread::sleep(Duration::from_millis(2)); + }; + let elapsed = started.elapsed(); + Ok(Run { + observed: Observed { + stdout: stdout.join().unwrap(), + stderr: stderr.join().unwrap(), + code: status.code(), + }, + elapsed, + }) + } +} + +fn thread_dump(pid: u32) -> String { + let mut lines = Vec::new(); + if let Ok(entries) = std::fs::read_dir(format!("/proc/{pid}/task")) { + for entry in entries.flatten() { + let base = entry.path(); + let read = |name: &str| { + std::fs::read_to_string(base.join(name)) + .unwrap_or_default() + .trim() + .to_string() + }; + let state = read("status") + .lines() + .find(|line| line.starts_with("State:")) + .unwrap_or("") + .to_string(); + lines.push(format!( + "tid={} comm={} wchan={} {}", + entry.file_name().to_string_lossy(), + read("comm"), + read("wchan"), + state + )); + } + } + lines.join("\n") +} + +fn drain(mut stream: impl Read + Send + 'static) -> std::thread::JoinHandle { + std::thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = stream.read_to_end(&mut buffer); + String::from_utf8_lossy(&buffer).into_owned() + }) +} + +#[derive(Clone, Debug)] +pub struct CapturedRequest { + pub head: String, + pub body: Vec, +} + +pub struct Receiver { + addr: SocketAddr, + requests: Arc>>, + connections: Arc, + stop: Arc, + thread: Option>, +} + +impl Receiver { + pub fn start(respond: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let held: Arc>> = Arc::default(); + let thread = { + let (requests, connections, stop) = + (requests.clone(), connections.clone(), stop.clone()); + std::thread::spawn(move || { + while !stop.load(Ordering::SeqCst) { + let Ok((stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(5)); + continue; + }; + connections.fetch_add(1, Ordering::SeqCst); + if respond { + let requests = requests.clone(); + std::thread::spawn(move || serve(stream, &requests)); + } else { + held.lock().unwrap().push(stream); + } + } + }) + }; + Self { + addr, + requests, + connections, + stop, + thread: Some(thread), + } + } + + pub fn addr(&self) -> SocketAddr { + self.addr + } + + pub fn endpoint(&self) -> String { + format!("http://{}", self.addr) + } + + pub fn connections(&self) -> usize { + self.connections.load(Ordering::SeqCst) + } + + pub fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + + pub fn wait_for_request(&self) -> bool { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if !self.requests().is_empty() { + return true; + } + std::thread::sleep(Duration::from_millis(10)); + } + false + } +} + +impl Drop for Receiver { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn serve(mut stream: TcpStream, requests: &Mutex>) { + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut buffer = Vec::new(); + let mut chunk = [0u8; 4096]; + let header_end = loop { + let Ok(read) = stream.read(&mut chunk) else { + return; + }; + if read == 0 { + return; + } + buffer.extend_from_slice(&chunk[..read]); + if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + break position + 4; + } + }; + let head = String::from_utf8_lossy(&buffer[..header_end]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + while buffer.len() < header_end + length { + let Ok(read) = stream.read(&mut chunk) else { + break; + }; + if read == 0 { + break; + } + buffer.extend_from_slice(&chunk[..read]); + } + if head.starts_with("post /v1/traces") { + requests.lock().unwrap().push(CapturedRequest { + head, + body: buffer[header_end..].to_vec(), + }); + } + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); +} + +pub fn closed_endpoint() -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + format!("http://{addr}") +} + +pub fn count_occurrences(haystack: &[u8], needle: &[u8]) -> usize { + haystack + .windows(needle.len()) + .filter(|window| *window == needle) + .count() +} + +pub fn contains(haystack: &[u8], needle: &str) -> bool { + count_occurrences(haystack, needle.as_bytes()) > 0 +} + +pub fn standalone_env(endpoint: &str) -> Vec<(&'static str, String)> { + vec![ + ("SCE_TELEMETRY", "standalone".to_string()), + ("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint.to_string()), + ] +} diff --git a/cli/tests/telemetry_default.rs b/cli/tests/telemetry_default.rs new file mode 100644 index 000000000..af49e55b4 --- /dev/null +++ b/cli/tests/telemetry_default.rs @@ -0,0 +1,143 @@ +mod support; + +use support::{ + closed_endpoint, contains, count_occurrences, standalone_env, Observed, Receiver, Sandbox, + NON_ROUTABLE_ENDPOINT, +}; + +const WORKLOADS: [&[&str]; 2] = [&["version"], &["doctor"]]; +const TRACE_SPAN_NAME_FIELD: &[u8] = b"\x2a\x0bsce.command"; + +fn baseline(sandbox: &Sandbox, args: &[&str]) -> Observed { + sandbox.run(args, &[]).observed +} + +fn assert_offline_parity(label: &str, endpoint: &str) { + let sandbox = Sandbox::new(); + for args in WORKLOADS { + let expected = baseline(&sandbox, args); + let enabled = sandbox.run(args, &standalone_env(endpoint)); + assert_eq!(enabled.observed, expected, "{label} {args:?}"); + assert!( + enabled.elapsed < support::HARD_TIMEOUT, + "{label} {args:?} {:?}", + enabled.elapsed + ); + } +} + +#[test] +fn telemetry_offline_parity_reachable_receiver() { + let receiver = Receiver::start(true); + assert_offline_parity("reachable", &receiver.endpoint()); + assert!(receiver.wait_for_request()); +} + +#[test] +fn telemetry_offline_parity_connection_refused() { + assert_offline_parity("refused", &closed_endpoint()); +} + +#[test] +fn telemetry_offline_parity_accept_but_never_respond() { + let receiver = Receiver::start(false); + assert_offline_parity("blackhole", &receiver.endpoint()); + assert!(receiver.connections() > 0); +} + +#[test] +fn telemetry_offline_parity_non_routable_address() { + assert_offline_parity("non-routable", NON_ROUTABLE_ENDPOINT); +} + +#[test] +fn sce_command_span_standalone_exports_one_root_span_without_credentials() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let run = sandbox.run(&["doctor"], &standalone_env(&receiver.endpoint())); + assert!(receiver.wait_for_request()); + + let requests = receiver.requests(); + assert_eq!(requests.len(), 1, "one batch for one command"); + let request = &requests[0]; + assert!(request + .head + .contains("content-type: application/x-protobuf")); + assert!(!request.head.contains("authorization")); + assert!(!request.head.contains("x-sce")); + assert_eq!( + count_occurrences(&request.body, TRACE_SPAN_NAME_FIELD), + 1, + "exactly one sce.command span" + ); + for key in ["sce.command.name", "sce.outcome", "doctor"] { + assert!(contains(&request.body, key), "{key}"); + } + let outcome = if run.observed.code == Some(0) { + "success" + } else { + "failure" + }; + assert!(contains(&request.body, outcome), "{outcome}"); + for other in ["sce.mutation_scope", "sce.worktree", "sce.db.operation"] { + assert!(!contains(&request.body, other), "{other}"); + } +} + +#[test] +fn sce_command_span_never_exports_arguments_paths_or_error_text() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let run = sandbox.run( + &[ + "config", + "validate", + "--config", + "/nonexistent-SECRETPATH-sce-config.json", + ], + &standalone_env(&receiver.endpoint()), + ); + assert_ne!(run.observed.code, Some(0)); + assert!(receiver.wait_for_request()); + + let body = &receiver.requests()[0].body; + assert_eq!(count_occurrences(body, TRACE_SPAN_NAME_FIELD), 1); + for key in ["sce.outcome", "failure", "sce.error.category", "runtime"] { + assert!(contains(body, key), "{key}"); + } + for leaked in [ + "SECRETPATH", + "nonexistent", + "validate", + "exception", + "stack", + ] { + assert!(!contains(body, leaked), "{leaked}"); + } +} + +#[cfg(not(feature = "telemetry-test-receiver"))] +#[test] +fn telemetry_test_receiver_inert_without_the_feature() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let other = Receiver::start(true); + for args in [&["version"][..], &["hooks", "pre-commit"][..]] { + let expected = baseline(&sandbox, args); + let run = sandbox.run( + args, + &[ + ("SCE_TELEMETRY", "test-receiver".to_string()), + ("SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", receiver.endpoint()), + ("SCE_TELEMETRY_TEST_LIFECYCLE_FILE", { + sandbox.marker_file("inert").to_string_lossy().into_owned() + }), + ("OTEL_EXPORTER_OTLP_ENDPOINT", other.endpoint()), + ], + ); + assert_eq!(run.observed, expected, "{args:?}"); + } + assert_eq!(receiver.connections(), 0); + assert_eq!(other.connections(), 0); + assert!(!sandbox.marker_file("inert").exists()); +} diff --git a/cli/tests/telemetry_shutdown.rs b/cli/tests/telemetry_shutdown.rs new file mode 100644 index 000000000..c0db17f68 --- /dev/null +++ b/cli/tests/telemetry_shutdown.rs @@ -0,0 +1,336 @@ +#![cfg(feature = "telemetry-test-receiver")] + +mod support; + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::Duration; + +use support::{ + closed_endpoint, standalone_env, Observed, Receiver, Sandbox, NON_ROUTABLE_ENDPOINT, +}; + +const WARMUP_RUNS: usize = 5; +const MEASURED_RUNS: usize = 30; +const FLUSH_BUDGET: Duration = Duration::from_secs(1); +const POST_COMMAND_SLACK: Duration = Duration::from_millis(250); +const END_TO_END_SLACK: Duration = Duration::from_millis(250); +const END_TO_END_MAX_SLACK: Duration = Duration::from_millis(500); +static TIMING_LOCK: Mutex<()> = Mutex::new(()); +const WORKLOADS: [&[&str]; 2] = [&["version"], &["doctor"]]; + +#[derive(Clone, Copy, Debug)] +enum Mode { + Reachable, + Refused, + Blackhole, + NonRoutable, +} + +struct Bounds { + p50: Duration, + p95: Duration, + max: Duration, +} + +impl Mode { + fn bounds(self) -> Bounds { + match self { + Self::Blackhole | Self::NonRoutable => Bounds { + p50: FLUSH_BUDGET + END_TO_END_SLACK, + p95: FLUSH_BUDGET + END_TO_END_SLACK, + max: FLUSH_BUDGET + END_TO_END_MAX_SLACK, + }, + Self::Reachable | Self::Refused => Bounds { + p50: Duration::from_millis(250), + p95: Duration::from_millis(400), + max: Duration::from_millis(750), + }, + } + } +} + +struct Target { + env: Vec<(&'static str, String)>, + _receiver: Option, +} + +fn target(mode: Mode) -> Target { + let loopback = |endpoint: String, receiver| Target { + env: vec![ + ("SCE_TELEMETRY", "test-receiver".to_string()), + ("SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", endpoint), + ], + _receiver: receiver, + }; + match mode { + Mode::Reachable => { + let receiver = Receiver::start(true); + loopback(receiver.endpoint(), Some(receiver)) + } + Mode::Refused => loopback(closed_endpoint(), None), + Mode::Blackhole => { + let receiver = Receiver::start(false); + loopback(receiver.endpoint(), Some(receiver)) + } + Mode::NonRoutable => Target { + env: standalone_env(NON_ROUTABLE_ENDPOINT), + _receiver: None, + }, + } +} + +fn percentile(sorted: &[Duration], percent: usize) -> Duration { + let rank = (sorted.len() * percent).div_ceil(100); + sorted[rank.saturating_sub(1).min(sorted.len() - 1)] +} + +fn markers(contents: &str) -> HashMap { + contents + .lines() + .filter_map(|line| { + let (name, nanos) = line.split_once(' ')?; + Some((name.to_string(), nanos.parse().ok()?)) + }) + .collect() +} + +fn post_command(markers: &HashMap) -> Duration { + let started = markers["command_complete"]; + let requested = markers["process_exit_requested"]; + assert!(markers["shutdown_begin"] >= started); + assert!(markers["shutdown_end"] >= markers["shutdown_begin"]); + assert!(requested >= markers["shutdown_end"]); + Duration::from_nanos(u64::try_from(requested - started).unwrap()) +} + +struct Measurement { + overhead: Vec, + post_command: Vec, +} + +struct SampleContext<'a> { + workload: &'a str, + mode: &'a str, + iteration: usize, + budget: Duration, +} + +fn load_diagnostics() -> String { + std::fs::read_to_string("/proc/loadavg").map_or_else( + |_| "loadavg unavailable".to_string(), + |load| load.trim().to_string(), + ) +} + +fn require_completed( + result: Result, + context: &SampleContext<'_>, +) -> support::Run { + result.unwrap_or_else(|failure| { + panic!( + "hard timeout: workload={} mode={} iteration={} flush_budget={:?} hard_timeout={:?} loadavg={}\n{failure}", + context.workload, + context.mode, + context.iteration, + context.budget, + support::HARD_TIMEOUT, + load_diagnostics(), + ) + }) +} + +fn assert_strict_max(sorted: &[Duration], bound: Duration, what: &str, summary: &str) { + let worst = *sorted.last().unwrap(); + assert!( + worst <= bound, + "{what}: maximum sample {worst:?} exceeds bound {bound:?} (loadavg={}); {summary}", + load_diagnostics(), + ); +} + +fn measure( + sandbox: &Sandbox, + args: &[&str], + target: &Target, + extra_env: &[(&'static str, String)], + label: &str, + budget: Duration, +) -> Measurement { + let _serial = TIMING_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut expected: Option = None; + let mut measurement = Measurement { + overhead: Vec::new(), + post_command: Vec::new(), + }; + for iteration in 0..WARMUP_RUNS + MEASURED_RUNS { + let context = SampleContext { + workload: args[0], + mode: label, + iteration, + budget, + }; + let disabled = require_completed(sandbox.try_run(args, &[]), &context); + let marker_file = sandbox.marker_file(&format!("{label}-{iteration}")); + let mut env = target.env.clone(); + env.extend(extra_env.iter().cloned()); + env.push(( + "SCE_TELEMETRY_TEST_LIFECYCLE_FILE", + marker_file.to_string_lossy().into_owned(), + )); + let enabled = require_completed(sandbox.try_run(args, &env), &context); + + let baseline = expected.get_or_insert_with(|| disabled.observed.clone()); + assert_eq!( + &disabled.observed, baseline, + "{label} {args:?} disabled drift" + ); + assert_eq!( + &enabled.observed, baseline, + "{label} {args:?} enabled drift" + ); + + let recorded = markers(&std::fs::read_to_string(&marker_file).unwrap()); + assert_eq!(recorded.len(), 4, "{recorded:?}"); + if iteration >= WARMUP_RUNS { + measurement + .overhead + .push(enabled.elapsed.saturating_sub(disabled.elapsed)); + measurement.post_command.push(post_command(&recorded)); + } + } + measurement.overhead.sort(); + measurement.post_command.sort(); + measurement +} + +fn assert_protocol(mode: Mode) { + let sandbox = Sandbox::new(); + let target = target(mode); + let bounds = mode.bounds(); + for args in WORKLOADS { + let label = format!("{mode:?}-{}", args[0]); + let measurement = measure(&sandbox, args, &target, &[], &label, FLUSH_BUDGET); + let summary = format!( + "{label}: overhead p50={:?} p95={:?} max={:?}; post-command p50={:?} p95={:?} max={:?}", + percentile(&measurement.overhead, 50), + percentile(&measurement.overhead, 95), + measurement.overhead.last().unwrap(), + percentile(&measurement.post_command, 50), + percentile(&measurement.post_command, 95), + measurement.post_command.last().unwrap(), + ); + println!("{summary}"); + assert!( + percentile(&measurement.overhead, 50) <= bounds.p50, + "{summary}" + ); + assert!( + percentile(&measurement.overhead, 95) <= bounds.p95, + "{summary}" + ); + assert_strict_max(&measurement.overhead, bounds.max, "overhead", &summary); + assert_strict_max( + &measurement.post_command, + FLUSH_BUDGET + POST_COMMAND_SLACK, + "post-command", + &summary, + ); + } +} + +#[test] +fn telemetry_shutdown_subprocess_reachable_receiver() { + assert_protocol(Mode::Reachable); +} + +#[test] +fn telemetry_shutdown_subprocess_connection_refused() { + assert_protocol(Mode::Refused); +} + +#[test] +fn telemetry_shutdown_subprocess_accept_but_never_respond() { + assert_protocol(Mode::Blackhole); +} + +#[test] +fn telemetry_shutdown_subprocess_non_routable_address() { + assert_protocol(Mode::NonRoutable); +} + +#[test] +fn telemetry_shutdown_subprocess_expired_budget_does_not_wait_for_a_stalled_exporter() { + let sandbox = Sandbox::new(); + let blackhole = target(Mode::Blackhole); + let budget = Duration::from_millis(200); + let extra = [("SCE_TELEMETRY_FLUSH_TIMEOUT_MS", "200".to_string())]; + let measurement = measure(&sandbox, &["version"], &blackhole, &extra, "expiry", budget); + let worst = *measurement.post_command.last().unwrap(); + println!("expiry: post-command max={worst:?}"); + assert_strict_max( + &measurement.post_command, + budget + POST_COMMAND_SLACK, + "post-command", + "expiry", + ); + assert!( + worst >= budget / 2, + "the stalled exporter should have consumed the budget, not returned early: {worst:?}" + ); + assert_strict_max( + &measurement.overhead, + budget + END_TO_END_MAX_SLACK, + "overhead", + "expiry", + ); +} + +#[test] +fn telemetry_shutdown_measurement_rejects_hard_timeout() { + let context = SampleContext { + workload: "version", + mode: "synthetic", + iteration: 7, + budget: FLUSH_BUDGET, + }; + let panic_message = |result: std::thread::Result<()>| { + let payload = result.expect_err("expected a failure"); + payload + .downcast_ref::() + .cloned() + .unwrap_or_default() + }; + + let message = panic_message(std::panic::catch_unwind(|| { + require_completed( + Err("sce [\"version\"] hung past the 5s hard timeout\nTHREAD DUMP".to_string()), + &context, + ); + })); + for needle in [ + "workload=version", + "mode=synthetic", + "iteration=7", + "flush_budget=1s", + "THREAD DUMP", + ] { + assert!(message.contains(needle), "{needle} missing from {message}"); + } + + let bound = Duration::from_millis(750); + let mut one_outlier = vec![Duration::from_millis(10); 29]; + one_outlier.push(bound + Duration::from_millis(1)); + assert!(percentile(&one_outlier, 50) <= bound); + assert!(percentile(&one_outlier, 95) <= bound); + let message = panic_message(std::panic::catch_unwind(|| { + assert_strict_max(&one_outlier, bound, "overhead", "synthetic"); + })); + assert!(message.contains("maximum sample"), "{message}"); + + let mut bounded = vec![Duration::from_millis(10); 29]; + bounded.push(bound); + assert_strict_max(&bounded, bound, "overhead", "synthetic"); +} diff --git a/cli/tests/telemetry_test_receiver.rs b/cli/tests/telemetry_test_receiver.rs new file mode 100644 index 000000000..ef9e1f8c4 --- /dev/null +++ b/cli/tests/telemetry_test_receiver.rs @@ -0,0 +1,126 @@ +#![cfg(feature = "telemetry-test-receiver")] + +mod support; + +use support::{contains, standalone_env, Observed, Receiver, Sandbox}; + +const MODE: &str = "test-receiver"; + +fn receiver_env(endpoint: &str) -> Vec<(&'static str, String)> { + vec![ + ("SCE_TELEMETRY", MODE.to_string()), + ("SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT", endpoint.to_string()), + ] +} + +fn baseline(sandbox: &Sandbox, args: &[&str]) -> Observed { + sandbox.run(args, &[]).observed +} + +#[test] +fn telemetry_test_receiver_exports_to_the_loopback_receiver_without_credentials() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let expected = baseline(&sandbox, &["version"]); + let run = sandbox.run(&["version"], &receiver_env(&receiver.endpoint())); + assert_eq!(run.observed, expected); + assert!(receiver.wait_for_request()); + let request = &receiver.requests()[0]; + assert!(!request.head.contains("authorization")); + assert!(contains(&request.body, "sce.command")); +} + +#[test] +fn telemetry_test_receiver_rejects_hostname_https_and_non_loopback_endpoints() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let port = receiver.addr().port(); + let expected = baseline(&sandbox, &["version"]); + for rejected in [ + format!("http://localhost:{port}"), + format!("https://127.0.0.1:{port}"), + format!("http://0.0.0.0:{port}"), + format!("http://127.0.0.1:{port}/v1/traces"), + format!("http://user@127.0.0.1:{port}"), + format!("127.0.0.1:{port}"), + "http://192.0.2.1:4318".to_string(), + ] { + let run = sandbox.run(&["version"], &receiver_env(&rejected)); + assert_eq!(run.observed, expected, "{rejected}"); + } + assert_eq!(receiver.connections(), 0); +} + +#[test] +fn telemetry_test_receiver_ignores_otel_endpoint_and_rejects_header_overrides() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let ambient = Receiver::start(true); + let mut env = receiver_env(&receiver.endpoint()); + env.push(("OTEL_EXPORTER_OTLP_ENDPOINT", ambient.endpoint())); + env.push(( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + format!("{}/v1/traces", ambient.endpoint()), + )); + sandbox.run(&["version"], &env); + assert!(receiver.wait_for_request()); + assert_eq!(ambient.connections(), 0); + + let rejected = Receiver::start(true); + let mut env = receiver_env(&rejected.endpoint()); + env.push(( + "OTEL_EXPORTER_OTLP_HEADERS", + "authorization=Bearer managed-secret".to_string(), + )); + let expected = baseline(&sandbox, &["version"]); + assert_eq!(sandbox.run(&["version"], &env).observed, expected); + assert_eq!(rejected.connections(), 0); +} + +#[test] +fn telemetry_test_receiver_never_touches_the_auth_database() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + sandbox.run(&["version"], &receiver_env(&receiver.endpoint())); + assert!(receiver.wait_for_request()); + let mut stack = vec![sandbox.path().to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).unwrap().flatten() { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().to_lowercase(); + assert!(!name.contains("auth"), "{}", path.display()); + if path.is_dir() { + stack.push(path); + } + } + } +} + +#[test] +fn telemetry_test_receiver_wins_over_standalone_and_inherited_otel_variables() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let standalone = Receiver::start(true); + let mut env = standalone_env(&standalone.endpoint()); + env.retain(|(key, _)| *key != "SCE_TELEMETRY"); + env.extend(receiver_env(&receiver.endpoint())); + sandbox.run(&["version"], &env); + assert!(receiver.wait_for_request()); + assert_eq!(standalone.connections(), 0); +} + +#[test] +fn telemetry_hook_export_gated_test_receiver_hook_reaches_only_loopback_receiver() { + let sandbox = Sandbox::new(); + let receiver = Receiver::start(true); + let inherited = Receiver::start(true); + for hook in [&["hooks", "pre-commit"][..], &["policy", "bash"][..]] { + let expected = baseline(&sandbox, hook); + let mut env = receiver_env(&receiver.endpoint()); + env.push(("OTEL_EXPORTER_OTLP_ENDPOINT", inherited.endpoint())); + let run = sandbox.run(hook, &env); + assert_eq!(run.observed, expected, "{hook:?}"); + } + assert!(receiver.wait_for_request()); + assert_eq!(inherited.connections(), 0); +} diff --git a/context/architecture.md b/context/architecture.md index 927e56149..3d2a60ccb 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -120,7 +120,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/main.rs` is the executable entrypoint (`sce`) and delegates to `app::run`. - `cli/src/cli_schema.rs` defines the clap-based CLI schema using derive macros for all top-level commands and subcommands, including the top-level `sync` command, and renders command-local help text for the `auth` command tree (`auth`, `auth login`, `auth logout`, `auth whoami`). - `cli/src/app.rs` provides the clap-based argument dispatch loop with deterministic help/setup execution, bare-command help routing for `sce auth` and `sce config`, successful unknown-command fallback to top-level or closest-parent help, centralized stream routing (`stdout` success payloads, `stderr` redacted diagnostics), stable class-based exit-code mapping (`2` parse, `3` validation, `4` runtime, `5` dependency), and stable class-based stderr diagnostic codes (`SCE-ERR-PARSE`, `SCE-ERR-VALIDATION`, `SCE-ERR-RUNTIME`, `SCE-ERR-DEPENDENCY`) with default `Try:` remediation injection when missing. -- The app runtime now moves through explicit startup phases in `cli/src/app.rs`: dependency bootstrapping (`perform_dependency_check`), startup context construction (`build_startup_context`), runtime initialization (`initialize_runtime`), directly awaited command parse/execute inside the telemetry polling scope (`run_command_lifecycle`, `parse_command_phase` plus `services::app_support::execute_command_phase`), and final output rendering through `services::app_support::render_run_outcome`. `AppRuntime` owns the concrete production logger, no-op telemetry runtime, filesystem ops, git ops, static `CommandRegistry`, and startup-diagnostic state across those phases; `RunOutcome` carries final render data with an optional generic logger implementing `services::observability::traits::Logger`, so render support can log classified errors without production-logger type coupling. If a telemetry implementation attempts to invoke the command action more than once, dispatch returns a runtime-classified error instead of panicking or reusing consumed arguments. +- The app runtime now moves through explicit startup phases in `cli/src/app.rs`: dependency bootstrapping (`perform_dependency_check`), startup context construction (`build_startup_context`), runtime initialization (`initialize_runtime`), directly awaited command parse/execute inside the telemetry polling scope (`run_command_lifecycle`, `parse_command_phase` plus `services::app_support::execute_command_phase`), and final output rendering through `services::app_support::render_run_outcome`. `AppRuntime` owns the concrete production logger, `RuntimeTelemetry` (no-op unless the opt-in standalone OTel runtime is selected after startup), filesystem ops, git ops, static `CommandRegistry`, and startup-diagnostic state across those phases; `RunOutcome` carries final render data with an optional generic logger implementing `services::observability::traits::Logger`, so render support can log classified errors without production-logger type coupling. If a telemetry implementation attempts to invoke the command action more than once, dispatch returns a runtime-classified error instead of panicking or reusing consumed arguments. - `AppContext` is the CLI's borrowed dependency view in `cli/src/app.rs`: it is generic over logger, telemetry, filesystem, and git capability implementations and stores references plus an optional `repo_root: Option` instead of owning `Arc` trait objects. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. Startup creates a context view over `AppRuntime`'s concrete production dependencies with `repo_root` set to `None`; command paths can derive repo-root-scoped context views through the `ContextWithRepoRoot` accessor trait / `AppContext::with_repo_root(...)`, which reuses the same borrowed dependencies while attaching the resolved root. Narrow accessor traits expose associated concrete capability types for logger, telemetry, fs, and git (`&Self::...`) plus repo-root access, so call sites can express capability requirements without erasing the borrowed dependencies back to trait objects; lifecycle providers consume the repo-root accessor rather than the full context type. - Command parse-time conversion and run-time handling are separated by an internal static `RuntimeCommand` seam. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` enum with variants for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync, plus a deterministic `CommandRegistry` name catalog populated by `build_default_registry()`. `parse_command_phase` in `cli/src/app.rs` delegates clap-output conversion to `cli/src/services/parse/command_runtime.rs`, which owns clap error classification, help rendering bridges, longest-valid-parent traversal for unknown command paths, and parsed-request-to-enum conversion while returning concrete enum values. Service-owned `command.rs` modules define command payload structs and generic execution methods with narrow context requirements: context-free commands accept any context, hooks requires logger access, setup/doctor require repo-root scoping, and central dispatch requires the union of logger plus repo-root-scoping capabilities. `services::app_support::execute_command_phase` emits lifecycle logs around `RuntimeCommand::execute_with_stderr(...)`; the enum performs the only central dispatch match and delegates business behavior to the service-owned command structs, with sync receiving the app-owned stderr writer for format-gated progress. - Startup observability bootstrapping in `cli/src/app.rs` still tolerates invalid default-discovered config files by continuing with degraded defaults plus `sce.config.invalid_config` warn-level logs, but the warning/logging work is now isolated behind the startup-context and runtime-initialization phases rather than one inline startup function. @@ -132,7 +132,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/config/mod.rs` is the config service facade and `sce config` orchestration surface (`show`, `validate`, `--help`), with bare `sce config` routed by `cli/src/app.rs` to the same help payload as `sce config --help`. Focused submodules own the implementation slices: `types.rs` owns shared config/runtime primitives, `schema.rs` owns generated schema embedding plus typed file parsing, `policy.rs` owns bash-policy semantic validation plus policy-specific formatting and runtime preset-catalog access for the Rust evaluator, `resolver.rs` owns deterministic config-file discovery, file-layer merging, explicit value precedence (`flags > env > config file > defaults` where flag-backed), shared auth-key resolution, observability-runtime resolution, attribution-hooks runtime gate and `agent_trace.auto_sync` resolution, database-retry config resolution and `DATABASE_RETRY_CONFIG` `OnceLock` initialization, default-discovered invalid-file degradation, strict invalid-discovered-layer errors for Agent Trace storage, and explicit-path fatal errors for `--config` / `SCE_CONFIG_FILE`, and private `render.rs` owns `sce config show` / `sce config validate` text and JSON output construction plus rendering-specific display-value helpers. The facade preserves existing `services::config` imports for startup/auth/hooks callers while delegating command execution to resolution plus rendering submodules. - `cli/src/services/output_format.rs` defines the canonical shared CLI output-format contract (`OutputFormat`) for supporting commands, with deterministic `text|json` parsing and command-scoped actionable invalid-value guidance. - `cli/src/services/config/types.rs` is the canonical owner for the shared runtime/config primitive seam used by the CLI: `LogLevel`, `LogFormat`, `SCE_LOG_LEVEL`, `SCE_LOG_FORMAT`, `SCE_LOG_DIR`, `DEFAULT_LOG_FILE_RETENTION_LIMIT`, and the shared bool parsing helpers used by both config resolution and observability bootstrap; `cli/src/services/config/mod.rs` re-exports those primitives through the facade. -- `cli/src/services/capabilities.rs` defines the current broad CLI capability traits consumed by the borrowed, compile-time-typed `AppContext`: `FsOps` with `StdFsOps` for filesystem operations and `GitOps` with `ProcessGitOps` for git command execution plus repository-root/hooks-directory resolution. Existing service internals do not consume these traits directly yet; command execution uses narrow accessors and repo-root-scoped context derivation. +- `cli/src/services/capabilities.rs` defines the current broad CLI capability traits consumed by the borrowed, compile-time-typed `AppContext`: `FsOps` with `StdFsOps` as a method-less filesystem capability marker and `GitOps` with `ProcessGitOps` for git command execution and availability checks. Existing service internals do not consume these traits directly yet; command execution uses narrow accessors and repo-root-scoped context derivation. - `cli/src/services/lifecycle.rs` defines the current compile-safe lifecycle seam. `ServiceLifecycle` has default no-op generic `diagnose`, `fix`, and `setup` methods over `C: HasRepoRoot`, with lifecycle-owned health, fix, and setup result types so the trait contract is not publicly anchored to doctor/setup module types or the full `AppContext` shape. The same module owns the static `LifecycleProvider` enum and shared `lifecycle_providers(include_hooks)` catalog/factory, returning providers in deterministic order (config → local_db → auth_db → agent_trace_db → hooks when requested); enum dispatch calls each concrete provider through generic context methods without boxed lifecycle-provider allocation or repo-root trait-object context erasure. Hooks exposes a `HooksLifecycle` provider in `cli/src/services/hooks/lifecycle.rs` for hook rollout diagnosis/fix/setup using lifecycle-owned health records plus the canonical required-hook installer. Config exposes a `ConfigLifecycle` provider in `cli/src/services/config/lifecycle.rs` for global/repo-local config validation and repo-local `.sce/config.json` bootstrap. local_db exposes a `LocalDbLifecycle` provider in `cli/src/services/local_db/lifecycle.rs` for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup. auth_db exposes an `AuthDbLifecycle` provider in `cli/src/services/auth_db/lifecycle.rs` for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup. agent_trace_db exposes an `AgentTraceDbLifecycle` provider in `cli/src/services/agent_trace_db/lifecycle.rs` for setup-time repository-scoped Agent Trace storage initialization when a repo root is available and repository Agent Trace DB path health/fix from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path; the former fallback was removed by the `retire-legacy-agent-trace-db` plan). Doctor runtime aggregates the full provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor report/fix records at the orchestration boundary; setup command aggregates the shared catalog for `setup` with hooks included only when requested and adapts hook setup outcomes before rendering setup-owned messages. - Agent Trace lifecycle setup resolves `agent_trace.repository_id` / `agent_trace.repository_remote`, creates/reuses checkout identity for diagnostics, and creates or migrates the repository-scoped DB through `agent_trace_storage::resolve_agent_trace_storage(...)`; hook runtime uses the same storage identity/path resolution and no-migration open path, with missing or stale schema failing open through the existing `Run 'sce setup'.` guidance. - `cli/src/services/auth_command/mod.rs` defines the implemented auth command surface for `sce auth login|logout|whoami`, including device-flow login, stored-credential validation/renewal through login with device-flow fallback, logout, and Control Plane `/me`-backed whoami rendering in text/JSON formats; text mode uses flat `Email`, `First Name`, `Last Name`, `Role`, `Permissions`, and `Organization Name` labels, with optional names and missing role/permissions/workspace values handled deterministically. Logged-out text returns exact login guidance and renewal reports retain the `login` operation label. `cli/src/services/auth_command/command.rs` owns the `AuthCommand` payload used by the static `RuntimeCommand` enum. There is no public renewal or status subcommand. @@ -152,7 +152,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `context/cli/agent-trace-auto-sync.md` documents the sync-owned, one-shot post-commit launcher boundary: it reuses `sce sync`, has no daemon or local retry machinery, and fails open when child startup cannot be completed; doctor reports this capability without invoking the launcher. - `cli/src/services/sync/progress.rs` owns the sync-local, consumer-typed progress seam: generic `ProgressReporter` supports event delivery plus explicit successful finalization, closure-based collectors, and a no-op implementation alongside the fixed `indicatif` stderr presentation adapter. `cli/src/services/sync/sync.rs` owns `SyncProgressEvent` and its payload semantics for the three remotely synchronized streams (`messages`, `parts`, `agent_traces`; `diff_traces` is captured locally but not synchronized), while `sync/command.rs` selects the terminal adapter for text and the no-op reporter for JSON. There is no top-level `cli/src/services/progress/` module; sync orchestration depends only on its sync-owned contract, so terminal-library details stay at the sync presentation boundary. - `sce sync [--format text|json]` is implemented: `cli/src/services/sync/sync.rs` resolves repository-scoped Agent Trace storage, authenticates against the control plane with stored WorkOS credentials, uses the config-resolved `control_plane_base_url` with baked default `https://sce.crocoderlab.dev`, calls the ingestion `/state` endpoint once, then starts the three remote `messages`/`parts`/`agent_traces` stream state machines concurrently (`diff_traces` remains a local-only compatibility report) via `AgentTraceExportReader` and a shared per-stream reconciliation engine. Batches and cursor refreshes remain sequential within each stream, while fixed stream order is retained for final and stream-completion reporting; `cli/src/services/sync/render_sync.rs` renders the converged `AgentTraceSyncReport` as concise per-stream text or `camelCase` JSON without a nested subcommand field (see `context/cli/sync-command.md`). Local DB bootstrap and setup-time repository-scoped Agent Trace DB initialization otherwise still flow through lifecycle providers aggregated by setup, while repository-scoped DB health/repair flows through the doctor surface. The former trace database inspection and nested sync surfaces are unavailable. -- `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helpers (`load_patch_from_json` for string input, `load_patch_from_json_bytes` for byte input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. +- `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helper (`load_patch_from_json` for string input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. - `cli/src/services/structured_patch.rs` defines the synchronous structured editor-hook derivation seam. It derives Claude `PostToolUse` `Write` structured-update hunks, `Write` `tool_input.content` create fallback, and `Edit` structured-patch payloads into canonical `ParsedPatch` values plus Claude session/tool metadata, returning deterministic skip reasons for unsupported events/tools/payload shapes. The module is pure and side-effect-free. It is wired into `sce hooks diff-trace` for Claude payload classification at intake (T04) and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing dispatch at read time (T05). - `cli/src/services/` contains module boundaries for command_registry, lifecycle, auth_command, config, setup, doctor, hooks, checkout identity, bash_policy, version, completion, help, patch, sync, SCE web URL helpers, shared database infrastructure, local DB adapters, encrypted auth DB adapters, and Agent Trace DB adapters with explicit trait seams for future implementations. `cli/src/services/checkout/` owns checkout ID file infrastructure (the per-checkout Agent Trace DB opener/path helper was removed by the `retire-legacy-agent-trace-db` plan); active setup/hooks use `agent_trace_storage` to establish checkout identity as diagnostics and initialize/open the repository-scoped DB, while `sce doctor` surfaces checkout identity facts plus credential-safe repository Agent Trace DB metadata. There is no checkout-scoped discovery or former trace inspection surface; any pre-migration `agent-trace-*.db` files on disk are never touched and no longer inspectable via the CLI. `cli/src/services/bash_policy.rs` owns both the CLI-agnostic evaluator logic and the hidden `sce policy bash` command adapter used by OpenCode, Claude, and Pi callers; Codex delegates in-process to the same evaluator for its native `PreToolUse(Bash)` response. `cli/src/services/command_registry.rs` defines the static `RuntimeCommand` enum, deterministic `CommandRegistry` name catalog, and `build_default_registry()` function for command dispatch metadata. Service-owned command modules own the runtime command payload structs for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync. - `cli/README.md` is the crate-local onboarding and usage source of truth for placeholder behavior, safety limitations, and roadmap mapping back to service contracts. @@ -231,15 +231,20 @@ caller: `Handle::try_current` + `spawn_blocking` (inline removal without a runtime). The task is fire-and-forget and holds no lock. A killed process can leave an orphaned `index-.lock`/temporary index file behind; this is an accepted - limitation (a sweep is a PR3 candidate). + limitation. PR3 audited it and adds no automatic sweeper: the temporary + index has no ownership or lock record, and a file name or age alone is not + evidence of abandonment, so a sweep is deferred to a separate change that + first defines a safe ownership protocol. - The guarded-shell supervisor (`cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`) runs the synchronous `Child::wait`/descriptor supervision. The worker owns the `ProtectedWorktree` and the guarded child, ends when the child exits or cancellation is observed, and keeps the protected lock until it returns. The - lifetime pipe is created close-on-exec atomically and the guarded child alone - receives the protected descriptors through `pre_exec`, so unrelated spawned - processes do not inherit them. + lifetime pipe is created close-on-exec atomically only where `pipe2` exists + (Linux); elsewhere it is `pipe` then `fcntl`, which leaves a small + fork/exec window that is an unresolved limitation (not fixed in PR3). The + guarded child alone clears close-on-exec on the protected descriptors in + `pre_exec`. Runtime shutdown waits for these workers; each is bounded by its lock timeout, file I/O or child process. No other production `spawn_blocking` or @@ -289,7 +294,7 @@ Investigations T08 (`turso default-features = false`) and T09 (isolating the with rationale in the benchmark doc. Final after-change numbers and remaining bottlenecks are captured in T11. -This phase establishes compile-safe extension seams with a dependency baseline (`anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `toml`, `tracing`, `turso`, `uuid`, plus target-specific keyring backends); `[dev-dependencies]` now carries `quint-connect`, dev/test-only, for the in-progress `mutation-cursor-quint-connect` model-based-testing harness. Per-user local Turso DB and Agent Trace DB bootstrap/health coverage now exist through setup/doctor flows; the user-invocable `sce sync` command is now fully implemented including rendering (see above), and broader runtime integrations remain deferred. +This phase establishes compile-safe extension seams with a dependency baseline (`anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `toml`, `tracing`, `turso`, `uuid`, plus target-specific keyring backends; the exact-pinned OTel crates `opentelemetry`, `opentelemetry_sdk`, `opentelemetry-otlp`, `tracing-opentelemetry`, `tracing-subscriber` were added with the standalone telemetry runtime); `[dev-dependencies]` now carries `quint-connect`, dev/test-only, for the in-progress `mutation-cursor-quint-connect` model-based-testing harness. Per-user local Turso DB and Agent Trace DB bootstrap/health coverage now exist through setup/doctor flows; the user-invocable `sce sync` command is now fully implemented including rendering (see above), and broader runtime integrations remain deferred. ## SCE plan/code role boundary diff --git a/context/cli/agent-trace-storage.md b/context/cli/agent-trace-storage.md index d26c8ced3..bbfaf237e 100644 --- a/context/cli/agent-trace-storage.md +++ b/context/cli/agent-trace-storage.md @@ -7,7 +7,6 @@ Module at `cli/src/services/agent_trace_storage/` (T04 of the `repository-scoped - `AgentTraceStorageContext { repository_root, explicit_repository_id, repository_remote }` — borrowed inputs mirroring the `agent_trace.repository_id` / `agent_trace.repository_remote` config keys; callers pass already-resolved configuration values. Active hook runtime and Agent Trace lifecycle setup/health resolve these values through the config service before constructing the storage context. - `ResolvedAgentTraceStorage { repository_identity, db_path, db, metadata }` — resolved repository identity (`ResolvedRepositoryIdentity` including source provenance), the repository-scoped DB path, the open `RepositoryAgentTraceDb`, and the typed `RepositoryMetadata { repository_id, source_instance_id }` produced by the same verification/initialization call that opened `db` (see [../sce/agent-trace-db.md](../sce/agent-trace-db.md) for the `source_instance_id` identity contract). - `resolve_agent_trace_storage(context)` — setup/lifecycle production entrypoint using the canonical state root from the default-path catalog; may run migrations, including migration `002`. -- `resolve_agent_trace_storage_at_state_root(context, state_root).await` — async setup/lifecycle resolution core against an explicit state root; used to exercise the full path without touching the real user state directory. - `resolve_agent_trace_storage_for_hook_runtime(context).await` — async hook-runtime production entrypoint using the canonical state root; never runs migrations, including migration `002`. - `resolve_agent_trace_storage_for_hook_runtime_at_state_root(context, state_root).await` — async hook-runtime resolution core against an explicit state root. @@ -17,6 +16,7 @@ Module at `cli/src/services/agent_trace_storage/` (T04 of the `repository-scoped 2. DB path from `default_paths::agent_trace_db_path_for_repository{,_at}`, which rejects empty or path-unsafe repository IDs (separators, `.`, `..`). 3. DB open splits by caller through `agent_trace_db::repository::RepositoryAgentTraceDb`, sharing steps 1–2 through an internal `open_storage_with` helper parameterized by the DB-opener; both resolver paths await async database operations: - Setup/lifecycle (`resolve_agent_trace_storage{,_at_state_root}`): a bounded concurrent-safe fast-path-then-migrate sequence — `open_without_migrations_at` + `ensure_schema_ready_for_hooks()` + `verify_or_initialize_repository_metadata(repository_id)`, falling back to migration-running `new_at` plus the same metadata validation. Directory creation rides on `TursoDb` parent-dir `create_dir_all`. If concurrent first open leaves all repository schema tables present but the one-file baseline migration record missing, the repository adapter records that missing metadata before rechecking readiness; otherwise the resolver retries the open/migrate sequence for a bounded window while SQLite/Turso locks clear. + - Maintenance (`resolve_existing_agent_trace_storage_for_maintenance` + `RepositoryAgentTraceDb::open_verified_existing_at`, used only by snapshot-ref reconciliation): a non-creating read-only Turso open (`TursoDb::open_existing_without_migrations_at`, multiprocess WAL preserved) that never creates the DB file or parent directory, never migrates, never calls the migration-metadata repair, and never runs `verify_or_initialize_repository_metadata`. It verifies schema readiness, metadata presence, `repository_id` equality, and a valid `source_instance_id` before returning, with typed `ExistingRepositoryDbError` (`Missing`, `Unreadable`, `IncompatibleSchema`, `MissingMetadata`, `RepositoryMismatch`). Plain `sce doctor` shares the open via `RepositoryAgentTraceDb::open_existing_schema_ready_at`. See [mutation-trace-ref-reconciliation.md](mutation-trace-ref-reconciliation.md). - Hook runtime (`resolve_agent_trace_storage_for_hook_runtime{,_at_state_root}`): `open_for_hooks_without_migrations_at` + `ensure_schema_ready_for_hooks()`, applying the same narrow concurrent-first-open metadata repair on failure but never falling back to `new_at`/migrations. A missing or baseline-only (pre-`002`) database fails with `ensure_schema_ready_for_hooks`'s existing `Run 'sce setup'.` guidance. Once readiness passes, `verify_or_initialize_repository_metadata(repository_id)` initializes `source_instance_id` when needed, matching setup's result for an already-set-up repository. `sce hooks` command call sites use this path via `open_agent_trace_db_for_hook_runtime` in `cli/src/services/hooks/mod.rs`. ## Never-touch on-disk boundary diff --git a/context/cli/async-persistence-guarantees.md b/context/cli/async-persistence-guarantees.md index 9ef09496e..f3b42eeeb 100644 --- a/context/cli/async-persistence-guarantees.md +++ b/context/cli/async-persistence-guarantees.md @@ -26,8 +26,34 @@ Each statement points to a test, a type-level argument or a named limitation. boundary lease until it ends, and guarded-shell descriptors are inherited only by the guarded child; see [adapter lifecycle](opencode-mutation-scope-adapter-lifecycle.md) and [external mutation guard](mutation-trace-external-mutation-guard.md). -- **Known limits:** large futures keep `#![recursion_limit = "256"]` pressure - (measured in PR3); the temporary-index cleanup limitation above stands. +- **Known limits:** + - Future size and `recursion_limit` (PR3 audit, Linux debug build, + `RUSTC_BOOTSTRAP=1 RUSTFLAGS=-Zprint-type-sizes`, disposable worktree): + the top-level `app::run` future is 14000 bytes and `main`'s `block_on` + future 14016; the chain `run_with_dependency_check` 13952, + `run_command_lifecycle` 13568, `execute_command_phase` 13168, + `SyncCommand::execute_with_stderr` 13072, `run_current_sync_without_progress` + 12784, `run_sync_async` 12040 and `join_three_to_completion` 11832 carries + almost all of it. The dependency's own `DatabaseSyncEngine::open_db` future + is 12704 bytes, so the size is dominated by a third-party future captured + inline and each wrapper adds roughly 50-200 bytes. No safe reduction exists + without boxing (out of scope). Building without the override fails with + "queries overflow the depth limit" (depth increased by 130 computing the + layout of `app::run`; the default limit is 128), so `#![recursion_limit = + "256"]` in `cli/src/main.rs` is kept. + - Temporary Git index: `TempIndexGuard` removes its file on completion or drop, + but a killed process can leave an orphaned `index-` file under the + runtime tmp directory. No automatic sweeper is added: there is no ownership + or lock record for the file and name or age alone is not evidence of + abandonment. A sweep needs a separate change that first defines that + protocol. + - Guarded-shell descriptor atomicity: the lifetime pipe is created + close-on-exec atomically with `pipe2` on Linux only; elsewhere `pipe` then + `fcntl` leaves a small fork/exec window. This race is an unresolved + limitation and is not fixed here. + - Blocking sites: the 14 production `spawn_blocking` sites match the + retained-worker list in [architecture](../architecture.md#application-execution-runtime); + none runs a database or Turso future. The ordinary Rust suites deleted during the async migration are not restored; correctness evidence is the focused regression set, the Quint MBT suite and surviving targeted tests. @@ -36,6 +62,6 @@ Each statement points to a test, a type-level argument or a named limitation. - **PR1** (complete): application-owned multi-thread Tokio runtime and static async command dispatch. - **PR2**: async Turso core and the callers it required (credentials, token storage, lifecycle, hooks, mutation trace, sync/export, setup/doctor); the Git/process/filesystem async work and cancellation protections above; transaction isolation, credential-rotation, terminal-sync, adapter-boundary and FD-inheritance fixes; removal of the DB runtime bridge and its blocking scopes. -- **PR3**: post-migration cleanup and runtime audit — redundant `async` wrappers, unused `run_with_retry_sync`, redundant `drop(storage)`, stale `#[allow(dead_code)]`, orphaned test support and unused seams, `join_three_to_completion` re-evaluation, large-future/recursion-limit measurement and an optional temporary-index sweep. Cosmetic and maintainability only. +- **PR3** (implementation complete; final `/validate` pending): post-migration cleanup and runtime audit. Verified-dead code and blanket dead-code allowances were deleted, redundant `async` wrappers, `run_with_retry_sync` and the redundant `drop(storage)` were removed, `join_three_to_completion` was retained, the `recursion_limit = "256"` override was kept, and no temporary-index sweeper was added. The final unsuppressed audit on Linux reports 32 binary-target and 27 test-target diagnostics, all covered by declaration-level `#[allow(dead_code, reason = ...)]` exceptions: the `diff_traces` compatibility surface (cites the 2026-10-01 ADR), the single-event mutation loaders with no production reader (the former ref-reconciliation exemptions were removed once `sce doctor --fix` wired the pass), cfg-gated `PlatformFamily` variants and `GuardError::UnsupportedPlatform`, and never-read `CoordinateOutcome`/`committed`/`completed` fields kept as suspected defects. Linux-only diagnostics do not prove cross-platform safety. Cosmetic and maintainability only. - **PR4**: OpenTelemetry (not implemented in PR2). diff --git a/context/cli/capability-traits.md b/context/cli/capability-traits.md index c919e0143..07cd66bd8 100644 --- a/context/cli/capability-traits.md +++ b/context/cli/capability-traits.md @@ -4,14 +4,12 @@ The CLI exposes broad capability traits in `cli/src/services/capabilities.rs` an ## Current contract -- `FsOps: Send + Sync` defines broad filesystem operations: `read_file`, `write_file`, `metadata`, and `exists`. -- `StdFsOps` is the production filesystem implementation and wraps `std::fs` / `Path::exists` with `anyhow` context on fallible operations. -- `GitOps: Send + Sync` defines broad git process capabilities: `run_command`, `resolve_repository_root`, `resolve_hooks_directory`, and `is_available`. +- `FsOps: Send + Sync` is a method-less filesystem capability marker; the unused read/write/metadata/exists operations were removed. +- `StdFsOps` is the production `FsOps` implementation. +- `GitOps: Send + Sync` defines broad git process capabilities: `run_command` and `is_available`. - `ProcessGitOps` is the production git implementation and shells out to `git`, returning stdout as UTF-8 text with command/directory context on failure. -- `resolve_repository_root` uses `git rev-parse --show-toplevel`. -- `resolve_hooks_directory` uses `git rev-parse --git-path hooks` and resolves relative hook paths against the provided repository root. - `AppRuntime` owns concrete `StdFsOps` and `ProcessGitOps` production dependencies alongside concrete observability dependencies; these dependencies are borrowed by `AppContext` rather than stored behind type-erased runtime containers. -- `AppContext` is generic over logger, telemetry, filesystem, and git capability implementations and borrows them from the runtime; it is passed through directly awaited static `RuntimeCommand::execute` (enum defined in `cli/src/services/command_registry.rs`) behind narrow accessor bounds. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. Its inherent `logger()` / `telemetry()` / `fs()` / `git()` helpers return the concrete generic references (`&L`, `&T`, `&F`, `&G`), and `HasLogger`, `HasTelemetry`, `HasFs`, and `HasGit` use associated capability types that return `&Self::{Capability}` instead of object-erased `&dyn ...` values. This is the current static-DI boundary: command and service call sites express narrow capability requirements while preserving concrete borrowed dependency types. `ContextWithRepoRoot` derives a repository-scoped context while preserving the borrowed runtime capability objects. +- `AppContext` is generic over logger, telemetry, filesystem, and git capability implementations and borrows them from the runtime; it is passed through directly awaited static `RuntimeCommand::execute` (enum defined in `cli/src/services/command_registry.rs`) behind narrow accessor bounds. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. Its inherent `logger()` helper returns the concrete generic reference (`&L`), and `HasLogger`, `HasTelemetry`, and `HasGit` use associated capability types that return `&Self::{Capability}`, while `HasFs` only declares its associated `Fs` type as a bound, instead of object-erased `&dyn ...` values. This is the current static-DI boundary: command and service call sites express narrow capability requirements while preserving concrete borrowed dependency types. `ContextWithRepoRoot` derives a repository-scoped context while preserving the borrowed runtime capability objects. - Current command execution bounds are capability-oriented: central dispatch requires logger access plus repo-root scoping, context-free commands accept any context, hooks require logger access, and setup/doctor require repo-root scoping. - Test-only `UnimplementedFsOps` and `UnimplementedGitOps` stubs are available under `capabilities::test_stubs` for tests that need to satisfy trait bounds before providing focused fakes. diff --git a/context/cli/config-precedence-contract.md b/context/cli/config-precedence-contract.md index 75c9f7b88..b541063c3 100644 --- a/context/cli/config-precedence-contract.md +++ b/context/cli/config-precedence-contract.md @@ -37,6 +37,8 @@ Resolved observability values that currently have no CLI flag layer follow the s 2. config file values (`log_format`, `log_to_file`, `log_dir`) 3. defaults (`log_format=text`, `log_to_file=true`; `log_dir=/sce/logs` through `default_paths::observability_log_dir()`, resolving on Linux to `$XDG_STATE_HOME/sce/logs` or `~/.local/state/sce/logs` when `XDG_STATE_HOME` is unset) +`SCE_TELEMETRY=standalone` (and, in feature-enabled test builds only, `test-receiver`) is an environment-only switch outside this precedence chain: it has no flag, config-file key, or default other than disabled, and repository config cannot enable it (see `../sce/cli-observability-contract.md`). `SCE_TELEMETRY_FLUSH_TIMEOUT_MS` (default 1000, accepted 1 to 5000, otherwise the default) bounds the final telemetry flush and is read only when telemetry is enabled. `TRACEPARENT` and `TRACESTATE` (W3C trace context) are likewise environment-only inputs read from the process environment when the `sce.command` span starts; they have no flag or config key and malformed values are ignored. + `log_to_file` is config-file/default only; unlike `log_dir`, it has no environment variable or CLI flag. Omitting either property does not produce a cross-property validation error: `log_to_file` defaults to `true`, and omitted `log_dir` resolves to the default location. At runtime, enabled file logging suppresses normal logger records on `stderr`; disabled file logging routes only error-level logger records to `stderr`. An explicit empty config value for `log_dir` is rejected by the generated schema. `log_file_retention_limit` intentionally has no environment or CLI-flag layer: diff --git a/context/cli/default-path-catalog.md b/context/cli/default-path-catalog.md index 2e4ef6e60..50c44cf6c 100644 --- a/context/cli/default-path-catalog.md +++ b/context/cli/default-path-catalog.md @@ -25,13 +25,12 @@ - `.opencode/`, `.opencode/opencode.json` - `.claude/` - `.pi/` -- `.git/`, `.git/hooks/`, `.git/COMMIT_EDITMSG` - `context/`, `context/plans/`, `context/decisions/`, `context/handovers/`, `context/tmp/`, `context/tmp/.gitignore` via `RepoPaths::context_tmp_gitignore_file()` ### Install paths - OpenCode plugin/catalog targets under `.opencode/` -- required git hook install targets under `.git/hooks/` +- required git hook install targets are resolved from git truth by the setup service, not by this catalog (the former `.git/`, hooks and `COMMIT_EDITMSG` path helpers were removed as unused) Compile-time payload locations are build-script-owned `OUT_DIR` paths rather than default runtime paths and are intentionally absent from this catalog. @@ -41,7 +40,7 @@ Compile-time payload locations are build-script-owned `OUT_DIR` paths rather tha - `cli/src/services/config/mod.rs` now resolves the default repo-local config path through `RepoPaths::sce_config_file()` during config discovery. - `cli/src/services/doctor/inspect.rs` now resolves the repo-local config path through `RepoPaths::sce_config_file()` for local-config health reporting and validation. - `cli/src/services/doctor/inspect.rs` also resolves OpenCode manifest/plugin/preset locations through shared `RepoPaths` and `InstallTargetPaths` accessors instead of owning those paths locally. -- `cli/src/services/setup/mod.rs` now resolves setup target directory names and required hook identifiers through `default_paths.rs` constants/accessors instead of owning those path literals locally. +- `cli/src/services/setup/mod.rs` now resolves setup target directory names through `default_paths.rs` constants/accessors instead of owning those path literals locally. - `cli/src/services/default_paths.rs` includes a regression test that scans non-test Rust source under `cli/src/` and fails when new centralized production path literals appear outside the default-path service. - Active hook runtime no longer resolves or writes collision-safe JSON artifacts under `context/tmp/`; `context/tmp/` remains a repo-relative scratch/session path owned by the default path catalog. - Active hook runtime and `cli/src/services/agent_trace_db/lifecycle.rs` resolve repository-scoped Agent Trace DB files through `agent_trace_storage` and `agent_trace_db_path_for_repository(repository_id)`. Outside a Git repository, lifecycle code returns an actionable "requires a Git repository" diagnostic instead of resolving any sentinel path. diff --git a/context/cli/doctor-ref-reconciliation.md b/context/cli/doctor-ref-reconciliation.md new file mode 100644 index 000000000..1d53a602c --- /dev/null +++ b/context/cli/doctor-ref-reconciliation.md @@ -0,0 +1,46 @@ +# Doctor snapshot-ref reconciliation + +`sce doctor --fix` is the only production caller of snapshot-ref reconciliation. +Plain `sce doctor` only reports whether a pass is recommended. Automatic +reclamation is not delivered; it is a documented follow-up (see +[`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md) and +[`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)). + +## Runtime facade + +`cli/src/services/mutation_trace/runtime/ref_doctor.rs` is the only crate-visible +surface doctor uses; it keeps the maintenance types private to `runtime`: + +- `run_explicit_reconciliation(repository_root)` wraps `reconcile_explicit` + (one `WorktreeLock` acquisition, verified-existing Agent Trace DB, state + recorded under the same lock) and returns a plain `ReconciliationFix`. +- `inspect_reconciliation_recommendation(repository_root)` derives the Git dir + from the filesystem, reads `/sce/ref-maintenance.json`, and applies the + pure `evaluate_recommendation`. It takes no lock and writes no state or ref. It + returns `None` unless reconciliation is recommended or the state is unusable. + +Doctor receives both as injected seams (`execute_doctor_with_lifecycle_providers`). + +## `--fix` reporting + +After the existing lifecycle, config and mutation-scope repairs, `--fix` always +runs one pass, regardless of the recommendation, failure streak, or timestamp +validity, and appends one fix-result row (category `mutation_scope_health`): + +| Pass result | Outcome | Detail | +| --- | --- | --- | +| `Completed` | `fixed` | deleted / retained / locally required counts | +| `CompletedStatePersistFailed` | `fixed` | counts plus a diagnostics-not-recorded warning | +| `Failed` | `failed` | typed kind (for example `agent_trace_db_missing`, `agent_trace_db_incompatible_schema`, `agent_trace_db_missing_metadata`, `agent_trace_db_repository_mismatch`, `malformed_ref`, `missing_required_pins`, `durable_roots_unavailable`, `delete_transaction_failed`) and message, plus any state warning | +| `Skipped(Busy)` | `skipped` | another boundary or pass holds the worktree lock; retry | + +A busy pass leaves state unchanged and is not a failure. While a pass runs, a +mutation boundary may wait up to the 10 s coordinator lock timeout. + +## Plain `sce doctor` + +When reconciliation is recommended (stale since the last success, an outstanding +failed attempt, invalid timestamps, or an unreadable state file) the text report +adds a `Snapshot ref reconciliation` section and the JSON report carries a +`ref_reconciliation` object (`null` otherwise). The recommendation does not depend +on whether a hook reminder was ever seen. diff --git a/context/cli/execution-glossary.md b/context/cli/execution-glossary.md index 609e2be93..54eda3c40 100644 --- a/context/cli/execution-glossary.md +++ b/context/cli/execution-glossary.md @@ -3,7 +3,8 @@ - `command loop`: The `clap` derive-based parser + dispatcher in `cli/src/cli_schema.rs`, `cli/src/services/parse/command_runtime.rs`, and `cli/src/app.rs` that routes `help`, `config`, `setup`, `doctor`, `auth`, `hooks`, `policy`, `sync`, `version`, and `completion`, executes implemented command flows, emits command-local help payloads for supported subcommand trees, resolves unknown command paths to the longest valid parent's help surface, and returns deterministic actionable errors for unknown options and other invalid invocation. - `RuntimeCommand seam`: Internal static command-execution abstraction in `cli/src/services/command_registry.rs` where clap-parsed commands are represented as a `RuntimeCommand` enum with variants for `Help`, `HelpText`, `Version`, `Completion`, `Auth`, `Config`, `Setup`, `Doctor`, `Hooks`, `Policy`, and `Sync`. The enum owns `name()` and async `execute(...)` dispatch, directly awaited under the application-level multi-thread Tokio runtime (distinct from retained DB-owned runtimes), delegating behavior to service-owned command payload structs while avoiding boxed `dyn RuntimeCommand` handles. Parsed request construction lives in `cli/src/services/parse/command_runtime.rs` when user-provided options or subcommands are required. - `logger trait boundary`: `services::observability::traits::Logger` mirrors the current observability logger API (`info`, `debug`, `warn`, `error`, `log_cli_error`) for generic command/runtime bounds and tests, with each method accepting `Option<&str>` session context for file routing; the concrete `services::observability::Logger` implements it and `NoopLogger` remains available for side-effect-free tests. -- `telemetry trait boundary`: `services::observability::traits::Telemetry` defines the `with_default_subscriber` command-lifecycle capability used by generic app-runtime bounds and tests. The generic async callback deliberately makes this capability non-object-safe and statically selected through `HasTelemetry::Telemetry`. Production `NoopTelemetry` awaits the action once and installs no subscriber. Future static telemetry must scope every poll using future-aware subscriber attachment or an explicitly chosen global subscriber; a thread-local guard across await is insufficient. -- `AppContext`: Generic borrowed dependency view in `cli/src/app.rs` passed through static command dispatch. `AppRuntime` owns the concrete production logger, `NoopTelemetry` (no tracing subscriber), filesystem, and git implementations; `AppContext` stores references to those dependencies plus an optional `repo_root: Option`, not owned `Arc` trait objects. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. The `repo_root` field is `None` at startup and command paths can derive a repo-root-scoped context with `AppContext::with_repo_root(...)` / `ContextWithRepoRoot`, preserving the borrowed runtime dependencies while attaching the resolved root. Narrow accessor traits (`HasLogger`, `HasTelemetry`, `HasFs`, `HasGit`, `HasRepoRoot`) let command and lifecycle call sites express capability requirements without depending on the full production context type; logger/telemetry/fs/git accessors use associated concrete capability types and return `&Self::{Capability}` rather than object-erased `&dyn ...` values. +- `telemetry trait boundary`: `services::observability::traits::Telemetry` defines the `with_default_subscriber` command-lifecycle capability used by generic app-runtime bounds and tests. The generic async callback deliberately makes this capability non-object-safe and statically selected through `HasTelemetry::Telemetry`. Production selects `NoopTelemetry` (awaits the action once, installs no subscriber) unless `SCE_TELEMETRY=standalone` selects `OtelTelemetry`, which scopes every poll by attaching a private dispatch to the command future (`ScopedDispatch`); a thread-local guard across await is insufficient. See `../sce/cli-observability-contract.md`. +- `AppContext`: Generic borrowed dependency view in `cli/src/app.rs` passed through static command dispatch. `AppRuntime` owns the concrete production logger, `RuntimeTelemetry` (`NoopTelemetry` by default, or the opt-in standalone `OtelTelemetry`), filesystem, and git implementations; `AppContext` stores references to those dependencies plus an optional `repo_root: Option`, not owned `Arc` trait objects. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. The `repo_root` field is `None` at startup and command paths can derive a repo-root-scoped context with `AppContext::with_repo_root(...)` / `ContextWithRepoRoot`, preserving the borrowed runtime dependencies while attaching the resolved root. Narrow accessor traits (`HasLogger`, `HasTelemetry`, `HasFs`, `HasGit`, `HasRepoRoot`) let command and lifecycle call sites express capability requirements without depending on the full production context type; logger/telemetry/fs/git accessors use associated concrete capability types and return `&Self::{Capability}` rather than object-erased `&dyn ...` values. See [application execution runtime](../architecture.md#application-execution-runtime), [capability traits](capability-traits.md), and [observability](../sce/cli-observability-contract.md). +- `standalone telemetry` / `hook-export gate`: Opt-in OTel mode enabled only by the environment switch `SCE_TELEMETRY=standalone` (never by repository config); it exports allowlisted `sce::otel` spans over OTLP/HTTP protobuf. Invocations classified as hooks by the typed `RuntimeCommand::is_hook_invocation` (`hooks`, `policy`), and unparseable invocations, resolve to `NoopTelemetry` before any exporter exists. Details: `../sce/cli-observability-contract.md`. diff --git a/context/cli/mutation-scope-hook-ingress.md b/context/cli/mutation-scope-hook-ingress.md index 561fc6718..1f7dc40ee 100644 --- a/context/cli/mutation-scope-hook-ingress.md +++ b/context/cli/mutation-scope-hook-ingress.md @@ -258,4 +258,5 @@ The Codex mapping and partial coverage are in [`codex-mutation-scope-integration - [Mutation-trace scope abandonment](mutation-trace-scope-abandonment.md) - [Mutation-trace protected worktree](mutation-trace-protected-worktree.md) - [Mutation-scope provenance](mutation-scope-provenance.md) +- [Mutation-scope ref-reconciliation advisory](mutation-scope-ref-advisory.md) - [Agent Trace hooks command routing](../sce/agent-trace-hooks-command-routing.md) diff --git a/context/cli/mutation-scope-ref-advisory.md b/context/cli/mutation-scope-ref-advisory.md new file mode 100644 index 000000000..b7bfd2c5b --- /dev/null +++ b/context/cli/mutation-scope-ref-advisory.md @@ -0,0 +1,95 @@ +# Mutation-scope ref-reconciliation advisory (hook trigger) + +The shared mutation-scope ingress (`cli/src/services/hooks/mutation_scope.rs`, +see [`mutation-scope-hook-ingress.md`](mutation-scope-hook-ingress.md)) emits a +lightweight, advisory-only signal that snapshot-ref reconciliation is +recommended. It never reconciles, lists refs, opens the Agent Trace DB, or spawns +Git. Reconciliation itself is operator-driven; this signal is only a reminder. + +## Trigger policy + +`drive_mutation_scope` takes an injected advisory seam (production: +`runtime::advise_after_completed_boundary`) and a generic diagnostic writer +`E: Write` (production: `std::io::stderr()`, supplied by +`run_mutation_scope_from_payload`). The seam runs exactly once, and only when all +of these hold: + +- the boundary is `Close` or `Flush`; +- the `coordinate_boundary(...)` result is `Ok` or + `CoordinateError::MarkerClearAfterCommit` (a durably completed boundary). + +It runs after the coordinator future has returned, so the coordinator's +`WorktreeLock` is already released. It never runs for `Start`, `Advance`, +`Abandon`, or a failed coordinate. The hook result is computed before the seam +runs and is never altered by it, including when the diagnostic write fails. The +advisory never creates, clears, or modifies the external-taint marker. + +## Outcomes, logging and diagnostics + +Every outcome is logged as `sce.hooks.mutation_scope.ref_reconciliation_advisory` +with an `outcome` field (plus `persistence_warning` when present). Actionable +outcomes are additionally written, one `SCE: ...` line each, to the diagnostic +writer. Stdout stays empty and never carries advisory text. + +| Outcome | Log | Diagnostic (stderr) | +|---|---|---| +| `Anchored` | debug | none | +| `NoAction` | debug | none | +| `Busy` | debug | none | +| `Advised` | warn | `SCE: Reconciliation is recommended. Run sce doctor --fix.` | +| `AdvisedDurabilityUncertain` | warn, with `persistence_warning` | the line above, then `SCE: Advisory-state durability could not be confirmed: .` | +| `StateWriteFailed` + `NotApplied` | warn, with `persistence_warning` | `SCE: Reconciliation advisory state could not be written: . Run sce doctor to check reconciliation status.` (replacement did not occur; previous state preserved; no advice) | +| `StateWriteFailed` + `DurabilityUncertain` | warn, with `persistence_warning` | `SCE: Reconciliation advisory-state durability could not be confirmed: . Run sce doctor to check reconciliation status.` (anchor or normalization write; replacement occurred, new state may be visible, crash durability unconfirmed; no advice) | +| `StateUnavailable` | warn | `SCE: Reconciliation advisory state is unavailable. Run sce doctor to check reconciliation status.` | + +`StateWriteFailed` means no confirmed successful advisory-state update, not +necessarily that atomic replacement failed; it carries the `PersistFailure` so the +two phases stay distinguishable. `AdvisedDurabilityUncertain` is reserved for a +due recommendation whose advice-state replacement occurred with uncertain +durability (the recommendation is still emitted). `StateUnavailable` means the +state could not be accessed or evaluated and is not a persistence-write outcome. + +The diagnostic does not depend on the logger (it is emitted with `logger = None`), +and `Logger::warn` and the observability defaults are unchanged: observability +routes only ERROR records to stderr, and only when file logging is off, so WARN +alone would not reach the operator. A write or flush error on the diagnostic +writer is ignored. Only this path writes the advisory diagnostic to stderr. + +## Advisory behavior and cadence + +`ref_advisory::advise_if_due` (`cli/src/services/mutation_trace/runtime/`) takes +no Git, DB, store, or snapshot parameters. It derives the Git dir from the +filesystem (no Git subprocess), makes one zero-wait `try_lock` of +`WorktreeLock`, and reads/writes the small per-worktree state file +`/sce/ref-maintenance.json` (capped, versioned, past-event timestamps +only, atomic staging-and-swap writes). It persists `last_advised` before +reporting `Advised`. A busy lock or unreadable state is reported, never an error. + +- **Time-based reminders:** the first use anchors the state. A stale state + becomes eligible after `RECONCILIATION_ADVISORY_AFTER` (24 h); further + stale-state reminders are suppressed for 24 h after `last_advised`. +- **Failure-triggered reminders:** an outstanding failed explicit reconciliation + is recommended immediately. A newly recorded failure (recorded after + `last_advised`) may be advised even inside the previous 24 h window; once + recorded, that reminder is suppressed. Repeated reminders for the same + unresolved failure follow the normal 24 h cadence. A successful explicit + reconciliation clears the failure-based recommendation. +- Anchoring and clock-skew normalization are the only other state writes, so + state is not written "at most once per window" unconditionally. +- `sce doctor` reports the recommendation from state regardless of whether a hook + reminder was ever delivered. + +## Delivery guarantee and limits + +- Best-effort, not exactly-once, and not guaranteed to be seen by a human. + `last_advised` is persisted before the diagnostic, so a crash or an unseen + stderr can still consume one reminder; the next reminder follows the cadence. +- Claude Code and Codex run the hook as a command and can capture its stderr, but + rendering of exit-0 hook stderr is up to the harness. The OpenCode plugin and + the Pi extension discard the child's stderr, so there the structured log file + and `sce doctor` are the supported surfaces. Changing the plugins is out of + scope here. +- Time-based, not evidence-based: orphan detection needs the Git/DB scan this + path deliberately avoids. +- Orphan pins are reclaimed only by an explicit reconciliation pass. Automatic + reclamation is not provided. diff --git a/context/cli/mutation-scope-runtime.md b/context/cli/mutation-scope-runtime.md index d07fbfdc8..d099e1384 100644 --- a/context/cli/mutation-scope-runtime.md +++ b/context/cli/mutation-scope-runtime.md @@ -33,7 +33,7 @@ The mechanics live in [`mutation-trace-runtime-coordinator.md`](mutation-trace-r `ExternalTaintOperation` crosses the boundary because it is part of `CoordinateError`'s public shape. It lives in `protected_worktree.rs` and is re-exported by `coordinator.rs` without making that module public. -Every runtime `mod` stays private, including `ProtectedWorktree`, its error, `WORKTREE_LOCK_TIMEOUT`, and `reconcile_worktree`. Adapters drive only the two entrypoints and never assemble the safety prefix. +Every runtime `mod` stays private, including `ProtectedWorktree`, its error, `WORKTREE_LOCK_TIMEOUT`, and `reconcile_explicit`. Adapters drive only the two entrypoints and never assemble the safety prefix. The re-exports are the intentional crate-visible seam consumed by the generic ingress; runtime internals stay private. The two re-export statements retain `#[allow(unused_imports)]` because no consumer names the completing types yet. diff --git a/context/cli/mutation-trace-agent-attribution.md b/context/cli/mutation-trace-agent-attribution.md index b714ae4df..ee0ab64fb 100644 --- a/context/cli/mutation-trace-agent-attribution.md +++ b/context/cli/mutation-trace-agent-attribution.md @@ -186,8 +186,6 @@ observable tree difference. committed `src/lib.rs` even with identical added text and line number; an exact nested path still attributes; multiple similar nested mutation paths stay independent of the target; a shared basename alone is not a match. -- `runtime/tests.rs` — a still-relevant event behind 128 newer events is never - loaded or reconstructed. - `hooks/mod.rs` (`mutation_attribution_e2e`) — real Git/DB: mutation-only `ai` without provenance; direct+mutation completion; a newer non-exclusive event keeps a line non-AI; adversarial linked-worktree isolation; the three diff --git a/context/cli/mutation-trace-external-mutation-guard.md b/context/cli/mutation-trace-external-mutation-guard.md index 58e70598e..a345c9525 100644 --- a/context/cli/mutation-trace-external-mutation-guard.md +++ b/context/cli/mutation-trace-external-mutation-guard.md @@ -1,6 +1,6 @@ # External-mutation guard -`run_external_mutation_guard`, in +`arm_external_mutation_guard` (which yields an `ArmedExternalMutationGuard` whose `exec` runs the shell), in `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, is a harness-neutral runtime primitive — **not** Pi-specific, despite being built alongside the Pi adapter to back Pi's `!`/`!!` `user_bash` — that lets a @@ -27,7 +27,7 @@ composition rather than attributing the ambiguous interval to AI. ```text caller starts the hidden supervisor with {"operation":"arm"} -> acquire ProtectedWorktree (WorktreeLock + ExternalTaintMarker, write-ahead) - -> create a Unix pipe (atomically CLOEXEC via pipe2 on Linux): supervisor + -> create a Unix pipe (CLOEXEC set atomically via pipe2 on Linux only; pipe+fcntl elsewhere leaves an unresolved fork/exec window): supervisor owns both ends while waiting -> durably establish the lifetime-token infrastructure -> write and flush {"status":"armed"} @@ -268,7 +268,7 @@ eventual explicit unlock. wrapper around the existing (still-private) `coordinate_protected`, plus ordinary OS process/fd mechanics. - No Windows support: the guard is Unix-only; a non-Unix build's - `run_external_mutation_guard` unconditionally returns + `arm_external_mutation_guard` unconditionally returns `GuardError::UnsupportedPlatform`. - No recovery beyond the guard's own crash semantics above — a stale, never-armed marker left by some other failure mode is out of scope here. diff --git a/context/cli/mutation-trace-external-taint.md b/context/cli/mutation-trace-external-taint.md index 0f1542f6d..5dbe935cb 100644 --- a/context/cli/mutation-trace-external-taint.md +++ b/context/cli/mutation-trace-external-taint.md @@ -176,7 +176,7 @@ window: the recovery is durable (cursor rebaselined, scope abandoned, revision advanced), the triggering boundary is unprocessed with no `MutationEvent`, the on-disk marker survives, and a later `coordinate()` inherits it and recovers conservatively again without resurrecting the abandoned scope. -`runtime/tests.rs`'s +The former `runtime/tests.rs` test `a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_recovery` proves an attributable `Advance` commits durably, the returned `MarkerClearAfterCommit` carries the matching `committed` outcome diff --git a/context/cli/mutation-trace-protocol.md b/context/cli/mutation-trace-protocol.md index 9653c37e7..83e949fbf 100644 --- a/context/cli/mutation-trace-protocol.md +++ b/context/cli/mutation-trace-protocol.md @@ -172,7 +172,7 @@ owns the lock, the external-taint fence, the caller-supplied DB provider, one Git snapshot, and a bounded CAS-retry loop; `store.rs` never remaps an existing `ScopeId`'s `actor_kind`/`worktree_id`; `protocol.rs` assumes referenced scopes already exist and stays free of any Git object, DB row, or CAS transaction -concept. `runtime::ref_reconciliation` +concept. `runtime::ref_reconciliation` / `runtime::ref_maintenance` ([`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)) is imperative durability maintenance *outside* the verified protocol — it never advances the cursor, chooses attribution, changes scope state, or creates a diff --git a/context/cli/mutation-trace-ref-maintenance-state.md b/context/cli/mutation-trace-ref-maintenance-state.md new file mode 100644 index 000000000..dd53b4067 --- /dev/null +++ b/context/cli/mutation-trace-ref-maintenance-state.md @@ -0,0 +1,81 @@ +# Snapshot-ref maintenance state and verified-existing DB opener + +Supporting contracts for [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md): the durable per-worktree maintenance state file shared by the hook advisory, `sce doctor`, and `reconcile_explicit`, and the dedicated Agent Trace DB opener reconciliation uses. Related: [`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md), [`doctor-ref-reconciliation.md`](doctor-ref-reconciliation.md), [`agent-trace-storage.md`](agent-trace-storage.md). + +## Verified-existing Agent Trace DB opener + +Reconciliation never uses the hook opener +(`open_agent_trace_db_for_hook_runtime`), which can create directories and files, +repair migration metadata, and insert repository metadata. The dedicated path is +`resolve_existing_agent_trace_storage_for_maintenance` plus +`RepositoryAgentTraceDb::open_verified_existing_at`, built on +`TursoDb::open_existing_without_migrations_at`: + +- The Turso open is non-creating and read-only (`read_only(true)` with + `experimental_multiprocess_wal(true)` preserved), so existence and open are one + OS call. A failed open is classified `Missing` only when the path is then + absent. No pre-existence check and no parent-directory creation. +- It never runs migrations, `repair_missing_repository_schema_migration_metadata` + or `verify_or_initialize_repository_metadata`. +- Before any ref listing it verifies schema readiness (all expected migrations), + metadata presence, stored `repository_id` against the resolved identity, and a + valid `source_instance_id` (`verify_existing_repository_metadata`). +- Typed failures `ExistingRepositoryDbError`: `Missing`, `Unreadable`, + `IncompatibleSchema`, `MissingMetadata`, `RepositoryMismatch`. They stay + downcastable from `ReconcileError::AgentTraceDbUnavailable`. + +Per-DB-state guarantees: a **missing** DB creates no DB file, parent directory, +WAL or shared-memory file and no ref is listed or deleted; an **existing invalid** +DB may incur incidental Turso WAL/shared-memory bookkeeping but no migration, +repair, initialization, or application-table write and no ref deletion; an +**existing valid** DB is verified before reconciliation with no initialization. +Logical immutability is the guarantee, not filesystem byte-identity. + +Limitations: Turso opens `-wal`/`-tshm` sidecars by path, so only a coherent +replacement of main file and sidecars is meaningful; the process-wide Turso +registry may share an already-open instance (the read-only flag is then not +enforced); a DB replaced between verification and the delete transaction is not +detected, the guard being that roots are read from the verified connection under +the held lock. + +Plain `sce doctor` shares the non-creating read-only open through +`RepositoryAgentTraceDb::open_existing_schema_ready_at`, which inspects schema +readiness without requiring metadata. + +## Maintenance state file + +`/sce/ref-maintenance.json`, beside `mutation-cursor.lock` and the +taint marker. Versioned, capped at `MAINTENANCE_STATE_MAX_BYTES` (4096), written +only under `WorktreeLock` by atomic staging-and-swap (temp file in the same +directory, rename, best-effort parent-directory sync; the filesystem is the +statically dispatched `StateFilesystem` seam). No in-memory registry or +process-global state; the wall clock is an injected `Fn() -> i64` (unix ms). + +Fields: past-event timestamps `anchor`, `last_success`, `last_attempt`, +`last_advised`; `last_attempt_outcome` (`completed` | `failed`); `last_report` +counts; `consecutive_failures`; `last_failure`. **No field is a deadline.** +Eligibility is computed at read time as `now − reference ≥ window`, so rereading +cannot extend it. Constants: `RECONCILIATION_ADVISORY_AFTER` 24 h, +`FUTURE_SKEW_TOLERANCE` 5 min. + +Transitions: explicit success sets `last_attempt` = `last_success` = now, stores +the report, and clears the failure streak and fields; explicit failure sets +`last_attempt`, outcome `failed`, increments `consecutive_failures`, and leaves +`last_success` untouched; lock contention changes nothing. A missing, unparsable, +wrong-version, or oversized file is treated as default and rewritten by the next +recorded operation; it never fails an explicit pass. + +Timestamps: a value strictly more than `FUTURE_SKEW_TOLERANCE` ahead of `now` is +invalid; one within the tolerance (inclusive) is valid, preserved byte-for-byte, +and evaluated as age 0. **Successful state-writing operations** (a successful +advisory write, a successful explicit record) normalize invalid timestamps once +under the lock and persist them, so afterwards every stored timestamp satisfies +`t ≤ now + FUTURE_SKEW_TOLERANCE`. **Read-only or unsuccessful operations** +(plain `sce doctor`, lock-busy passes, unreadable state, a `NotApplied` write, an +interrupted write) never mutate the file, evaluate invalid timestamps +conservatively as eligible/recommended, and never report them as normalized. + +`evaluate_recommendation(state, now)` is the pure function shared by the advisory +and read-only doctor. An outstanding failed explicit pass is recommended +independently of staleness and of `last_advised`. + diff --git a/context/cli/mutation-trace-ref-reconciliation.md b/context/cli/mutation-trace-ref-reconciliation.md index dc74874cb..6c53a0809 100644 --- a/context/cli/mutation-trace-ref-reconciliation.md +++ b/context/cli/mutation-trace-ref-reconciliation.md @@ -1,131 +1,108 @@ -# Mutation-cursor snapshot-ref reconciliation (`runtime::ref_reconciliation`) - -`cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` is a -conservative, per-worktree maintenance pass that removes orphaned / unreferenced -SCE-owned snapshot pins **within the ref namespace of a checkout identity a -current worktree still owns** while retaining every tree any current or -historical durable mutation-cursor state in the repository still references. -Built by the `mutation-cursor-ref-reconciliation` plan -(`context/plans/mutation-cursor-ref-reconciliation.md`). It is deliberately -**not** a guarantee that SCE snapshot refs never accumulate: a namespace whose -checkout identity no current worktree derives is beyond its reach (see -[Scope: an owned checkout-identity namespace only](#scope-an-owned-checkout-identity-namespace-only)). - -`GitSnapshotService::pin_tree` is **create-only per invocation** — a crash, -failed transition, or other interrupted `coordinate()` path can leave a -`refs/sce/mutation-cursor//` pin with no corresponding -durable root. Reconciliation is the reclamation step for exactly that state. It -is **not** a bound on storage growth: every retained `mutation_trace_events` -row keeps its `before_tree` / `after_tree` as durable roots, so a successful -`A → B → C → D` history keeps all four pins. Bounding historical snapshot -storage needs a separate future retention/compaction lifecycle this module does -not design. - -The design is deliberately asymmetric and this bias governs every rule below: -**keeping an unnecessary ref costs disk; deleting a required ref destroys -durable evidence.** False retention is acceptable; false deletion is not. - -`mod ref_reconciliation;` is private in `runtime/mod.rs` (like `coordinator`); -`reconcile_worktree` is `pub` only within `runtime` — no `pub(crate)` re-export, -no hook / command / `diff_traces` wiring yet (invocation timing is the -harness-wiring PR's). - -## Entry point and identity - -```rust -pub fn reconcile_worktree( - repository_root: &Path, - open_db: impl FnOnce() -> anyhow::Result, -) -> Result -``` - -A one-line delegation to `pub(super) fn reconcile_worktree_inner(.., -on_lock_contention: impl FnOnce())` — the deterministic test seam (mirroring -`coordinate` / `coordinate_inner`), visible only within `runtime`. - -Like `coordinate()`, it never accepts a `WorktreeId`, `TreeId`, or ref name and -never opens the DB itself: identity is -`repository_root → resolve_git_dir → read_checkout_id`, `open_db` is -caller-supplied. `read_checkout_id → Ok(None)` returns -`Ok(SkippedNoCheckoutIdentity)` — an observable skip, **not** a zero-count -report: nothing inventoried, `open_db` never called, no ref touched, lock -released, no identity created or recovered; it makes **no** claim the repository -holds no SCE refs for a prior checkout identity (those are the unowned -namespaces below). `Err` is `ReconcileError::CheckoutIdentity` (a corrupt id, not an absent one). - -## Scope: an owned checkout-identity namespace only - -Reconciliation reclaims orphan / unreferenced pins **only** under -`refs/sce/mutation-cursor//` where `` is a checkout identity a **current -worktree still derives** (`resolve_git_dir → read_checkout_id`). The unsupported -case is identity-based, not path-based: - -```text -refs/sce/mutation-cursor//* — does a current worktree derive ? - yes → active namespace (reclaimed here under the WorktreeLock) - no → unowned namespace (no per-worktree pass can ever reach it) -``` - -High-frequency harness traffic against a worktree with a **stable** checkout -identity — where interrupted `coordinate()` runs leave orphan pins — is exactly -the active-namespace case this pass covers. - -A namespace becomes **unowned** whenever no current worktree can derive its -checkout id. Two lifecycles cause this: - -**Case A — deleted linked worktree.** `git worktree remove W` deletes W's -worktree-specific git dir, so its `/sce/checkout-id` is gone; the -`refs/sce/mutation-cursor//*` it created survive in the shared repository -ref namespace. - -**Case B — checkout-identity metadata loss / recreation.** A **present** -worktree's `/sce/checkout-id` disappears (id `A`). `reconcile_worktree` -returns `SkippedNoCheckoutIdentity` and does nothing. A later -`get_or_create_checkout_id` (e.g. the next `coordinate()`) mints a **new** id -`B`; the worktree now operates as `B` while `refs/.../A/*` are unowned. This is -**not** normal operation: it is a metadata-loss / recreation lifecycle the -maintenance model must handle conservatively — reconciliation never recreates -`A` or adopts its namespace. - -**Harness gate.** A persistent / current-worktree harness whose **checkout -identity stays stable** relies on this pass for active-namespace orphan cleanup -and is storage-cleanup complete for this module's scope. Any lifecycle that can -**retire, replace, lose, or recreate** checkout identities (ephemeral linked -worktrees among them, but not only those) can leave unowned namespaces and needs -the future repository-scoped operation below. This is a scope limit, not a bug, -orthogonal to the "reconciliation ≠ historical retention policy" boundary above. - -### Future work: repository-scoped unowned-namespace reconciliation - -Not implemented here — this module adds **no** repository-global namespace scan -and **no** repository-global ref deletion. Recorded shape: - -```text -enumerate refs/sce/mutation-cursor//* (git for-each-ref on the namespace) - ↓ -active checkout ids (enumerate current worktrees → read each one's checkout-id) - ↓ -unowned ids (namespace present, no current worktree derives it) - ↓ -for each unowned namespace, each pinned tree T: - T ∈ durable_roots(repository) → retain - T ∉ durable_roots(repository) → safe deletion candidate -``` +# Mutation-cursor snapshot-ref reconciliation (`runtime::ref_reconciliation`, `runtime::ref_maintenance`) + +Snapshot-ref reconciliation is a conservative, per-worktree maintenance pass that +removes orphaned SCE-owned snapshot pins under +`refs/sce/mutation-cursor//` while retaining every tree any current +or historical durable mutation-cursor state in the repository still references. +It is built by the `mutation-cursor-ref-reconciliation` plan and wired by the +`mutation-cursor-ref-reconciliation-wiring` plan +(`context/plans/mutation-cursor-ref-reconciliation-wiring.md`). + +`GitSnapshotService::pin_tree` is **create-only per invocation**: a crash, failed +transition, or other interrupted `coordinate()` path can leave a pin with no +corresponding durable root. Reconciliation is the reclamation step for exactly +that state. It is **not** a bound on storage growth: every retained +`mutation_trace_events` row keeps its `before_tree` / `after_tree` as durable +roots, so a successful `A → B → C → D` history keeps all four pins. Bounding +historical snapshot storage needs a separate retention/compaction lifecycle that +does not exist. + +The design is deliberately asymmetric: **keeping an unnecessary ref costs disk; +deleting a required ref destroys durable evidence.** False retention is +acceptable; false deletion is not. + +## Who runs it + +Reconciliation is **explicit only**. `sce doctor --fix` is the single production +caller (see [`doctor-ref-reconciliation.md`](doctor-ref-reconciliation.md)). +Orphan pins are reclaimed only when an operator runs it. **Automatic reclamation +is not delivered** and nothing may claim otherwise; it is a follow-up (see +[Deferred follow-ups](#deferred-follow-ups)). + +The mutation-hook completion path performs no reconciliation, ref listing, DB +I/O, or Git subprocess. After a durably completed `Close`/`Flush` it only runs a +lightweight advisory check that may record "reconciliation is recommended" in a +small state file (see [`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md)). +Inline reconciliation was rejected because `git for-each-ref`, DB open, +repository-wide durable-root queries, and a started `git update-ref` cannot be +bounded, the Tokio runtime joins in-flight blocking workers, and the pass holds +`WorktreeLock`, which a coordinator waits on for at most 10 s before failing the +boundary. + +## Module layout and entrypoints + +All runtime modules are private to `mutation_trace::runtime`; only the +`ref_doctor` facade and the advisory seam are re-exported `pub(crate)`. + +| Module | Role | +| --- | --- | +| `ref_reconciliation.rs` | The single reconciliation algorithm, `reconcile_with_held_lock` | +| `ref_maintenance.rs` | `reconcile_explicit`: the production maintenance entrypoint | +| `ref_advisory.rs` | `advise_if_due`: hook-path advisory | +| `maintenance_state.rs` | `/sce/ref-maintenance.json` state, normalization, atomic writes, `evaluate_recommendation` | +| `ref_doctor.rs` | Plain-data facade used by `sce doctor` | + +### Single lock acquisition + +`WorktreeLock` is a non-reentrant advisory file lock. The algorithm is split so +each maintenance operation acquires it **exactly once**: + +- `reconcile_with_held_lock` borrows the held `&WorktreeLock`, never acquires, + and derives deletion leases from `lock.lease()`. +- `reconcile_explicit(repository_root) -> ExplicitOutcome` acquires the lock once + with the bounded 10 s `EXPLICIT_LOCK_TIMEOUT`, resolves the repository identity + and canonical Agent Trace DB path itself, opens the DB only through the + verified-existing opener, runs `reconcile_with_held_lock`, records the result + in the state file while still holding the same lock, then drops it. It takes + no opener parameter; the injection seam is private to `ref_maintenance`. +- `advise_if_due` takes one zero-wait `try_lock`, has no Git/DB/store + parameters, and never reaches reconciliation. +- `reconcile_with_held_lock` is the only reconciliation entrypoint; it never + acquires the lock. The former test-only lock-acquiring wrappers were removed. + +Mutating Git work receives `lock.lease()` and moves it into the `spawn_blocking` +worker, so dropping the caller future cannot release the file lock while a ref +deletion is running. + +### Concurrency contract + +An explicit pass holds the lock for its whole duration, because the +pin-then-root ordering that makes deletion safe requires inventory, root reads, +and deletion not to interleave with `coordinate()`'s pin-then-CAS. A mutation +boundary arriving meanwhile waits up to `WORKTREE_LOCK_TIMEOUT` (10 s) and then +fails **before durable completion** with the existing lock-timeout error. This is +an operator-initiated trade-off of `sce doctor --fix`. The hook advisory never +waits on the lock; contention yields `Busy`. + +## Worktree identity + +Identity is Git-topology-derived (`resolve_worktree_id`): `main` for the primary +worktree, `worktrees/` for a linked worktree. Failure is +an `Err`. The former checkout-identity service was removed (see +[`checkout-identity.md`](checkout-identity.md)); there is no skip-on-missing-identity +outcome. `ReconcileError::CheckoutIdentity` remains the error variant for a +worktree identity that cannot be resolved. + +The dedicated DB opener and the `ref-maintenance.json` state file are documented in +[`mutation-trace-ref-maintenance-state.md`](mutation-trace-ref-maintenance-state.md). -It must inherit this module's guarantees: the repository-wide durability -invariant (`delete /T` only if `T ∉ durable_roots(repository)`, read -through `load_all_tree_roots()`) because an unowned namespace may still hold the -only SCE ref protecting historical `mutation_trace_events` trees other tooling -needs; the false-retention-over-false-deletion bias; and a hard prohibition on -the shortcut "checkout id is unowned → delete its whole namespace". It is a -separate PR (a repository-global scan and cross-worktree active-id inventory this -module omits), gated behind the same harness-wiring work as invocation timing. ## Two invariants -Conflating them — deciding deletion from the target worktree's roots alone — is -the cross-worktree safety bug this design avoids: linked worktrees share one -object database, so an `A`-owned ref can be the last SCE ref protecting a tree only `B` durably requires. +Conflating them, i.e. deciding deletion from the target worktree's roots alone, +is the cross-worktree safety bug this design avoids: linked worktrees share one +object database, so an `A`-owned ref can be the last SCE ref protecting a tree +only `B` durably requires. ```mermaid flowchart TD @@ -142,111 +119,131 @@ flowchart TD ds -- "retained" --> keep["keep the pin"] ``` -- **Local consistency** (`load_tree_roots(W)`) is strictly per-worktree. A - missing pin in some *other* worktree never makes `W`'s pass fail — that would - let one worktree's degradation block maintenance everywhere. -- **Deletion safety** (`load_all_tree_roots()`) is repository-wide. `W/T` is - deleted only when `T` is in **no** worktree's durable root set. If `B` durably - needs `T` and `A` also has a `T` pin, `A` retains it as accidental backup - reachability for `B`'s degraded state. +- **Local consistency** (`load_tree_roots(W)`) is per-worktree and checked before + any deletion. If it fails the pass deletes nothing and returns + `MissingRequiredPins`. A missing pin in some *other* worktree never makes `W`'s + pass fail. +- **Deletion safety** (`load_all_tree_roots()`) is repository-wide: `delete(W, T)` + only if `T` is in **no** worktree's durable root set. A pin another worktree + requires is retained. + +Claimed: these two invariants. **Not claimed:** that every repository-wide durable +root is pinned. Reconciliation does not repair, recreate, or verify missing pins +belonging to other worktrees. A DB `TreeId` is a *logical* durability requirement, not itself a Git reachability edge: it obliges reconciliation to keep at least one SCE ref -protecting that tree, and that retained ref supplies physical Git reachability. -Each root-set query is [one SQL statement over one DB -snapshot](mutation-trace-store.md), which keeps a concurrent atomic -`cursor T → X` + `event T → X` commit on another worktree from tearing the -repository-wide read — no repository-global lock is needed. - -## Locking - -Reconciliation holds the **same** `/sce/mutation-cursor.lock` -`WorktreeLock` that `coordinate()` holds across `pin → CAS → return`, acquired -via `worktree_lock::acquire_inner` and bounded by the module-owned -`RECONCILIATION_LOCK_TIMEOUT` (`Duration::from_secs(10)`, matching -`runtime::protected_worktree`'s private `WORKTREE_LOCK_TIMEOUT` by intent, not a -shared constant — there is no semantic reason the two must stay identical, so -each module owns its own). -Mutual exclusion on that one file makes the pin → DB-CAS race structurally -impossible: the reconciler's inventory → diff → delete runs wholly before -`coordinate()` takes the lock (nothing pinned yet) or wholly after it releases -it (tree committed → durable root → retained; never committed → true orphan → -deletable). The lock stays per-worktree; only the durable-root *read* is repository-wide. +protecting that tree. Each root-set query is [one SQL statement over one DB +snapshot](mutation-trace-store.md), so a concurrent atomic `cursor T → X` + +`event T → X` commit on another worktree cannot tear the repository-wide read. + +## Locking and the pin → CAS boundary + +The pass holds the **same** `/sce/mutation-cursor.lock` that +`coordinate()` holds across `pin → CAS → return`. Mutual exclusion on that one +file makes the pin → DB-CAS race structurally impossible: the inventory → diff → +delete runs wholly before `coordinate()` takes the lock (nothing pinned yet) or +wholly after it releases it (tree committed → durable root → retained; never +committed → true orphan → deletable). The lock stays per-worktree; only the +durable-root *read* is repository-wide. ## Algorithm and error contract -Every step runs under the lock, and every fallible step maps to one dedicated -`ReconcileError` variant — there is no `Other` catch-all: +Every step runs under the lock and every fallible step maps to one dedicated +`ReconcileError` variant; there is no catch-all: | Step | Error on failure | | --- | --- | | `resolve_git_dir` | `GitDir` | -| `acquire_inner(RECONCILIATION_LOCK_TIMEOUT)` | `Lock(WorktreeLockError)` | -| `read_checkout_id` → `Err` (corrupt id) | `CheckoutIdentity` | -| `open_db()` provider | `AgentTraceDbUnavailable` | +| lock acquisition (`EXPLICIT_LOCK_TIMEOUT`) | `Lock(WorktreeLockError)`; surfaced as `Skipped(Busy)` by `reconcile_explicit` | +| worktree identity resolution | `CheckoutIdentity` | +| verified-existing DB open | `AgentTraceDbUnavailable` (downcastable `ExistingRepositoryDbError`) | | `GitSnapshotService::new` | `SnapshotService` | | `list_pins` → `PinInventoryError::Git` | `PinInventory` | | `list_pins` → `PinInventoryError::MalformedRef` | `MalformedPin { ref_name, reason }`, delete nothing | -| `load_tree_roots` / `load_all_tree_roots` | `DurableRoots` | +| `load_tree_roots` / `load_all_tree_roots` | `DurableRoots` (durable-root queries belong to migration `004_mutation_trace_protocol`) | | a target-worktree root has no pin | `MissingRequiredPins { missing }`, delete nothing | | `delete_pins` transaction | `DeleteTransaction`, delete nothing (atomic-or-nothing) | `AgentTraceDbUnavailable` here is a **maintenance** error only: reconciliation never arms `ExternalTaintMarker`, calls `protocol::*`, or writes a `mutation_trace_*` row, and never becomes -`CoordinateError::AgentTraceDbUnavailable` — no mutation boundary is being -coordinated (contrast [`mutation-trace-external-taint.md`](mutation-trace-external-taint.md)). +`CoordinateError::AgentTraceDbUnavailable` (contrast +[`mutation-trace-external-taint.md`](mutation-trace-external-taint.md)). +`delete_pins` is the only ref-deletion path; there is no `git gc`, object +removal, or independent deletion. + +## Outcomes and report + +`reconcile_with_held_lock` returns `Result` +where `ReconciliationOutcome` has the single variant `Reconciled(ReconciliationReport)`. + +`ReconciliationReport { local_required, retained, deleted }`: +`local_required = load_tree_roots(W).len()`, `deleted` = stale pins removed, +`retained = actual.len() − deleted`. `retained == local_required` is not an +invariant; only `report.local_required ≤ report.retained` holds. -## Outcome and report +`reconcile_explicit` returns the typed, never-merged `ExplicitOutcome`: -Both entrypoints return `Result` — -`Reconciled(ReconciliationReport)` | `SkippedNoCheckoutIdentity`, the skip an -`Ok` (never `Err`) and distinct from a zero-work `Reconciled(.. { deleted: 0 })`. +- `Completed(report)` +- `CompletedStatePersistFailed { report, warning }`: the cleanup stands and is + reported as done +- `Failed { error, state_warning: Option }`: keeps both the + reconciliation error and any persistence warning +- `Skipped(Busy)`: lock contention; state untouched; not a failure -`ReconciliationReport { local_required: usize, retained: usize, deleted: usize }` -— `local_required = load_tree_roots(W).len()`, `deleted` = stale pins removed, -`retained = actual.len() − deleted`. `retained == local_required` is **not** an -invariant (a pin another worktree needs counts toward `retained` only); for -`Reconciled(report)` the only relation is `report.local_required ≤ report.retained`, -and `SkippedNoCheckoutIdentity` carries no report. +`StatePersistWarning` is `PreviousStateUnreadable(io::Error)` (update not +attempted) or `WriteFailed(PersistFailure)`. `PersistFailure::NotApplied` means +the atomic replacement did not occur and the previous state is preserved; +`PersistFailure::DurabilityUncertain` means the rename occurred (new state may be +visible) but crash durability could not be confirmed. A completed ref deletion is +never rolled back. ## Model boundary Ref reconciliation is imperative durability maintenance **below** the verified -`spec/mutation_cursor.qnt` protocol — it never advances the cursor, chooses -attribution, changes scope state, or creates a `MutationEvent`, so no Quint -change is warranted. It deletes only SCE-owned refs, never Git objects, and runs -no `git gc` / `git prune` / `git reflog expire`; Git reclaims unreachable -objects on its own schedule. - -## Testing boundary - -`ref_reconciliation.rs`'s inline `#[cfg(test)] mod tests` uses a RAII `Fixture` -(`tempfile::TempDir`, real `git init` repo, checkout id via -`get_or_create_checkout_id`, a schema `RepositoryAgentTraceDb` beside the -worktree so it never perturbs a captured tree) with raw-SQL row seeders, -following the filesystem-touching inline-test precedent (`context/patterns.md`). -Coverage: orphan pin deleted (with/without a worktree row); current-cursor pin -retained without a referencing event; historical `before`/`after` pins retained -after the cursor advances; a pin another worktree durably requires retained -(the `retained > local_required` case); `MissingRequiredPins` fail-closed; a -malformed / symbolic ref fail-closed; idempotence; refs deleted without object -reclamation; and the no-checkout-identity skip (`SkippedNoCheckoutIdentity`; -`open_db` never called; the pin ref byte-identical across the skip). - -`runtime/tests.rs` holds the cross-module integration suite (T09), driven -through the public `reconcile_worktree` (and, for setup, `coordinate`) against -real Git and a real `RepositoryAgentTraceDb`: active-worktree orphan deletion; -current-cursor retention with no referencing event; historical `before`/`after` -retention through a real `A → B → C → D` `coordinate()` history; idempotence; -linked-worktree isolation and cross-worktree degraded-tree retention -(byte-identical pins); a missing required pin failing closed despite another -worktree pinning the same tree; a malformed / symbolic ref failing closed; no -protocol/DB/marker write; no object reclamation; the no-checkout-identity skip; -and two deterministic lock-race regressions (no sleeps): -`reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it` -(the generic `WorktreeLock` happens-before edge) and -`reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` -(the same edge across the real `coordinate()` `pin → store CAS` path, test-only). - -See also: [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md), [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md) (`list_pins` / `delete_pins`), [`mutation-trace-store.md`](mutation-trace-store.md) (`load_tree_roots` / `load_all_tree_roots`), [`mutation-trace-protocol.md`](mutation-trace-protocol.md). +`spec/mutation_cursor.qnt` protocol. It never advances the cursor, chooses +attribution, changes scope state, or creates a `MutationEvent`, so no Quint or +`protocol.rs` change was made. It deletes only SCE-owned refs, never Git objects. + +## Testing + +Coverage lives in `runtime/ref_maintenance/tests.rs` and its submodules +(`support`, `coordination`, `worktrees`, `cross_process`, `fail_closed`, +`state_recovery`), the advisory and state unit tests, `git_snapshot` cancellation +tests, `hooks::mutation_scope::advisory_trigger_tests`, and +`app::doctor_reconciliation_cli_tests`. Race and ordering scenarios use +deterministic coordination (barriers, parked phase hooks via `ReconcilePhase`, +the worker-entered seam, injected clocks), not sleeps. The plan's coverage map +records each scenario's verification status and the guard-removal results. + +## Deferred follow-ups + +Not implemented and not to be described as behavior: + +- **Automatic out-of-band reclamation.** Binding prerequisites: (1) a + lifecycle-owned execution point with defined ownership, lifetime, scheduling, + and failure recovery (for example an explicitly invoked `sce` maintenance + command run by a user-managed scheduler, or a future shared session-end + mechanism), never a detached task, daemon, fire-and-forget subprocess, or + timeout-cancelled Git/DB work; (2) a durable next-attempt reservation persisted + under `WorktreeLock` **before** opening the DB, listing pins, or loading + durable roots, bounded, never interpreted as success, superseded by the success + cadence or exponential failure backoff, and bypassed by explicit repair, with + the automatic pass skipped if the reservation cannot be persisted; (3) + distinct outcomes for completed, completed-but-state-not-persisted, failed, and + skipped/deferred; (4) bounded or split lock-hold analysis against the 10 s + coordinator timeout; (5) the same invalid-timestamp normalization as above. +- **Retired-worktree cleanup.** The pass only inventories the namespace of a + currently resolvable worktree, so refs owned by removed linked worktrees + survive. A repository-scoped operation would need active-worktree inventory, + repository-wide durable-root retention (`delete /T` only if `T` is + outside `durable_roots(repository)`, never "namespace unowned → delete all"), + and protection against worktree removal/recreation races. +- **Evidence-based advice.** The advisory is time-based; detecting actual orphans + requires the Git/DB scan kept off the hook path. +- **Historical retention.** Trees referenced by retained mutation events are + never deleted; bounding history needs a separate event-retention and + snapshot-compaction policy. + +See also: [`mutation-trace-ref-maintenance-state.md`](mutation-trace-ref-maintenance-state.md), [`doctor-ref-reconciliation.md`](doctor-ref-reconciliation.md), [`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md), [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md), [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md) (`list_pins` / `delete_pins`), [`mutation-trace-store.md`](mutation-trace-store.md) (`load_tree_roots` / `load_all_tree_roots`), [`mutation-trace-protocol.md`](mutation-trace-protocol.md). diff --git a/context/cli/mutation-trace-runtime-coordinator.md b/context/cli/mutation-trace-runtime-coordinator.md index 07683b292..b0845aee5 100644 --- a/context/cli/mutation-trace-runtime-coordinator.md +++ b/context/cli/mutation-trace-runtime-coordinator.md @@ -13,8 +13,8 @@ submodule is declared privately in `runtime/mod.rs`, which re-exports `coordinate` and `abandon_scope` at `pub(crate)` — reachable crate-wide, contract in [`mutation-scope-runtime.md`](mutation-scope-runtime.md). `coordinate()` and `abandon_scope()` are now driven by the generic `sce hooks mutation-scope` CLI -ingress. `reconcile_worktree` stays `runtime`-internal and unwired, and the -mutation runtime does not itself insert into `diff_traces`. Concrete Claude Code, +ingress. Snapshot-ref reconciliation stays `runtime`-internal and runs only through +`sce doctor --fix`, and the mutation runtime does not itself insert into `diff_traces`. Concrete Claude Code, Codex, and OpenCode adapters are wired to the seam (OpenCode reachable in production via a generated plugin installed by `sce setup`); Pi remains future work. @@ -30,11 +30,11 @@ The per-worktree runtime lock, the isolated Git snapshot service (which also derives `WorktreeId` from Git topology), the coordinator's internal protocol-integration pipeline, and the public, lock-wrapped `coordinate()` entrypoint that owns the external-taint fence and DB provider around that -pipeline all exist, with cross-module integration tests in `runtime/tests.rs` -exercising the public API end to end. Only harness/command wiring remains. +pipeline all exist (the former cross-module integration tests in `runtime/tests.rs` were removed; +see Testing boundary). Only harness/command wiring remains. - `cli/src/services/mutation_trace/runtime/worktree_lock.rs` — - `WorktreeLock::acquire(git_dir: &Path, timeout: Duration) -> + `WorktreeLock::acquire_async(git_dir: &Path, timeout: Duration) -> Result` opens/creates `/sce/mutation-cursor.lock` and polls `std::fs::File::try_lock()` on a 100ms interval against the caller-supplied bounded `timeout`, rather @@ -106,16 +106,16 @@ exercising the public API end to end. Only harness/command wiring remains. `refs/sce/mutation-cursor//...` ref. This derivation reads only Git's own state — no file is created or read under `/sce/`, and no identity is generated or persisted anywhere. -- `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` — the - conservative per-worktree snapshot-ref maintenance pass — `reconcile_worktree` - / `pub(super) reconcile_worktree_inner` return `Result` (`ReconciliationOutcome` = `Reconciled(ReconciliationReport)` - | `SkippedNoCheckoutIdentity`). Under the worktree's `WorktreeLock` it deletes - only pins whose tree is a durable root of **no** worktree, fails closed if any - local root lacks a pin, and writes no `mutation_trace_*` row or taint marker - (only the namespace of a checkout id a current worktree still derives — a - namespace no current worktree owns, via a deleted worktree or checkout-id - metadata loss/recreation, is future repository-scoped work). Full contract in +- `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` and + `ref_maintenance.rs` — the conservative per-worktree snapshot-ref maintenance + pass. `reconcile_with_held_lock` is the single lock-held algorithm and + `reconcile_explicit` (single `WorktreeLock` acquisition, verified-existing + Agent Trace DB, state recorded under the same lock) is its only production + entrypoint, called by `sce doctor --fix`. It deletes only pins whose tree is a + durable root of **no** worktree, fails closed if any local root lacks a pin, + and writes no `mutation_trace_*` row or taint marker. Only the namespace of the + currently resolvable worktree is inventoried; refs owned by removed linked + worktrees survive (deferred follow-up). Full contract in [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md). - `cli/src/services/mutation_trace/runtime/protected_worktree.rs` — the shared safety prefix every runtime entrypoint runs behind (`ProtectedWorktree`: @@ -199,9 +199,12 @@ The runtime lock guards every runtime entrypoint's critical section (external-taint marker arming/clearing, snapshot capture where applicable, worktree/scope materialization, recovery, and the CAS retry loop): `coordinate()` and `abandon_scope()` acquire it before their protected work and -hold it until the call returns. The separate `ref_reconciliation::reconcile_worktree` -pass acquires this same lock before inventorying pins, reading durable roots, -or deleting refs, with its own bounded timeout. `/sce/mutation-cursor.lock` +hold it until the call returns. The separate `ref_maintenance::reconcile_explicit` +pass acquires this same lock exactly once, before inventorying pins, reading +durable roots, or deleting refs, with a bounded 10 s wait; a boundary arriving +meanwhile waits up to `WORKTREE_LOCK_TIMEOUT` and then fails before durable +completion. The hook-path advisory takes the lock with a zero-wait `try_lock` +only after `coordinate()` has returned. `/sce/mutation-cursor.lock` remains worktree-specific because `git_dir` itself is worktree-specific for linked worktrees (`resolve_git_dir` resolves each worktree's own `--absolute-git-dir`), so each worktree has an independent critical section. @@ -217,7 +220,7 @@ On-disk layout so far: (runtime::git_snapshot writes here directly) -└── refs/sce/mutation-cursor// (runtime::git_snapshot, create-only per invocation; orphan/unreferenced pins reclaimed by runtime::ref_reconciliation only for a checkout id a current worktree still derives, every pin for a current or historical durable mutation-cursor root retained; a namespace no current worktree owns — deleted worktree or checkout-id metadata loss/recreation — is unreachable, future repository-scoped work) +└── refs/sce/mutation-cursor// (runtime::git_snapshot, create-only per invocation; orphan/unreferenced pins reclaimed by runtime::ref_maintenance only through sce doctor --fix and only for the namespace of a currently resolvable worktree, every pin for a current or historical durable mutation-cursor root retained; refs owned by removed linked worktrees are unreachable, future repository-scoped work) ``` ## Testing boundary @@ -289,28 +292,15 @@ durable, the boundary unprocessed with no `MutationEvent`, the marker still present, and a later `coordinate()` re-recovering off it; another proves an attributable `Advance` that commits then fails its trailing `marker.clear()` surfaces `MarkerClearAfterCommit` with the matching committed outcome. The -`after_load` seam is exercised by the reconciliation pin→CAS lock-race regression -([`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)), pausing a real `coordinate()` between `pin` and CAS. +The `after_load` seam was used by the removed reconciliation pin→CAS lock-race regression and is unused while that suite is gone. -`runtime/tests.rs` is `runtime`'s own `#[cfg(test)] mod tests`, holding -cross-module integration tests that drive the public `coordinate()` and -`abandon_scope()` APIs against real Git repositories (`git init`, `git worktree -add`) and real temp-file `RepositoryAgentTraceDb`s, following the same -unique-temp-path precedent: two linked worktrees of one repository (different -`git_dir` → different lock paths → different `WorktreeId`s) are proven -independently locked by holding one worktree's `WorktreeLock` across a -synchronous `coordinate()` call for the other and observing that call return -`Ok` before the held guard is dropped — a shared lock could not be acquired -while the guard is alive, and no wall-clock timing is used. Each call is handed -a provider closure that opens the one shared repository-scoped DB path -(`coordinate()` never resolves the DB), and both distinct worktree rows then -coexist in it. A full failure/recovery cycle and the cross-runtime abandonment -regressions also run through the public entrypoints. The same module covers the -public `reconcile_worktree` integration suite and the -`coordinate_inner` / `reconcile_worktree_inner` / `abandon_scope_inner` -lock-race seams, including orphan reclamation, durable-root retention, -missing-pin fail-closed behavior, malformed refs, linked-worktree scoping, and -the real coordinator and abandonment CAS races. +There is no `runtime/tests.rs` cross-module integration suite. Coverage of +`coordinate()` / `abandon_scope()` rests on the Quint MBT suite in +`mutation_trace/mbt`, the `mutation_trace` unit tests and the inline +`worktree_lock.rs` / `git_snapshot.rs` tests. Reconciliation has its own +production-path and concurrency suite under `ref_maintenance/tests/` (see +[`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)), +including tests that drive the real coordinator against a parked pass. ## Status @@ -319,7 +309,7 @@ pipeline, the public `coordinate()` entrypoint (prefix → DB provider → pipel → `complete()` on success), and the second `abandon_scope()` entrypoint sharing that prefix are all implemented and both `pub(crate)` re-exported from `runtime/mod.rs` ([`mutation-scope-runtime.md`](mutation-scope-runtime.md)), -with `runtime/tests.rs` covering `coordinate()` end to end and both entrypoints +with the former `runtime/tests.rs` end-to-end coverage removed (see Testing boundary) and both entrypoints driven together; an inherited external-taint marker is overlaid onto `database_failure` recovery on the next invocation. The generic `sce hooks mutation-scope` CLI ingress now drives both entrypoints diff --git a/context/cli/mutation-trace-scope-abandonment.md b/context/cli/mutation-trace-scope-abandonment.md index 247b0c089..9993f2cda 100644 --- a/context/cli/mutation-trace-scope-abandonment.md +++ b/context/cli/mutation-trace-scope-abandonment.md @@ -163,7 +163,7 @@ whole transition back (below); and a `clear()` failure returning A `pub(super) abandon_scope_inner(.., after_load)` seam (mirroring `coordinate_inner`) fires once per CAS attempt after the projection loads, so a -test — in this module or in `runtime/tests.rs` — can land a competing write +test in this module can land a competing write inside the CAS window. It is invisible outside `runtime`; production passes a no-op. @@ -188,8 +188,8 @@ exists, and the external-taint marker is still armed — the general rule that ### Cross-runtime regressions -`runtime/tests.rs` ([`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md#testing-boundary)) -drives `coordinate()` and `abandon_scope()` together over real `git init` / +The former `runtime/tests.rs` cross-module suite was removed ([`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md#testing-boundary)); it +drove `coordinate()` and `abandon_scope()` together over real `git init` / `git worktree add` repositories and one real repository-scoped Agent Trace DB, covering what a single-module test with no real Git cannot: diff --git a/context/cli/mutation-trace-snapshot-service.md b/context/cli/mutation-trace-snapshot-service.md index 2ba710f5d..e14684a31 100644 --- a/context/cli/mutation-trace-snapshot-service.md +++ b/context/cli/mutation-trace-snapshot-service.md @@ -119,8 +119,8 @@ second, read-only caller: the bounded mutation-history attribution consumer transition and reads baseline file content through its own `TreeReadSource` trait (see [`mutation-trace-agent-attribution.md`](mutation-trace-agent-attribution.md)). -`list_pins` / `delete_pins` are consumed by the per-worktree -ref-reconciliation maintenance pass. +`list_pins` / `delete_pins` are consumed only by the per-worktree +ref-reconciliation maintenance pass, which runs only through `sce doctor --fix`: pins orphaned by crashed or failed `coordinate()` paths are reclaimed when an operator runs it, and automatic reclamation does not exist. `delete_pins` is the only ref-deletion path and moves the `WorktreeLockLease` into its blocking worker, so dropping the caller future cannot release the lock while `git update-ref` runs. See [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md). ## Testing boundary diff --git a/context/cli/mutation-trace-store.md b/context/cli/mutation-trace-store.md index eec5790fe..5f19aea2e 100644 --- a/context/cli/mutation-trace-store.md +++ b/context/cli/mutation-trace-store.md @@ -136,7 +136,7 @@ worktree is still an `Err` there. cold-path, read-only queries — siblings of `load_mutation_event`, never reached from `load_worktree` or a hook-boundary path — that expose the set of Git tree SHAs the mutation-cursor protocol still durably depends on, for the -per-worktree ref-reconciliation pass (see +per-worktree ref-reconciliation pass, which runs only through `sce doctor --fix` (the single-event loaders `load_mutation_event` / `load_mutation_event_active_scopes` and their SQL/row-mapping helpers have no production reader and carry declaration-level `dead_code` allows saying so; attribution history uses `load_mutation_event_page`; see [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md)). - `load_tree_roots(worktree) -> BTreeSet` returns one worktree's roots: diff --git a/context/cli/patch-service.md b/context/cli/patch-service.md index 20d8ae307..4ede0648b 100644 --- a/context/cli/patch-service.md +++ b/context/cli/patch-service.md @@ -38,7 +38,6 @@ All types derive `Clone, Debug, Deserialize, Eq, PartialEq, Serialize` and suppo Storage-agnostic helpers for reconstructing `ParsedPatch` from serialized JSON content: - `load_patch_from_json(input: &str) -> Result` — loads a `ParsedPatch` from a JSON string; callers who have already read JSON from a database or file can pass the string directly -- `load_patch_from_json_bytes(input: &[u8]) -> Result` — loads a `ParsedPatch` from JSON bytes; convenient when the caller has raw bytes (for example, from a database BLOB column or file read) rather than a UTF-8 string Both functions wrap `serde_json::from_str`/`serde_json::from_slice` and map serde errors to actionable `PatchLoadError` messages. `PatchLoadError` carries a `message` field describing why the JSON payload could not be reconstructed into a valid `ParsedPatch`. @@ -96,7 +95,7 @@ builder, pending state, or schema migration is introduced. | Operation | Wired into | Notes | |-----------|-----------|-------| | `parse_patch` | Hook runtime, Agent Trace DB recent-row parsing, tests | Consumed by `post-commit` capture flow and `recent_diff_trace_patches` to parse stored raw `diff_traces.patch` text | -| `load_patch_from_json` / `load_patch_from_json_bytes` | Storage-agnostic JSON reconstruction callers | Reconstructs serialized `ParsedPatch` JSON when callers already have JSON payloads; not used for raw `diff_traces.patch` text in `recent_diff_trace_patches` | +| `load_patch_from_json` | Storage-agnostic JSON reconstruction callers | Reconstructs serialized `ParsedPatch` JSON when callers already have JSON payloads; not used for raw `diff_traces.patch` text in `recent_diff_trace_patches` | | `intersect_patches` | Post-commit hook runtime | Combines recent patches then intersects with current commit patch | | `combine_patches` | Post-commit hook runtime | Combines chronological recent patches before intersection | diff --git a/context/cli/repository-identity.md b/context/cli/repository-identity.md index e14b35b7a..890d78a49 100644 --- a/context/cli/repository-identity.md +++ b/context/cli/repository-identity.md @@ -12,8 +12,6 @@ The root module (`mod.rs`) performs no I/O: it never opens databases, reads Git - `canonicalize_remote_url(raw) -> Result` — canonicalization without hashing. - `derive_repository_id(canonical_identity) -> String` — domain-separated SHA-256 hex. - `REPOSITORY_ID_HASH_DOMAIN` — the `b"sce-repository-id-v1\0"` prefix constant. -- `repository_dir_segment(canonical_identity) -> String` — pure human-readable on-disk directory segment `-`, where `slug` is the lowercased canonical identity with non-alphanumeric runs collapsed to a single `-` and leading/trailing `-` trimmed, and `short` is the first 4 hex chars of `SHA256(canonical_identity)` with **no** domain prefix (deliberately distinct from `repository_id`, which keeps its `sce-repository-id-v1\0` prefix). All-non-alphanumeric input slugs to empty and the segment falls back to just `short`. Display/layout helper only — the authoritative identity stays `repository_id`. Not yet wired into path construction (T02 of `human-readable-repo-db-directory`). -- `RepositoryIdentity::dir_segment(&self) -> String` — convenience wrapper over `repository_dir_segment` for `self.canonical_identity`. ## Canonicalization rules (remote URLs) diff --git a/context/context-map.md b/context/context-map.md index 479207d19..3e8d620c9 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -16,7 +16,7 @@ Feature/domain context: - `context/cli/repository-identity.md` (repository identity module in `cli/src/services/repository_identity/`: pure scheme-neutral `host[:port]/path` canonicalization for SCP/`ssh://`/HTTPS/`git://` remote URLs with credential stripping, hostname lowercasing, default-port removal, and query/fragment/trailing-`.git` cleanup, trim-only explicit-identity handling, `sha256("sce-repository-id-v1\0" + canonical_identity)` repository IDs, credential-safe fieldless errors, plus the `resolve` runtime submodule applying explicit-config-then-configured-remote precedence with `git config --get remote..url` lookup, `RepositoryIdentitySource` provenance, and `.sce/config.json`-guidance resolution errors that never echo URLs; consumed by the T04 `agent_trace_storage` resolver) - `context/cli/agent-trace-storage.md` (async repository-scoped Agent Trace storage resolver in `cli/src/services/agent_trace_storage/`: `AgentTraceStorageContext` inputs mirroring the `agent_trace.*` config keys, shared degradation of invalid default-discovered config before identity fallback while explicit selections remain fatal, `ResolvedAgentTraceStorage` carrying repository identity + `/sce/repos//agent-trace.db` path + open `RepositoryAgentTraceDb` + typed `RepositoryMetadata`, `resolve_agent_trace_storage{,_at_state_root}` setup/lifecycle entrypoints with idempotent concurrent-safe first open via bounded fast-path-then-migrate retry plus narrow one-file schema migration-metadata repair and repository metadata validation, a separate no-migration `resolve_agent_trace_storage_for_hook_runtime{,_at_state_root}` pair that high-frequency hook callers use exclusively and that never runs migration `002` or any migration, path-unsafe repository ID rejection in `default_paths::agent_trace_db_path_for_repository{,_at}`, strict never-touch boundary for any pre-migration checkout-scoped/global DB files, and active hook/runtime plus Agent Trace lifecycle setup call-site consumption after T08) - `context/cli/checkout-identity.md` (checkout identity infrastructure, `cli/src/services/checkout/` and its `/sce/checkout-id` UUIDv7 storage, was removed by the `remove-checkout-id` plan; documents the removal plus the never-touch on-disk handling of any pre-existing checkout-id/pre-migration checkout-scoped DB files left behind) -- `context/cli/patch-service.md` (standalone patch domain model, parser, JSON load helpers, and set operations in `cli/src/services/patch.rs` for in-memory parsed unified-diff representation, capturing only touched lines plus minimal per-file/per-hunk metadata, supporting both `Index:` SVN-style and `diff --git` git-style formats, with `ParseError` for actionable malformed-input diagnostics, `PatchLoadError`/`load_patch_from_json`/`load_patch_from_json_bytes` for storage-agnostic JSON reconstruction, `intersect_patches` for target-shaped overlap with exact-match-first and historical `kind`+`content` fallback semantics plus matched-constructed-line `session_id` and matched-constructed-hunk `model_id` provenance inheritance, and `combine_patches` for ordered patch combination with later-wins conflict resolution plus winning-hunk `model_id` provenance inheritance; repository structured-row reconstruction supplies persisted hunk-model and canonical touched-line-session provenance before these operations; `parse_patch`, `intersect_patches`, and `combine_patches` are consumed by the active post-commit hook runtime) +- `context/cli/patch-service.md` (standalone patch domain model, parser, JSON load helpers, and set operations in `cli/src/services/patch.rs` for in-memory parsed unified-diff representation, capturing only touched lines plus minimal per-file/per-hunk metadata, supporting both `Index:` SVN-style and `diff --git` git-style formats, with `ParseError` for actionable malformed-input diagnostics, `PatchLoadError`/`load_patch_from_json` for storage-agnostic JSON reconstruction, `intersect_patches` for target-shaped overlap with exact-match-first and historical `kind`+`content` fallback semantics plus matched-constructed-line `session_id` and matched-constructed-hunk `model_id` provenance inheritance, and `combine_patches` for ordered patch combination with later-wins conflict resolution plus winning-hunk `model_id` provenance inheritance; repository structured-row reconstruction supplies persisted hunk-model and canonical touched-line-session provenance before these operations; `parse_patch`, `intersect_patches`, and `combine_patches` are consumed by the active post-commit hook runtime) - `context/cli/structured-patch-service.md` (Claude structured editor-hook derivation in `cli/src/services/structured_patch.rs`, including `Write` structured-update hunks, `Write` `tool_input.content` create fallback, `Edit` structured patches, deterministic skip reasons, `ParsedPatch` output semantics, Rust golden fixture coverage, and repository read-time enrichment that assigns persisted row `model_id` to each hunk and canonical row `session_id` to each touched line) - `context/cli/styling-service.md` (CLI text-mode output styling with `owo-colors`, TTY/`NO_COLOR` policy, shared helper API for human-facing surfaces including sync completion markers, and per-column right-to-left RGB gradient banner rendering) - `context/cli/agent-trace-auto-sync.md` (setup-enabled post-commit Agent Trace synchronization with explicit-false opt-out plus doctor readiness reporting: a newly created repo-local config supplies the explicit `true` opt-in while omitted runtime values resolve to `false`; the existing `sce sync` command is launched once through the current executable after local persistence, with detached null-standard-stream behavior, fail-open startup, canonical managed-block proof of hook readiness, stable text/JSON enabled/disabled/not-ready/not-applicable states, no daemon/queue/high-frequency trigger, and retryability through manual sync and control-plane cursor authority) @@ -24,24 +24,27 @@ Feature/domain context: - `context/cli/async-persistence-guarantees.md` (verified guarantees and named limits of the directly awaited Turso persistence migration — exclusive `&mut` transactions, deferred rollback, credential-rotation and terminal-sync behavior, lease/descriptor protection — plus PR1–PR4 staging and the PR3 cleanup ledger) - `context/cli/agent-trace-sync-command.md` (composed local-to-control-plane `sce sync` architecture: the `hooks/plugins → repository Agent Trace DB → AgentTraceExportReader → sce sync → HTTPS + WorkOS Bearer → control plane` data flow, the `sce auth login` / `cd ` / `sce sync` user flow, the no-local-cursor/no-`agent-trace-sync.db`/no-Turso-Sync/no-`BridgeLock`/no-local-DWH invariants, the four-local-capture-streams versus three-remote-streams boundary with its `diff_traces` release gate, and `401`/`409`/ambiguous-batch-failure recovery semantics) - `context/cli/config-precedence-contract.md` (implemented `sce config` show/validate command contract, deterministic `flags > env > config file > defaults` resolution order, focused `config/resolver.rs` ownership for config discovery/merge/runtime precedence plus default-discovered invalid-file degradation, focused `config/render.rs` ownership for `show`/`validate` text+JSON output construction, canonical `$schema` acceptance for startup-loaded `sce/config.json` files, shared auth-key env/config/optional baked-default support starting with `workos_client_id`, shared runtime resolution for flat logging observability keys including config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` with `/sce/logs` defaulting plus config-file/default-only positive `log_file_retention_limit`, config-file-only `agent_trace.repository_id`/`agent_trace.repository_remote` repository-identity keys with default remote `origin`, `agent_trace.auto_sync` resolution with omitted-value fallback `false`, explicit-false opt-out, and setup-written explicit `true` bootstrap behavior for the post-commit trigger boundary, the catalog-derived `integrations.optional_workflows` optional-workflow selection key, JSON-pointer-prefixed schema-validation errors, canonical Pkl-generated `sce/config.json` schema ownership plus CLI embedding/reuse contract including `policies.attribution_hooks.enabled` default-true/explicit-false opt-out metadata, config-file selection order, `show` provenance output, and trimmed `validate` output contract) -- `context/cli/capability-traits.md` (current broad CLI capability seam in `cli/src/services/capabilities.rs`, including `FsOps`/`StdFsOps`, `GitOps`/`ProcessGitOps`, git root/hooks resolution behavior, compile-time-typed borrowed AppContext wiring with scoped borrowing across directly awaited commands with associated-type narrow capability accessors plus `ContextWithRepoRoot` repo-root-scoped context derivation, generic command execution bounds, and test-only unimplemented stubs; current service internals do not consume fs/git traits until later lifecycle migration tasks) +- `context/cli/capability-traits.md` (current broad CLI capability seam in `cli/src/services/capabilities.rs`, including `FsOps`/`StdFsOps` (method-less marker), `GitOps`/`ProcessGitOps` (command execution and availability), compile-time-typed borrowed AppContext wiring with scoped borrowing across directly awaited commands with associated-type narrow capability accessors plus `ContextWithRepoRoot` repo-root-scoped context derivation, generic command execution bounds, and test-only unimplemented stubs; current service internals do not consume fs/git traits until later lifecycle migration tasks) - `context/cli/service-lifecycle.md` (current async lifecycle seam in `cli/src/services/lifecycle.rs`, including default no-op `ServiceLifecycle` diagnose/fix/setup methods against narrow `HasRepoRoot`, lifecycle-owned health/fix/setup result types with generic setup messages, doctor/setup adapter boundaries, the static `LifecycleProvider` enum catalog/dispatcher, hook/config/local_db/auth_db/agent_trace_db lifecycle providers including setup-time repository-scoped Agent Trace DB initialization plus checkout identity diagnostics, implemented doctor aggregation over diagnose/fix providers, and implemented setup aggregation over `setup` providers in order config → local_db → auth_db → agent_trace_db → hooks when requested) - `context/cli/mutation-trace-protocol.md` (pure, dependency-free `cli/src/services/mutation_trace/` refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol: the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — plus cross-action sequence/invariant tests and a module-level Quint refinement matrix are all complete (`types.rs`, `protocol.rs`, `mod.rs`, `tests.rs`, registered with `#[allow(dead_code)]`); opaque-newtype identity refinement decisions vs. the Quint model's bounded enums, and the `coordinator.rs` target end-state seam this layout leaves room for but does not create — `store.rs`, `runtime/git_snapshot.rs`, and `runtime/coordinator.rs` (including its public `coordinate()` entrypoint) now exist, built out by the `mutation-cursor-store-persistence` and `mutation-cursor-runtime-coordinator` plans respectively; `protocol.rs` is invoked only through the runtime, and the runtime is now driven by the generic `sce hooks mutation-scope` CLI ingress and a first Claude Code adapter driver, registered by `sce setup` and reachable by a real Claude Code session, now proven against real Git repositories and a real Agent Trace DB by the `claude-mutation-scope-integration` plan's T08 regressions) - `context/cli/mutation-trace-runtime-materialization.md` (how the finite, pre-populated Quint `SCOPES`/`WORKTREES` universes map onto the Rust protocol's unbounded identity space: the coordinator/store layer materializes scope and worktree state, scope identity is immutable once materialized, and a missing scope is distinct from a `NeverSeen` scope; `protocol.rs` only transitions already-known identities) -- `context/cli/mutation-trace-agent-attribution.md` (causal mutation-lineage attribution: direct target-shaped coverage is resolved first, then the newest-128 events for the invoking worktree (bounded also by a commit attribution cut — `revision <= latest_mutation_event_revision` captured under the worktree lock) are replayed oldest-to-newest as one ordered sequence of tree transitions. `lineage.rs` is a pure module propagating per-line `LineProvenance` (`Unknown` / `MutationAi{scope}` / `MutationNonAi`) forward through structurally-applied hunks: context carries, removed is permanently deleted, added takes only the introducing transition's origin, replacement never transfers by text. A committed line is AI only if an AI event's line survives every later transition — including a history-gap reload and the unobserved latest-tree→commit-tree tail — into the committed tree; anything unproven stays `Unknown`. Historical patches are never matched against the committed patch. `attribution.rs` keeps only `exclude_direct_coverage` + `patch_for_locations`. The `runtime/mutation_attribution.rs` consumer drives it over `MutationEventPageSource` + `TreeReadSource` (`diff_trees` + `file_at_tree`) seams with conservative fail-closed reloads; wired into post-commit via the read-only `runtime::resolve_post_commit_mutation_ai_patch` entrypoint — existing checkout identity only, direct-only fallback on absent identity/history/unavailable cut, no identity creation, no mutation-cursor write, `diff_traces` / `post_commit_patch_intersections` unchanged; see `agent-trace-hooks-command-routing.md` and `agent-trace-minimal-generator.md` — with real Git/DB post-commit regressions in `cli/src/services/hooks/mod.rs` and a real 128/129-horizon regression in `runtime/tests.rs`) +- `context/cli/mutation-trace-agent-attribution.md` (causal mutation-lineage attribution: direct target-shaped coverage is resolved first, then the newest-128 events for the invoking worktree (bounded also by a commit attribution cut — `revision <= latest_mutation_event_revision` captured under the worktree lock) are replayed oldest-to-newest as one ordered sequence of tree transitions. `lineage.rs` is a pure module propagating per-line `LineProvenance` (`Unknown` / `MutationAi{scope}` / `MutationNonAi`) forward through structurally-applied hunks: context carries, removed is permanently deleted, added takes only the introducing transition's origin, replacement never transfers by text. A committed line is AI only if an AI event's line survives every later transition — including a history-gap reload and the unobserved latest-tree→commit-tree tail — into the committed tree; anything unproven stays `Unknown`. Historical patches are never matched against the committed patch. `attribution.rs` keeps only `exclude_direct_coverage` + `patch_for_locations`. The `runtime/mutation_attribution.rs` consumer drives it over `MutationEventPageSource` + `TreeReadSource` (`diff_trees` + `file_at_tree`) seams with conservative fail-closed reloads; wired into post-commit via the read-only `runtime::resolve_post_commit_mutation_ai_patch` entrypoint — existing checkout identity only, direct-only fallback on absent identity/history/unavailable cut, no identity creation, no mutation-cursor write, `diff_traces` / `post_commit_patch_intersections` unchanged; see `agent-trace-hooks-command-routing.md` and `agent-trace-minimal-generator.md` — with real Git/DB post-commit regressions in `cli/src/services/hooks/mod.rs`) - `context/cli/mutation-trace-revision-refinement.md` (the Quint `revision: int` → Rust `WorktreeState::revision: u64` bounded-integer refinement: the private `next_revision` checked-arithmetic helper `commit`/`taint`/`abandon`/`recover` all route through instead of a raw `+ 1`, so a worktree at `revision: u64::MAX` is a guarded no-op/rejection rather than a silent wrap to `0`) - `context/cli/mutation-trace-quint-connect.md` (`#[cfg(test)]`-only Quint Connect model-based-testing harness in `cli/src/services/mutation_trace/mbt/` continuously checking `protocol.rs` against `spec/mutation_cursor.qnt`: the verification-only `mbtAction`/`MbtAction` record-payload transport excluded from comparison, the operation-identity-vs-`MbtStutter` distinction on guarded/no-op branches with its two deterministic regressions, finite ID mapping, the AC5 comparable-state field list, `randomPrepare` staying a single `step` branch, deterministic/generated (500×30, seed-reproducible) test coverage, and the two Nix checks — generic `checks.cli-tests` and dedicated `checks.mutation-trace-quint-connect` — that both require the pinned Quint binary plus the top-level `spec/` directory in `workspaceSrc`'s Nix fileset) - `context/cli/mutation-trace-store.md` (durable persistence for the mutation-cursor protocol in `cli/src/services/mutation_trace/store.rs`, built by the `mutation-cursor-store-persistence` plan: the one-directional `protocol.rs` -> `DurableTransition::between` (pure structural diff) -> `store.rs` (SQL translation) -> `RepositoryAgentTraceDb` boundary; migration `004_mutation_trace_protocol.sql`'s five tables; migration `006_mutation_trace_health_invariant.sql`'s transactional rebuild of the worktree and event tables with a cross-column `tainted`/`failure_kind` health `CHECK`, `failure_kind`-authoritative legacy normalization, and a re-runnable SQL body, plus the shared fail-closed `validate_health_encoding` used by all three health-row decoders; `AttemptState`/`external_taint` non-persistence; the 8-byte big-endian `BLOB` revision encoding plus explicit non-`Debug` enum codecs; the bounded hot-path `load_worktree` read vs. the cold-path `load_mutation_event` and descending, exact-worktree, cursor-paged `load_mutation_event_page` reader capped at 32 rows; the public cold-path single-row `load_scope` scope seam that returns one `mutation_trace_scopes` row as `Option` without widening into a projection and, unlike `load_worktree`, never adjudicates worktree identity (a cross-worktree scope is returned as-is, and the mismatch is the caller's decision); the cold-path read-only `load_tree_roots` (one worktree) / `load_all_tree_roots` (repository-wide) durable-tree-SHA queries for ref reconciliation, each a single-statement `UNION` of `cursor_tree`/`before_tree`/`after_tree` read from one DB snapshot; `commit`'s single-`BEGIN IMMEDIATE` CAS batch via `TursoDb::execute_transactional_cas_batch`, distinguishing `Conflict`/retryable-transient/deterministic-`Err` outcomes; and the store's non-goals — no Git/filesystem I/O, no attribution/boundary-kind decisions, no retry-after-`Conflict` loop, no row deletion) - `context/cli/mutation-scope-provenance.md` (the insert-once `mutation_trace_scope_provenance` table from migration `005_mutation_scope_provenance.sql` and its `MutationTraceStore::register_scope_provenance` / `load_scope_provenance` seams: `scope_id -> session_id + model_id?` metadata *about* an established scope, stored beside but never inside protocol state — outside `ProtocolState` and the CAS batch, and never an input to `IneligibleUnscoped` / `AiExclusive` / `AiContended`; `session_id` is immutable identity and `model_id` immutable first-observed metadata, so the first persisted row wins in both directions and only a differing `session_id` for an existing `scope_id` errors; the supported `MutationTraceStore` write seam requires an existing owning `mutation_trace_scopes` row and never creates one implicitly, enforced in the store rather than by a `FOREIGN KEY`, leaving `004_mutation_trace_protocol.sql` unchanged; creation is additionally admission-bounded in the runtime — a row may be created only while the durable scope is `NeverSeen`, so after admission absent provenance stays absent permanently and a later `Start` replay cannot backfill it, while an existing row is still validated on every provenance-carrying `Start`; producers own canonicalization — both Codex and Claude adapters supply provenance today, with Claude resolving exact `(cc_, agent_id)` model state at admission and degrading unavailable models to `NULL`) -- `context/cli/mutation-trace-runtime-coordinator.md` (imperative-shell runtime layer in `cli/src/services/mutation_trace/runtime/`, built by the `mutation-cursor-runtime-coordinator` plan: the per-worktree OS advisory lock, `runtime::worktree_lock::WorktreeLock::acquire(git_dir, timeout)` at `/sce/mutation-cursor.lock`, bounded `try_lock()` polling with a distinct matchable timeout error and RAII release, and its distinction from the separate checkout-identity-creation lock; the isolated Git snapshot service `runtime::git_snapshot::GitSnapshotService` (documented in `mutation-trace-snapshot-service.md`); and `runtime::coordinator`'s protocol-integration pipeline (`RuntimeBoundary`/`CoordinateOutcome`/`CoordinateError`, the load → recover-if-needed → prepare/commit CAS-retry loop, and its own bounded snapshot-failure taint-retry loop) plus the public `coordinate(repository_root, boundary, open_db)` entrypoint that runs the shared `runtime::protected_worktree` prefix (`WorktreeLock` -> external-taint write-ahead fence -> `WorktreeId`, extracted so a second entrypoint cannot drift from it and documented in `mutation-trace-protected-worktree.md`), invokes the caller-supplied `open_db` provider so DB acquisition falls inside that fence, drives the pipeline under one held lock, and clears the marker through `ProtectedWorktree::complete()` only on success; `runtime/tests.rs` covers the public `coordinate()` API end to end against real linked worktrees and a real Agent Trace DB; an inherited external-taint marker is overlaid onto `protocol::database_failure` recovery on the next invocation, against the single captured snapshot and re-injected across a losing recovery CAS; the private `runtime::ref_reconciliation` per-worktree snapshot-ref maintenance pass (documented in `mutation-trace-ref-reconciliation.md`) shares the same `WorktreeLock`; `coordinate()` and `abandon_scope()` are now `pub(crate)` re-exported from `runtime/mod.rs` (documented in `mutation-scope-runtime.md`) while the runtime submodules themselves stay private; a generic `sce hooks mutation-scope` CLI ingress now drives both entrypoints, joined by current Claude Code and Codex adapter drivers (registered by `sce setup`/`sce setup --codex` and reachable) and an OpenCode adapter driver (lifecycle + recovery wired, and now generated as the final `sce-mutation-scope.ts` plugin and installed by `sce setup`), documented in `mutation-scope-runtime.md`) +- `context/cli/mutation-trace-runtime-coordinator.md` (imperative-shell runtime layer in `cli/src/services/mutation_trace/runtime/`, built by the `mutation-cursor-runtime-coordinator` plan: the per-worktree OS advisory lock, `runtime::worktree_lock::WorktreeLock::acquire_async(git_dir, timeout)` at `/sce/mutation-cursor.lock`, bounded `try_lock()` polling with a distinct matchable timeout error and RAII release, and its distinction from the separate checkout-identity-creation lock; the isolated Git snapshot service `runtime::git_snapshot::GitSnapshotService` (documented in `mutation-trace-snapshot-service.md`); and `runtime::coordinator`'s protocol-integration pipeline (`RuntimeBoundary`/`CoordinateOutcome`/`CoordinateError`, the load → recover-if-needed → prepare/commit CAS-retry loop, and its own bounded snapshot-failure taint-retry loop) plus the public `coordinate(repository_root, boundary, open_db)` entrypoint that runs the shared `runtime::protected_worktree` prefix (`WorktreeLock` -> external-taint write-ahead fence -> `WorktreeId`, extracted so a second entrypoint cannot drift from it and documented in `mutation-trace-protected-worktree.md`), invokes the caller-supplied `open_db` provider so DB acquisition falls inside that fence, drives the pipeline under one held lock, and clears the marker through `ProtectedWorktree::complete()` only on success; an inherited external-taint marker is overlaid onto `protocol::database_failure` recovery on the next invocation, against the single captured snapshot and re-injected across a losing recovery CAS; the private `runtime::ref_reconciliation` per-worktree snapshot-ref maintenance pass (documented in `mutation-trace-ref-reconciliation.md`) shares the same `WorktreeLock`; `coordinate()` and `abandon_scope()` are now `pub(crate)` re-exported from `runtime/mod.rs` (documented in `mutation-scope-runtime.md`) while the runtime submodules themselves stay private; a generic `sce hooks mutation-scope` CLI ingress now drives both entrypoints, joined by current Claude Code and Codex adapter drivers (registered by `sce setup`/`sce setup --codex` and reachable) and an OpenCode adapter driver (lifecycle + recovery wired, and now generated as the final `sce-mutation-scope.ts` plugin and installed by `sce setup`), documented in `mutation-scope-runtime.md`) - `context/cli/mutation-trace-protected-worktree.md` (the shared safety prefix every mutation-cursor runtime entrypoint runs behind, in `cli/src/services/mutation_trace/runtime/protected_worktree.rs`, extracted from `coordinate()` by the `mutation-scope-runtime-integration` plan so a second entrypoint cannot drift from it: `ProtectedWorktree::acquire(repository_root)` running the safety-critical fixed order resolve `git_dir` → `WorktreeLock` (module-owned 10s `WORKTREE_LOCK_TIMEOUT`) → `ExternalTaintMarker::exists()` → `persist()` (fence armed write-ahead of every fallible step that follows, including DB acquisition) → `get_or_create_checkout_id` as `WorktreeId`; the `worktree_id()` / `inherited_external_taint()` / consuming `complete()` surface, where `complete()` clears the marker under the still-held lock and is the only thing that ever clears it while `Drop` releases only the lock; and the one-variant-per-prefix-step `ProtectedWorktreeError` (`GitDirResolution` | `LockAcquisition` | `ExternalTaintMarker { operation, source }` | `CheckoutIdentity`) each entrypoint maps onto its own error surface — `coordinate()` onto exactly the `CoordinateError` variants that step produced before the extraction) - `context/cli/mutation-trace-scope-abandonment.md` (the mutation-cursor runtime's second protected entrypoint in `cli/src/services/mutation_trace/runtime/scope_runtime.rs`, built by the `mutation-scope-runtime-integration` plan and the first production call site for `protocol::abandon`: `abandon_scope(repository_root, scope, open_db) -> Result` retires a scope whose final worktree boundary was never observed, sharing `coordinate()`'s `ProtectedWorktree` prefix but deliberately capturing **no** Git snapshot, pin, diff, reconciliation, scope registration, or worktree initialization — so abandonment is not a `RuntimeBoundary` and needs no Quint change; the classify-before-transition order that recovers what `protocol::abandon`'s uniform guarded no-op cannot report (`load_scope` first for the missing-row and cross-worktree cases the projection seam treats as errors, then `load_worktree` for the `NeverSeen` / terminal / `Active` split); the `Abandoned` / `AlreadyTerminal` / `RecoveryRequired` outcomes and the `InheritedExternalTaint` | `MissingScope` | `NeverSeenScope` | `MissingWorktreeState` recovery reasons; the inherited-marker short-circuit that returns before the DB provider is ever invoked; the fence-completion rule that clears the marker only for a settled abandonment or proven-terminal no-op and leaves it armed for every recovery-required outcome and every error, with `MarkerClearAfterCompletion` carrying the already-settled outcome; the CAS retry bounded by the coordinator's shared `MAX_CAS_RETRY_ATTEMPTS`, settling on a competitor's terminal status rather than overwriting it; and the deliberate false-negative tradeoff whereby a missing or `NeverSeen` target forces conservative strong recovery that may abandon unrelated live scopes, because attribution safety outranks preserving potentially valid evidence) -- `context/cli/mutation-scope-runtime.md` (the crate-visible mutation-trace runtime seam and the lifecycle contract every current or future harness adapter must uphold, recorded by the `mutation-scope-runtime-integration` plan: the ten `pub(crate)` re-exports in `runtime/mod.rs` (`coordinate`, `RuntimeBoundary`, `StartProvenance`, `CoordinateOutcome`, `CoordinateError`, `ExternalTaintOperation`, `abandon_scope`, `AbandonScopeOutcome`, `AbandonRecoveryReason`, `AbandonScopeError`), with `ExternalTaintOperation` riding through `coordinator.rs`'s own `pub use` because `CoordinateError::ExternalTaintMarker` carries it — so the type becomes crate-visible without `protected_worktree` becoming a public module — while every `mod` declaration stays private and `ProtectedWorktree`/`ProtectedWorktreeError`/`WORKTREE_LOCK_TIMEOUT`/`reconcile_worktree` stay internal, kept clippy-clean by `#[allow(unused_imports)]` on the re-exports per the `services/style.rs` precedent rather than a placeholder consumer; and the adapter obligations themselves — a scope is one independently mutation-capable execution so a concurrent main agent and subagent need distinct `ScopeId`s, `Start`/`Advance`/`Close` semantics including that only `Start` may carry the optional `StartProvenance`, registered after scope registration and before the protocol commits and never inside `ProtocolState` or the CAS transition, with that registration conditional on the `ScopeState` `register_scope` returns so a provenance row may only be created while the scope is `NeverSeen` (an admission-time snapshot that a post-admission replay cannot backfill) while an existing row is still validated on every provenance-carrying `Start`, that a failed tool still requires `Advance` (the boundary is an observation, not a successful edit) and that a `ScopeId` is never reused after a terminal status (the `NeverSeen` guard silently refuses to reactivate it), `abandon_scope()` requiring positive staleness evidence and never inferring it from `ActorKind`, the `abandon` → `coordinate(Start(successor))` sequence with what each outcome implies for it including that a failed abandonment must never be treated as a safely started successor, that abandonment is not a `RuntimeBoundary` and needs no Quint change, the D1 tradeoff whereby a missing or `NeverSeen` target deliberately forces conservative strong recovery that may invalidate unrelated live scopes because attribution safety outranks preserving potentially valid evidence, and the attribution boundary that `AiExclusive(scope)` means scope exclusivity only and is not standalone proof that no human edited the worktree; a generic `sce hooks mutation-scope` ingress now drives the seam, joined by current Claude Code and Codex adapter drivers (both registered by `sce setup` and reachable) and an OpenCode adapter driver (full lifecycle + recovery, now generated as the final `sce-mutation-scope.ts` plugin and registered by `sce setup`, so reachable by a real session) — a Pi adapter driver now also exists (`cli/src/services/hooks/pi_mutation_scope/`, hidden `sce hooks pi-mutation-scope`) and is driven by the canonical generated Pi extension in ordinary Pi sessions; its human `user_bash` path uses the separate guarded external-mutation supervisor) +- `context/cli/mutation-scope-runtime.md` (the crate-visible mutation-trace runtime seam and the lifecycle contract every current or future harness adapter must uphold, recorded by the `mutation-scope-runtime-integration` plan: the ten `pub(crate)` re-exports in `runtime/mod.rs` (`coordinate`, `RuntimeBoundary`, `StartProvenance`, `CoordinateOutcome`, `CoordinateError`, `ExternalTaintOperation`, `abandon_scope`, `AbandonScopeOutcome`, `AbandonRecoveryReason`, `AbandonScopeError`), with `ExternalTaintOperation` riding through `coordinator.rs`'s own `pub use` because `CoordinateError::ExternalTaintMarker` carries it — so the type becomes crate-visible without `protected_worktree` becoming a public module — while every `mod` declaration stays private and `ProtectedWorktree`/`ProtectedWorktreeError`/`WORKTREE_LOCK_TIMEOUT`/`reconcile_explicit` stay internal, kept clippy-clean by `#[allow(unused_imports)]` on the re-exports per the `services/style.rs` precedent rather than a placeholder consumer; and the adapter obligations themselves — a scope is one independently mutation-capable execution so a concurrent main agent and subagent need distinct `ScopeId`s, `Start`/`Advance`/`Close` semantics including that only `Start` may carry the optional `StartProvenance`, registered after scope registration and before the protocol commits and never inside `ProtocolState` or the CAS transition, with that registration conditional on the `ScopeState` `register_scope` returns so a provenance row may only be created while the scope is `NeverSeen` (an admission-time snapshot that a post-admission replay cannot backfill) while an existing row is still validated on every provenance-carrying `Start`, that a failed tool still requires `Advance` (the boundary is an observation, not a successful edit) and that a `ScopeId` is never reused after a terminal status (the `NeverSeen` guard silently refuses to reactivate it), `abandon_scope()` requiring positive staleness evidence and never inferring it from `ActorKind`, the `abandon` → `coordinate(Start(successor))` sequence with what each outcome implies for it including that a failed abandonment must never be treated as a safely started successor, that abandonment is not a `RuntimeBoundary` and needs no Quint change, the D1 tradeoff whereby a missing or `NeverSeen` target deliberately forces conservative strong recovery that may invalidate unrelated live scopes because attribution safety outranks preserving potentially valid evidence, and the attribution boundary that `AiExclusive(scope)` means scope exclusivity only and is not standalone proof that no human edited the worktree; a generic `sce hooks mutation-scope` ingress now drives the seam, joined by current Claude Code and Codex adapter drivers (both registered by `sce setup` and reachable) and an OpenCode adapter driver (full lifecycle + recovery, now generated as the final `sce-mutation-scope.ts` plugin and registered by `sce setup`, so reachable by a real session) — a Pi adapter driver now also exists (`cli/src/services/hooks/pi_mutation_scope/`, hidden `sce hooks pi-mutation-scope`) and is driven by the canonical generated Pi extension in ordinary Pi sessions; its human `user_bash` path uses the separate guarded external-mutation supervisor) - `context/cli/mutation-scope-hook-ingress.md` (the one harness-neutral CLI ingress that drives the mutation-scope runtime, in `cli/src/services/hooks/mutation_scope.rs`, built by the `mutation-scope-hook-ingress` plan: the hidden `sce hooks mutation-scope` command routing through the normal `cli_schema::HooksSubcommand::MutationScope` → `convert_hooks_subcommand_request` → `services::hooks::HookSubcommand::MutationScope` → `run_hooks_subcommand_in_repo` stack, reading one normalized JSON lifecycle object from STDIN; the strict `parse_mutation_scope_payload` contract supporting exactly `start`/`advance`/`close`/`flush`/`abandon` with a local `MutationScopePayload` transport enum, `claude_code`/`codex`/`opencode`/`pi` actor mapping, non-blank `scope_id`/`event_id`, exact per-operation key sets, an optional `provenance` object accepted on `start` only (a required non-blank `session_id`, an optional `model_id` where an absent key and an explicit `null` both mean no model, and no other key), and a dedicated hard rejection for any `worktree_id` key; the operation mapping to `RuntimeBoundary::Start`/`Advance`/`Close`/`Flush` through `coordinate()` or a direct `abandon_scope()` call, forwarding `ScopeId`/`EventId`/`ActorKind` and `start`'s optional `StartProvenance` verbatim because `EventId` equality is the runtime replay/idempotency key and provenance values arrive already canonical; identity ownership — the adapter owns `scope_id`/`event_id`/`actor_kind`, SCE owns `worktree_id`/Git tree identities/revisions/attempt IDs, and worktree identity is derived only by the runtime from the invoking checkout; the lazy `FnOnce` DB provider reusing `open_agent_trace_db_for_hook_runtime` so DB acquisition stays inside the runtime's protected-worktree ordering; the non-fail-open error classification by durable completion — malformed payload or any pre-completion `CoordinateError`/`AbandonScopeError` → `CliError`/non-zero, while `CoordinateError::MarkerClearAfterCommit` and `AbandonScopeError::MarkerClearAfterCompletion` are treated as durable success with empty stdout, the marker-cleanup failure logged via `sce.hooks.mutation_scope.marker_clear_after_durable_completion`, and the transition not retried; the empty-stdout success contract; abandonment ownership keeping no-snapshot semantics; and the generic-ingress vs harness-adapter boundary — a Claude Code adapter driver (`cli/src/services/hooks/claude_mutation_scope/`) and a Codex adapter driver (`cli/src/services/hooks/codex_mutation_scope/`, hidden `sce hooks codex-mutation-scope`) now exist and are registered by `sce setup`, both consuming this seam's own `pub(crate)` in-process entrypoint; an OpenCode adapter driver (`cli/src/services/hooks/opencode_mutation_scope/`, hidden `sce hooks opencode-mutation-scope`) also consumes it with a full `Start`/`Close`/`Abandon` lifecycle and checkout-local recovery state, and is now driven in production by the generated `sce-mutation-scope.ts` plugin installed last by `sce setup`; a Pi adapter driver (`cli/src/services/hooks/pi_mutation_scope/`, hidden `sce hooks pi-mutation-scope`) also consumes it now, with `(session-id, tool-call-id)`→`ScopeId` derivation and getppid()/`/proc`-start-time-based stale-process recovery; the canonical generated Pi extension drives it in ordinary sessions — see `context/cli/pi-mutation-scope-integration.md`) - `context/cli/claude-mutation-scope-integration.md` (the first concrete harness lifecycle adapter, `cli/src/services/hooks/claude_mutation_scope/`, hidden command `sce hooks claude-mutation-scope`, built by the `claude-mutation-scope-integration` plan: one independently mutation-capable Claude tool execution = one SCE mutation `ScopeId` (a session/prompt/main agent/subagent is never a scope); the `classify_tool` table (mutation-capable including unknown names, read-only `Read`/`Glob`/`Grep`/`WebFetch`/`WebSearch`/`AskUserQuestion`, `Agent` = delegation) plus the model-only `is_explicit_background_shell` predicate; the length-prefixed hash-free `cc-tool-v1|n=|s=..|a=..|t=..` `ScopeId` keyed on a monotonic checkout-local `attempt_seq` (never reused after terminal) with deterministic `|start` / `|close` `EventId`s; the `/sce/claude-mutation-scope-state.json` bookkeeping store (never attribution evidence, never synced) with its own separate lock never held across a seam call; `PreToolUse` write-ahead `pending_start` → seam `start` → `active` and its fail-closed Claude `permissionDecision: "deny"` on any failure (never `allow`, detail logged via `sce.hooks.claude_mutation_scope.pre_tool_use_fail_closed`); `PostToolUse`/`PostToolUseFailure` → `close`, with `pending_start`+terminal → abandon-not-late-start (D11) and failed-`close` → abandon-not-replay (D12); the abandonment cleanup signals (`PermissionDenied`, `Stop`/`StopFailure`, `UserPromptSubmit`, `SubagentStop`, `SessionEnd`, best-effort `WorktreeRemove` — the last two not observed to fire on Claude Code `2.1.258`); the `recovery_pending` barrier that denies new mutation-capable `PreToolUse` until quiescent then runs one seam `flush`; raw hook `cwd` (or `worktree_path` for `WorktreeRemove`) as authoritative repository root with no adapter-constructed `WorktreeId`; `run_in_background = true` handling — native Claude background Bash is operationally supported but unsupported for mutation attribution (allowed in `PreToolUse` before repository resolution with no mutation scope, adapter attempt, or lifecycle tracking, per the effective decision `context/decisions/2026-10-02-claude-background-bash-untracked.md`), while background `PowerShell` stays denied — and the separate self-detaching-descendant unsupported boundary (D20, with T04's Git-observable evidence — full detail split out to [`claude-mutation-scope-background-execution.md`](cli/claude-mutation-scope-background-execution.md) for the repository's per-file line budget); the ten unmatched `sce setup` registrations; the strict `claude_mutation_scope → hooks::mutation_scope → mutation_trace::runtime` dependency direction through the single `run_mutation_scope_from_payload` seam import (T05); admission-time exact model-state snapshot into `ScopeProvenance`; real Git/Agent Trace persistence coverage shared with the Codex path; and, added by the `doctor-mutation-scope-fix` plan's T04, a third `pending_abandon` attempt phase durably persisted before any abandon seam call — established evidence that abandonment has already been decided, distinct from "may still be running" — with every attempt a broad cleanup signal (`Stop`/`UserPromptSubmit`/`SubagentStop`/`SessionEnd`/`WorktreeRemove`) matches now marked `pending_abandon` in one write before any of their seam calls run, so one attempt's seam failure can never leave a batch sibling without its own retryable evidence; health classification detail split out to [`claude-mutation-scope-health.md`](cli/claude-mutation-scope-health.md)) - `context/cli/claude-mutation-scope-background-execution.md` (native Claude background Bash is operationally supported but unsupported for mutation attribution: `Bash(run_in_background=true)` is allowed in `PreToolUse` with no mutation scope, adapter attempt, lifecycle tracking, or attribution guarantee — effective record `context/decisions/2026-10-02-claude-background-bash-untracked.md` — its later terminal hooks are no-ops, and the accepted A/B contamination consequence is pinned by a regression; background `PowerShell` stays denied with the existing deny text; the T01 `2.1.284` probe result with lifecycle completion still unproven; and the separate self-detaching-descendant unsupported boundary D20 — Git-observable evidence that a foreground `setsid` command's detached descendant can still mutate the repository after `PostToolUse` returns — detail split out of `claude-mutation-scope-integration.md` for the repository's per-file line budget) - `context/cli/claude-mutation-scope-health.md` (the `doctor-mutation-scope-health` plan's Claude task, extended by the `doctor-mutation-scope-fix` plan's T04: `classify_health` in `cli/src/services/hooks/claude_mutation_scope/health.rs` maps `recovery_pending`/`attempts` onto `healthy | recovering | blocked | invalid` — `recovery_pending == false` is `healthy` regardless of live attempts; `recovery_pending == true` with `attempts` empty is `recovering` (the barrier's own next-`PreToolUse` flush clears it); `recovery_pending == true` with `attempts` non-empty is `blocked`, the exact incident shape this classifier exists to surface, since nothing in the ordinary hook lifecycle retries, removes, flushes, or clears from that shape; a read/parse failure is `invalid`. T04 adds a separate, `doctor`-invoked repair path (`assess_repairability`/`repair_blocked` in `health.rs`/`lifecycle.rs`, wired into `sce doctor --fix` by the same plan's T05) for the `blocked` shape: `AutoFixable` only when every currently persisted attempt is already `pending_abandon` (no established-abandon-intent `pending_start`/`active` attempt present); `repair_blocked` re-reads and re-proves that same condition inside one lock-protected read-only state transaction before releasing the lock and retrying each attempt's already-established seam `abandon` call independently, clearing the `recovery_pending` barrier only once no attempts remain — this classifier's own read-only behavior is unaffected by that repair path's existence) -- `context/cli/mutation-trace-ref-reconciliation.md` (the conservative per-worktree snapshot-ref reconciliation pass in `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`, built by the `mutation-cursor-ref-reconciliation` plan: `reconcile_worktree(repository_root, open_db)` → `pub(super) reconcile_worktree_inner(.., on_lock_contention)`, both returning `ReconciliationOutcome` (`Reconciled(ReconciliationReport { local_required, retained, deleted })` for a pass that ran | `SkippedNoCheckoutIdentity` — an `Ok`, not an `Err` — when no current checkout identity could be derived), a variant-per-fallible-step `ReconcileError` with no `Other`, and the module-owned `RECONCILIATION_LOCK_TIMEOUT`; the two-invariant model — a strictly per-worktree fail-closed local-consistency check via `load_tree_roots(W)` vs. a repository-wide deletion-safety set via `load_all_tree_roots()` so an `A`-owned ref is retained whenever any worktree still durably needs its tree — run entirely under the same `/sce/mutation-cursor.lock` `WorktreeLock` `coordinate()` holds, with the repository-wide read kept coherent by being one SQL statement / one DB snapshot rather than a repository-global lock; deletes only SCE-owned refs via one atomic `git update-ref --no-deref --stdin`, writes no `mutation_trace_*` row, never arms `ExternalTaintMarker`, runs no `git gc`; imperative durability maintenance below the verified Quint protocol; reclaims orphan/unreferenced refs only for the namespace of a checkout id a current worktree still derives — a namespace no current worktree owns (identity-based: a deleted linked worktree, or checkout-id metadata loss followed by `get_or_create_checkout_id` minting a fresh id on a still-present worktree) is beyond every per-worktree pass and left to a recorded future repository-scoped unowned-namespace operation, so the current pass does not bound all orphan-ref growth; `reconcile_worktree` has no `pub(crate)` re-export and no harness/command wiring yet) +- `context/cli/mutation-trace-ref-reconciliation.md` (the conservative per-worktree snapshot-ref reconciliation pass, built by the `mutation-cursor-ref-reconciliation` plan and wired by `mutation-cursor-ref-reconciliation-wiring`: the single lock-held algorithm `reconcile_with_held_lock` in `ref_reconciliation.rs`, the explicit production entrypoint `reconcile_explicit` in `ref_maintenance.rs` (one `WorktreeLock` acquisition, verified-existing Agent Trace DB, typed `ExplicitOutcome`), the documented 10 s concurrency contract, the `ref-maintenance.json` state file (past-only timestamps, skew normalization, atomic writes), the D4 verified-existing DB opener, the two-invariant durability model (per-worktree fail-closed local consistency vs repository-wide deletion safety, never a claim that every repository-wide root is pinned), and the deferred follow-ups (automatic out-of-band reclamation with its binding prerequisites, retired-worktree cleanup, evidence-based advice, historical retention); reconciliation runs only through `sce doctor --fix` and is never automatic) +- `context/cli/mutation-trace-ref-maintenance-state.md` (supporting contracts for snapshot-ref reconciliation: the per-worktree `/sce/ref-maintenance.json` state file with past-only timestamps, `FUTURE_SKEW_TOLERANCE` normalization on successful writes only, atomic staging-and-swap writes, the pure `evaluate_recommendation`, and the non-creating verified-existing Agent Trace DB opener with typed `ExistingRepositoryDbError` and per-DB-state guarantees) +- `context/cli/mutation-scope-ref-advisory.md` (the advisory-only hook trigger in `cli/src/services/hooks/mutation_scope.rs` built by T03 of the `mutation-cursor-ref-reconciliation-wiring` plan: a lightweight `advise_if_due` check after a durably completed `Close`/`Flush`, logged and, when actionable, written as an `SCE:` line to an injectable stderr writer (best-effort delivery), with the per-worktree `ref-maintenance.json` state; no reconciliation, Git, or DB work on the hook path) +- `context/cli/doctor-ref-reconciliation.md` (the `sce doctor --fix` caller of snapshot-ref reconciliation built by T04 of the `mutation-cursor-ref-reconciliation-wiring` plan: the `runtime::ref_doctor` facade, typed fix-result rows for completed/completed-with-persist-warning/failed/busy passes, and the read-only recommendation section/`ref_reconciliation` JSON object in plain `sce doctor`) - `context/cli/mutation-trace-snapshot-service.md` (the isolated Git snapshot and ref-pinning service `runtime::git_snapshot::GitSnapshotService` in `cli/src/services/mutation_trace/runtime/git_snapshot.rs`: `new` resolving an absolute `git_dir`, `capture_tree` snapshotting staged/unstaged/untracked/deleted worktree state into the repository's normal object database via a throwaway temp index, `pin_tree` protecting a durable tree with a create-only idempotent **direct** `refs/sce/mutation-cursor//` ref, `diff_trees` emitting `patch.rs`-parseable raw diff text; plus the callerless reconciliation substrate — worktree-scoped `list_pins` inventory returning `Result, PinInventoryError>` that rejects a symbolic ref inside the namespace (mutation-cursor pins are direct refs) as `MalformedRef`, matchable separately from a `git for-each-ref` execution failure, and conditional-atomic `delete_pins` running one `git update-ref --no-deref --stdin` transaction of SHA-conditioned deletes — no-dereference so an inventory→delete direct-ref→symref race cannot escape the inventoried namespace, plus a fail-closed pre-check — that aborts whole if any ref changed since inventory; `REF_NAMESPACE` + `pin_ref_prefix` as the single source of truth for the pin path) - `context/cli/mutation-trace-external-taint.md` (the worktree-local mutation-cursor durability boundary in `cli/src/services/mutation_trace/runtime/external_taint.rs`, built by the `mutation-cursor-external-taint` plan: the `ExternalTaintMarker` primitive — `new(git_dir)`/`exists()`/`persist()`/`clear()` over an empty file at `/sce/mutation-cursor-tainted` whose existence is its entire state, `checkout::persist_checkout_id_inner`-style durability (`sync_data` plus best-effort `#[cfg(unix)]` parent-dir `sync_all`), idempotent persist/clear, `NotFound`-on-clear as success, no `Drop` deletion — as the concrete runtime refinement of the abstract `ProtocolState.external_taint`; armed by the reshaped `coordinate()` entrypoint write-ahead after the `WorktreeLock` and before Agent Trace DB acquisition (a caller-supplied DB provider closure), cleared only on a successful `CoordinateOutcome`, with dedicated fail-closed pre-commit `CoordinateError::ExternalTaintMarker` (`Inspect`/`Persist` only)/`AgentTraceDbUnavailable` variants plus a post-commit `MarkerClearAfterCommit { source, committed }` that carries the durable outcome so a failed trailing clear never hides a committed `MutationEvent`; an inherited marker seeds an invocation-local `external_taint_pending` flag that overlays `protocol::database_failure` onto each freshly loaded projection so `recover` runs once against the captured snapshot, held across a losing recovery CAS and cleared once it lands) - `context/sce/cli-exit-code-contract.md` (stable class-based `sce` process exit-code contract in `cli/src/app.rs`, so automation can branch on failure category without parsing error text) @@ -51,7 +54,8 @@ Feature/domain context: - `context/sce/cli-version-command-contract.md` (implemented `sce version` contract for deterministic human and machine-readable runtime identification) - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) -- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, and the trimmed `sce config validate` status-only validation surface) +- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior, the allowlisted tracing boundaries (Boundary A event-ID registry and classification, Boundary B `sce::otel` policy), plus the opt-in standalone OTel runtime (`SCE_TELEMETRY=standalone`, typed hook-export gate, `ScopedDispatch` per-poll scoping, `BoundaryBExporter`), the `NoopTelemetry`-backed default telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, and the trimmed `sce config validate` status-only validation surface) +- `context/sce/hook-telemetry-benchmark.md` (T05 hook-latency benchmark of the D6 `telemetry-test-receiver` build across disabled/reachable/blackhole/refused modes, recorded build command and methodology, candidate 100 ms hook flush budget, and the statement that production hook export stays gated until a user-owned enablement policy lands) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) - [Context workflow rules](sce/context-workflow-rules.md) (canonical bootstrap, ongoing context maintenance, task synchronization, hygiene, discoverability, and feature-existence rules) @@ -85,7 +89,7 @@ Feature/domain context: - `context/sce/agent-trace-post-rewrite-local-remap-ingestion.md` (current post-rewrite no-op baseline plus historical remap-ingestion reference) - `context/sce/agent-trace-rewrite-trace-transformation.md` (current post-rewrite no-op baseline plus historical rewrite-transformation reference) - `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited async `execute`/`query`/`query_map` methods using the shared Turso adapter) -- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven async constructor/open-connect retry via `run_with_retry`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven async operation retry for `execute`/`query`/`query_values`/`query_map` (retry-policy budget, not a wall-clock bound), per-`DbSpec` Turso busy timeout on connect (1000 ms for the Agent Trace DB, unset for other databases), the Agent Trace-only outer write-contention retry for replay-safe write units including the opt-in `execute_idempotent_write` and its structured contention-exhaustion error plus `sce.agent_trace_db.contention_exhausted` tracing event, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and async concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`; no adapter runtime bridge) +- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven async constructor/open-connect retry via `run_with_retry`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven async operation retry for `execute`/`query_map` (retry-policy budget, not a wall-clock bound), per-`DbSpec` Turso busy timeout on connect (1000 ms for the Agent Trace DB, unset for other databases), the Agent Trace-only outer write-contention retry for replay-safe write units including the opt-in `execute_idempotent_write` and its structured contention-exhaustion error plus `sce.agent_trace_db.contention_exhausted` tracing event, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and async concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`; no adapter runtime bridge) - `context/sce/agent-trace-db-write-contention-evidence.md` (measured Agent Trace DB write-contention behavior under the busy-timeout/outer-retry/contention-deadline contract: the `lock_contention_tests.rs` suite with its lock-budget boundary, strict N=2–4 in-process and real-hook gates and N=8 stress characterization, policy history from 500 / 1250 to the current 1000 / 2250 defaults, the supported-load failure that motivated the tuning, before-vs-after comparison, and the bounded-policy exhaustion failure mode) - `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, async encrypted token-storage persistence through a Result-valued Tokio OnceCell, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) - `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by the fresh multi-statement baseline schema plus additive `source_instance_id` and `claude_model_state` migrations, with `repository_metadata` carrying both `repository_id` and a concurrency-safe atomic-claim `source_instance_id` (physical database identity, independent of `repository_id`), narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, parts, and the non-exported exact-scope Claude model-state register, repository-level recent diff-trace reads without checkout filtering including persisted hunk-model plus canonical touched-line-session enrichment for structured rows, async on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) @@ -124,7 +128,7 @@ Additional mutation-scope integration context: - `context/cli/opencode-mutation-scope-health.md` (the `doctor-mutation-scope-health` plan's T04: `classify_health` in `cli/src/services/hooks/opencode_mutation_scope/health.rs` maps the adapter's `RecoveryState`/`AttemptPhase` combinations onto `healthy | recovering | blocked | invalid`, proven by driving the real dispatch/recovery functions — `Clear` + `PendingAbandon` is `invalid`; any state containing a `PendingStart` attempt is `blocked` (except that `Clear` + `PendingAbandon` retains `invalid` precedence), because no boundary sweeps a stale `PendingStart` on behalf of an unrelated call, unlike Pi's dead-owner sweep or Codex's same-lane retry, so a crashed owning process leaves it a permanent wedge for new admissions even while a `Pending`/`Flushing` recovery for another attempt is also outstanding, since `resolve_recovery` never retires `PendingStart`; `Pending`/`Flushing` with no `PendingStart` outstanding is `recovering`; `Clear` with ordinary `Active`/no attempts is `healthy`; any read/parse failure is `invalid`) - `context/cli/pi-mutation-scope-integration.md` (the fourth concrete harness adapter, `cli/src/services/hooks/pi_mutation_scope/`, hidden command `sce hooks pi-mutation-scope`, driven by the canonical generated extension in ordinary Pi sessions: Pi's `bash`/`edit`/`write` tracked-tool allowlist with `tool_call` as the single universal pre-execution gate for every tool including `bash`; the `pi-tool-v1|n=|s=:|c=:` `ScopeId` whose checkout-local monotonic attempt sequence stops a reused `toolCallId` from ever reactivating a terminal scope; the `PendingStart`→`Executed`→`Closed`/`PendingAbandon` attempt-phase machine with no `Active` phase, because `PendingStart` is Pi's normal resting state for an entire in-flight execution (`tool_execution_start` fires unconditionally before the fail-closed `tool_call` gate and proves nothing; `tool_result` is the sole `Executed` evidence, and `tool_execution_end` with no preceding `tool_result` abandons rather than closes); why admission's fail-closed "uncertain attempt" check covers only a lingering `PendingAbandon` or non-`Clear` recovery state and never a sibling's `PendingStart`, unlike the other three adapters, so concurrent Pi tool calls stay distinct live scopes; a positive-process-death-only stale-`PendingStart`/`Executed` recovery (`getppid()`-captured owner plus Linux `/proc`-start-time PID-reuse proofing, no TTL/sweep), run on every tracked Start admission by independently inspecting persisted attempts rather than only ones matching the incoming key, and adapter reconciliation with the external-mutation guard's forced recovery, both added by a later task in this same plan; and the harness-neutral `external-mutation-guard` mechanism this task also built for Pi's `user_bash`, detailed separately in `context/cli/mutation-trace-external-mutation-guard.md`) - `context/cli/pi-mutation-scope-health.md` (the `doctor-mutation-scope-health` plan's T05: `classify_health` in `cli/src/services/hooks/pi_mutation_scope/health.rs` maps the adapter's `RecoveryState`/`AttemptPhase`/owner-liveness combinations onto `healthy | recovering | blocked | invalid`, proven by driving the real dispatch/recovery functions — unlike Codex and OpenCode, Pi's investigation found no persisted shape reachable through ordinary production dispatch that classifies `blocked`: `admit_tracked_attempt` never gates admission on a sibling's `PendingStart`/`Executed` phase (only on a co-existing `PendingAbandon`, itself proven unreachable under `Clear`), and the D10 dead-owner sweep (`reconcile_stale_owners`) unconditionally retires a dead-owner `PendingStart`/`Executed` attempt on the next tracked `tool_call` from any session, so such an attempt is `recovering`, not `blocked`; `Pending`/`Flushing` are always `recovering` regardless of attempt composition, since a recovery-capable fresh tracked Start — one whose key is not already represented by a nonterminal attempt — can claim and retry the flush, not lane- or key-scoped like Codex, though a duplicate Start for an already-tracked `PendingStart`/`Executed` key is instead reused idempotently before the recovery-state gate and so does not itself advance recovery; `Clear` with a `PendingAbandon` attempt and any read/parse failure are `invalid`) -- `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral, not-Pi-specific D13 external-mutation supervisor process, `run_external_mutation_guard` in `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only: acquires a `ProtectedWorktree`, reports it armed, spawns the human shell command as its own child in its own process group with a `dup()`-before-`spawn()` fd handed to the child so the `WorktreeLock` flock survives the supervisor's own death via the child's independent descriptor, finishes exclusively on the shell's own `wait()` (never on a cancel/control-channel signal), and only then forces `database_failure`+`recover` on the already-held worktree and calls `ProtectedWorktree::complete()` — reusing existing runtime primitives with zero `protocol.rs`/Quint change; called by the canonical Pi extension for human `user_bash`) +- `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral, not-Pi-specific D13 external-mutation supervisor process, `arm_external_mutation_guard`/`ArmedExternalMutationGuard::exec` in `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only: acquires a `ProtectedWorktree`, reports it armed, spawns the human shell command as its own child in its own process group with a `dup()`-before-`spawn()` fd handed to the child so the `WorktreeLock` flock survives the supervisor's own death via the child's independent descriptor, finishes exclusively on the shell's own `wait()` (never on a cancel/control-channel signal), and only then forces `database_failure`+`recover` on the already-held worktree and calls `ProtectedWorktree::complete()` — reusing existing runtime primitives with zero `protocol.rs`/Quint change; called by the canonical Pi extension for human `user_bash`) Working areas: diff --git a/context/glossary.md b/context/glossary.md index a5e17a2e8..344ccdce6 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -39,6 +39,8 @@ - `checkout identity`: Stable UUIDv7 identifier assigned to a cloned repository or linked Git worktree, stored in `/sce/checkout-id` (never committed) and resolved via `git rev-parse --git-dir`. The identity is created or reused by `sce setup` through `AgentTraceDbLifecycle::setup()` and also auto-created by hook runtime when `sce setup` has not been run. Checkout identity is now diagnostic metadata for repository-scoped Agent Trace storage; it does not select the active DB and is not stored on Agent Trace rows. Any pre-migration per-checkout DB files at `/sce/agent-trace-{checkout_id}.db` are never touched by SCE and are no longer inspectable via the CLI (the checkout-scoped DB surface was removed by the `retire-legacy-agent-trace-db` plan). See `context/cli/checkout-identity.md`. - `repository identity`: Stable identity of a logical Git repository used to select the active repository-scoped Agent Trace DB path `/sce/repos//agent-trace.db` through the `agent_trace_storage` resolver. Resolved by `cli/src/services/repository_identity/` with precedence: explicit `agent_trace.repository_id` config value, then the URL of the configured Git remote (`agent_trace.repository_remote`, default `origin`), otherwise an actionable `.sce/config.json` error. Remote URLs canonicalize to a scheme-neutral, credential-free `host[:port]/path` form so equivalent SSH/SCP/HTTPS remotes converge, and the repository ID is `sha256("sce-repository-id-v1\0" + canonical_identity)` hex. Distinct from `checkout identity`, which stays per clone/worktree for diagnostics. Setup's named-remote preflight uses a strict lookup that returns `Ok(None)` only for a missing/empty URL, preserves other failures as runtime errors, and discards the URL, retaining only the remote name in technical failure sources. See `context/cli/repository-identity.md`. - `refresh single-flight guard`: Client-owned async coordination for Agent Trace control-plane authentication. Concurrent callers whose stored access token is expired, or whose request rejected the same token, serialize only refresh-and-save work, re-check credentials after acquiring the guard, and reuse the token saved by the first refresher; valid-token requests do not acquire the guard. The guard is held by the spawned task that performs renewal and save, so caller cancellation cannot release it mid-persistence; `wait_for_pending_refresh` acquires it as a completion barrier before `sce sync` returns. +- `snapshot-ref reconciliation`: Explicit, per-worktree maintenance pass (`runtime::ref_maintenance::reconcile_explicit`, run only by `sce doctor --fix`) that deletes orphaned SCE snapshot pins under `refs/sce/mutation-cursor//` while retaining every tree in any worktree's durable roots; never automatic. See `context/cli/mutation-trace-ref-reconciliation.md`. +- `ref-reconciliation advisory`: Lightweight hook-path check (`advise_if_due`) after a durably completed `Close`/`Flush` that may record and print a "reconciliation is recommended" reminder; it never reconciles, lists refs, opens the DB, or spawns Git. See `context/cli/mutation-scope-ref-advisory.md`. - `repository-scoped Agent Trace DB`: Active Agent Trace storage shape where one logical Git repository maps to `/sce/repos//agent-trace.db`. The current seam is `RepositoryAgentTraceDb = TursoDb` in `cli/src/services/agent_trace_db/repository.rs`, backed by the fresh multi-statement `001_repository_schema` baseline plus the additive `002_repository_source_instance_id` migration, with `repository_metadata` (`repository_id` plus `source_instance_id`) plus repository-level trace tables, no `checkout_id` columns, and typed repository-level insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, and parts. Hook runtime, Agent Trace setup/lifecycle, and `sce sync` resolve repository-scoped storage through `agent_trace_storage`. This is the sole Agent Trace DB adapter; the checkout-scoped adapter and former trace inspection surface were removed by the `retire-legacy-agent-trace-db` plan. - `source_instance_id`: Physical-database identity column on `repository_metadata`, independent of the logical `repository_id`. Added by the additive `002_repository_source_instance_id` migration (existing/placeholder rows default to an empty string); generated once per physical `agent-trace.db` by application code (`generate_source_instance_id()`, UUID v4 today) and validated with `is_valid_source_instance_id()` (non-empty once trimmed) — never generated in SQL and never derived from `repository_id`, remote URL, checkout ID, filesystem path, hostname, or user/workspace identity. `RepositoryAgentTraceDb::verify_or_initialize_repository_metadata` claims it with a concurrency-safe `UPDATE ... WHERE source_instance_id = ''`, so concurrent first opens of the same physical database converge on one winner and an already-valid value is never overwritten; the value stays stable across reopen and repeated `sce setup` runs. Two independently created databases for the same logical repository (for example two clones) get different `source_instance_id` values. See `context/sce/agent-trace-db.md`. - `checkout registry` (removed): The central JSON registry at `/sce/checkout-registry.json` was removed in the `remove-checkout-registry` plan. The later trace inspection surface was also removed; current checkout IDs are diagnostic metadata only and are not discovered as command targets. Any pre-migration `/sce/agent-trace-*.db` files remain untouched. See `context/cli/checkout-identity.md`. @@ -70,7 +72,7 @@ - `auth login stored-credential renewal`: The `sce auth login` behavior that first validates every stored credential through the existing non-forced token path, preserves valid credentials, refreshes expired credentials, and falls back to device authorization when credentials are absent or renewal fails. Renewal reports retain `login` labels in text and JSON; credential renewal is not exposed as a public subcommand. The auth command boundary treats missing-credential logout/whoami calls as successful state queries, classifies authenticated Control Plane failures as `UserError::NotAuthenticated`, token-storage plus `AuthError::Io`/`Storage` failures as `UserError::AuthStorageUnavailable` (`auth.storage_unavailable`), and approved user-facing rendering/prompt fallbacks as `UserError::UnexpectedFailure` (`general.unexpected_failure`); fixed catalog messages expose no storage or implementation details while technical sources remain available for structured observability, and remaining auth-command failures use `UnexpectedFailure` with preserved technical sources. - `command surface contract`: The current top-level command/help catalog split where `cli/src/cli_schema.rs` owns the real clap-backed command metadata (top-level purpose text plus help visibility for `auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, and `completion`) and `cli/src/command_surface.rs` consumes that catalog for the custom banner/help surface plus known-command classification, while still adding the synthetic `help` row. - `top-level help visibility metadata`: Per-command `show_in_top_level_help` metadata in `cli/src/cli_schema.rs` that controls whether a known command appears in `sce`, `sce help`, and `sce --help` without affecting direct invocation; the current hidden top-level commands are `hooks` and `policy`, while `auth` is visible, and `cli/src/command_surface.rs` renders the curated top-level help list from that shared metadata. -- `sce dependency baseline`: Current crate dependency set declared in `cli/Cargo.toml` (`anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `indicatif`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `turso`, `uuid`, plus target-specific keyring backends), validated through normal compile/test coverage. `[dev-dependencies]` now carries `quint-connect` (dev/test-only, for the in-progress mutation-cursor Quint Connect model-based-testing harness; see `context/plans/mutation-cursor-quint-connect.md`). +- `sce dependency baseline`: Current crate dependency set declared in `cli/Cargo.toml` (`anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `indicatif`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `turso`, `uuid`, the pinned OTel crates, plus target-specific keyring backends), validated through normal compile/test coverage. `[dev-dependencies]` now carries `quint-connect` (dev/test-only, for the in-progress mutation-cursor Quint Connect model-based-testing harness; see `context/plans/mutation-cursor-quint-connect.md`). - `progress reporter contract`: Library-independent, consumer-typed sync seam in `cli/src/services/sync/progress.rs` where `ProgressReporter` carries an arbitrary consumer event type, supports explicit successful finalization and closure-based collectors, and provides `NoopProgressReporter` for callers without human progress output. The sync-owned `SyncProgressEvent` in `cli/src/services/sync/sync.rs` carries lifecycle, accepted-batch, and stream-completion payloads for the three concurrent remote Agent Trace streams (`messages`, `parts`, `agent_traces`); `services::sync::progress` also owns the fixed `indicatif` terminal adapter and focused contract tests, while `sync/command.rs` selects text versus JSON behavior. No top-level `services::progress` module exists. - `remote sync streams`: The three Agent Trace streams `sce sync` uploads to the control plane — `messages`, `parts`, `agent_traces` — as distinct from the four local capture streams, which also include the local-only `diff_traces`. `streams.diffTraces` in `sce sync --format json` is a compatibility no-op entry echoing the server cursor, not a synchronized stream. See `context/cli/agent-trace-sync-command.md`. - `local Turso adapter`: Module in `cli/src/services/local_db/mod.rs` that defines `LocalDbSpec` and exposes `LocalDb` as a `TursoDb` alias. It resolves the canonical local DB path with `local_db_path()`, currently declares zero migrations, and inherits retry-backed `new()`, `execute()`, `query()`, and `query_map()` behavior from the shared generic adapter. @@ -81,10 +83,10 @@ - `post-commit Agent Trace auto-sync readiness`: The doctor report fact that explains whether the enabled post-commit trigger is ready without invoking it. Doctor compares the installed `post-commit` hook's SCE managed block using the same currency semantics as setup and resolves config-file-only `agent_trace.auto_sync` with source metadata. JSON states are `ready`, `disabled`, `not_ready`, and `not_applicable`; explicit disable is healthy, while existing hook problems continue to own overall readiness and remediation. See [automatic Agent Trace synchronization](cli/agent-trace-auto-sync.md) and [doctor human text](sce/doctor-human-text-contract.md). - `structured patch service`: Pure synchronous Rust service in `cli/src/services/structured_patch.rs` that derives supported structured editor hook payloads into canonical `ParsedPatch` values. The current implemented source is Claude `PostToolUse` payloads for `Write` creates and `Edit` structured patches; wired into `sce hooks diff-trace` for Claude payload classification at intake and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing, where persisted row `model_id` is assigned to every hunk and persisted canonical row `session_id` to every touched line before downstream reconstruction. - `MutationLineage` / `causal mutation-lineage attribution`: `cli/src/services/mutation_trace/lineage.rs` is a pure module holding, per repo path, a vector of `(content, LineProvenance)` where `LineProvenance` is `Unknown`, `MutationAi { scope_id }`, or `MutationNonAi`. `MutationLineage::apply(patch, origin)` advances it one reconstructed tree transition at a time, applying hunks structurally: context lines carry provenance forward, removed lines are deleted permanently, added lines take only the introducing `TransitionOrigin` (`MutationAi` for healthy untainted `AiExclusive`, `MutationNonAi` for any other recorded event, `Unobserved` → `Unknown` for gap reloads and the latest-tree→commit-tree tail); a structurally inconsistent transition is a `LineageError` and the caller reloads that file to an all-`Unknown` baseline. Historical mutation patches are never matched independently against the committed patch — that earlier `MutationPatchEvidence` / strict-matcher model let stale evidence resurrect and is removed. `attribution.rs` now only supplies `exclude_direct_coverage` (drop directly covered lines) and `patch_for_locations`. The store supplies events through an exact-worktree-scoped `mutation-event page` ordered by descending 8-byte big-endian revision, continued with an exclusive revision cursor, capped at 32 rows; plus `latest_mutation_event_revision` for the commit attribution cut. The `bounded mutation-history consumer` (`resolve_bounded_mutation_attribution` in `cli/src/services/mutation_trace/runtime/mutation_attribution.rs`) loads the newest ≤128 in-cut events through `MutationEventPageSource` + `TreeReadSource` (`diff_trees` + `file_at_tree`) seams, replays them oldest-to-newest from a conservative baseline through the tail into `commit_tree`, then projects surviving provenance onto the committed patch; it is current-worktree-only and timestamp-independent, owns `MAX_MUTATION_ATTRIBUTION_EVENTS = 128` (event 128 may contribute, 129 is never loaded), tracks database (`loaded_pages`/`loaded_rows`) apart from Git (`inspected_events`/`reconstructed_events`/`gap_resets`) work, and records a `mutation-attribution barrier` on any page-query or reconstruction/tail failure while still returning conservative results. See [mutation-trace store](cli/mutation-trace-store.md) and [mutation-trace Agent Trace attribution](cli/mutation-trace-agent-attribution.md). -- `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace(...)` under `metadata.sce`, carrying `version` (sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`) and `line_changes` (exact `{ai,mixed,unknown}` × `{added,removed}` `u64` touched-line attribution counts derived from canonical `post_commit_patch` hunks, reusing each hunk's existing `Conversation.contributor.type` classification with no independent second classification pass, `#[serde(default)]` for backward-compatible deserialization of pre-existing payloads); the whole object is schema-validated with the rest of the payload and persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. +- `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace_from_evidence(...)` under `metadata.sce`, carrying `version` (sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`) and `line_changes` (exact `{ai,mixed,unknown}` × `{added,removed}` `u64` touched-line attribution counts derived from canonical `post_commit_patch` hunks, reusing each hunk's existing `Conversation.contributor.type` classification with no independent second classification pass, `#[serde(default)]` for backward-compatible deserialization of pre-existing payloads); the whole object is schema-validated with the rest of the payload and persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. - `ScopeProvenance`: Optional insert-once metadata keyed by a mutation `ScopeId`, storing canonical SCE `session_id` and nullable first-observed `model_id`. It is registered only at `Start` admission while the owning scope is `NeverSeen`, remains outside `ProtocolState` and the CAS transition, and is resolved during mutation projection. `ScopeId` proves mutation ownership; `ScopeProvenance` describes the owning scope. Model/session provenance is observational and never changes `AiExclusive`/`AiContended`/`IneligibleUnscoped` protocol decisions. -- `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. -- `external-mutation guard` / `external-mutation supervisor`: `run_external_mutation_guard` (`cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only), the harness-neutral, not-Pi-specific mechanism that lets a human-initiated shell command (Pi's `!`/`!!` `user_bash`) mutate a worktree without ever creating a mutation scope: it acquires the same `ProtectedWorktree` guard `coordinate()` uses, reports it armed, itself spawns the human shell as its own child (never signaled or stopped by control-channel closure — only by an explicit cancel request, which it enacts by signaling the shell's own process group), finishes exclusively on the shell's real process termination, and only then forces the existing `database_failure`+`recover` composition on the already-held worktree via `coordinate_on_held_worktree` before clearing the marker with `ProtectedWorktree::complete()`. See [`mutation-trace-external-mutation-guard.md`](cli/mutation-trace-external-mutation-guard.md). +- `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace_from_evidence(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. +- `external-mutation guard` / `external-mutation supervisor`: `arm_external_mutation_guard` + `ArmedExternalMutationGuard::exec` (`cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only), the harness-neutral, not-Pi-specific mechanism that lets a human-initiated shell command (Pi's `!`/`!!` `user_bash`) mutate a worktree without ever creating a mutation scope: it acquires the same `ProtectedWorktree` guard `coordinate()` uses, reports it armed, itself spawns the human shell as its own child (never signaled or stopped by control-channel closure — only by an explicit cancel request, which it enacts by signaling the shell's own process group), finishes exclusively on the shell's real process termination, and only then forces the existing `database_failure`+`recover` composition on the already-held worktree via `coordinate_on_held_worktree` before clearing the marker with `ProtectedWorktree::complete()`. See [`mutation-trace-external-mutation-guard.md`](cli/mutation-trace-external-mutation-guard.md). - `Claude diff-trace attribution`: Diff-trace enrichment rule where one Claude `PostToolUse` event resolves its model with `direct > exact transcript > exact session/agent state > bridge-derived chain state > NULL`: direct top-level/nested metadata first, then that event's `transcript_path` matched by `tool_use_id` to an assistant envelope's `tool_use.id`, then one exact `(cc_, agent_id)` lookup in local `claude_model_state`. Only for a raw structured Claude payload in the exact main-session scope (`agent_id` empty), a state miss then falls back to `bridge session correlation` across the transcript's bridge-linked chain; when that resolves, persistence writes a `claude_model_state` row for the current session (`source="bridge_inherited"`) — the one diff-trace resolution path that writes state — and uses that model, so later traces of the same session resolve from their own exact-scope row. Model sources receive one `claude/` normalization step, ephemeral agent context is never exported, and subagents do not inherit main-session state. - `bridge session correlation`: Claude-specific local correlation using the `bridgeSessionId` in leading transcript records to relate a Claude session's transcript to its sibling transcripts sharing that ID. Selection resolves the newest `claude_model_state` observation across all bridge-linked chain members by `observed_at_ms` (deterministic session-ID tie-break), not the chain origin and not the most recently modified sibling transcript. Discovery is bounded (leading records only) and fail-open, uses existing exact-scope `claude_model_state` rows without persisting the bridge ID, and does not claim authoritative session ordering. One selection rule serves both call sites — the `diff-trace` state-miss path and the `sce hooks claude-model-state` `SessionStart` path — with no most-recently-modified-sibling pick left in the code. See [the read-path bridge-seeding decision](decisions/2026-09-10-claude-bridge-seeding-on-diff-trace.md) and [the bridge-session inheritance decision](decisions/2026-09-08-claude-bridge-session-model-inheritance.md). - `DiffTraceInsert`: Insert payload in `cli/src/services/agent_trace_db/mod.rs` carrying `time_ms`, tool-prefixed `session_id`, `patch`, `model_id`, `tool_name`, nullable `tool_version`, and `payload_type` for parameterized writes to the `diff_traces` table; `payload_type` uses `PAYLOAD_TYPE_PATCH` (`"patch"`) for `OpenCode` unified-diff payloads and `PAYLOAD_TYPE_STRUCTURED` (`"structured"`) for `Claude` `PostToolUse` structured payloads. @@ -101,13 +103,13 @@ - `__sce_migrations`: Per-database migration metadata table created by async shared adapter migration helpers; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. - `CLI generated migration manifest`: Build-time Rust source at `OUT_DIR/generated_migrations.rs` written by `cli/build.rs` from immediate `cli/migrations//*.sql` directories after staging SQL under `OUT_DIR/static/migrations`; constants are named from the database directory (for example `AGENT_TRACE_REPOSITORY_MIGRATIONS`, `AUTH_MIGRATIONS`), sorted by the numeric filename prefix before `_`, and embed staged SQL via `include_str!`. - `sync command deferral` (historical): Former plan/state note that a user-invocable sync command was deferred to `0.4.0`; superseded first by nested `sce trace sync` and now by top-level `sce sync` (see `context/cli/sync-command.md`). Local DB bootstrap and setup-time repository-scoped Agent Trace DB initialization still flow through lifecycle providers aggregated by the setup command, hook runtime still keeps a lazy repository Agent Trace DB fallback for repositories where setup has not run or schema metadata is incomplete, and DB health/repair still flows through the doctor surface. -- `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, async `run_with_retry`, sync `run_with_retry_sync`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. Async database construction and operations use `run_with_retry`; `run_with_retry_sync` remains for pure blocking contexts. +- `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, async `run_with_retry` and non-cancelling `run_with_retry_elapsed`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. Async database construction and operations use `run_with_retry`; there is no synchronous variant. - `setup service orchestration`: Setup execution logic in `cli/src/services/setup/command.rs` that resolves the repository root, always ensures the durable-context baseline, then for normal modes derives a repo-root-scoped `AppContext` from the runtime command context, dispatches `setup` through the static lifecycle provider catalog (config → local_db → auth_db → agent_trace_db → hooks when requested), handles interactive target selection for config asset installation, and emits deterministic success messaging per target. - `setup target flags` and `setup mode contract`: Mutually-exclusive `sce setup` target selectors (`--opencode`, `--claude`, `--pi`, `--codex`, `--all`) force non-interactive `SetupMode::NonInteractive(SetupTarget)` when exactly one is provided; otherwise `SetupMode::Interactive` is the default. `--all` expands to opencode+claude+pi+codex and replaced the removed `--both` flag. - `setup interactive target prompt`: `inquire::Select` flow in `cli/src/services/setup/mod.rs` (`InquireSetupTargetPrompter`) that presents OpenCode, Claude, Pi, Codex, and All (OpenCode + Claude + Pi + Codex) when `sce setup` runs without target flags. - `setup dispatch outcome`: Execution model in `cli/src/services/setup/mod.rs` (`SetupDispatch`) where setup either proceeds with a selected/non-interactive target or exits as cancelled without file changes. Interactive dispatch also carries independent `agent_trace.auto_sync` and `policies.attribution_hooks.enabled` booleans from Yes-default confirmations; non-interactive dispatch carries neither, allowing persistence to preserve existing keys and omissions. - `setup embedded asset manifest`: Compile-time generated file index emitted by `cli/build.rs` into `OUT_DIR/setup_embedded_assets.rs`, embedding bytes from Pkl-generated `OUT_DIR/pkl-generated/config/.{opencode,claude,pi}/**` plus staged `OUT_DIR/static/hooks/**` as deterministic normalized relative-path entries consumed by `cli/src/services/setup/mod.rs`; `OPENCODE_EMBEDDED_ASSETS`, `CLAUDE_EMBEDDED_ASSETS`, and `PI_EMBEDDED_ASSETS` all back live setup targets. The manifest also carries `CODEX_EMBEDDED_ASSETS`, embedding Codex's two Pkl-generated output roots (`config/.agents/**`, `config/.codex/**`) merged by `cli/build.rs` into a build-time-only `OUT_DIR/pkl-generated/config/codex-target/` staging tree so its relative-path entries keep their `.agents/`/`.codex/` prefixes; `SetupTarget::Codex` now backs it as a fourth live setup target via `sce setup --codex`/`--all`, installing directly at the repository root (via `InstallTargetPaths::codex_target_dir()`) since its asset paths already carry their own output-root prefix, unlike the other three targets' single-subdirectory destinations. -- `setup required-hook embedded assets`: Setup-service accessors in `cli/src/services/setup/mod.rs` (`iter_required_hook_assets`, `get_required_hook_asset`) that expose canonical embedded templates for `pre-commit`, `commit-msg`, and `post-commit` without runtime config reads. +- `setup required-hook embedded assets`: Setup-service accessor in `cli/src/services/setup/mod.rs` (`iter_required_hook_assets`) that expose canonical embedded templates for `pre-commit`, `commit-msg`, and `post-commit` without runtime config reads. - `SCE managed block`: The CLI-presence check plus `sce hooks ` invocation in each canonical hook template (`cli/assets/hooks/{pre-commit,commit-msg,post-commit}`), delimited by `# >>> sce managed block (do not edit) >>>` / `# <<< sce managed block <<<` comment markers so the same block content can be embedded inside a foreign hook without disturbing content around it (see `context/sce/setup-githooks-hook-asset-packaging.md`). The block propagates an available `sce` command's exit status by capturing `$?` and calling `exit` explicitly rather than by `exec`, so it terminates the script deterministically even when appended after other content. A pure merge module computes hook install bytes against this marker pair (see `setup hook-merge seam`); both `sce setup --hooks` install (see `setup required-hook install orchestration`) and `sce doctor` hook inspection decide currency against this marker pair rather than whole-file byte comparison, so a hook a repository has extended around the block still reports current. - `setup hook-merge seam`: Pure module `cli/src/services/setup/hook_merge.rs`, covering `pre-commit`, `commit-msg`, and `post-commit`. `merge_or_create_hook(existing: Option<&[u8]>, canonical: &[u8], hook_name: &str) -> Result` returns `canonical` verbatim (`HookMergeKind::Created`) when no hook exists; otherwise it locates the `SCE managed block` marker pair by exact line match. A hook already carrying a balanced marker pair identical to the canonical block returns its bytes unchanged (`AlreadyCurrent`); one whose block differs gets that block spliced in place between the same marker lines, leaving surrounding content untouched (`ManagedBlockReplaced`); a marker-free hook containing the legacy pre-marker guidance URL (`https://sce.crocoder.dev/docs/getting-started#install-cli`) is treated as SCE-owned wholesale and replaced entirely with `canonical` (also `ManagedBlockReplaced`); any other marker-free hook is foreign and kept as an exact byte prefix with the canonical block appended after it (`AppendedToForeign`). An unbalanced or partial marker pair is a hard, deterministic error naming `hook_name`, with no bytes returned. For the `AppendedToForeign` case, `HookMerge.unreachable_block_advisory` is set when the foreign hook's last non-blank, non-comment line sits at zero indentation and starts with `exec ` or `exit` — a narrow heuristic (no shell parsing) flagging that the appended block would never run. This module is pure and filesystem-free per "Unit testing in Nix sandbox"; required-hook install calls it (see `setup required-hook install orchestration`), and doctor hook inspection (`cli/src/services/hooks/lifecycle.rs`, `cli/src/services/doctor/inspect.rs`) also calls it, reporting a hook `Current` only when merging the canonical template into its on-disk bytes is a no-op — including treating an unbalanced or partial marker pair as `Stale` rather than `Unknown`, so `sce doctor --fix` repairs it. - `setup required-hook install orchestration`: Setup-service flow in `cli/src/services/setup/mod.rs` (`install_required_git_hooks`, backed by the rename-injectable `install_required_git_hooks_with_rename`) that resolves repository root + effective hooks directory via git truth, then for each hook computes the bytes to stage with the `setup hook-merge seam` (`hook_merge::merge_or_create_hook`) instead of writing the canonical asset verbatim, reports deterministic per-hook outcomes (`Installed`, `Updated`, `Skipped`) against that merged content plus the executable bit, enforces executable permissions, sets `RequiredHookInstallResult.unreachable_block_advisory` (rendered as a named advisory line) when an appended block would be unreachable, and uses the `setup atomic-swap` policy (see `setup atomic-swap`) — staged content is renamed directly over an existing hook without unlinking it first — with deterministic recovery guidance on swap failure. @@ -121,7 +123,8 @@ - `sce exit-code class contract`: Stable top-level process exit-code mapping owned by `cli/src/app.rs` (`0` success, `2` parse failure, `3` validation failure, `4` runtime failure, `5` dependency failure) so automation can branch on failure class without parsing text errors. - `sce stderr error-code taxonomy`: Stable internal failure diagnostic classes emitted by `cli/src/app.rs` (`SCE-ERR-PARSE`, `SCE-ERR-VALIDATION`, `SCE-ERR-RUNTIME`, `SCE-ERR-DEPENDENCY`) via styled `Error []: ...` stderr formatting; expected catalog failures emit only their redacted, unstyled catalog message. - `class-default Try guidance injection`: `cli/src/app.rs` diagnostic behavior that appends `Try:` remediation text by failure class only for internal failures when an error message does not already include `Try:` guidance. -- `sce observability baseline`: App-runtime logging contract in `cli/src/services/observability.rs` and `cli/src/app.rs` with config-resolved observability inputs, deterministic env-over-config-file-over-default precedence for non-flag logging keys, default-backed `log_dir` / `SCE_LOG_DIR` file routing by machine-local date plus optional session filename partitioning, creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle `event_id` values, tracing for all emitted records, normal logger records routed only to files when `log_to_file=true`, and only error-level logger records routed to `stderr` when `log_to_file=false`. +- `sce observability baseline`: App-runtime logging contract in `cli/src/services/observability.rs` and `cli/src/app.rs` with config-resolved observability inputs, deterministic env-over-config-file-over-default precedence for non-flag logging keys, default-backed `log_dir` / `SCE_LOG_DIR` file routing by machine-local date plus optional session filename partitioning, creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle `event_id` values, allowlisted tracing for all emitted records (registry-classified `event_id` and `log_level` only), normal logger records routed only to files when `log_to_file=true`, and only error-level logger records routed to `stderr` when `log_to_file=false`. +- `tracing Boundary A` / `tracing Boundary B`: Boundary A (`cli/src/services/observability/tracing_boundary.rs`) limits existing `sce`-target tracing events to a registry-classified `event_id` (`TRACING_EVENT_IDS`; unknown or dynamic IDs become `sce.unclassified`), typed `log_level`, enumerated `operation`/`db_name`, counters, and constants; Boundary B (`otel_policy.rs`) is the allowlist for the OTel export surface (target `sce::otel`, enumerated names, counters, closed enum values, no exception text), enforced by `BoundaryBExporter` in `otel_runtime.rs`. Trace-context propagation (`trace_context.rs`) makes a valid W3C `TRACEPARENT`/`TRACESTATE` the remote parent of `sce.command`; absent or malformed values yield an independent root. - `general unexpected user error`: `UserError::UnexpectedFailure` (`general.unexpected_failure`) catalog entry with the fixed sentence `An unexpected error occurred. Check the log files for more details.`. `sce sync` uses it for non-authentication and non-credential-storage failures while preserving the technical source for observability; the message exposes no path or implementation details. - `sce stdout/stderr contract`: App-level stream routing contract in `cli/src/app.rs` where command success payloads are emitted on stdout only, while redacted user-facing diagnostics and text-mode `sce sync` progress are emitted on stderr; JSON sync emits no human progress. - `SCE_LOG_LEVEL`: Optional runtime env key for `sce` observability threshold; allowed values are `error`, `warn`, `info`, and `debug`, defaulting to `error` when unset. @@ -142,9 +145,9 @@ - `sce shell completion contract`: Deterministic CLI completion contract where `sce completion --shell ` emits parser-aligned Bash/Zsh/Fish completion scripts for current top-level commands and supported options/subcommands. - `CLI default path catalog`: Canonical production path-ownership contract in `cli/src/services/default_paths.rs`; it is the shared owner for non-test production CLI path definitions, covering per-user persisted paths plus repo-relative, install, hook, and context-path definitions used by production CLI code; build-time payload paths are `OUT_DIR`-owned and excluded. - `SCE web URL owner`: Rust service module at `cli/src/services/agent_trace.rs`; owns `SCE_WEB_BASE_URL = "https://sce.crocoder.dev"` and helper functions for SCE-owned URL construction, including Agent Trace conversation URLs, persisted Agent Trace trace URLs, Agent Trace session URLs, and setup-created versioned config schema URLs. It does not own the `control_plane_base_url` used by `sce sync`, whose baked default is `https://sce.crocoderlab.dev`. -- `CLI capability traits`: Broad capability seam in `cli/src/services/capabilities.rs` consumed by the borrowed, compile-time-typed `AppContext`. `FsOps`/`StdFsOps` wrap filesystem operations and `GitOps`/`ProcessGitOps` wrap git process execution plus repository-root and hooks-directory resolution; current service internals do not consume them directly yet. -- `FsOps`: Filesystem capability trait in `cli/src/services/capabilities.rs` with `read_file`, `write_file`, `metadata`, and `exists`, implemented in production by `StdFsOps`. -- `GitOps`: Git capability trait in `cli/src/services/capabilities.rs` with `run_command`, `resolve_repository_root`, `resolve_hooks_directory`, and `is_available`, implemented in production by `ProcessGitOps`. +- `CLI capability traits`: Broad capability seam in `cli/src/services/capabilities.rs` consumed by the borrowed, compile-time-typed `AppContext`. `FsOps`/`StdFsOps` is a method-less filesystem capability marker and `GitOps`/`ProcessGitOps` wrap git command execution and availability; current service internals do not consume them directly yet. +- `FsOps`: Filesystem capability trait in `cli/src/services/capabilities.rs` with no methods (the previous unused file operations were removed), implemented in production by `StdFsOps`. +- `GitOps`: Git capability trait in `cli/src/services/capabilities.rs` with `run_command` and `is_available`, implemented in production by `ProcessGitOps`. - `SCE default path policy seam`: Canonical path resolver in `cli/src/services/default_paths.rs` that owns config/state/cache root resolution through an internal `roots` helper seam, named default paths, and an explicit inventory for the current default persisted artifacts (`global config`, `auth tokens`); named DB/log paths include `auth DB`, `local DB`, `Agent Trace DB`, and the default observability log directory. On Linux those defaults resolve to `$XDG_CONFIG_HOME/sce/config.json`, `$XDG_STATE_HOME/sce/auth/tokens.json`, `$XDG_STATE_HOME/sce/auth.db`, `$XDG_STATE_HOME/sce/local.db`, `$XDG_STATE_HOME/sce/repos//agent-trace.db`, and `$XDG_STATE_HOME/sce/logs`, with platform-equivalent `dirs` fallbacks elsewhere. The same module is also the canonical owner for broader production CLI path definitions, including repo-local `context/tmp/` scratch/session access, and is protected by a regression test that fails when new non-test production path literals are introduced outside `default_paths.rs`. - `cli config precedence contract`: Deterministic runtime value resolution in `cli/src/services/config/resolver.rs` with precedence `flags > env > config file > defaults`; `log_dir` resolves through `SCE_LOG_DIR` > config-file `log_dir` > `/sce/logs`. Config discovery order is `--config`, `SCE_CONFIG_FILE`, then default discovered global+local paths, with the canonical schema declaration accepted in parsed config files. - `shared runtime/config primitives seam`: Canonical ownership in `cli/src/services/config/types.rs` for the CLI's shared observability/config enums (`LogLevel`, `LogFormat`, `LogFileMode`), request/response primitives (`ConfigSubcommand`, `ConfigRequest`, `ReportFormat`), source metadata types (`ValueSource`, `ConfigPathSource`, `LoadedConfigPath`, `ResolvedValue`, `ResolvedOptionalValue`), resolved runtime config types (`ResolvedAuthRuntimeConfig`, `ResolvedObservabilityRuntimeConfig`, `ResolvedHookRuntimeConfig`), the `NAME` constant, observability env-key constants, and shared bool parsing helpers; re-exported through `cli/src/services/config/mod.rs` via `pub use types::*` so downstream modules continue importing through `services::config` unchanged. @@ -222,19 +225,16 @@ - `ParseError`: Error type in `cli/src/services/patch.rs` produced when `parse_patch` encounters malformed input (missing file headers, invalid hunk headers, missing `@@` delimiters); carries an actionable `message` field - `FileChangeKind`: Enum in `cli/src/services/patch.rs` classifying a file change as `Added`, `Modified`, `Deleted`, or `Renamed`; serialized as `snake_case` JSON strings - `TouchedLineKind`: Enum in `cli/src/services/patch.rs` classifying a touched line as `Added` or `Removed`; serialized as `snake_case` JSON strings; derives `Hash` to support set-based intersection operations -- `PatchLoadError`: Error type in `cli/src/services/patch.rs` produced when `load_patch_from_json` or `load_patch_from_json_bytes` encounters invalid JSON or a payload that does not match the expected `ParsedPatch` structure; carries an actionable `message` field +- `PatchLoadError`: Error type in `cli/src/services/patch.rs` produced when `load_patch_from_json` encounters invalid JSON or a payload that does not match the expected `ParsedPatch` structure; carries an actionable `message` field - `load_patch_from_json`: Public function in `cli/src/services/patch.rs` that reconstructs a `ParsedPatch` from a JSON string; storage-agnostic entrypoint for callers who have already read serialized JSON content from a database, file, or other source -- `load_patch_from_json_bytes`: Public function in `cli/src/services/patch.rs` that reconstructs a `ParsedPatch` from JSON bytes; bytes-oriented counterpart to `load_patch_from_json` for callers working with raw byte data - `intersect_patches`: Public function in `cli/src/services/patch.rs` that computes target-shaped touched-line intersection between two `ParsedPatch` values; takes `constructed_patch` and `post_commit_patch` as inputs, matches files by post-change path identity (exact `new_path` equality or absolute-vs-relative suffix-equivalent path segments), prefers exact touched-line matching by `kind` + `line_number` + `content`, falls back to historical matching by `kind` + `content` when line numbers drift across intermediate edits, and returns a `ParsedPatch` shaped from `post_commit_patch` file/hunk ranges while inheriting result-hunk `model_id` from matched `constructed_patch` hunk provenance when available; consumed by the active post-commit hook runtime - `combine_patches`: Public function in `cli/src/services/patch.rs` that merges multiple `ParsedPatch` values into one deterministic result with later-input-wins semantics; groups files by `new_path`, deduplicates touched lines by identity (`kind` + `line_number` + `content`) with later patches winning, preserves file metadata and hunk metadata from the last contributing patch, orders files by first encounter and hunks by `old_start`; consumed by the active post-commit hook runtime before intersection - `HunkContributor`: Enum in `cli/src/services/agent_trace.rs` classifying a `post_commit_patch` hunk from the union of direct AI coverage (`intersection_patch = intersect_patches(direct_patch, post_commit_patch)`) and any mutation-derived AI coverage supplied via `AgentTraceEvidence.mutation_ai_patch`: `Ai` (every touched line covered), `Mixed` (a non-empty proper subset covered), `Unknown` (no line covered); with no mutation evidence this reduces exactly to the direct-only slot rule (`Ai` exact line-by-line match, `Mixed` same slot different content, `Unknown` no matching `old_start` slot); serialized as `snake_case` JSON strings - `Conversation`: Struct in `cli/src/services/agent_trace.rs` representing one per-hunk entry in the minimal agent-trace payload, carrying a nested `contributor` object (`type` plus optional `model_id`) plus `ranges`, where the current implementation emits exactly one `{ start_line, end_line }` entry derived from the `post_commit_patch` hunk - `TraceFile`: Struct in `cli/src/services/agent_trace.rs` representing one per-file entry in the minimal agent-trace payload, carrying `path` (from `post_commit_patch`'s `new_path`) plus `conversations` (one per `post_commit_patch` hunk) - `AgentTraceVcs`: Top-level VCS metadata struct in `cli/src/services/agent_trace.rs` carrying `type` and `revision`; builder behavior maps `type` from caller metadata (`AgentTraceMetadataInput.vcs_type`, enum-backed) and maps revision from caller metadata when VCS metadata is present. -- `AgentTrace`: Top-level struct in `cli/src/services/agent_trace.rs` representing the minimal agent-trace payload, carrying top-level `version` (fixed to `0.1.0`, strict numeric `x.y.z`), `id` (UUIDv7 string derived from the same commit-time moment used for `timestamp` in `build_agent_trace(...)`), `timestamp` (caller-provided commit timestamp via `AgentTraceMetadataInput.commit_timestamp`, validated as RFC 3339), optional `vcs` (`Option`, omitted from serialized JSON when `None`), and `files` (`Vec`, one per `post_commit_patch` file); `serde`-serializable with `snake_case` field naming -- `classify_hunk`: Public function in `cli/src/services/agent_trace.rs` implementing the direct-only slot rule (retained primitive; the builder itself classifies through the internal combined direct+mutation line-coverage rule): matches a `post_commit_patch` hunk against `intersection_patch` hunks on `old_start` slot, returning `HunkContributor::Ai` for exact line-by-line match, `Mixed` for same-slot-but-different-content, or `Unknown` when no matching slot exists +- `AgentTrace`: Top-level struct in `cli/src/services/agent_trace.rs` representing the minimal agent-trace payload, carrying top-level `version` (fixed to `0.1.0`, strict numeric `x.y.z`), `id` (UUIDv7 string derived from the same commit-time moment used for `timestamp` in `build_agent_trace_from_evidence(...)`), `timestamp` (caller-provided commit timestamp via `AgentTraceMetadataInput.commit_timestamp`, validated as RFC 3339), optional `vcs` (`Option`, omitted from serialized JSON when `None`), and `files` (`Vec`, one per `post_commit_patch` file); `serde`-serializable with `snake_case` field naming - `AgentTraceMetadataInput`: Metadata input struct in `cli/src/services/agent_trace.rs` that carries `commit_timestamp` (RFC 3339 commit-time value used as `AgentTrace.timestamp`), `commit_revision` (mapped to `AgentTrace.vcs.revision` when VCS metadata is emitted), and optional `vcs_type` (`Option`, mapped to `AgentTrace.vcs.type` and controlling whether top-level `vcs` is emitted); `AgentTraceVcsType` is the schema-aligned `Git`/`Jj`/`Hg`/`Svn` enum serialized as `git`/`jj`/`hg`/`svn`. -- `build_agent_trace`: Public function in `cli/src/services/agent_trace.rs` — the direct-only compatibility path — that delegates to `build_agent_trace_from_evidence` with an empty `mutation_ai_patch`: computes `intersection_patch = intersect_patches(constructed_patch, post_commit_patch)`, iterates over `post_commit_patch` files and hunks, classifies each hunk, validates `AgentTraceMetadataInput.commit_timestamp` as RFC 3339, derives UUIDv7 `AgentTrace.id` from that same commit-time moment, and returns `Result` with top-level metadata fields plus one `Conversation` per `post_commit_patch` hunk; consumed by the active post-commit hook flow, with no standalone `sce agent-trace` command surface. `build_agent_trace_from_evidence(AgentTraceEvidence { direct_patch, mutation_ai_patch }, post_commit_patch, metadata)` is the separated-evidence entrypoint: identical top-level metadata behavior, combined direct+mutation hunk classification, deduplicated direct+mutation session links, and a contributor model only when the present evidence sources agree; top-level `tool` remains derived from the direct intersection only. - `agent-trace plugin diff extraction seam`: Helper `extractDiffTracePayload` in `config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts` that accepts a typed `message` event and returns `{ sessionID, diff, time, model_id }` only for user-role messages with non-empty `summary?.diffs`; it joins present object-entry `patch` fields with `\n`, skips entries without `patch`, returns `undefined` when no usable patches remain, uses `Date.now()` for `time`, and builds `model_id` as `providerID/modelID` from `event.properties.info.model`. - `get_or_create_encryption_key`: Public keyring-backed helper in `cli/src/services/db/encryption_key.rs` that retrieves or generates a 64-character hex encryption key from the OS credential store (macOS Keychain, Linux Secret Service via zbus, Windows Credential Store); uses `keyring_core::Entry` with service name `"sce"` and the database name as username. Actively consumed by `EncryptedTursoDb::new()` via the shared adapter constructor. - `conversation-trace mixed batch`: Rust `sce hooks conversation-trace` STDIN contract accepting `{ payloads: [{ type: "message" | "message.part", ... }] }` with top-level `type` ignored and malformed-item skipping. See `context/sce/agent-trace-hooks-command-routing.md` and `context/sce/agent-trace-db.md`. diff --git a/context/overview.md b/context/overview.md index f502e29fc..1fca904a4 100644 --- a/context/overview.md +++ b/context/overview.md @@ -45,7 +45,7 @@ for `/validate`. See This repository maintains shared assistant configuration for OpenCode, Claude, Pi, and Codex from a single canonical Pkl authoring source. One typed workflow catalog owns the six workflows' shared identity and target routing metadata, while canonical workflow/phase modules own behavior and migrated package-local documents, and target renderers own formatting. Generated target layouts are ephemeral: repository builds consume a pre-Cargo generated payload through `SCE_CLI_GENERATED_INPUT_DIR`, crates.io and Flatpak stage packaging-only fallbacks, and `config/.opencode`, `config/.claude`, `config/.pi`, `config/.agents`, and the generated working-tree SCE config schema are not committed; versioned SCE config schema snapshots live under `schema/v/`. The catalog also marks a workflow `optional` — currently only `brownfield` — which changes nothing about generation and is projected into a generated `config/optional-workflows.json` manifest for install-time consumers. `nix run .#pkl-check-generated` preserves its exact 142-path artifact, metadata/package, phase-reference, internal-reference, optional-workflow-manifest, workflow-orchestration, OpenCode-permission, required-path, and forbidden-path checks while delegating deterministic payload production and inventories to the shared generated-input producer; `nix flake check` runs the same contract. The target matrix contains one manual OpenCode profile plus Claude, Pi, and Codex; the former automated OpenCode profile has been removed. A fourth Pkl renderer, `config/pkl/renderers/codex-content.pkl`, also consumes the same canonical workflow composition to emit skills-only Codex output under `config/.agents/skills/**` (no per-target frontmatter, matching Pi, and no command/prompt layer). Each of the six catalog workflow skills (not `sce-decision`, which has no user-facing entrypoint on any target) additionally carries a Codex-only `agents/openai.yaml`, rendered by `config/pkl/renderers/codex-metadata.pkl` from the shared catalog's `title`/`description` plus an authored `default_prompt`, with `policy.allow_implicit_invocation: false` so these stateful lifecycle workflows activate only from explicit `$sce-` invocation or Codex's `/skills` discovery, never from conversational relevance alone. The Codex renderer also emits a Codex hook registration file (`config/.codex/hooks.json`) and its install-guidance helper (`config/.codex/hooks/run-sce-or-show-install-guidance.sh`). The existing conversation/diff registrations (`UserPromptSubmit`, `Stop`, `PreToolUse` for `Bash`, `PostToolUse` for `apply_patch`) route through `sce hooks codex` and retain their fail-open behavior where designed. The separate mutation-scope registrations route through `sce hooks codex-mutation-scope`: `PreToolUse` and `PostToolUse` use matcher `^(Bash|apply_patch)$`, while `Stop`, `Interrupt`, `SubagentStop`, and `SessionEnd` omit matcher and are unmatched. The tracked mutation `PreToolUse` bootstrap fails closed if Git-root resolution, helper or `sce` discovery, adapter startup, runtime `Start`, or Bash-policy evaluation cannot establish attribution; mutation-scope `PostToolUse` and cleanup hooks do not use this bootstrap behavior. Delegation, MCP, and unknown tools do not match these generated mutation Pre/Post registrations. `sce setup --codex` (and `--all`) now installs it as a fourth `SetupTarget`; setup merges the user-owned `.codex/hooks.json` through the shared structural Codex hook-config service, preserving unrelated valid handlers and rejecting malformed or Codex-invalid documents before the atomic swap. `sce doctor` diagnoses that file per required registration (`PresentAndCurrent`/`Missing`/`Stale`, or `Malformed` for the whole document) rather than by whole-document equality, reporting the mutation-scope registrations under distinct `#(mutation-scope)` rows separate from the `sce hooks codex` rows, and separately reports whether Codex has actually marked each structurally current registration trusted and whether its effective hook-discovery policy allows project hooks, by reading (never writing) Codex's own `$CODEX_HOME/config.toml` and policy state; `sce doctor --fix` repairs structurally unhealthy registrations through the same merge service but never touches trust or policy state (see `context/sce/doctor-human-text-contract.md`). `sce hooks codex` now exists as a typed dispatcher (`cli/src/services/hooks/codex/`): it parses the raw hook JSON into a `CodexHookEvent` and classifies `(hook_event_name, tool_name)` into `UserPromptSubmit`, `Stop`, `PreToolUse(Bash)`, `PostToolUse(apply_patch)`, or a fail-open `NoOp` fallthrough covering every other combination — including `apply_patch` under `PreToolUse`. `UserPromptSubmit` and `Stop` each capture one `messages`/`parts` row (`role="user"`/`role="assistant"`) into the repository Agent Trace DB under the idempotent `cx_` session prefix, `PreToolUse(Bash)` delegates to the existing Bash policy engine, and `PostToolUse(apply_patch)` parses Codex's own `apply_patch` text, resolves paths from the event `cwd` against the real Git root into safe repository-relative paths, normalizes Add/Update evidence into an SCE unified diff under deterministic event-scoped synthetic line identities derived from `tool_use_id`, and persists it as one `diff_traces` row when non-empty (see `context/sce/codex-integration-runtime.md`) — while invalid cwd/path resolution and malformed STDIN payloads fail open with empty stdout, reported model IDs are preserved without fabricated provider prefixes, and missing/blank apply_patch sessions produce no evidence. Delete-File operations and Bash filesystem effects do not become `diff_traces` in this existing conversation/diff pipeline, and its apply_patch evidence has separate operation limitations. Separately, the Codex mutation-scope adapter tracks `Bash` and `apply_patch` executions as `TrackedMutation` scopes; `AiExclusive` means tracked-scope exclusivity, not that Bash authored every mutation in the interval. -It also includes a Rust CLI (`sce`) for Shared Context Engineering workflows: auth, config inspection, setup, doctor, Agent Trace hooks and synchronization, bash-policy evaluation, and repository-scoped Agent Trace storage infrastructure. See `context/architecture.md` for module-level boundaries and `context/context-map.md` for the full domain file index. A new pure, dependency-free `cli/src/services/mutation_trace/` module now exists as a Rust refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol; it currently carries domain types and the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — is registered with `#[allow(dead_code)]`. A separate `cli/src/services/mutation_trace/store.rs` persistence layer (built out by the `mutation-cursor-store-persistence` plan) now provides a real, CAS-guarded database call site against the repository-scoped Agent Trace DB, and a `cli/src/services/mutation_trace/runtime/` coordinator layer (built out by the `mutation-cursor-runtime-coordinator` plan) adds the per-worktree advisory lock, the isolated Git snapshot/ref-pinning service, and a public `coordinate()` entrypoint that drives the protocol against a real worktree under that lock, plus a per-worktree `ref_reconciliation` maintenance pass (built out by the `mutation-cursor-ref-reconciliation` plan) that, only for the namespace of a checkout id a current worktree still derives, reclaims orphaned SCE-owned snapshot refs under that same lock while retaining every tree any durable mutation-cursor state still references (a namespace no current worktree owns — via a deleted linked worktree or checkout-id metadata loss/recreation — is out of reach and left to future repository-scoped work). That runtime now has a second protected entrypoint alongside `coordinate()`: `cli/src/services/mutation_trace/runtime/scope_runtime.rs`'s `abandon_scope()` (built out by the `mutation-scope-runtime-integration` plan) retires a scope whose final worktree boundary was never observed — a dead agent process leaves no `Close` behind — sharing `coordinate()`'s extracted `runtime/protected_worktree.rs` safety prefix but deliberately capturing no Git snapshot, so abandonment is not a `RuntimeBoundary` and needs no Quint change. Both entrypoints are now reachable crate-wide through nine `pub(crate)` re-exports in `runtime/mod.rs` while every module behind them stays private, and the lifecycle contract every current or future harness adapter must uphold is recorded in `context/cli/mutation-scope-runtime.md`. The protocol runtime entrypoints (`coordinate()` / `abandon_scope()`) are now driven by one harness-neutral CLI ingress, the hidden `sce hooks mutation-scope` command (`context/cli/mutation-scope-hook-ingress.md`), which reads a single normalized JSON lifecycle object from STDIN, strictly parses it into exactly `start`/`advance`/`close`/`flush`/`abandon`, maps it to a `RuntimeBoundary` (`start`/`advance`/`close`/`flush`) or an `abandon_scope()` call forwarding `scope_id`/`event_id`/`actor_kind` verbatim (any `worktree_id` key is refused — worktree identity is derived by the runtime from the invoking checkout), and invokes the runtime with a lazy DB provider closure so DB acquisition stays inside the protected-worktree ordering. Unlike the fail-open `diff-trace`/`conversation-trace` intakes, this ingress never discards a valid boundary: it classifies results by durable completion, returning a non-zero `CliError` for a malformed payload or a pre-completion runtime error while treating the two carried-outcome variants (`MarkerClearAfterCommit` / `MarkerClearAfterCompletion` — the durable transition succeeded and only the trailing external-taint marker cleanup failed) as empty-stdout success without retrying the transition. Every successful execution emits empty stdout and writes `mutation_trace_*` rows only. It is the generic transport seam. Concrete harness lifecycle adapters for Claude Code and Codex are wired to it in-process via a `pub(crate)` seam rather than by re-invoking the ingress command; they map each harness's lifecycle events onto one mutation `ScopeId` per mutation-capable tool execution with write-ahead fail-closed `Start`, terminal `Close`, cleanup, recovery barriers, and harness-specific unsupported-boundary handling. A Pi adapter driver (`cli/src/services/hooks/pi_mutation_scope/`) also now consumes this same `pub(crate)` in-process seam through its hidden `sce hooks pi-mutation-scope` command and canonical generated Pi extension, with a `PendingStart`→`Executed`→`Closed`/`PendingAbandon` attempt-phase lifecycle keyed on Pi's `tool_result` event rather than `tool_execution_start` (see `context/cli/pi-mutation-scope-integration.md`); ordinary Pi sessions now reach it, and its human `user_bash` path uses the separate guarded external-mutation supervisor. Codex execution identity and `ScopeId`/`EventId` derivation are owned by `context/cli/codex-mutation-scope-integration.md`. A separate **read-only** consumer of the same module's persisted mutation-event history is wired, though: the post-commit Agent Trace flow calls `mutation_trace::runtime::resolve_post_commit_mutation_ai_patch(...)` to attribute committed touched lines that direct `diff_traces` evidence does not cover, by replaying the newest 128 events for the invoking worktree (bounded also by a commit attribution cut captured under the worktree lock) oldest-to-newest as one causal per-line provenance lineage rather than matching historical patches independently, resolving the worktree's *existing* checkout identity only, creating no identity and writing no mutation-cursor state, and never inserting into `diff_traces` or `post_commit_patch_intersections` (see `context/cli/mutation-trace-agent-attribution.md` and `context/sce/agent-trace-hooks-command-routing.md`). See also `context/cli/mutation-trace-protocol.md`, `context/cli/mutation-trace-runtime-coordinator.md`, `context/cli/mutation-trace-ref-reconciliation.md`, `context/cli/mutation-trace-protected-worktree.md`, `context/cli/mutation-trace-scope-abandonment.md`, `context/cli/mutation-scope-runtime.md`, `context/cli/mutation-scope-hook-ingress.md`, `context/cli/claude-mutation-scope-integration.md`, and `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral process supervisor that lets Pi's `user_bash` mutate a worktree, guarded by the same `WorktreeLock`, without ever creating a scope). +It also includes a Rust CLI (`sce`) for Shared Context Engineering workflows: auth, config inspection, setup, doctor, Agent Trace hooks and synchronization, bash-policy evaluation, and repository-scoped Agent Trace storage infrastructure. See `context/architecture.md` for module-level boundaries and `context/context-map.md` for the full domain file index. A new pure, dependency-free `cli/src/services/mutation_trace/` module now exists as a Rust refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol; it currently carries domain types and the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — is registered without a module-wide dead-code allowance (only declaration-level, reasoned exceptions remain, and snapshot-ref reconciliation is wired explicitly through `sce doctor --fix`; automatic reclamation is a deferred follow-up). A separate `cli/src/services/mutation_trace/store.rs` persistence layer (built out by the `mutation-cursor-store-persistence` plan) now provides a real, CAS-guarded database call site against the repository-scoped Agent Trace DB, and a `cli/src/services/mutation_trace/runtime/` coordinator layer (built out by the `mutation-cursor-runtime-coordinator` plan) adds the per-worktree advisory lock, the isolated Git snapshot/ref-pinning service, and a public `coordinate()` entrypoint that drives the protocol against a real worktree under that lock, plus a per-worktree `ref_reconciliation` maintenance pass (built out by the `mutation-cursor-ref-reconciliation` plan) that, only for the namespace of a checkout id a current worktree still derives, reclaims orphaned SCE-owned snapshot refs under that same lock while retaining every tree any durable mutation-cursor state still references (a namespace no current worktree owns — via a deleted linked worktree or checkout-id metadata loss/recreation — is out of reach and left to future repository-scoped work). That runtime now has a second protected entrypoint alongside `coordinate()`: `cli/src/services/mutation_trace/runtime/scope_runtime.rs`'s `abandon_scope()` (built out by the `mutation-scope-runtime-integration` plan) retires a scope whose final worktree boundary was never observed — a dead agent process leaves no `Close` behind — sharing `coordinate()`'s extracted `runtime/protected_worktree.rs` safety prefix but deliberately capturing no Git snapshot, so abandonment is not a `RuntimeBoundary` and needs no Quint change. Both entrypoints are now reachable crate-wide through nine `pub(crate)` re-exports in `runtime/mod.rs` while every module behind them stays private, and the lifecycle contract every current or future harness adapter must uphold is recorded in `context/cli/mutation-scope-runtime.md`. The protocol runtime entrypoints (`coordinate()` / `abandon_scope()`) are now driven by one harness-neutral CLI ingress, the hidden `sce hooks mutation-scope` command (`context/cli/mutation-scope-hook-ingress.md`), which reads a single normalized JSON lifecycle object from STDIN, strictly parses it into exactly `start`/`advance`/`close`/`flush`/`abandon`, maps it to a `RuntimeBoundary` (`start`/`advance`/`close`/`flush`) or an `abandon_scope()` call forwarding `scope_id`/`event_id`/`actor_kind` verbatim (any `worktree_id` key is refused — worktree identity is derived by the runtime from the invoking checkout), and invokes the runtime with a lazy DB provider closure so DB acquisition stays inside the protected-worktree ordering. Unlike the fail-open `diff-trace`/`conversation-trace` intakes, this ingress never discards a valid boundary: it classifies results by durable completion, returning a non-zero `CliError` for a malformed payload or a pre-completion runtime error while treating the two carried-outcome variants (`MarkerClearAfterCommit` / `MarkerClearAfterCompletion` — the durable transition succeeded and only the trailing external-taint marker cleanup failed) as empty-stdout success without retrying the transition. Every successful execution emits empty stdout and writes `mutation_trace_*` rows only. It is the generic transport seam. Concrete harness lifecycle adapters for Claude Code and Codex are wired to it in-process via a `pub(crate)` seam rather than by re-invoking the ingress command; they map each harness's lifecycle events onto one mutation `ScopeId` per mutation-capable tool execution with write-ahead fail-closed `Start`, terminal `Close`, cleanup, recovery barriers, and harness-specific unsupported-boundary handling. A Pi adapter driver (`cli/src/services/hooks/pi_mutation_scope/`) also now consumes this same `pub(crate)` in-process seam through its hidden `sce hooks pi-mutation-scope` command and canonical generated Pi extension, with a `PendingStart`→`Executed`→`Closed`/`PendingAbandon` attempt-phase lifecycle keyed on Pi's `tool_result` event rather than `tool_execution_start` (see `context/cli/pi-mutation-scope-integration.md`); ordinary Pi sessions now reach it, and its human `user_bash` path uses the separate guarded external-mutation supervisor. Codex execution identity and `ScopeId`/`EventId` derivation are owned by `context/cli/codex-mutation-scope-integration.md`. A separate **read-only** consumer of the same module's persisted mutation-event history is wired, though: the post-commit Agent Trace flow calls `mutation_trace::runtime::resolve_post_commit_mutation_ai_patch(...)` to attribute committed touched lines that direct `diff_traces` evidence does not cover, by replaying the newest 128 events for the invoking worktree (bounded also by a commit attribution cut captured under the worktree lock) oldest-to-newest as one causal per-line provenance lineage rather than matching historical patches independently, resolving the worktree's *existing* checkout identity only, creating no identity and writing no mutation-cursor state, and never inserting into `diff_traces` or `post_commit_patch_intersections` (see `context/cli/mutation-trace-agent-attribution.md` and `context/sce/agent-trace-hooks-command-routing.md`). See also `context/cli/mutation-trace-protocol.md`, `context/cli/mutation-trace-runtime-coordinator.md`, `context/cli/mutation-trace-ref-reconciliation.md`, `context/cli/mutation-trace-protected-worktree.md`, `context/cli/mutation-trace-scope-abandonment.md`, `context/cli/mutation-scope-runtime.md`, `context/cli/mutation-scope-hook-ingress.md`, `context/cli/claude-mutation-scope-integration.md`, and `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral process supervisor that lets Pi's `user_bash` mutate a worktree, guarded by the same `WorktreeLock`, without ever creating a scope). The generated `/next-task` workflow persists task-level context-synchronization lifecycle state in each plan (`pending`, `synced`, or `blocked`) so unresolved task synchronization debt survives a session boundary and gates new implementation. Successful `/next-task` execution hands task synchronization an explicit, pre-edit-Git-baseline-relative changed-file list plus implementation, verification, done-check, plan-update, and context-impact evidence, recorded directly on the completed task (`Completed`, `Files changed`, `Result`, `Verify`, `Context impact`, `Context synchronization`); the five-file root context pass remains mandatory. A later-session sync-debt retry reads that same completed task record directly from the plan by plan path and task ID, with no separate persisted synchronization handoff. `/validate` is validation-only: it runs final checks, writes the Validation Report, and reports `validated`, `failed`, or `blocked` without plan-level context synchronization. @@ -60,7 +60,7 @@ The generated `/next-task` workflow persists task-level context-synchronization - **Setup behavior choices:** interactive setup asks independent Yes-default confirmations for post-commit Agent Trace auto-sync and commit-attribution trailers, persists the answers under `agent_trace.auto_sync` and `policies.attribution_hooks.enabled`, and leaves existing behavior keys/omissions untouched during non-interactive setup. The runtime resolver and hook contracts remain authoritative (see `context/sce/setup-repo-local-config-bootstrap.md`). - **Install channels:** repo-flake Nix, Cargo, npm, and source-built Flatpak (`dev.crocoder.sce`); Homebrew deferred (see `context/sce/cli-first-install-channels-contract.md`). -The CLI crate currently depends on `anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `indicatif`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `turso`, and `uuid`, with target-specific keyring backend dependencies for Linux/FreeBSD, macOS, and Windows. `quint-connect` is the CLI's first dev-only dependency, added for the in-progress `mutation-cursor-quint-connect` model-based-testing harness. +The CLI crate currently depends on `anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `indicatif`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `turso`, `uuid`, and the exact-pinned OTel stack (`opentelemetry`, `opentelemetry_sdk`, `opentelemetry-otlp`, `tracing-opentelemetry`, `tracing-subscriber`), with target-specific keyring backend dependencies for Linux/FreeBSD, macOS, and Windows. `quint-connect` is the CLI's first dev-only dependency, added for the in-progress `mutation-cursor-quint-connect` model-based-testing harness. Its command loop is implemented with `clap` derive-based argument parsing and `anyhow` error handling. Top-level help displays an ASCII art "SCE" banner with a per-column right-to-left color gradient (cyan to magenta when color is enabled, plain ASCII when disabled) above a slim command list without implemented/placeholder labels; `auth` is visible while `hooks` and `policy` remain directly invocable but hidden. The real top-level command catalog/help-visibility contract is centralized in `cli/src/cli_schema.rs` and consumed by `cli/src/command_surface.rs` for custom banner/help rendering plus known-command classification. The runtime includes implemented auth flows (`auth login|logout|whoami`), with authenticated whoami reading the Control Plane `GET /me` profile and rendering flat email/name/role/permissions/organization labels, optional names and missing values handled deterministically, and exact login guidance returned when logged out, alongside config inspection/validation, setup orchestration, doctor diagnosis/repair, attribution-only hooks, shell completion, and the top-level `sync` command. Unknown command paths are successful help requests: top-level unknown tokens use the normal top-level help payload and nested unknown tokens use the closest valid parent's help surface, while unknown options and other parse/validation failures retain their existing errors. Parse-time command conversion plus run-time command handling flow through the internal `RuntimeCommand` seam in `cli/src/app.rs`. The current doctor presentation contract supersedes the earlier output-shape scaffolding wording above: human text uses the compact Environment/Repository/Integrations hierarchy with healthy rows collapsed and unhealthy branches expanded, while JSON retains complete path, identity, problem, and fix-result detail. See `context/sce/doctor-human-text-contract.md`. The command loop now enforces a stable exit-code contract in `cli/src/app.rs`: `2` parse failures, `3` invocation validation failures, `4` runtime failures, and `5` dependency startup failures. Auth command orchestration routes authenticated Control Plane authentication failures, token-storage failures, and approved user-facing fallback failures through typed `CliError::User` catalog entries (`NotAuthenticated`, `AuthStorageUnavailable`, `NotGitRepository`, and `UnexpectedFailure`) while retaining technical sources for observability; missing-credential logout/whoami state queries succeed with their documented text/JSON reports, and internal auth failures remain runtime errors. @@ -77,7 +77,7 @@ The setup service also provides repository-root install orchestration: it resolv The CLI now also applies baseline security hardening for reliability-driven automation: diagnostics/logging paths use deterministic secret redaction,`sce setup --hooks --repo ` canonicalizes and validates repository paths before execution, and setup write flows run explicit directory write-permission probes before staging/swap operations. The config service now provides deterministic runtime config resolution with explicit precedence (`flags > env > config file > defaults`), strict config-file validation (`$schema`, `log_level`, `log_format`, `log_to_file`, `log_dir`, `workos_client_id`, and nested `policies.bash`, `policies.attribution_hooks.enabled`, plus `policies.database_retry` with per-DB `connection_open`/`query` retry policy specs and Agent Trace-only `agent_trace_db.busy_timeout_ms`/`contention_deadline_ms`), deterministic default discovery/merge of global+local config files (`${config*root}/sce/config.json`then`.sce/config.json`with local override, where`config_root` comes from the shared default-path seam with XDG/`dirs::config_dir()` config-root resolution), defaults for the resolved observability value set (`log_level=error`, `log_format=text`, `log_dir=/sce/logs`), shared auth-key resolution with optional baked defaults starting at `workos_client_id`, first-class bash-policy preset/custom parsing with deterministic conflict and duplicate-prefix validation, custom-policy `satisfied_by`wrapper exemption (a policy does not fire when the matched command was unwrapped from a declared wrapper such as`nix shell nixpkgs#ripgrep`), and a canonical Pkl-authored `sce/config.json`JSON Schema generated beneath Cargo`OUT_DIR`and embedded by`cli/src/services/config/mod.rs`for both`sce config validate`and doctor-time config checks. Runtime startup config loading keeps parity with that schema by accepting its`$schema`declaration in repo-local and global config files, so startup commands such as`sce version`no longer fail before dispatch on that field; the canonical declaration is versioned as`"https://sce.crocoder.dev/v/config.json"` using the CLI release version; this schema URL is separate from the `https://sce.crocoderlab.dev` baked default used by `sce sync` for control-plane ingestion. App-runtime observability now consumes flat logging keys through the shared resolver, so env values still override config-file values while config files provide deterministic fallback for `log_dir`; positive-integer `log_file_retention_limit` uses config-file/default precedence, defaults to `10`, and controls creation-triggered cleanup for primary and v2 log files; `sce config show` reports resolved observability/auth/policy values with provenance, while `sce config validate` is now a trimmed validation surface that reports only pass/fail plus validation errors or warnings in text and JSON modes. The canonical preset catalog and matching contract live in `config/pkl/base/bash-policy-presets.pkl` and `context/sce/bash-tool-policy-enforcement-contract.md`. Invalid default-discovered config files now also degrade gracefully at startup: `sce` keeps running with degraded observability defaults, logs `sce.config.invalid_config` warnings, and reserves hard failures for explicit `--config` / `SCE_CONFIG_FILE` targets or other truly invalid runtime observability inputs. -`cli/src/services/config/mod.rs` is now a module facade that declares focused config submodules (`types`, `schema`, `policy`, `resolver`, private `render`, `command`, and `lifecycle`), re-exporting `pub use types::*`and`pub(crate) use schema::validate_config_file`. Shared config primitive ownership is delegated to `cli/src/services/config/types.rs`; schema loading and file parsing to `cli/src/services/config/schema.rs`; bash-policy semantic validation and policy-specific formatting to `cli/src/services/config/policy.rs`; runtime discovery/precedence to `cli/src/services/config/resolver.rs`; and `sce config show`/`sce config validate`text+JSON output construction to`cli/src/services/config/render.rs`. Downstream modules continue importing through `services::config`unchanged. The CLI now has a generic borrowed`AppContext`dependency view in`cli/src/app.rs`; `AppRuntime`owns concrete production logger/telemetry/fs/git dependencies, and command execution receives context views that borrow those dependencies plus an optional`repo_root: Option`. `AppContext::with_repo_root(...)`/`ContextWithRepoRoot`derives repo-root-scoped views while preserving the borrowed runtime dependencies, and command execution is generic over associated-type narrow accessor traits where practical. The broad capability seam lives in`cli/src/services/capabilities.rs`, where `FsOps`/`StdFsOps`wrap filesystem operations and`GitOps`/`ProcessGitOps`wrap git process execution plus repository-root/hooks-directory resolution. +`cli/src/services/config/mod.rs` is now a module facade that declares focused config submodules (`types`, `schema`, `policy`, `resolver`, private `render`, `command`, and `lifecycle`), re-exporting `pub use types::*`and`pub(crate) use schema::validate_config_file`. Shared config primitive ownership is delegated to `cli/src/services/config/types.rs`; schema loading and file parsing to `cli/src/services/config/schema.rs`; bash-policy semantic validation and policy-specific formatting to `cli/src/services/config/policy.rs`; runtime discovery/precedence to `cli/src/services/config/resolver.rs`; and `sce config show`/`sce config validate`text+JSON output construction to`cli/src/services/config/render.rs`. Downstream modules continue importing through `services::config`unchanged. The CLI now has a generic borrowed`AppContext`dependency view in`cli/src/app.rs`; `AppRuntime`owns concrete production logger/telemetry/fs/git dependencies, and command execution receives context views that borrow those dependencies plus an optional`repo_root: Option`. `AppContext::with_repo_root(...)`/`ContextWithRepoRoot`derives repo-root-scoped views while preserving the borrowed runtime dependencies, and command execution is generic over associated-type narrow accessor traits where practical. The broad capability seam lives in`cli/src/services/capabilities.rs`, where `FsOps`/`StdFsOps` form a method-less filesystem marker and `GitOps`/`ProcessGitOps` wrap git command execution and availability. The shared default path service in`cli/src/services/default_paths.rs`is now the canonical owner for production CLI path definitions. It resolves per-user config/state/cache roots through a dedicated internal`roots`seam, exposes the current persisted-artifact inventory (global config and auth tokens), and also defines named DB paths (auth DB, local DB, Agent Trace DB) plus the repo-relative, install, hook, and context-path accessors consumed across current CLI production code. Non-test production modules should consume this shared catalog instead of hardcoding owned path literals. No default cache-backed persisted artifact currently exists, so cache-root resolution remains available without speculative cache-path features and no legacy default-path fallback is supported. The Rust CLI also centralizes SCE-owned web URI construction in`cli/src/services/agent_trace.rs`, with `SCE_WEB_BASE_URL`as the single Rust owner for`https://sce.crocoder.dev` and helpers consumed by Agent Trace conversation URLs, Agent Trace persisted trace URLs, Agent Trace session URLs, and setup-created repo-local config schema URLs. The config resolver separately owns `control_plane_base_url` and its `https://sce.crocoderlab.dev` baked sync default; the two URL owners must not be conflated. The current user-facing synchronization entrypoint is `sce sync`; references to the former nested spelling in historical records do not describe an available command. @@ -117,7 +117,7 @@ The current supported automated release target matrix is `x86_64-unknown-linux-m The CLI now also includes an approved operator-environment doctor contract documented in `context/sce/agent-trace-hook-doctor.md`; the runtime includes `sce doctor --fix` parsing/help, stable problem/fix-result reporting, canonical hook-repair reuse, bounded doctor-owned local-DB directory bootstrap for the missing SCE-owned DB parent path, and target-scoped integration inventory: Claude reports `Plugins`, `Commands`, and `Skills`; OpenCode reports `Plugins`, `Agents`, `Commands`, and `Skills`; Pi reports `Extensions`, `Prompts`, and `Skills`; and Codex reports `Skills` and `Hooks` (see `context/sce/doctor-human-text-contract.md`). Its non-launching post-commit Agent Trace auto-sync fact reports enabled/current, explicit disabled, not-ready, and not-applicable states using canonical managed-block currency and resolved configuration; existing hook remediation and readiness semantics remain unchanged. The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support local or remote sync-mode opens, parent-directory creation, connection setup, synchronous query helpers, embedded migration execution, and shared DB lifecycle helpers. Auth DB persistence uses encrypted `AuthDb = EncryptedTursoDb` and token storage persists credentials through the `auth_credentials` table. Agent Trace persistence uses the sole `RepositoryAgentTraceDb = TursoDb` adapter at `/sce/repos//agent-trace.db`, with a one-file repository schema for `repository_metadata`, `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes/triggers, and no `checkout_id` columns on trace rows. The checkout-scoped `AgentTraceDb = TursoDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook runtime writes nullable event-local diff-trace attribution without a `session_models` API/table dependency. The hooks command surface now also supports concrete runtime subcommand routing (`pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`, `conversation-trace`, and `claude-model-state`) with deterministic argument/STDIN validation; `session-model` is no longer supported. Claude settings install the model-state command for both `SessionStart` and `PostModelSwitch`; compatibility smoke against Claude Code 2.1.251 and immediately older 2.1.250 showed that the older client safely ignores the unknown event, so installation is unconditional without a raised minimum or capability gate. Current runtime behavior keeps commit-msg attribution enabled by default unless explicitly opted out: the attribution gate enables canonical trailer insertion in `commit-msg` only when the staged-diff AI-overlap preflight confirms AI/editor evidence (no trailer is appended when the preflight finds no overlap or encounters any error); `pre-commit`/`post-rewrite` remain deterministic no-ops, `post-commit` requires validated `--remote-url`, threads that URL into the Agent Trace flow, prints it to stderr, remains the active bounded recent-diff-trace intersection path, and after successful Agent Trace persistence optionally launches the detached sync-owned `sync --format json` child when config-file-only `agent_trace.auto_sync` is true; `diff-trace` is the active intake path for parsed STDIN `{ sessionID, diff, time, model_id?, tool_name, tool_version }` payload persistence with optional `model_id`, required non-empty `tool_name`, required nullable/non-empty `tool_version`, `direct > exact transcript > exact session/agent state > bridge-derived chain state > NULL` Claude `model_id` plus direct `tool_version` values (exact local state only; no generic session-model fallback or cache), required `u64` millisecond `time`, same-tool-idempotent stored `session_id` prefixing (`oc_` for OpenCode, `cc_` for Claude, `pi_` for Pi), non-lossy AgentTraceDb `time_ms` conversion, and no parsed-payload artifact fallback. This behavior is documented in `context/sce/agent-trace-hooks-command-routing.md`. The removed `sce hooks claude-capture` raw capture route is documented in `context/sce/claude-raw-hook-capture.md` as a removed feature. - The setup service now also exposes deterministic required-hook embedded asset accessors (`iter_required_hook_assets`, `get_required_hook_asset`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. + The setup service now also exposes deterministic required-hook embedded asset accessor (`iter_required_hook_assets`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. The setup service now also includes required-hook install orchestration (`install_required_git_hooks`) that resolves repository root and effective hooks path from git truth, computes the bytes to stage by merging the canonical hook template with any existing hook (preserving a foreign hook's content as an exact prefix with the SCE managed block appended, or bringing an SCE-owned block current in place) rather than writing canonical bytes verbatim, enforces deterministic per-hook outcomes (`Installed`/`Updated`/`Skipped`) against that merged content, surfaces a deterministic advisory when an appended block would be unreachable, and uses a unified atomic-swap policy that renames staged content directly over existing hooks without unlinking them first, with deterministic recovery guidance on swap failures; this behavior is documented in `context/sce/setup-githooks-install-flow.md`. The setup command parser/dispatch now also supports composable setup+hooks runs (`sce setup --opencode|--claude|--pi|--codex|--all --hooks`) plus hooks-only mode (`sce setup --hooks` with optional `--repo `), enforces deterministic compatibility validation (`--repo` requires `--hooks`; target flags remain mutually exclusive), and emits deterministic setup/hook outcome messaging (`installed`/`updated`/`skipped`); this behavior is documented in `context/sce/setup-githooks-cli-ux.md`. diff --git a/context/patterns.md b/context/patterns.md index 8eacf8926..ccdcec3ca 100644 --- a/context/patterns.md +++ b/context/patterns.md @@ -118,7 +118,7 @@ - Keep production CLI path ownership centralized in `cli/src/services/default_paths.rs`; new non-test path literals or path-shape definitions should be added there as named accessors/constants instead of becoming new path owners in other modules. - Keep SCE-owned web URI construction centralized in `services::agent_trace`; production Rust code should use its helpers instead of repeating `https://sce.crocoder.dev`, host-only variants, or derived path prefixes. -- Prefer localized `#[allow(dead_code)]` on intentionally shared path/setup helper items over file-level dead-code suppression so lint scope stays narrow while keeping catalog seams available to tests and future consumers. +- Dead-code exceptions are declaration-level `#[allow(dead_code, reason = "...")]` only, naming the compatibility, deferral or cross-target reason; never file-wide or module-wide. Unused helpers with no such reason are deleted, not suppressed. - For early CLI foundation tasks, keep the real top-level command catalog/help metadata centralized in one canonical seam (`cli/src/cli_schema.rs` in the current architecture) and let custom top-level help renderers consume that seam instead of maintaining a second parallel command list. - Keep top-level help intentionally curated: command visibility on `sce`, `sce help`, and `sce --help` may differ from parser availability when a command should remain directly invocable but temporarily hidden from operator-facing help. - Keep wrapper-only help rows or banner rendering logic outside the clap catalog, but do not duplicate the real command visibility/purpose metadata in those renderers. @@ -155,7 +155,7 @@ - Route local Turso access through service adapters so command handlers do not expose low-level `turso` API details. New Turso-backed services should build on `cli/src/services/db/mod.rs` (`DbSpec` + `TursoDb`, or `EncryptedTursoDb` when at-rest encryption is required) for runtime, connection, per-database `__sce_migrations` tracking, and migration infrastructure, then expose domain-specific async methods from their own service modules. - For current local DB flows, route initialization through the dedicated adapter (`cli/src/services/local_db/mod.rs`) and invoke it from approved orchestration surfaces such as setup or doctor rather than exposing a partial user command before its contract is approved. - For Turso-backed services with setup/doctor ownership, add service-owned lifecycle providers that reuse shared DB path-health and parent-bootstrap helpers, then register them through `lifecycle_providers()` instead of adding command-local database checks. -- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` for Turso and other async operations; use sync `run_with_retry_sync` only for pure blocking contexts where a Tokio sleep/timeout future cannot be awaited. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. Opt an Agent Trace write into the write-contention retry only when its whole retry unit is replay-safe: a complete `BEGIN IMMEDIATE` → `COMMIT` transaction primitive, or a single statement made idempotent by its SQL (`ON CONFLICT DO NOTHING`, a guarded `UPDATE`) routed through `TursoDb::execute_idempotent_write`; append-only or last-writer statements stay on generic `execute()`. +- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` (cancels the attempt when the timeout elapses) for Turso and other async operations, and `run_with_retry_elapsed` (measures elapsed time without cancelling the attempt) for operations that must lend an exclusive borrow; there is no synchronous retry variant. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. Opt an Agent Trace write into the write-contention retry only when its whole retry unit is replay-safe: a complete `BEGIN IMMEDIATE` → `COMMIT` transaction primitive, or a single statement made idempotent by its SQL (`ON CONFLICT DO NOTHING`, a guarded `UPDATE`) routed through `TursoDb::execute_idempotent_write`; append-only or last-writer statements stay on generic `execute()`. - For SCE operator-health commands, prefer deterministic local diagnostics over implicit pass/fail behavior: report the inspected environment scope, stable problem categories, severity/fixability classes, actionable remediation text, and any path/location facts needed to repair the issue; when repair mode exists, keep outcome vocabulary deterministic and idempotent (`cli/src/services/doctor/mod.rs`, with focused diagnosis/render/fix helpers under `cli/src/services/doctor/`). - For service-owned operator health, keep command modules as thin aggregators over `ServiceLifecycle` providers once a lifecycle slice is wired: providers own diagnosis/fix problem production through narrow capability accessors, while command-specific report builders preserve existing output facts and rendering contracts. - Keep static lifecycle provider-list construction centralized in the lifecycle service layer so doctor/setup choose provider inclusion without maintaining parallel concrete provider lists. @@ -194,7 +194,7 @@ - Pure unit tests should test in-memory logic, parsing, validation, and data transformations without external dependencies; prefer mocking/faking external dependencies over creating real filesystem or database state when a fake is available and sufficient. - In-memory database tests (e.g., `LocalDatabaseTarget::InMemory`) are acceptable for unit tests since they don't touch the filesystem. -- A filesystem-touching `#[cfg(test)] mod tests` inline in the module under test is an established, Nix-sandbox-safe pattern in this codebase when every path is a unique, process-local path under `TMPDIR` (itself a writable per-build directory inside the Nix sandbox). Prefer an RAII `tempfile::TempDir` fixture, which cleans itself up even when a test panics — as `cli/src/services/mutation_trace/store.rs`'s `RepositoryAgentTraceDb`-backed tests, `runtime/tests.rs`, `runtime/protected_worktree.rs`, `runtime/scope_runtime.rs`, `runtime/mutation_attribution.rs`'s real-store-and-`GitSnapshotService` seam test, and `cli/src/services/hooks/mod.rs`'s `mutation_attribution_e2e` real Git + repository-DB post-commit regressions do. Manually composed unique `std::env::temp_dir()` paths are the older form of the same pattern and remain in `cli/src/services/checkout/mod.rs`'s checkout-identity-lock tests and in `runtime/coordinator.rs`, `runtime/worktree_lock.rs`, and `runtime/external_taint.rs`; they leak their directory on a panicking test, so new tests should use `TempDir`. This is not integration-test-only territory in this repository. +- A filesystem-touching `#[cfg(test)] mod tests` inline in the module under test is an established, Nix-sandbox-safe pattern in this codebase when every path is a unique, process-local path under `TMPDIR` (itself a writable per-build directory inside the Nix sandbox). Prefer an RAII `tempfile::TempDir` fixture, which cleans itself up even when a test panics — as `cli/src/services/mutation_trace/store.rs`'s `RepositoryAgentTraceDb`-backed tests, `runtime/protected_worktree.rs`, `runtime/scope_runtime.rs`, `runtime/mutation_attribution.rs`'s real-store-and-`GitSnapshotService` seam test, and `cli/src/services/hooks/mod.rs`'s `mutation_attribution_e2e` real Git + repository-DB post-commit regressions do. Manually composed unique `std::env::temp_dir()` paths are the older form of the same pattern and remain in `cli/src/services/checkout/mod.rs`'s checkout-identity-lock tests and in `runtime/coordinator.rs`, `runtime/worktree_lock.rs`, and `runtime/external_taint.rs`; they leak their directory on a panicking test, so new tests should use `TempDir`. This is not integration-test-only territory in this repository. - Use integration tests instead of this inline pattern where the behavior genuinely cannot be made deterministic and isolated inside the Nix sandbox this way. - Do not depend on a shared or ambient path (`$HOME`, a fixed `/tmp` file name, the repository's own working tree) from a unit test; each test must construct its own unique, self-cleaning path. - When a unit test needs behavior that cannot be made Nix-sandbox-safe this way (for example, network access, or state shared across the whole test binary), delete it from the unit-test suite and reintroduce that coverage later as an integration test instead of keeping ignored tests in-tree. diff --git a/context/plans/cli-async-persistence-cleanup-pr3.md b/context/plans/cli-async-persistence-cleanup-pr3.md new file mode 100644 index 000000000..9bcb7c97d --- /dev/null +++ b/context/plans/cli-async-persistence-cleanup-pr3.md @@ -0,0 +1,303 @@ +# Plan: cli-async-persistence-cleanup-pr3 + +## Change summary + +PR3 is the dead-code elimination and post-async cleanup that follows PR2 (PR #301, branch `remove-tokio-db-runtimes`, still open at head `f8e9c748`, itself based on `single-tokio-runtime`). PR2 removed the database-owned Tokio runtimes and migrated persistence to native async; its Deferred cleanup ledger in `context/plans/cli-async-turso-persistence-pr2.md` hands PR3 the redundant `async` wrappers, `run_with_retry_sync`, the redundant `drop(storage)`, stale `#[allow(dead_code)]` annotations, orphaned test support, the `join_three_to_completion` re-evaluation, the large-future/`recursion_limit = "256"` measurement and the temporary-index limitation. + +An external investigation reported 114 compiler-reported unused items once dead-code suppressions are removed. That report is a seed investigation, not an authoritative inventory and not a deletion quota: its symbol-by-symbol listing is not committed to the repository, and its figures (114 items, 35 files with suppressions, 37 occurrences, 117 later `rg` matches) may use different counting conventions. **T01's reproducible, source-grounded compiler inventory, taken on the exact PR2 baseline, is the authoritative baseline for PR3.** T01 keeps the seed's classification leads (Category A retained `diff_traces` compatibility surface; Category B unwired mutation-cursor ref reconciliation and its dependencies; Category C superseded Agent Trace, repository identity, setup, sync and retry helpers; Category D unused database, path, doctor, hook, capability, configuration and mutation-trace helpers; and fields written but never read) and classifies every reproduced item. T02 deletes the verified Category C/D code and its suppressions, T03 applies the retain-and-defer disposition to the reconciliation cluster, T04 simplifies the remaining async layers, T05 audits blocking sites, future size and temporary-index cleanup, and T06 re-runs the unsuppressed audit and prepares the validation and PR4 handoff. Full validation is a separate `/validate` gate after T06. + +**Stacking.** PR3 is a stacked PR. The working branch `dead-code-blues` currently sits exactly at the PR2 head (`f8e9c748`), so every PR3 diff and review is taken against `remove-tokio-db-runtimes` (`git diff remove-tokio-db-runtimes...HEAD`), never against `main`, so the PR2 migration is not re-reviewed. If PR2 merges first, rebase onto its integration branch and keep the same commit range. + +**Reconciliation default: retain and defer.** Evidence in the current code and context: `pin_tree` is called only from `coordinator.rs`, nothing in production deletes pins, and `ref_reconciliation` has no caller (`mod ref_reconciliation;` is private; `context/cli/mutation-trace-ref-reconciliation.md` says invocation timing belongs to a later "harness-wiring PR"). The implementation and its integration path are already planned in `context/plans/mutation-cursor-ref-reconciliation.md`, and a separate `ref-rec` branch carries related reconciliation work. So pins left by crashed or failed `coordinate()` paths accumulate until a manual `git update-ref -d`; this documented consequence is accepted. PR3 therefore retains the reconciliation implementation and the helper APIs it needs, narrows suppressions to justified declarations, records that it remains unwired, and does not wire it. No new approval and no new ADR are required for this default; durable context and a reference to the existing reconciliation plan suffice. A new user or architecture decision is needed only if T01 finds material new evidence that invalidates retain-and-defer. + +## Acceptance criteria + +How this plan is proven complete. The final `/validate` gate (run after T01–T06 are complete and synchronized) executes these checks and the Full validation list below; T01–T06 prepare evidence and run only focused checks for their own changes, and none claims full repository validation. + +- [x] AC1: T01's reproduced, symbol-level compiler inventory is the authoritative dead-code baseline: it identifies the exact PR2 baseline commit, was produced from unsuppressed compiler diagnostics in an isolated worktree covering the binary and test targets and accounting for `cfg` and supported platforms, records the actual number of unique unused declarations, counts suppression attributes separately from `rg` matches, and explains differences from the seed report's 114 items where evidence exists. Every inventory row has file, category (A/B/C/D or field), usage evidence and disposition. If the original full report is located it is retained as supporting audit evidence; otherwise the plan states that the reproduced inventory supersedes it. No rows were invented to reach 114. + - Validate: inspect the inventory recorded in T01's completion record (or the file it names under `context/`); confirm the baseline commit, per-target diagnostic counts, unique-declaration count, suppression-attribute count versus `rg` match count versus files-with-suppressions count, the explanation of differences from the seed figures, and that each row has symbol, file, production/test/cross-platform/generated usage, decision/context requirement, category, disposition and risk. +- [x] AC2: Verified Category C/D dead code, written-but-never-read fields and obsolete tests are deleted together with their dead callers, constructors, imports and suppressions, and only rows classified safe-to-remove were deleted; no deletion was made on Linux-only evidence for platform-gated code. + - Validate: `git diff --stat remove-tokio-db-runtimes...HEAD -- cli/src`; each deleted symbol has an inventory row with a `delete` disposition; platform-gated symbols (macOS/Windows variants in `default_paths.rs`, `GuardError::UnsupportedPlatform`) were checked against `cfg` and supported targets rather than the Linux diagnostic. +- [x] AC3: The `diff_traces` compatibility contract from `context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md` is intact. These surfaces are intentionally unused during the staged compatibility window, not accidentally dead, and need no production callers: `ingest_diff_traces` and `read_diff_traces_after` still exist; `AgentTraceDiffTraceExportRow` and its SQL (`SELECT_DIFF_TRACES_AFTER_SQL`) and mapping (`diff_trace_export_row_from_turso`) dependencies remain; `IngestionStream::DiffTraces` keeps its wire representation; `/state.cursors.diffTraces` remains required; `streams.diffTraces` remains in `sce sync --format json` as a zero-upload entry (`uploaded: 0`, `batches: 0`, `initialCursor == finalCursor == state.cursors.diffTraces`); the remote sync path uploads only `messages`, `parts` and `agent_traces` and does not read local `diff_traces` rows; local `diff_traces` capture and consumers are unchanged. Any dead-code exceptions for these declarations are declaration-level and cite the decision. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n "ingest_diff_traces|read_diff_traces_after|AgentTraceDiffTraceExportRow|SELECT_DIFF_TRACES_AFTER_SQL|diff_trace_export_row_from_turso|DiffTraces|diffTraces" cli/src` finds each definition and the wire-name/cursor handling, and `git diff remove-tokio-db-runtimes...HEAD -- cli/src` shows no signature, serialization or `/state` DTO change for them; the existing sync tests that assert the three-stream upload, the required `cursors.diffTraces` decode and the zero-upload `streams.diffTraces` JSON entry pass (names confirmed via `-- --list`; zero matches do not count); local hook-ingress, patch-reconstruction, post-commit intersection and attribution consumers of `diff_traces` show no diff. A symbol-existence grep alone is not sufficient. +- [x] AC4: Ref reconciliation is retained and deferred by default: the implementation and required helper APIs are preserved, suppressions are narrowed to justified declarations, it remains unwired, and PR3 introduces no new production behavior for it (no caller, hook, command or doctor wiring). Durable context records that it is unwired, that orphaned snapshot pins can accumulate, and references `context/plans/mutation-cursor-ref-reconciliation.md` as the intended integration path. A different disposition exists only if T01 recorded material new evidence and a user or architecture decision followed. + - Validate: inspect the T03 record; `rg -n "reconcile_worktree" cli/src` shows no non-test caller outside `ref_reconciliation.rs`; `context/cli/mutation-trace-ref-reconciliation.md` and `context/cli/mutation-trace-snapshot-service.md` state the retained-and-unwired status and the orphaned-pin consequence; no duplicate ADR was added to restate an already accepted decision. +- [x] AC5: No blanket or file-wide `#[allow(dead_code)]` remains that hides unrelated declarations; every remaining exception is declaration-level with a stated compatibility, deferral or cross-target reason; no `#[allow(unused)]`, unrelated warning suppression or weakened lint setting was added. + - Validate: `rg -n "allow\(dead_code\)|allow\(unused" cli/src` (via nix) lists only the justified sites recorded in T06; no `#![allow(dead_code)]` at file level; `git diff remove-tokio-db-runtimes...HEAD -- cli/Cargo.toml cli/src/main.rs flake.nix` shows no lint weakening. +- [x] AC6: Two verification modes are kept distinct. In diagnostic mode (isolated worktree, suppressions temporarily removed, binary and test targets compiled), a nonzero exit caused solely by expected dead-code diagnostics is an expected audit outcome, and any other compiler failure is investigated rather than counted as dead code. In normal verification mode (retained narrowly scoped suppressions restored), the binary and test targets compile, clippy passes with warnings denied, and the normal validation checks pass. The final diagnostic-mode audit reports only known, documented retained declarations. + - Validate: the T06 diagnostic-mode output is recorded with its build-target and platform context and every diagnostic maps to a documented exception; no claim of cross-platform safety is derived from the Linux build alone; normal-mode results come from `/validate`. +- [x] AC7: Retry, cancellation, transaction-ownership, adapter/worktree-lease and terminal-sync/credential-persistence behavior is unchanged; `run_with_retry` (cancelling timeout) and `run_with_retry_elapsed` (elapsed measurement, no cancellation) remain distinct; `run_with_retry_sync` is gone only if it had no active consumer. + - Validate: `/validate` runs the complete Rust suite, including `db::transaction_cancellation_tests`, `agent_trace_db::transaction_tests`, the sync `terminal_stream` tests and the Quint MBT suite in `cli/src/services/mutation_trace/mbt`; `git diff remove-tokio-db-runtimes...HEAD -- cli/src/services/resilience.rs` shows no policy change. +- [x] AC8: `join_three_to_completion`/`JoinSlot` is either simplified without changing cooperative stopping, in-flight batch completion, credential-persistence protection, first-observed-error selection, non-`Send` progress reporting or concurrent execution, or is retained with a recorded reason; no `try_join!` cancellation of siblings, no extra spawned tasks. + - Validate: inspect the `cli/src/services/sync/sync.rs` diff and the T04 record; `terminal_stream` tests pass in `/validate`. +- [x] AC9: No dynamic dispatch, `BoxFuture`/`dyn Future`, or nested Tokio runtime is introduced. T05 measured large-future size and `recursion_limit = "256"` pressure (a reduction is not assumed possible; "no reduction available" is an acceptable recorded outcome), and the temporary-index (`TempIndexGuard`) finding is documented; the recursion-limit override is removed only if the pinned compiler and the full checks pass without it. + - Validate: `git diff remove-tokio-db-runtimes...HEAD -- cli/src` searched for `dyn `, `BoxFuture`, `Runtime::new`, `block_on` finds nothing new; the T05 record states the measured future sizes, the measurement method and the decisions on `#![recursion_limit]` and on temporary-index cleanup (default: no automatic sweeper). +- [x] AC10: No broad test restoration occurred (no deleted ordinary test suite was restored for coverage parity) and the Rust diff is net smaller. + - Validate: `git diff --shortstat remove-tokio-db-runtimes...HEAD -- cli/src` shows deletions exceeding insertions; no previously deleted test module or fixture is re-added (`git diff --diff-filter=A --name-only remove-tokio-db-runtimes...HEAD -- cli`). +- [x] AC11: Durable context and docs match the code, including any docs that claim guarded-shell descriptor creation is atomic on every supported platform; the macOS `pipe()`/`fcntl()` race is described as an unresolved limitation, not fixed. + - Validate: inspect `context/cli/mutation-trace-external-mutation-guard.md` and `context/cli/async-persistence-guarantees.md` against `external_mutation_guard.rs` (`pipe2` on Linux, `pipe`+`fcntl` fallback elsewhere); `context/cli/default-path-catalog.md`, `capability-traits.md`, `repository-identity.md`, `agent-trace-sync-command.md` and the context map no longer describe deleted symbols. +- [x] AC12: Validation ownership is respected: T06 records which checks must run during final validation but does not claim they have passed; the plan is marked fully validated only after `/validate` records actual results, including failures. + - Validate: `/validate`'s report lists each Full validation command with its real outcome. + +### Full validation + +Owned by the separate final `/validate` gate (repository workflow `sce-validate`), run only after T01–T06 are complete and every task is `synced`. Record actual results, including failures. Do not mark PR3 fully validated because T06 is complete. + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` (complete Rust suite) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` +- `nix flake check` +- `nix build .#ci-checks` +- `nix run .#pkl-check-generated` (only if a Pkl or `config/lib` input was touched; generated-input and platform checks relevant to the diff) +- `git diff --check` + +### Verification modes + +- **Diagnostic mode (T01, T06, optionally T05):** in an isolated `git worktree`, temporarily remove dead-code suppressions, compile the non-test binary and the test targets (`cargo test --no-run`), capture warnings/errors, identify unused declarations, and record diagnostics with their build-target and platform context. A nonzero exit caused solely by expected dead-code diagnostics (the project compiles with `-D warnings`) is an expected audit outcome, not a verification failure. Other compiler failures are investigated, not counted as dead code. Discard the worktree safely (`git worktree remove`), never with destructive restore commands against the main worktree. +- **Normal verification mode (T02–T05 focused checks and `/validate`):** with the intentionally retained, narrowly scoped suppressions restored, the binary and test targets compile, clippy passes with warnings denied and the normal validation checks pass. Unrelated warnings are never suppressed and compiler lint settings are never weakened. + +### Context sync + +- `context/cli/async-persistence-guarantees.md` (PR3 staging, known limits: future size, temporary index, macOS descriptor race wording) +- `context/architecture.md`, `context/patterns.md` (only where deleted seams are described) +- `context/cli/mutation-trace-ref-reconciliation.md`, `mutation-trace-snapshot-service.md`, `mutation-trace-store.md`, `mutation-trace-runtime-coordinator.md` (retained-and-unwired status, orphaned-pin consequence, cluster ownership and link to `context/plans/mutation-cursor-ref-reconciliation.md`) +- `context/cli/default-path-catalog.md`, `capability-traits.md`, `repository-identity.md`, `agent-trace-sync-command.md`, `sync-command.md`, `cli-command-surface.md`, `service-lifecycle.md` (only for deleted symbols they name) +- `context/context-map.md`, `context/overview.md`, `context/glossary.md` when entries change +- No new ADR for the default retain-and-defer reconciliation disposition (already covered by the accepted plan and context); a new ADR only if T01 finds material new evidence that forces a different architectural decision + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src` dead-code and suppression cleanup; deleting verified-dead production code and the tests/fixtures that only exercised it; `resilience.rs`, `db/mod.rs`, `sync/sync.rs`, service forwarding and hook/lifecycle helper simplification; measurement/audit of futures, `spawn_blocking` sites and temporary Git indexes; durable context updates for the above. +- **Out of scope:** the macOS guarded-shell `pipe()`/`fcntl()` race fix; implementing or wiring ref reconciliation; new mutation-cursor semantics; database schema migrations; retry policy changes; OpenTelemetry (PR4); broad test-suite restoration; new generic execution or persistence abstractions; removal of the `diff_traces` compatibility surface (separate planned phase); full repository validation (owned by the final `/validate` gate). +- **Constraints:** the baseline is PR2 head `f8e9c748` on `remove-tokio-db-runtimes`; all diffs are measured against that branch. All tools run through Nix per `AGENTS.md` (`nix develop -c ./scripts/run-cli-cargo.sh ...`, `nix shell nixpkgs#ripgrep -c rg`); no host `rg`, `cargo`, `python3`. Reproduction and the unsuppressed build run in a disposable `git worktree`/copy, never with destructive restore commands (`git checkout .`, `git reset --hard`) against the main worktree. Repository conventions apply: no code comments except documented-exception reasons required by this plan's suppression policy (user preference is comment-free code, so an exception's reason lives in the attribute's `reason = "..."` string or the owning context doc rather than prose comments — T01 confirms the repo's lint level accepts `reason`), exact-pinned dependency versions unchanged, minimal focused tests only. +- **Constraints (preservation):** keep static dispatch and associated-type capability boundaries; do not add dynamic dispatch; preserve fail-closed recovery in mutation-scope adapters; do not delete platform-gated code on Linux-only evidence; written-but-never-read state fields are removed only after checking serialization, derives, Debug, Drop, tests and protocol transitions. +- **Non-goal:** preserving unused APIs for hypothetical future flexibility, or converting dead production code to `#[cfg(test)]` merely to keep obsolete tests alive. +- **Non-goal:** meeting a deletion count or reproducing the seed report's 114; the reproduced inventory is the baseline and no rows are invented to reach any number. +- **Non-goal:** restoring deleted ordinary test suites for coverage parity. +- **Non-goal:** deleting the reconciliation cluster to shrink the dead-code count, or wiring it into production in PR3. +- **Constraint (compatibility):** the `diff_traces` compatibility contract in `context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md` is preserved; intentionally unused compatibility declarations are retained with narrow exceptions rather than deleted. + +## Assumptions + +- The seed report's figures (114 items, 35 files, 37 occurrences) and the later `rg` count of 117 `allow(dead_code)` matches (35 files on the PR2 head) may use different counting conventions (attributes versus annotated declarations versus compiler-reported items). The original symbol-by-symbol listing is not committed and cannot be assumed reproducible; T01 establishes the authoritative counts and does not invent rows. +- Since PR2 is open and `dead-code-blues` equals the PR2 head, no rebase is needed now; the PR base for review is `remove-tokio-db-runtimes`. +- `reason = "..."` on `#[allow(dead_code, reason = ...)]` is acceptable under the pinned toolchain; if not, the exception is documented in the owning context file and named in T06's audit table instead of in a code comment (rests on the user's recorded no-comments-in-code preference). +- Per the user's prefs, final verification is the `/validate` gate using the commands in Full validation; T02–T06 use only narrow focused checks (`cargo test` filtered by names confirmed via `-- --list`; zero-match filters never count as passing). +- The reconciliation default (retain and defer) follows the existing context statement that invocation timing is a later harness-wiring decision, the existing plan `context/plans/mutation-cursor-ref-reconciliation.md` and the "false retention costs disk, false deletion destroys evidence" bias recorded in `mutation-trace-ref-reconciliation.md`. + +## Task stack + +- [x] T01: `Reproduce the authoritative dead-code baseline and classify it` (status:done) + - Task ID: T01 + - Scope: In — (1) identify the exact PR2 baseline commit (`git rev-parse remove-tokio-db-runtimes`; expected `f8e9c748`) and confirm `git diff remove-tokio-db-runtimes...HEAD -- cli/src` is empty; (2) in a disposable `git worktree`, in diagnostic mode, temporarily remove every dead-code suppression and reproduce the unsuppressed compiler diagnostics, repeating until the diagnostic set is stable; (3) inspect the non-test binary target and the test targets (`cargo test --no-run`) separately and record per-target diagnostics; (4) account for conditional compilation and supported platforms (`cfg(target_os)`, Windows/macOS variants, features) rather than treating the Linux build as complete; (5) produce a symbol-level inventory with file, category, usage evidence (production binary, tests, other supported target/platform, generated code or build tooling, context/ADR requiring retention) and disposition (delete, retain with reason, decision-needed) plus risk; (6) record the actual number of unique unused declarations; (7) count suppression attributes, `rg` matches and files-with-suppressions as separate figures; (8) explain differences from the seed report's 114/35/37/117 figures where evidence is available; if the original full report can be located retain it as supporting audit evidence, otherwise state explicitly that the reproduced inventory supersedes it, and do not invent rows to reach 114. The seed leads guide classification: Category A intentionally retained `diff_traces` compatibility surface (protected list per the accepted decision); Category B unwired mutation-cursor ref reconciliation and its dependencies; Category C superseded Agent Trace, repository identity, setup, sync and retry helpers; Category D unused database, path, doctor, hook, capability, configuration and mutation-trace helpers; and fields written but never read (with serialization/derive/Debug/Drop/protocol checks and a separate list of suspected logic defects). Also record the full reconciliation dependency graph (used only by `ref_reconciliation` and its tests versus also by production, and whether any material new evidence invalidates retain-and-defer), baseline measurements (test list names via `-- --list`, LOC of `cli/src`, per-category removable LOC estimate, future/`recursion_limit` baseline method, current `spawn_blocking` sites), and `run_with_retry_sync` consumers. The inventory is recorded as an appendix of this plan's T01 completion record (not under `context/tmp`). Out — any production or test modification in the real worktree, any deletion, wiring, or lint change, and counting expected dead-code diagnostics as failures. + - Dependencies: none + - Done when: the baseline commit, per-target diagnostics, unique-declaration count, separate suppression/match/file counts and the explanation of differences from the seed figures are recorded; every reproduced item has a row with category and disposition; counts of delete-now / Category A / retained-for-tests-or-cross-platform / decision-needed are recorded; the reconciliation graph, the `run_with_retry_sync` consumer check and the D-category platform variants are resolved; the disposable worktree is removed without touching unrelated changes; the main worktree is unchanged. + - Verify: in the disposable worktree, `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml` and `... test --manifest-path cli/Cargo.toml --no-run` with suppressions removed (nonzero exit from dead-code diagnostics alone is expected; any other failure is investigated); `git status --short` in the main worktree is unchanged; `nix shell nixpkgs#ripgrep -c rg -c "allow\(dead_code\)" cli/src` figures recorded. + - Completed: 2026-10-08 + - Files changed: `context/plans/cli-async-persistence-cleanup-pr3.md` (this record and the T01 inventory appendix); no `cli/` changes + - Result: Authoritative baseline reproduced on PR2 head `f8e9c748` (`cli/src` diff vs `remove-tokio-db-runtimes` empty; branch HEAD `2f073f98` adds only context plan commits). 147 unique unused declarations on the binary target (114 compiler warnings, several grouping multiple symbols), 127 symbol entries on the test target (100 warnings). Classified (corrected after PR review): 5 Category A, 30 Category B, 4 cross-platform retains, 7 fields retained pending defect review, 101 delete-now. See "T01 inventory appendix". + - Verify: unsuppressed `build` and `test --no-run` ran in a disposable worktree with `RUSTFLAGS=--cap-lints warn` (so the `-D warnings` lint level reports every diagnostic instead of aborting); both exited 0 with the diagnostic counts above; main worktree `git status --short` empty before and after; disposable worktree removed; `rg` suppression figures recorded in the appendix; `#[allow(dead_code, reason = "...")]` compiles under `-D warnings` on rustc 1.95.0 (open question resolved). + - Post-review correction (PR #303 review): two inventory rows were corrected without a new compiler run or any claimed test execution. (1) `run_external_mutation_guard` stays a delete of the production wrapper only; its two guarded-shell regression tests are preserved by adaptation, not deleted. (2) `WorktreeLock::acquire` moved from Category B to D (delete in T02): callers verified from source are `acquire_async` and `acquire_inner_async` (git_snapshot, mutation_attribution, protected_worktree, ref_reconciliation) while `acquire` has none; `acquire_inner` stays as the blocking implementation of `acquire_inner_async`. Totals: Category B 31 to 30, delete-now 100 to 101; 101+5+30+4+7 = 147 reconciles to the unique-declaration count. Compiler diagnostic counts (114 / 147 / 100 / 127) are unchanged. + - Context impact: none to root context; the appendix is plan-local evidence consumed by T02-T06. + - Context synchronization: synced +- [x] T02: `Delete verified Category C/D dead code and its suppressions` (status:done) + - Task ID: T02 + - Scope: In — only items T01 classified safe to remove (`delete` disposition with usage evidence), in small module-grouped commits: Agent Trace DB convenience inserts; unused `default_paths.rs` helpers/constants (only platform variants T01 proved unused on every supported target); doctor, mutation-scope adapter, capabilities (`FsOps`/`GitOps` methods and the file-wide `#![allow(dead_code)]`), `db/mod.rs` `QueryRows`/`query`/`query_values`, `HasFs::fs`, `AppContext` accessors, config/command-registry/lifecycle/storage-resolver helpers; superseded Category C items T01 verified against their originating plans/ADRs (agent-trace generator and validators, `repository_identity` dir-segment helpers, setup hook-asset and prompt helpers, `run_current_sync*`/`SyncCommand::execute`, `load_patch_from_json_bytes`, the unused `run_external_mutation_guard` production wrapper and its re-exports (its two guarded-shell regression tests are preserved, see below), the unused synchronous `WorktreeLock::acquire` wrapper, `GuardError::UnsupportedPlatform` only if no supported target needs it); genuinely dead fields (decided per T01's field table); dead tests and test support that only exercised deleted code; the `#[allow(dead_code)]` attributes those removals leave unneeded, then a second diagnostic-mode pass to catch newly exposed unreachable items. Out — Category A items (including all `diff_traces` compatibility declarations), the Category B reconciliation cluster, `run_with_retry_sync` and async wrappers (T04), any state field whose non-use may hide a missing invariant (recorded as a suspected defect instead), any row not classified safe-to-remove, restoring tests. + - Dependencies: T01 + - Done when: all T02-owned deletion candidates are removed, including their unused imports, constructors, callers and suppressions. Items explicitly assigned to later tasks, including `run_with_retry_sync` in T04, remain tracked and do not block T02 completion; broad/file-wide suppressions in the touched files are gone; the production and test targets build in normal mode with `-D warnings`; Category A symbols are untouched; each commit is one module/ownership group; suspected logic defects are listed in the completion record, not deleted; `guarded_shell_supervision_leaves_the_runtime_worker_responsive` and `guarded_shell_streams_output_and_finishes_with_exit_code` still exist in `external_mutation_guard.rs`, drive the real guarded-shell supervisor through `arm_external_mutation_guard(...).await` and `ArmedExternalMutationGuard::exec(...).await` (any `#[cfg(test)]` helper is test-only, not a production abstraction), preserve `GuardEvent::Armed` timing, output event order/contents, exit-code and external-taint marker cleanup assertions and the single-worker Tokio responsiveness assertion, use no mocks bypassing the supervisor, and pass; `acquire_inner_async` and `acquire_inner` are retained unchanged (no change to polling, timeouts or cancellation protection). + - Verify: focused normal-mode checks only: `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml`; `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml ` for touched modules after confirming names with `-- --list`; `git diff --check`; explicitly run both guarded-shell tests by name (confirmed via `-- --list`; a zero-match filter never counts) after the wrapper is removed. + - Completed: 2026-10-08 + - Files changed: `cli/src/**` (about 60 source files across app, agent_trace, agent_trace_db, agent_trace_storage, capabilities, command_registry, config, db, default_paths, doctor, error, hooks, lifecycle providers, mutation_trace, observability, output_format, patch, repository_identity, setup, sync); deleted orphan fixtures `cli/src/services/agent_trace/fixtures/{average_age,file_rename,hello_world,mixed_change,poem_edit,poem_write,text_file_lifecycle}_reconstruction/`; `context/plans/cli-async-persistence-cleanup-pr3.md`. Working tree diff vs HEAD: 116 files, 111 insertions, 3343 deletions. + - Result: Deleted all T02-owned T01 `delete` rows with their callers, constructors, imports, tests and suppressions: Agent Trace DB inserts, `agent_trace.rs` validators/`classify_hunk`/`hunks_match_exactly`/`build_agent_trace` (+ seven golden tests and fixtures), `default_paths` helpers/constants, doctor chain (`inspect_repository_hooks`, `collect_global_state_health`, `collect_hook_health`, `inspect_hook_content_state`, `display_node`, `domain`/`group`/`branch`, `validate_config_file` dependency field and its generic), capabilities `FsOps`/`GitOps` methods (`FsOps` is now a marker trait), `db/mod.rs` `QueryRows`/`query`/`query_values`/encrypted `query`, `AppContext`/`HasFs` accessors, `parse_env` x2 and `from_env_lookup`, `CliError::user` (tests construct `CliError::User { .. source: None }`), `ServiceLifecycle::id` and its five impls, mutation-scope `is_subagent`/`is_clear`/`mark_recovery_pending`/`arm_recovery`/`arm_and_begin_recovery_flush`/MCP helpers, codex `tool_response` field and `resolve_codex_patch_path`, `OutputFormat::parse`, `load_patch_from_json_bytes`, repository-identity dir-segment helpers, setup hook-asset/prompt helpers, `run_current_sync`/`run_current_sync_with_progress`/`SyncCommand::execute`, `run_external_mutation_guard` (production wrapper, non-unix stub and re-exports), `WorktreeLock::acquire`, `MutationLineage::reset_file`/`tracked_paths`, `is_terminal` x2 and `scope_actor`. The two guarded-shell tests were preserved via a test-only `arm_and_exec_guard` helper driving `arm_external_mutation_guard(...).await` and `ArmedExternalMutationGuard::exec(...).await`. Blanket/file-wide dead-code allows were removed everywhere (including `capabilities.rs`, four `*_mutation_scope/mod.rs`, `generated_migrations`, `agent_trace_storage`, `agent_trace_sync`, `bash_policy`, `repository_identity`); the unsuppressed binary and test builds now warn only on retained items. Retained with declaration-level `reason` allows: Category A `diff_traces` declarations (cite the ADR), `PlatformFamily` and `GuardError::UnsupportedPlatform` (cross-platform), `run_with_retry_sync` (T04 removes). Retained under the pre-existing module-level allow on `mutation_trace` in `services/mod.rs`, now carrying a reason (T03 narrows it): the Category B reconciliation cluster, `WorktreeLockInner::path`, and the suspected-defect fields. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` passed (exit 0, normal mode, binary and test targets); unsuppressed `RUSTFLAGS="--cap-lints warn"` build and `test --no-run` (second diagnostic pass, Linux only) report only the retained items above plus a pre-existing test-only `mbt/model.rs` `IgnoredAny` field warning that also exists at baseline; `test ... guarded_shell` ran both guarded-shell tests (2 passed, names confirmed by output); a filtered run of `agent_trace::`, `mutation_trace::lineage`, `mutation_trace::tests`, `error::`, `observability::`, `repository_identity` passed (126 passed, 0 failed); `git diff --check` clean. The full suite was not run (reserved for `/validate`); test count is now 344 (baseline 354). + - Deviations and notes: tests deleted with their code — seven `build_agent_trace` golden tests (evidence-based golden tests remain), `reset_file`, deleted-file `tracked_paths` assertion test, `dir_segment` test; `scope_status_live_and_terminal_table` was reduced to `scope_status_live_table`. `AppContext` fields `fs`/`git` and the `HasFs` associated type remain because `app_support` bounds still use them. `acquire_inner_async` and `acquire_inner` are unchanged. The `hook_dir` module and related `default_paths` constants were deleted after confirming no other users. Suspected logic defects (kept, not deleted): `CoordinateOutcome` fields `worktree_id`, `observed_tree`, `revision`, `evaluation`, `mutation_event`, and the `committed`/`completed` error-payload fields. + - Context impact: yes — context docs naming deleted symbols (default-path catalog, capability traits, repository identity, agent-trace sync command, patch/agent-trace docs) need verification and update. + - Context synchronization: synced +- [x] T03: `Retain and defer ref reconciliation; narrow its dependency-cluster suppressions` (status:done) + - Task ID: T03 + - Scope: In — default disposition is **retain and defer**; no approval gate. (1) Inspect the actual dependency graph from T01. (2) Preserve the reconciliation implementation and the helper APIs it requires. (3) Replace any file-wide suppression with declaration-level exceptions limited to cluster members used solely by reconciliation (`ref_reconciliation`, `list_pins`/`delete_pins*`/`run_ref_mutation`/`assert_pins_are_unchanged_direct_refs`/`PinnedRef`/`PinInventoryError`, `load_mutation_event*`/`load_tree_roots`/`load_all_tree_roots`/`reconstruct_boundary`/`decode_boundary_kind`, and the `WorktreeLockInner::path` field decision; `acquire_inner_async` and `acquire_inner` are live and untouched, and `WorktreeLock::acquire` is no longer in this cluster), each with a `reason` naming the deferral and `context/plans/mutation-cursor-ref-reconciliation.md` as the integration path; delete only members T01 proves are not part of the reconciliation graph; resolve the unused `WorktreeLock` `path` field per T01's field analysis. (4) Record in the ref-reconciliation, snapshot-service, store and runtime-coordinator context that reconciliation remains unwired and that orphaned snapshot pins can accumulate (pins created by `pin_tree` in `coordinator.rs`, no production deleter), and reference the existing reconciliation plan and the `ref-rec` branch as the intended integration path. (5) Keep production wiring outside PR3. Only if T01 recorded material new evidence that invalidates retain-and-defer: stop, record the evidence as a blocker and require a user or architecture decision (and an ADR only then). Do not create an ADR merely to restate the already accepted decision. Correct inaccurate context found on the way (including descriptor-atomicity claims). Out — deleting the reconciliation cluster to reduce the dead-code count, wiring reconciliation into any command/hook/doctor, adding behavior, new retention/compaction design, the macOS race fix. + - Dependencies: T01, T02 + - Done when: retain-and-defer is recorded in durable context with the unwired status, orphaned-pin consequence and plan reference; no reconciliation code is deleted or wired; exceptions are declaration-level with reasons; the `WorktreeLock` `path` field is resolved per T01. + - Verify: `rg -n "reconcile_worktree" cli/src` shows no new caller; targeted tests in `ref_reconciliation`/`git_snapshot`/`worktree_lock` run (names via `-- --list`; zero matches do not count); `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`; context inspection against the code. + - Completed: 2026-10-08 + - Files changed: `cli/src/services/mod.rs`, `cli/src/services/mutation_trace/{store.rs,mbt/model.rs}`, `cli/src/services/mutation_trace/runtime/{coordinator,git_snapshot,ref_reconciliation,scope_runtime,worktree_lock}.rs`, `context/cli/mutation-trace-{ref-reconciliation,runtime-coordinator,snapshot-service,store}.md`, `context/plans/cli-async-persistence-cleanup-pr3.md` + - Result: Removed the module-wide `#[allow(dead_code)]` on `pub mod mutation_trace`. Unsuppressed build then reported 26 warnings, all now covered by declaration-level `#[allow(dead_code, reason = ...)]`: the reconciliation cluster (`ref_reconciliation.rs` x6, `git_snapshot.rs` x10, `store.rs` x13) cites `context/plans/mutation-cursor-ref-reconciliation.md`; the suspected-defect fields (`CoordinateOutcome` x5, `committed`, `completed`) cite this plan; `mbt/model.rs` `WireModelState` (test-only, previously hidden by the module allow) is allowed because `deny_unknown_fields` requires the verification-only fields. `WorktreeLockInner::path` was never read, is outside the reconciliation graph, and was deleted (`acquire_inner_async`/`acquire_inner` untouched). No reconciliation code was deleted or wired. Context: added a "retained and unwired" status with the orphaned-pin consequence to the ref-reconciliation, snapshot-service, store and runtime-coordinator docs; corrected them where they described removed tests (`ref_reconciliation` inline tests, `runtime/tests.rs`). + - Verify: `clippy --all-targets --all-features -- -D warnings` passed (exit 0); `rg reconcile_worktree cli/src` outside `ref_reconciliation.rs` found nothing; `test mutation_trace::runtime` ran 11 passed (includes the `git_snapshot` pin tests; `ref_reconciliation` has no tests, a zero-match filter so not counted as coverage); `cargo fmt` applied; `git diff --check` clean. Full suite not run (reserved for `/validate`). + - Deviations and notes: other docs still mention the removed `runtime/tests.rs` (`mutation-trace-agent-attribution.md`, `mutation-trace-scope-abandonment.md`, `mutation-trace-external-taint.md`, `context/patterns.md`, `context/context-map.md`); left for T06 context cleanup. + - Context impact: yes — reconciliation-status docs updated above; remaining stale `runtime/tests.rs` references noted for T06. + - Context synchronization: synced +- [x] T04: `Simplify post-async wrappers and re-evaluate sync joining` (status:done) + - Task ID: T04 + - Scope: In — remove `run_with_retry_sync` (if T01 confirmed no consumer) and any duplicate helper logic it exposes, extracting shared validation/error formatting only if simpler while preserving the distinct semantics of `run_with_retry` (cancelling timeout: the attempt future is cancelled when the timeout elapses) and `run_with_retry_elapsed` (elapsed measurement, non-cancelling); remove redundant `async { ... }.await` wrappers (for example the storage scope in `sync/sync.rs`), redundant `drop(storage)`, trivial forwarding layers and unnecessary temporaries in `db/mod.rs`, sync, hook and lifecycle helpers; evaluate `join_three_to_completion`/`JoinSlot` and simplify only if cooperative stopping, in-flight completion, credential-persistence protection, first-observed-error selection, non-`Send` progress and concurrent execution are all preserved, otherwise retain and record why. Out — retry policy changes, collapsing the cancelling and non-cancelling retry variants, `try_join!` if it cancels siblings, spawned tasks, new async abstractions, dynamic dispatch, new tests unless a replacement introduces an untested invariant. + - Dependencies: T02 + - Done when: the listed leftovers are removed or recorded as retained with a reason; retry timeout/backoff behavior is unchanged in observable semantics and the cancelling versus non-cancelling distinction is intact; the join decision and rationale are recorded; no nested runtime, `BoxFuture` or `dyn` is introduced. + - Verify: focused normal-mode checks: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml terminal_stream` (confirm names with `-- --list`; zero matches do not count), `... agent_trace_db::transaction_tests`, `... db::transaction_cancellation_tests`; clippy with `-D warnings`; `git diff --check`. + - Completed: 2026-10-08 + - Files changed: `cli/src/services/resilience.rs`, `cli/src/services/sync/sync.rs`, `cli/src/services/db/mod.rs`, `context/plans/cli-async-persistence-cleanup-pr3.md` + - Result: Deleted `run_with_retry_sync` (zero consumers) with its `std::thread` import; `run_with_retry` (cancelling timeout) and `run_with_retry_elapsed` (non-cancelling elapsed measurement) are untouched and still distinct. In `sync/sync.rs` removed the redundant `async { ... }.await` storage scope and `drop(storage)` (storage still outlives the client and drops at function end) and the three redundant `async move { ... }.await` wrappers in the ingest closures. In `db/mod.rs` flattened the redundant `async { ... }.await` wrappers in `ensure_migrations_table`, `is_migration_applied` and `apply_migration`. Net 42 insertions, 135 deletions. + - Join decision: `join_three_to_completion`/`JoinSlot` retained unchanged. `try_join!` would cancel siblings on the first error, which breaks the in-flight batch completion and credential-persistence protection; `join!` would not give first-observed-error selection after all siblings finish; the hand-rolled poller keeps cooperative stopping, non-`Send` progress reporting and concurrent execution without spawned tasks, and is covered by the three `services::sync::sync::tests` cases. No simpler combinator preserves all of those behaviors. + - Verify: `clippy --all-targets --all-features -- -D warnings` passed (exit 0); `test terminal_stream` 2 passed; `test services::sync` 3 passed (terminal-stream stop, discard-after-termination, in-flight credential save); `test agent_trace_db::transaction_tests` 2 passed; `test db::transaction_cancellation_tests` 1 passed; `cargo fmt` applied; `git diff --check` clean; added lines contain no `dyn`, `BoxFuture`, `block_on` or `Runtime::new`. Full suite not run (reserved for `/validate`). + - Deviations and notes: other `async { ... }` blocks in `db/mod.rs` (inside retry operations) were left because they scope borrows or cancellation, not mere forwarding. + - Context impact: minor — `context/cli/async-persistence-guarantees.md` may mention `run_with_retry_sync` or the retained join helper; verify. + - Context synchronization: synced +- [x] T05: `Audit blocking sites, future size and temporary Git indexes` (status:done) + - Task ID: T05 + - Scope: In — review each remaining `spawn_blocking` site against the PR2 retained-worker table in `context/architecture.md`; measure future size and `recursion_limit` pressure for `main.rs`/`app.rs` and the largest async dispatch functions (for example compiler future-size diagnostics or `size_of_val` in a disposable worktree) and record the method and numbers; determine why `#![recursion_limit = "256"]` is needed; measure large-future complexity rather than assuming a reduction is possible, and take only safe reductions of nested async composition or captured state if the measurements show one (recording "no safe reduction" is an acceptable outcome); remove the override only if the pinned compiler and the checks pass without it; assess `TempIndexGuard` (`git_snapshot.rs`) and decide: no automatic sweeper unless a safe ownership/lock protocol already exists (filename or age alone is never evidence of abandonment), otherwise document and defer to a separate PR; fix documentation that overstates guarded-shell descriptor atomicity and describe the macOS `pipe()`/`fcntl()` race as an unresolved limitation, not fixed. Out — new concurrency/ownership protocols, a temp-index sweeper without safe ownership, the macOS race fix, `BoxFuture`/`dyn Future`, test restoration. + - Dependencies: T04 + - Done when: measurements and the keep/remove decision for the recursion limit, the temporary-index decision, and any retained blocking complexity are recorded in durable context; any code change is a safe reduction with passing focused checks; the macOS race is described as an open limitation. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml` (and the same build without the override in a disposable worktree if removal is attempted); clippy with `-D warnings`; context inspection. + - Completed: 2026-10-08 + - Files changed: `context/architecture.md`, `context/cli/async-persistence-guarantees.md`, `context/cli/mutation-trace-external-mutation-guard.md`, `context/plans/cli-async-persistence-cleanup-pr3.md`; no `cli/` changes + - Result: Audit only, no code change. The 14 `spawn_blocking` sites match the architecture retained-worker list. Future sizes measured with `RUSTC_BOOTSTRAP=1 RUSTFLAGS=-Zprint-type-sizes` on a Linux debug build in a disposable worktree: `app::run` 14000 bytes, `main` `block_on` future 14016, the dispatch chain 13952 down to `join_three_to_completion` 11832, with the dependency `DatabaseSyncEngine::open_db` future alone at 12704, so the size is dominated by a third-party future and wrappers add about 50-200 bytes each; no safe reduction without boxing. Removing `#![recursion_limit = "256"]` fails with "queries overflow the depth limit" (increased by 130 computing the layout of `app::run`), so the override is kept. `TempIndexGuard`: no automatic sweeper (no ownership or lock record; name or age is not evidence of abandonment); documented and deferred. Docs that claimed the guarded-shell pipe is created close-on-exec atomically now say atomicity holds only on Linux (`pipe2`) and the `pipe`+`fcntl` window elsewhere is an unresolved limitation. + - Verify: normal-mode `build` passed; `clippy --all-targets --all-features -- -D warnings` passed; the no-override build in the disposable worktree failed as described above and the worktree was removed; `git diff --check` clean; context inspected against `external_mutation_guard.rs` (`pipe2` on Linux, `pipe`+`fcntl` otherwise). Full suite not run (reserved for `/validate`). + - Context impact: yes — `architecture.md`, `async-persistence-guarantees.md` and the external-mutation-guard doc updated above. + - Context synchronization: synced +- [x] T06: `Final cleanup audit and validation preparation` (status:done) + - Task ID: T06 + - Scope: In — in diagnostic mode on a disposable worktree, reproduce the final unsuppressed dead-code diagnostic audit and record that each remaining diagnostic maps to a documented declaration-level exception (Category A `diff_traces`, Category B reconciliation, any cross-target cases); classify and list all remaining suppression sites with reasons; verify the `diff_traces` compatibility declarations and contract (AC3) are intact; record removed symbol and LOC counts (before/after, against T01's inventory) and the intentionally retained code; review cross-platform limitations (Linux-only diagnostics never support a cross-platform safety claim); update durable context (`context/cli/async-persistence-guarantees.md` PR3 staging/known limits, context map and overview); prepare the PR4 (OpenTelemetry) handoff and the reconciliation follow-up reference to `context/plans/mutation-cursor-ref-reconciliation.md`; record which tests and checks must run during final validation by pointing to this plan's Full validation section rather than duplicating it. T06 may run focused checks needed to verify its own changes (for example normal-mode build and the targeted regression filters) but must not run or claim the full repository validation. Out — new cleanup beyond T01's inventory, suppression changes beyond what the audit shows, any new behavior, running the Full validation suite, marking the plan validated. + - Dependencies: T03, T05 + - Done when: the audit table, LOC/declaration deltas, retained exceptions, cross-platform limitations, deferred work, PR4 handoff and the list of checks reserved for `/validate` are recorded; no new unexplained suppression exists; the completion record states that full validation has not yet run. + - Verify: focused only: diagnostic-mode build and `--no-run` test build in a disposable worktree; normal-mode `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml`; `nix shell nixpkgs#ripgrep -c rg -n "allow\(dead_code\)|allow\(unused" cli/src`; `git diff --check`. + - Completed: 2026-10-08 + - Files changed: `context/plans/cli-async-persistence-cleanup-pr3.md`, `context/cli/async-persistence-guarantees.md`, `context/cli/mutation-trace-{agent-attribution,external-taint,scope-abandonment}.md`, `context/context-map.md`, `context/overview.md`, `context/patterns.md`; no `cli/` changes + - Result: Final diagnostic audit (Linux x86_64, rustc 1.95.0, disposable worktree at `e45e7747`, all 44 `allow(dead_code ...)` attributes in 10 files stripped, `RUSTFLAGS=--cap-lints warn`): the binary target reports 32 warnings and the test target 27 (including a test-only `mbt/model.rs` `WireModelState` warning). Every diagnostic maps to a documented declaration-level exception. Audit table: (1) Category A `diff_traces` compatibility: `AgentTraceDiffTraceExportRow`, `SELECT_DIFF_TRACES_AFTER_SQL`, `read_diff_traces_after`, `diff_trace_export_row_from_turso` (`agent_trace_export/mod.rs`), `ingest_diff_traces` (`control_plane.rs`), all citing the ADR and all present with unchanged signatures. (2) Category B reconciliation, citing `context/plans/mutation-cursor-ref-reconciliation.md`: `ref_reconciliation.rs` x6, `git_snapshot.rs` x10 (`FOR_EACH_REF_PIN_FORMAT`, pin helpers, `PinnedRef`, `PinInventoryError`, `parse_pin_line`, `pin_ref_prefix`) and `store.rs` x13; `RECONCILIATION_LOCK_TIMEOUT` is the only one also flagged in the test target. (3) Cross-platform: `PlatformFamily::{Macos, Windows, Other}`, `GuardError::UnsupportedPlatform`. (4) Suspected-defect fields: `CoordinateOutcome::{worktree_id, observed_tree, revision, evaluation, mutation_event}`, `committed`, `completed`. No unexplained suppression exists; the remaining non-dead-code allows are clippy or `unused_imports` pre-existing exceptions. Deltas against PR2 (`remove-tokio-db-runtimes`): `cli/src` 66 files, 301 insertions, 1569 deletions; LOC 56620 to 55352 (-1268); unique unused declarations 147 to 32 binary-target diagnostics; `dead_code` allow attributes 119 matching lines in 35 files to 44 attributes in 10 files; no file-wide `#![allow(dead_code)]` remains; tests 354 to 344. No test module or fixture was added back (`git diff --diff-filter=A --name-only` over `cli` lists only already-reviewed paths: none are test modules or fixtures). No `dyn`, `BoxFuture`, `block_on` or `Runtime::new` was added in the `cli/src` diff; `Cargo.toml` and `flake.nix` carry no lint changes, and the `main.rs` change only drops a dead-code allow. Context: removed stale `runtime/tests.rs` references from the scope-abandonment, agent-attribution, external-taint, context-map and patterns docs, removed the stale module-wide dead-code claim from `overview.md`, and recorded the PR3 outcome and known limits in `async-persistence-guarantees.md`. + - Cross-platform limits: only the Linux build was compiled. macOS/Windows/non-unix code (`PlatformFamily` variants, `GuardError::UnsupportedPlatform`, the macOS `pipe()`/`fcntl()` descriptor race) was reviewed by reading and is not proven by this audit. + - Deferred work and PR4 handoff: (a) wiring ref reconciliation and a pin-retention or compaction design (plan `context/plans/mutation-cursor-ref-reconciliation.md`, `ref-rec` branch); orphaned snapshot pins accumulate until then. (b) The `TempIndexGuard` sweeper needs an ownership or lock protocol. (c) The macOS descriptor race fix. (d) Removal of the `diff_traces` compatibility surface (separate phase). (e) Resolving the suspected-defect fields. PR4 (OpenTelemetry) starts from a single application-owned Tokio runtime with directly awaited persistence; the dispatch future is about 14 KB with `recursion_limit = "256"` required, so instrumentation that nests more async layers should re-measure. + - Verify: diagnostic-mode `build` and `test --no-run` in the disposable worktree both exited 0 with the diagnostics above, and the worktree was removed; normal-mode `build` passed; `rg` listing of `allow(dead_code` and `allow(unused` recorded above; `git diff --check` clean. **Full validation has not run.** `/validate` must run the commands in this plan's Full validation section (complete Rust suite, clippy with `-D warnings`, `nix flake check`, `nix build .#ci-checks`, `git diff --check`; `pkl-check-generated` only if Pkl or `config/lib` inputs changed). + - Context impact: yes — updated as listed under Files changed. + - Context synchronization: synced + +- [x] Final gate (not an implementation task): `/validate` + - Dependencies: T01–T06 complete and `synced` + - Scope: run the repository's existing `sce-validate` workflow against this plan's acceptance criteria and the Full validation commands; record actual results, including failures, in the validation report. + - Done when: the validation report records every Full validation command outcome and each AC1–AC12 result. +## Open questions + +- None blocking. Reconciliation defaults to retain-and-defer without a gate; T01 raises a decision only if it finds material new evidence that invalidates it. +- Whether the original 114-item report can be located is unknown; T01 treats the reproduced inventory as authoritative either way. +- The no-comments-in-code preference conflicts with "every retained exception should explain its reason"; this plan carries reasons in `#[allow(..., reason = "...")]` or the owning context doc. T01 confirms the pinned toolchain accepts `reason`. + +## T01 inventory appendix + +**Baseline.** Branch `dead-code-blues` HEAD `2f073f98`; PR2 base `remove-tokio-db-runtimes` = `f8e9c748`; `git diff remove-tokio-db-runtimes...HEAD -- cli/src` is empty. Toolchain rustc 1.95.0. Platform: Linux x86_64 only; `cfg` for macOS/Windows/non-unix code was reviewed by reading, not compiled. + +**Method.** Disposable `git worktree` (since removed): deleted every `#[allow(dead_code)]`, `#![allow(dead_code)]` and `#[cfg_attr(not(test), allow(dead_code))]`, and dropped `dead_code, ` from two combined allows in `style.rs` (35 files, 119 deleted and 2 modified lines). Built with `RUSTFLAGS=--cap-lints warn` so every lint is reported in one pass. The nonzero-exit-from-lints outcome did not occur because of the cap; no non-dead-code compiler error appeared. + +**Counts.** + +| Figure | Value | +| --- | --- | +| Binary-target warnings | 114 (equals the seed's 114) | +| Binary-target unique unused declarations | 147 (warnings group multiple symbols) | +| Test-target warnings / symbol entries | 100 / 127 | +| Declarations live only through tests | 19 (9 outside reconciliation, 10 pin helpers) | +| `rg "allow\(dead_code"` matching lines | 119 (117 are `allow(dead_code)`, 2 combined in `style.rs`) | +| Files with suppressions | 35 | +| File-wide `#![allow(dead_code)]` | 5 (`capabilities.rs`, four `*_mutation_scope/mod.rs`) | + +The seed's 114 matches the binary-target warning count, not the declaration count; its 37 occurrences and 117 `rg` matches are not reproduced by any single count here (117 is the bare-attribute line count). The original symbol listing was not found, so this inventory supersedes it. No rows were invented. + +**Disposition totals.** delete-now 101 (corrected from 100); Category A retain 5; Category B retain-and-defer 30 (corrected from 31); cross-platform retain 4; fields retained pending defect review 7; decision-needed 0. Test baseline: 354 tests via `-- --list`. LOC of `cli/src`: 56620. + +**Inventory (grouped by file; all symbols listed).** + +| File | Symbols | Cat | Usage evidence | Disposition | Risk | +| --- | --- | --- | --- | --- | --- | +| app.rs | `fs`, `fs`/`git`/`repo_root` (AppContext) | D | no callers in prod or tests | delete | low | +| agent_trace.rs | `FileRead`, `InvalidJson`, `validate_agent_trace_json`, `validate_agent_trace_file`, `classify_hunk`, `hunks_match_exactly` | C | unused in prod and tests | delete | low | +| agent_trace.rs | `build_agent_trace` | C | tests only (`agent_trace/tests.rs`) | delete with its tests | low | +| agent_trace_db/mod.rs, repository.rs | `insert_message_with`, `insert_part_with`, `insert_message`, `insert_part` | D | only each other | delete | low | +| agent_trace_export/mod.rs, agent_trace_sync/control_plane.rs | `AgentTraceDiffTraceExportRow`, `SELECT_DIFF_TRACES_AFTER_SQL`, `read_diff_traces_after`, `diff_trace_export_row_from_turso`, `ingest_diff_traces` | A | protected by the 2026-10-01 diff_traces ADR | retain, declaration-level exception citing ADR | none | +| agent_trace_storage/mod.rs | `resolve_agent_trace_storage_at_state_root` | D | no callers | delete | low | +| capabilities.rs | `read_file`, `write_file`, `metadata`, `exists`, `resolve_repository_root`, `resolve_hooks_directory` | D | trait methods with no callers; file-wide allow | delete (check trait impls/fakes) | medium | +| command_registry.rs, sync/command.rs | `execute` x2 | C | no callers (T02 confirms against originating plan) | delete | low | +| config/types.rs | `parse_env` x2 | D | referenced only by unused `from_env_lookup` and tests of themselves (T02 re-verifies) | delete | low | +| db/mod.rs | `QueryRows`, `query`, `query_values`, `query` | D | no callers | delete | low | +| default_paths.rs | `PersistedArtifactRootKind`, `PersistedArtifactId`, `PersistedArtifactLocation`, `persisted_artifact_locations`, `cache_root`, `git_dir`, `git_hooks_dir`, `git_hook_file`, `git_commit_editmsg`, `opencode_plugin_target`, `skill_tile_relative_path`, `pre_commit_hook_path`, `commit_msg_hook_path`, `post_commit_hook_path`, constants `GIT`, `GIT_COMMIT_EDITMSG`, `HOOKS`, `PRE_COMMIT`, `COMMIT_MSG`, `POST_COMMIT`, `OPENCODE_DIR`, `PLUGIN_FILE`, `PLUGIN_MANIFEST_ENTRY`, `AGENTS_DIR`, `SKILL_DEFINITION` | D | defined and referenced only inside the unused helpers; constants `PRE_COMMIT`/`COMMIT_MSG`/`POST_COMMIT` also appear in `setup/mod.rs` (T02 re-verifies after setup deletion) | delete | low | +| default_paths.rs | `PlatformFamily::Macos`, `Windows`, `Other` | D | constructed only under `cfg(target_os)` in `current_platform_family`; matched in `encryption_key.rs` | retain (cross-platform, not provable on Linux) | high if deleted | +| doctor/mod.rs, inspect.rs, types.rs | field `validate_config_file`, `inspect_repository_hooks`, `collect_global_state_health`, `collect_hook_health`, `inspect_hook_content_state`, `display_node`, `domain`, `group`, `branch` | D | dead chain; the field is read only by the dead functions | delete the chain together | medium | +| error.rs | `user` | D | tests only | delete with tests | low | +| hooks/claude_mutation_scope/events.rs, codex_mutation_scope/events.rs | `is_subagent` x2, `MCP_TOOL_NAME_PREFIX`, `is_mcp_tool_name` | D | no callers | delete | low | +| hooks/*_mutation_scope/state.rs | `mark_recovery_pending`, `arm_recovery`, `arm_and_begin_recovery_flush`, `is_clear` x3 | D | no callers (T02 reviews recovery semantics before deleting) | delete (review recovery semantics) | medium | +| hooks/codex/mod.rs | field `tool_response` | field | serde payload field, never read | delete after checking serde derive and `deny_unknown_fields` | low | +| hooks/codex/apply_patch/path.rs | `resolve_codex_patch_path` | C | no callers | delete | low | +| lifecycle.rs | `id` | D | no callers | delete | low | +| mutation_trace/lineage.rs | `reset_file`, `tracked_paths` | D | tests only | delete with tests | low | +| mutation_trace/runtime/coordinator.rs | fields `worktree_id`, `observed_tree`, `revision`, `evaluation`, `mutation_event` of `CoordinateOutcome`; `committed` | field | never read; error payload carries outcome | retain, suspected defect list | medium | +| mutation_trace/runtime/scope_runtime.rs | field `completed` | field | never read; same error-payload pattern | retain, suspected defect list | medium | +| mutation_trace/runtime/external_mutation_guard.rs | `UnsupportedPlatform` | C | constructed under `cfg(not(unix))` | retain (cross-platform) | high if deleted | +| mutation_trace/runtime/external_mutation_guard.rs | `run_external_mutation_guard` | C | production wrapper with test-only callers; re-exported in `runtime/mod.rs` (post-review correction) | delete the production convenience wrapper (unix `unix_impl` and `cfg(not(unix))` stub) and its re-exports; preserve `guarded_shell_supervision_leaves_the_runtime_worker_responsive` and `guarded_shell_streams_output_and_finishes_with_exit_code` by adapting them to `arm_external_mutation_guard(...).await` and `ArmedExternalMutationGuard::exec(...).await` (a `#[cfg(test)]` helper is allowed) | low | +| mutation_trace/runtime/git_snapshot.rs | `FOR_EACH_REF_PIN_FORMAT`, `list_pins`, `delete_pins`, `delete_pins_inner`, `run_ref_mutation`, `assert_pins_are_unchanged_direct_refs`, `PinnedRef`, `PinInventoryError`, `parse_pin_line`, `pin_ref_prefix` | B | used by `ref_reconciliation` and tests only | retain, narrowed exception (T03) | none | +| mutation_trace/runtime/ref_reconciliation.rs | `RECONCILIATION_LOCK_TIMEOUT`, `ReconciliationReport`, `ReconciliationOutcome`, `ReconcileError`, `reconcile_worktree`, `reconcile_worktree_inner` | B | no caller outside the module (`mod ref_reconciliation;` private) | retain, narrowed exception (T03) | none | +| mutation_trace/runtime/worktree_lock.rs | `WorktreeLock::acquire` (sync wrapper; post-review correction) | D | no production, test or supported-platform callers; reconciliation calls `acquire_inner_async`, not `acquire` | delete in T02 (unused synchronous convenience wrapper) | low | +| mutation_trace/runtime/worktree_lock.rs | field `WorktreeLockInner::path` | B | never read; classified separately from the wrapper | retain; resolved in T03 | low | +| mutation_trace/store.rs | `decode_boundary_kind`, `SELECT_MUTATION_EVENT_SQL`, `SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL`, `SELECT_TREE_ROOTS_BY_WORKTREE_SQL`, `SELECT_ALL_TREE_ROOTS_SQL`, `load_mutation_event`, `load_tree_roots`, `load_all_tree_roots`, `load_mutation_event_active_scopes`, `tree_root_row_from_turso`, `MutationEventRow`, `mutation_event_row_from_turso`, `reconstruct_boundary` | B | only store-internal chain plus reconciliation | retain, narrowed exception (T03) | none | +| mutation_trace/types.rs | `is_terminal` x2, `scope_actor` | D | no production callers; `is_terminal` also exercised in `mutation_trace/tests.rs` (T02 re-verifies) | delete | low | +| observability.rs | `from_env_lookup` | D | no callers | delete | low | +| output_format.rs | `parse` | D | no callers | delete | low | +| patch.rs | `load_patch_from_json_bytes` | C | no callers | delete | low | +| repository_identity/mod.rs | `dir_segment`, `DIR_SEGMENT_SHORT_HASH_LEN`, `repository_dir_segment`, `slugify`, `derive_short_hash` | C | superseded directory-segment scheme; non-method items used by tests only | delete with tests | low | +| resilience.rs | `run_with_retry_sync` | C | zero consumers anywhere in `cli/src` | delete in T04 | low | +| setup/mod.rs, prompt.rs | `RequiredHookAsset`, `get_required_hook_asset`, `setup_prompt_target_label_with_color_policy`, `setup_prompt_title_with_color_policy` x2 | C | superseded setup helpers | delete | low | +| sync/sync.rs | `run_current_sync`, `run_current_sync_with_progress` | C | no callers | delete | low | + +**Reconciliation graph and decision.** `reconcile_worktree` has no caller outside `ref_reconciliation.rs`; its dependency cluster (30 rows after the post-review correction) is used only by itself, `pin_tree` remains called from `coordinator.rs`, and no production code deletes pins. No new evidence invalidates retain-and-defer; no decision is needed. + +**Defects and checks recorded for later tasks.** +- Suspected logic defects (not for deletion): `CoordinateOutcome` fields and the `committed`/`completed` error payloads are carried but never read, so error variants may drop information callers could report. +- `run_with_retry_sync` has zero consumers; T04 owns removal. +- 19 declarations are live only via tests; T02 deletes their tests with them, and T03 keeps the pin helpers. +- `spawn_blocking` baseline (14 sites): `git_snapshot.rs` x2, `protected_worktree.rs` x2, `worktree_lock.rs`, `external_mutation_guard.rs`, `mutation_scope_lock.rs` x3, `db/mod.rs` (encryption key), and the four `*_mutation_scope/state.rs` readers. +- Future-size baseline method (for T05): `#![recursion_limit = "256"]` is in `cli/src/main.rs`; measure with compiler future-size diagnostics in a disposable worktree. +- Platform limits: only the Linux build was compiled. `PlatformFamily` variants and `UnsupportedPlatform` are cfg-gated and must not be deleted on Linux evidence. +- Stale unrelated worktree `.../f4f0ae83-.../scratchpad/wt-t01` at `f8e9c748` exists from another session; left untouched. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-08 + +### Commands run + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (356 passed, 0 failed; includes `db::transaction_cancellation_tests`, `agent_trace_db::transaction_tests`, `services::sync::sync` terminal-stream tests, the new `sync::contract_tests`, and the `mutation_trace::mbt` Quint MBT suite) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` -> exit 0 +- `nix flake check` -> exit 0 (all checks passed; aarch64-darwin, aarch64-linux, x86_64-darwin omitted as incompatible systems) +- `nix build .#ci-checks` -> exit 0 +- `nix run .#pkl-check-generated` -> not run (no Pkl or `config/` input in `git diff remove-tokio-db-runtimes`) +- `git diff --check` -> exit 0 + +### Success-criteria verification + +- [x] AC1: T01 appendix records baseline `f8e9c748`, per-target counts (114/147 binary, 100/127 test), suppression/match/file counts, differences from the seed, and per-row dispositions -> inspected appendix +- [x] AC2: `git diff --shortstat remove-tokio-db-runtimes -- cli/src` -> 67 files, +665/-1575; platform-gated `PlatformFamily` and `GuardError::UnsupportedPlatform` retained +- [x] AC3: all listed declarations present with unchanged wire/DTO shapes; the working-tree diff touches only `control_plane.rs`, `render_sync.rs` and `sync.rs`, adding contract tests (required and out-of-range `cursors.diffTraces` decode, zero-upload `streams.diffTraces` JSON entry, `sync_uploads_only_three_streams_and_reports_zero_upload_diff_traces`) that pass in the full suite; the only non-test signature change is a generic credential-store parameter on private `run_sync_async`/`sync_one_stream` +- [x] AC4: no `reconcile_worktree` caller outside `ref_reconciliation.rs`; ref-reconciliation and snapshot-service docs state retained-and-unwired, the orphaned-pin consequence and the `mutation-cursor-ref-reconciliation` plan; no new ADR +- [x] AC5: no `#![allow(dead_code)]` remains; every `dead_code` allow in the diff carries a `reason`; no lint changes in `cli/Cargo.toml`, `flake.nix` +- [x] AC6: T06 records diagnostic-mode output (Linux x86_64, rustc 1.95.0; 32 binary / 27 test diagnostics mapped to documented exceptions); normal-mode build, clippy and tests pass above +- [x] AC7: full suite, transaction and terminal-stream tests pass; `resilience.rs` diff only removes `run_with_retry_sync` +- [x] AC8: `join_three_to_completion`/`JoinSlot` retained with a recorded reason (T04); sync tests pass +- [x] AC9: no added `dyn`, `BoxFuture`, `Runtime::new` or `block_on` in the `cli/src` diff (grep of added lines empty); T05 records future sizes (~14 KB), method, `recursion_limit = "256"` keep decision and temp-index deferral +- [x] AC10: deletions (1575) exceed insertions (665); `git diff --diff-filter=A --name-only remove-tokio-db-runtimes -- cli` is empty +- [x] AC11: `rg build_agent_trace context` no longer hits `context/sce/agent-trace-minimal-generator.md` (now describes `build_agent_trace_from_evidence`); macOS `pipe()`/`fcntl()` limitation documented as unresolved +- [x] AC12: T06 recorded that full validation had not run; this report lists each command with its real result + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Only the Linux build was compiled; macOS/Windows/non-unix code and the macOS descriptor race are unverified +- Orphaned snapshot pins accumulate until ref reconciliation is wired +- Fixes for the earlier AC3/AC11 failure are uncommitted in the working tree +- Historical plans under `context/plans/` still mention `build_agent_trace` diff --git a/context/plans/mutation-cursor-ref-reconciliation-wiring.md b/context/plans/mutation-cursor-ref-reconciliation-wiring.md new file mode 100644 index 000000000..0483ce975 --- /dev/null +++ b/context/plans/mutation-cursor-ref-reconciliation-wiring.md @@ -0,0 +1,537 @@ +# Plan: mutation-cursor-ref-reconciliation-wiring + +## Change summary + +Make the existing mutation-cursor snapshot-ref reconciler (`runtime/ref_reconciliation.rs`, async `reconcile_worktree`) operational. Today it is "retained and unwired": nothing in production calls it, so pins left by interrupted `coordinate()` paths accumulate forever. + +This plan wires reconciliation in two deliberately different ways: + +- **Actual reconciliation (Git and DB I/O) runs only through `sce doctor --fix`.** A new runtime maintenance entrypoint performs exactly one `WorktreeLock` acquisition, verifies and opens the existing repository Agent Trace DB without side effects, runs the existing reconciliation algorithm and atomic deletion transaction, and records a diagnostic result. +- **The mutation-hook completion path (`Close`/`Flush`) performs only a lightweight advisory check.** It never lists refs, opens the DB, or runs reconciliation. It records that reconciliation is recommended in a small durable state file at most once per advisory window for time-based staleness, and may record again earlier only for a newly recorded explicit-reconciliation failure (see AC5); `sce doctor` (read-only) reports the recommendation and `sce doctor --fix` acts on it. + +**Fully automatic reclamation is not delivered by this PR.** No lifecycle-owned, noncritical-path execution point exists today (see D2), and inline reconciliation on the tool-completion path cannot be bounded. Automatic out-of-band reconciliation is a documented follow-up PR with stated prerequisites. Nothing in this plan or in the synchronized context may claim that orphan pins are reclaimed automatically. + +The change extends existing behavior and preserves the reconciliation algorithm, `GitSnapshotService`, `MutationTraceStore`, `WorktreeLock`, the atomic SHA-conditional ref deletion transaction, the protocol, the Quint model, and mutation-attribution rules. It must never delete a tree required by any repository-wide durable root and must never change the success/failure outcome of a mutation boundary. + +Review history (all resolutions are binding on T01–T06): + +*Round 1 (six findings)* + +1. **Single lock acquisition (P1).** `reconcile_worktree_inner` already acquires `WorktreeLock` (`ref_reconciliation.rs:172`). The reconciler is split into a lock-acquiring wrapper and a shared lock-held implementation. Each maintenance operation acquires the lock exactly once. See D1. +2. **Hook latency (P1).** Resolved definitively in round 2: no reconciliation on the hook path. See D2. +3. **Failure state (P2).** Failures are durably recorded as diagnostics with a failure streak. Round 2 removed automatic retry scheduling because no automatic full pass exists. See D3. +4. **Corrected AC3 (P2).** Two invariants are claimed, not "every durable root is pinned": local consistency (fail closed, delete nothing) and repository-wide deletion safety (never remove another worktree's protecting pin). Reconciliation never repairs another worktree's missing pins. +5. **PR metadata (P2).** The real PR is #305, head branch `ref-recon`, base `fix-logging`. +6. **Database-opening contract (P2).** The existing hook opener is not read-only. Reconciliation uses a dedicated verified-existing-DB opener with no create/migrate/repair/initialize side effects. See D4. + +*Round 2 (three findings)* + +7. **Unbounded inline latency (P1).** A cooperative budget checked between phases cannot bound `git for-each-ref`, DB open, repository-wide root queries, `git update-ref`, or Tokio blocking-worker shutdown, and reconciliation holds the lock that the coordinator waits on for 10 s. Selected: advisory-only hook trigger, reconciliation through `sce doctor --fix`, automatic execution deferred to a follow-up. The 250 ms budget, size/cost gates, and deferral machinery are removed. See D2. +8. **Repeated scans after state-persistence failure (P2).** The reservation-before-expensive-work requirement applies to any *automatic* full pass. Because this PR has none, no reservation infrastructure is built; the requirement is recorded as a binding prerequisite of the follow-up. The advisory path emits a recommendation only after the state write succeeds, and `Completed` vs `CompletedStatePersistFailed` stay distinct for explicit passes. See D2/D3. +9. **Far-future timestamps (P2).** The state file stores only past-event timestamps and never a deadline; eligibility is computed as `now − timestamp ≥ window`, so rereading cannot extend a deadline. A timestamp more than `FUTURE_SKEW_TOLERANCE` ahead of `now` is invalid and is normalized once under `WorktreeLock` and persisted; a timestamp within the tolerance is valid and treated as age 0. See D3. + +*Round 3 (four findings; plan-only corrections, D1–D4 and the T01–T06 structure unchanged)* + +10. **External-taint marker assertion (P2).** `CoordinateError::MarkerClearAfterCommit` means the boundary committed but clearing the marker failed, so the marker may legitimately remain armed. The advisory must never create, clear, or modify the marker; it is not required to leave it unarmed. Normal completion and `MarkerClearAfterCommit` are asserted separately (AC4, S18a, S18b). `coordinator.rs`, `ProtectedWorktree` and external-taint recovery semantics are not changed. +11. **Advisory contention vs coordinator contention (P2).** The advisory runs only after `coordinate()` has returned, so an explicit pass holding `WorktreeLock` can make `coordinate()` itself fail (pre-commit lock timeout) before the advisory is ever reached. Contention is now tested as three distinct scenarios: direct advisory contention (S15), coordinator contention (S16), and the post-coordinate handoff race (S33). See AC12. +12. **Git-ref assertions (P2).** `coordinate()` legitimately captures and pins trees. Tests no longer expect refs to be unchanged across `Close`/`Flush`; they assert that the advisory itself performs no ref creation, update, deletion, or inventory, by comparing against a coordinate-only baseline or an injected advisory seam. See AC2, AC5, S14. +13. **Future-skew contradiction (P3).** D3 accepts timestamps up to `FUTURE_SKEW_TOLERANCE` (5 min) ahead of the injected `now` as age 0, so "no stored timestamp is ever in the future" was wrong. Replaced with the exact tolerance invariant and seven explicit clock cases (S24). See AC13. + +Stacking: + +- Repository: `crocoder-dev/shared-context-engineering` +- Parent: PR #304, branch `fix-logging`. PR #304 is not modified by this plan. +- This work: PR #305, head branch `ref-recon`, base `fix-logging`. No other branch or PR is created. +- Verified at authoring time: PR #304 head `0be74aad5bea5070828846d3277041a4649c9842`; PR #305 head is whatever `ref-recon` points to (it advances with every plan or implementation commit, so T01 re-records the then-current heads rather than relying on a value written here). +- If #304 advances, rebase `ref-recon` onto its latest head and record the exact SHA in the T01 audit findings. + +## Authoring-time observations (to be re-verified by T01) + +Facts the design relies on. T01 must re-read each cited location and either confirm or flag it as invalidating a later task. + +- `ref_reconciliation.rs:159-243` `reconcile_worktree_inner` acquires `WorktreeLock` via `acquire_inner_async` with a fixed 10 s timeout (`:172`), resolves the worktree ID, calls `open_db`, lists pins, loads local and repository-wide durable roots, and deletes through `delete_pins(lock.lease(), …)` (`:233`). The `lock` local is dropped on return. +- `worktree_lock.rs`: the lock is a non-reentrant advisory file lock (`File::try_lock`) on `/sce/mutation-cursor.lock`; `acquire_inner` polls every 100 ms until the deadline, so a zero timeout is exactly one `try_lock` attempt. `WorktreeLockLease` is an `Arc` clone of the lock inner; the file unlocks only when the last clone drops. +- `git_snapshot.rs:281-300` `run_ref_mutation_inner` moves the `WorktreeLockLease` into the `spawn_blocking` closure (`let _lease = lease;`). A dropped caller future therefore cannot release the file lock while the blocking `git update-ref` worker runs. `delete_pins_inner` has a post-preflight seam but `run_ref_mutation` passes a no-op `on_worker_entered`. +- `protected_worktree.rs:14` `WORKTREE_LOCK_TIMEOUT` is 10 s: a competing `coordinate()` fails its boundary (before durable completion, with the existing lock-timeout error) if any holder keeps the lock longer than that. +- `protected_worktree.rs:178-184` `abandon_after_spawn_without_unlock` leaves the lock held for an external-guard child; other lock users must skip or wait per their own policy. +- `hooks/mutation_scope.rs:356-378`: `classify_coordinate` maps `Ok(_)` and `MarkerClearAfterCommit` to `Ok(String::new())` and every other `CoordinateError` to a boundary failure. `drive_mutation_scope` returns after `coordinate_boundary(...).await`, so the coordinate-held lock is released before any code placed after it runs. +- `hooks/runtime.rs:53-69` `open_agent_trace_db_for_hook_runtime` → `agent_trace_storage/mod.rs:66-74, 195-207` `open_repository_db_for_hook_runtime`: (a) `TursoDb::open_without_migrations_at` (`db/mod.rs:856-860`) calls `ensure_db_parent_dir` and `turso::Builder::new_local(...).build()`, which can create parent directories and an empty DB file; (b) on schema-not-ready it calls `repair_missing_repository_schema_migration_metadata` (`agent_trace_db/repository.rs:136`), which inserts the baseline migration row; (c) it then calls `verify_or_initialize_repository_metadata` (`repository.rs:171`), which inserts repository metadata when absent. It avoids schema migrations but is **not** read-only. +- Pin count is not bounded by orphan count: every retained historical tree root keeps its pin (`A → B → C → D` retains all four), so inventory and durable-root load cost grow with history. A full pass is O(history), not O(orphans). +- Tokio runtime shutdown joins in-flight `spawn_blocking` workers, so dropping a future (for example through `tokio::time::timeout`) does not shorten process latency for work already in a blocking worker. +- `resolve_git_dir` and `resolve_worktree_id` spawn `git` subprocesses (`runtime/git_snapshot.rs`). Any hook-path advisory that called them would add a new subprocess to the completion path. +- Candidate execution points for out-of-band work found in the tree: the Git `post-commit` subcommand (`hooks/commit_hooks.rs:114`), per-harness session-end/stop events (for example `hooks/claude_mutation_scope/lifecycle.rs`, `hooks/codex/stop.rs`, Codex session-end fixtures), `sce setup` lifecycle, and the doctor command (`doctor/mod.rs`, `doctor/fixes.rs`). None is a lifecycle-owned, noncritical-path executor that is shared across harnesses; see D2. + +## Design decisions + +- **D1 — Single lock acquisition and ownership (round 1).** + - `ref_reconciliation.rs` is split into: (a) `reconcile_worktree` / `reconcile_worktree_inner`, kept with unchanged signatures and behavior, which acquire `WorktreeLock` once and delegate; and (b) a new lock-held implementation `reconcile_with_held_lock` that borrows the held `WorktreeLock`, never calls `acquire_inner_async`, and derives deletion leases from the held lock via `lock.lease()`. + - There are exactly two maintenance operations, each with its own single lock acquisition and neither calling the other: + - `reconcile_explicit` — acquires the lock once with the existing bounded 10 s wait, then runs (b), then records the diagnostic result in the state file while still holding the same lock. It never calls the lock-acquiring wrapper (a). + - `advise_if_due` — hook-path advisory; acquires the lock once with a zero timeout (single `try_lock`), reads/normalizes/writes the state file, releases. It never calls (a) or (b) and has no DB or Git parameters. + - State reads that decide a write and the write itself happen under the same single acquisition. An unlocked read is permitted only as an advisory fast exit (and for read-only `sce doctor` reporting) and is always rechecked under the lock before any write. + - The `WorktreeLock` stays owned by the explicit pass for its whole duration. Mutating Git work receives `lock.lease()` and, as today, moves it into the blocking worker, so cancellation of the caller future cannot release the lock while a Git ref mutation worker is still running. + - The hook ingress invokes `advise_if_due` only after `coordinate_boundary(...)` has returned, so the mutation coordinator's own lock is never held during the advisory. + - No reentrant lock, no lock-passing between processes, no global lock registry. + - Documented concurrency contract: an explicit pass holds the lock for its whole duration because the pin-then-root ordering that makes deletion safe requires it (inventory, root reads, and deletion must not interleave with `coordinate()`'s pin-then-CAS). A mutation boundary arriving meanwhile waits up to `WORKTREE_LOCK_TIMEOUT` (10 s) and then fails *before durable completion* with the existing lock-timeout error. This is an operator-initiated, documented trade-off of `sce doctor --fix`; the hook-path advisory never waits on the lock. +- **D2 — Hook-path trigger is advisory-only; reconciliation is explicit (round 2, resolves P1).** + - *Why inline reconciliation is rejected.* The hook completion path cannot bound `git for-each-ref`, DB open, repository-wide durable-root queries (they scale with historical evidence), or a started `git update-ref` (which must complete safely), and Tokio shutdown waits for blocking workers. A size cap, a historical-cost gate, or a cooperative between-phase budget only predicts cost; none gives an upper bound. Wrapping any step in a timeout would either abort unsafe Git work or fail to shorten latency. Additionally, reconciliation holds `WorktreeLock`, so a slow pass can push an unrelated boundary past its 10 s lock timeout. Cooperative budgets, caps, cost ceilings, deferral windows, and any 250 ms promise are therefore **removed from the design**, and no replacement timeout is introduced. + - *Execution-point investigation.* Alternatives considered and rejected (T01 re-verifies each claim): + 1. Git `post-commit` subcommand — synchronous inside `git commit`, which an agent tool can invoke, so it is the same critical path with the same unbounded work. + 2. Per-harness session-end/stop events — not part of the shared mutation-scope ingress, differ per harness (adapter changes are out of scope), still synchronous hook processes that block session shutdown, and have no uniform lifecycle owner. + 3. `sce setup` / lifecycle service — install-time, not recurring. + 4. Detached Tokio task, fire-and-forget subprocess, background daemon — prohibited (a short-lived hook process cannot own them; no supervision, ownership, or failure recovery exists). + 5. `sce doctor --fix` — operator-owned, foreground, can report its true result. **Selected.** + No suitable lifecycle-owned noncritical-path mechanism exists in the current architecture. + - *Selected design.* + - `Close`/`Flush` (durably completed, including the `MarkerClearAfterCommit` case, after `coordinate_boundary` returned) calls `advise_if_due`, a synchronous function with this complete I/O budget: one capped read (≤ `MAINTENANCE_STATE_MAX_BYTES`) of the state file, at most one zero-wait `try_lock`, at most one atomic write of that small file, all local. It performs no `git` subprocess, no DB open or query, no ref listing, no `spawn_blocking`, no await point after the lock is taken. It is a bounded set of small local file operations of the same class the hook path already performs for the lock and taint marker; it is not claimed to be hard real-time, and any unexpected failure is swallowed into a logged outcome. + - The advisory must not add a new Git subprocess. It obtains the worktree's Git dir without spawning Git (T01 determines whether it can be derived from the filesystem — `.git` directory, or `gitdir:` file for linked worktrees — and proves equality with `resolve_git_dir` for the main and a linked worktree, or whether `coordinate`'s already-resolved value can be reused without protocol changes). **Decision rule:** if neither is possible without a new Git subprocess, the hook trigger is dropped entirely and `sce doctor` computes the recommendation from the state file alone; AC1/AC5 are then satisfied with zero hook callers and T03 reduces to documenting that outcome. This fallback needs no user confirmation because it only removes a convenience signal. + - `advise_if_due` outcomes: absent/invalid-version/oversized/corrupt state → write an anchor and return `Anchored` (no advice). Reference time (`last_success`, else `anchor`) older than `RECONCILIATION_ADVISORY_AFTER` and no valid advice within the same window → persist `last_advised`, then (only after the write succeeded and its persistence was confirmed) return `Advised`. `AdvisedDurabilityUncertain`: the atomic replacement happened (the advisory state may be visible) but crash durability could not be confirmed; an actionable diagnostic is emitted and durability is never claimed. `StateWriteFailed` means no confirmed successful advisory-state update and preserves the `PersistFailure`: with `NotApplied` the replacement did not occur and the previous bytes are intact; with `DurabilityUncertain` (only for an `Anchored` or `NoAction` write, i.e. an anchor or normalization) the replacement occurred, the new state may be visible, crash durability is unconfirmed, and no advice is emitted. A due `Advised` decision with `NotApplied` is `StateWriteFailed`; with `DurabilityUncertain` it is `AdvisedDurabilityUncertain` (an actual recommendation was due and the advice-state replacement occurred). `StateWriteFailed` never emits a recommendation and no scan runs. `StateUnavailable`: the maintenance state cannot be accessed or evaluated (unresolvable Git dir, unreadable state, lock I/O error); it is not a write outcome, emits no advice, and has no retry loop. `NoAction` (nothing due, no write) and `Busy` (zero-wait lock contention, skipped, not an error) emit no actionable advisory. + - *Cadence.* The normal time-based reminder for an unresolved condition repeats at most once per `RECONCILIATION_ADVISORY_AFTER` (24 h) window per worktree. The one exception is a newly recorded explicit reconciliation failure (`last_attempt` newer than `last_advised`): it may trigger one immediate advisory inside an otherwise active 24 h window, is suppressed once `last_advised` records it, and then follows the normal 24 h cadence until a successful explicit pass clears it. The advisory is therefore not unconditionally one-per-window. + - `sce doctor` (read-only) reports whether reconciliation is recommended (last success age, last attempt/failure streak, last advice) computed from the state file as a pure function; `sce doctor --fix` runs `reconcile_explicit` regardless of any state. + - *Deferred to a follow-up PR (explicitly not in this plan).* Fully automatic out-of-band reconciliation. Binding prerequisites for that PR: (1) a lifecycle-owned execution point with defined ownership, lifetime, scheduling and failure recovery (for example an explicitly invoked `sce` maintenance command run by a user-managed scheduler, or a future shared session-end mechanism) — never a detached task, daemon, or timeout-cancelled Git operation; (2) a **durable next-attempt reservation persisted under `WorktreeLock` before opening the DB, listing pins, or loading durable roots**, bounded, never interpreted as success, superseded by the success cadence after success or exponential failure backoff after failure, and bypassed by explicit repair; if the reservation cannot be persisted the automatic pass is skipped and the maintenance-state failure is reported; (3) distinct outcomes for completed, completed-but-state-not-persisted, failed, and skipped/deferred; (4) bounded or split lock-hold analysis against the 10 s coordinator timeout; (5) the same invalid-timestamp normalization as D3. T06 records this follow-up in context. + - *Trade-offs.* (+) hook completion path has no unbounded work and no new failure mode; correct under every ordering with `coordinate()`; no new infrastructure without a consumer. (−) orphan pins are reclaimed only when an operator runs `sce doctor --fix`; the recommendation is time-based (staleness since last success), not evidence-based, because detecting orphans needs the Git/DB scan this design keeps off the hook path. +- **D3 — Durable maintenance state: diagnostics, no stored deadlines (rounds 1–2).** + - One small JSON file per worktree at `/sce/ref-maintenance.json` (versioned, capped at `MAINTENANCE_STATE_MAX_BYTES`, 4 KiB), beside the lock and taint marker. Fields (all optional past-event unix-ms timestamps unless noted): `anchor`, `last_success`, `last_attempt`, `last_advised`; `last_attempt_outcome` (`completed` | `failed`); `last_report` (retained/deleted/local-required counts); `consecutive_failures` (integer); `last_failure` (short classification plus bounded message). + - **No field is ever used as a deadline.** Eligibility is computed at read time as `now − reference ≥ window` (saturating at 0). Re-reading therefore cannot move a deadline. The stored-timestamp invariant is scoped to **successful state-writing operations**: after a successful write, every stored timestamp `t` satisfies `t ≤ now + FUTURE_SKEW_TOLERANCE`. It is not claimed after read-only or unsuccessful operations (see the two operation classes below). A value in `(now, now + FUTURE_SKEW_TOLERANCE]` is valid, is preserved byte-for-byte (not rewritten), and is evaluated as age 0; it can delay eligibility by at most the tolerance and cannot slide, because it is never refreshed by reads. + - **Two operation classes.** + - *Read-only or unsuccessful operations* — plain `sce doctor`; a lock-busy advisory or explicit pass; a failed state read; a failed state write; a process interrupted before atomic replacement. These never mutate maintenance state (a failed-before-rename write, `NotApplied`, or an interrupted write leaves the previous bytes intact; unreadable state is `StateUnavailable`, not a failed write), so they may observe preexisting invalid timestamps and cannot guarantee normalization. They must evaluate invalid timestamps conservatively (treated as eligible/recommended, never as age 0 and never as a reason to defer), must not postpone an advisory or explicit repair indefinitely, and must not report invalid state as successfully normalized. + - *Successful state-writing operations* — a successful `advise_if_due` write and a successful `reconcile_explicit` state record. While holding `WorktreeLock` they normalize invalid timestamps and persist the normalized state atomically, so that every stored timestamp satisfies the tolerance after the successful write; valid near-future timestamps are preserved as specified above; repeated reads never move eligibility forward; subsequent invocations, including after a process restart, observe the persisted normalization. + - Writes happen only under `WorktreeLock`, via atomic staging-and-swap (temp file in the same directory, then rename; pattern identified in T01) so a crash leaves either the old or the new file. No in-memory registry or process-global state. + - Transitions: advisory — anchor and `last_advised` only. Explicit success — `last_attempt` = `last_success` = now, `last_report` set, `consecutive_failures` = 0, failure fields cleared. Explicit failure (DB unavailable/invalid, malformed ref, missing required pins, durable-root read error, delete-transaction error) — `last_attempt` = now, outcome `failed`, `consecutive_failures` += 1, `last_failure` set, `last_success` untouched. Lock contention (including explicit lock timeout) — **no state change**; never classified as a failure. + - **Invalid timestamp normalization (deterministic).** A timestamp is invalid when it is strictly more than `FUTURE_SKEW_TOLERANCE` (5 min) after the injected `now` (far-future values, or clock rollback beyond the tolerance); a timestamp exactly equal to `now + FUTURE_SKEW_TOLERANCE` is valid. A valid timestamp ahead of `now` (within the tolerance) is treated as age 0, so a small clock rollback never makes an advisory eligible by itself. Under `WorktreeLock`, `advise_if_due` and `reconcile_explicit` normalize once, as part of their successful state write: invalid `last_success`/`last_attempt`/`last_advised`/`anchor` values are cleared; if `last_success` and `anchor` are both unusable the state is **immediately eligible** for advice, so the advisory persists `last_advised = now` and returns `Advised` in the same locked operation, and a fresh `anchor = now` is written. The normalized state is persisted before returning, so a second read sees valid timestamps, performs no rewrite, and does not shift or postpone eligibility; once the injected clock advances, ordinary window-based eligibility resumes. If the lock is busy or the normalizing write fails before the rename (`NotApplied`), nothing is persisted and the invalid bytes stay as they were; if it fails after the rename (`DurabilityUncertain`) the normalized state may already be visible but is never claimed durable. In every case the in-memory evaluation still treats invalid values conservatively (eligible), so a later invocation can normalize or re-read them safely. Plain `sce doctor` never writes; it evaluates invalid timestamps as "recommended" with an "invalid maintenance timestamps" note and leaves the file byte-identical. `sce doctor --fix` runs regardless of timestamp validity and rewrites valid state afterwards. + - Recovery: a missing, unparsable, wrong-version, or oversized file is treated as default and rewritten by the next recorded operation; it never fails an explicit pass. + - Result distinction for explicit passes (typed outcome `ExplicitOutcome`): `Completed(report)`; `CompletedStatePersistFailed { report, warning }` — the cleanup stands and is reported as done, with a typed `StatePersistWarning`; `Failed { error, state_warning: Option }` — a failed pass keeps both the reconciliation error and any persistence warning; `Skipped(Busy)`. Never merged. `StatePersistWarning` is `PreviousStateUnreadable(io::Error)` (the update was not attempted) or `WriteFailed(PersistFailure)`, where `PersistFailure::NotApplied` means the atomic replacement did not occur and the previous state is preserved, and `PersistFailure::DurabilityUncertain` means the replacement occurred (the new state may be visible) but crash durability could not be confirmed. The doctor facade preserves this as `StateWarning::{NotApplied, DurabilityUncertain, PreviousStateUnreadable}` with a stable kind, an optional phase, and a cause; it reports "Maintenance-state update was not applied." / "Maintenance-state durability could not be confirmed." and never claims that diagnostics were not recorded when they may have been. Advisory outcomes: `Anchored | NoAction | Advised | AdvisedDurabilityUncertain | Busy | StateWriteFailed | StateUnavailable`. + - Time comes from an injected wall clock (`Fn() -> i64` unix ms) passed as a generic parameter for deterministic tests. +- **D4 — Verified-existing Agent Trace DB opener (round 1).** + - Side-effect guarantees are stated per database state. **Missing DB:** Turso is invoked in non-creating read-only mode (`read_only(true)` with multiprocess WAL), so that open cannot create the database; no DB file, parent directory, WAL, or shared-memory file is created; no Git refs are listed or deleted. **Existing but invalid DB:** opening the existing file is permitted and incidental Turso WAL/shared-memory bookkeeping is permitted if unavoidable (documented); logical immutability is required — no schema migration, no migration-metadata repair, no repository-metadata initialization, no application-table INSERT/UPDATE/DELETE, no Git-ref deletion — and verification fails with the matching typed error. **Existing valid DB:** verify repository identity and schema readiness before reconciliation, preserve the existing multiprocess-WAL configuration, allow only the database access reconciliation needs, and run no initialization, repair, or migration. + - Contract for reconciliation: (1) never create a DB file or any parent directory (the non-creating read-only Turso open fails for a missing DB, including one removed between a preliminary check and the open); (2) never run migrations; (3) never call `repair_missing_repository_schema_migration_metadata`; (4) never call `verify_or_initialize_repository_metadata` or otherwise insert/update repository metadata; (5) verify repository identity (stored `repository_id` equals the identity resolved for this checkout, `source_instance_id` valid) and schema readiness (baseline plus the migration providing durable mutation roots) **before** any ref listing or deletion; (6) any failure returns an error with refs untouched, as a maintenance error that never arms `ExternalTaintMarker` and never becomes `CoordinateError::AgentTraceDbUnavailable`. + - The existing hook opener cannot satisfy (1), (3), or (4), and changing it would alter every hook caller. The smallest satisfying change is a dedicated opener in `agent_trace_storage/mod.rs` (for example `resolve_existing_agent_trace_storage_for_maintenance`) plus one read-only verification method on `RepositoryAgentTraceDb` (for example `verify_existing_repository_metadata(repository_id)`) wrapping the existing `select_repository_metadata_row` without insert. The opener relies on no separate existence check: the Turso open itself is non-creating and read-only (`TursoDb::open_existing_without_migrations_at`), and a failed open is classified `Missing` only when the path is then absent. Plain `sce doctor` shares that open through `RepositoryAgentTraceDb::open_existing_schema_ready_at`, which inspects schema readiness without reading or requiring repository metadata. Incidental WAL/shared-memory bookkeeping on an existing DB is an accepted limitation: the guarantee is logical immutability (schema objects and migration/metadata rows unchanged) and, for non-database files, byte identity; full filesystem byte-identity for valid Turso databases is not claimed. + - Typed failure kinds (small enum, converted to `anyhow` at the existing `open_db` seam): `Missing`, `Unreadable`, `IncompatibleSchema`, `MissingMetadata`, `RepositoryMismatch`. The existing hook opener and all its callers are unchanged. + - Considered and rejected: opening the DB before taking the lock to shorten lock-hold. It would require a second lock acquisition just to record an open failure, violating the one-acquisition-per-operation rule in D1; the lock-first order of the existing reconciler is kept and the D1 concurrency contract documents the consequence. + +## Acceptance criteria + +How this plan is proven complete. `/validate` runs these checks; no task performs final validation. + +- [x] AC1: Reconciliation has one real non-test **explicit** caller (`sce doctor --fix`) and no automatic reconciliation caller; the mutation-scope ingress has at most an advisory caller that cannot reach reconciliation. + - Validate: `nix shell nixpkgs#ripgrep -c rg "reconcile|advise_if_due" cli/src --glob '!*test*'` shows the reconciliation entrypoint referenced only from `doctor/` (and its definition module), and `advise_if_due` (if present per the D2 decision rule) referenced from `hooks/mutation_scope.rs`; the hook module imports no reconciliation, `GitSnapshotService`, store, or DB-opener symbol for this purpose. +- [x] AC2: `sce doctor --fix` deletes an orphan snapshot ref; `Close`/`Flush` never deletes. + - Validate: T05 S1 drives the doctor fix path against a real Git repo and asserts the orphan `refs/sce/mutation-cursor//` is gone; S14 asserts the advisory path creates, updates, deletes, and inventories no refs (ref changes made by `coordinate()` itself, such as snapshot pins, are legitimate and are not attributed to maintenance). This plan does **not** claim automatic orphan reclamation. +- [x] AC3: A successful pass establishes two distinct invariants, and claims nothing stronger. + - **Local consistency:** `durable_roots(W) ⊆ pinned_trees(W)` is checked before any deletion for the target worktree `W`. If it fails, the pass deletes nothing and returns `MissingRequiredPins`. + - **Repository-wide deletion safety:** `delete(W, T) ⇒ T ∉ durable_roots(repository)`. A pin is removed only when its tree is outside every worktree's durable roots, so reconciliation never removes an existing SCE pin that protects another worktree's required tree. + - Reconciliation does **not** repair, recreate, or verify missing pins belonging to other worktrees, and does not guarantee that every repository-wide durable root is pinned. + - Validate: T05 S2, S3, S4, S5 and S13 pass. S2/S3 prove preservation; S4 proves local fail-closed; S13 proves another worktree's missing pin is neither repaired nor treated as a failure of `W`'s pass. +- [x] AC4: Hook-path maintenance (advisory) cannot change the outcome of an already durably committed mutation boundary. + - Validate: T05 S17, S18a, S18b and S33 pass: every advisory outcome (`Anchored`, `NoAction`, `Advised`, `AdvisedDurabilityUncertain`, `Busy`, `StateWriteFailed`, `StateUnavailable`, corrupt/invalid state, seam error) and a failing diagnostic-output writer leave the hook result identical to the coordinate-only baseline. The advisory never creates, clears, or modifies the external-taint marker. After normal completion the marker state is whatever the existing coordinator leaves (cleared) and is unchanged by the advisory (S18a). `MarkerClearAfterCommit` remains a durably completed boundary with a successful hook result; the marker state resulting from the failed clear is preserved by the advisory, and the next real runtime invocation still takes the existing conservative inherited-taint recovery path (S18b). `MarkerClearAfterCommit` is never treated as equivalent to a successful clear. +- [x] AC5: The mutation-hook completion path performs no reconciliation, ref listing, DB I/O, or new Git subprocess, and its maintenance-state writes follow a precise cadence. Time-based reminders: a stale state becomes eligible after `RECONCILIATION_ADVISORY_AFTER` (24 h) and further stale-state reminders are suppressed for 24 h after `last_advised`. Failure-triggered reminders: an outstanding failed explicit reconciliation is recommended immediately; a newly recorded failure (`last_attempt` newer than `last_advised`) may be advised even inside the previous 24 h window and is suppressed once recorded; repeated reminders for the same unresolved failure follow the normal 24 h cadence; a successful explicit reconciliation clears the failure-based recommendation. Anchoring and clock-skew normalization are the only other writes. Actionable reminders reach an explicit diagnostic channel (stderr) in addition to structured logging; delivery is best-effort and neither exactly-once nor guaranteed to be seen by a human (T03 remediation, context/cli/mutation-scope-ref-advisory.md). + - Validate: T05 S14, S15, S19 and S20 pass (S14: the advisory performs no Git-ref creation, update, deletion, or inventory, no reconciliation, and no DB I/O; coordinator ref effects are legitimate and distinguished from it); `rg` over the advisory module shows no import of `GitSnapshotService`, `MutationTraceStore`, DB openers, `resolve_git_dir`, `resolve_worktree_id`, `spawn_blocking`, or `Command`. +- [x] AC6: Contention, cancellation, and failed maintenance never violate the pin-to-CAS safety boundary. + - Validate: T05 S6, S7 and S12 pass using deterministic synchronization (no sleep-based proof): concurrent `coordinate()` cannot race pin deletion; dropping the explicit-pass future during a ref-deletion worker does not release the lock until the worker finishes; each operation completes without self-deadlock. +- [x] AC7: No new unbounded in-memory registry, dynamic dispatch, detached background worker, timeout-cancelled Git work, or nested runtime. + - Validate: `git diff fix-logging...HEAD` inspection plus `nix shell nixpkgs#ripgrep -c rg "dyn Future|BoxFuture|tokio::spawn|Runtime::new|block_on|tokio::time::timeout|JoinHandle|abort\(|static .*(Mutex|RwLock|OnceLock|HashMap)" ` returns no new hits. +- [x] AC8: Mutation-cursor and Quint model-based tests remain valid without protocol changes. + - Validate: `git diff fix-logging...HEAD --stat` shows no change to `protocol.rs` or `spec/mutation_cursor.qnt`; the existing `mbt` tests pass. +- [x] AC9: `sce doctor` stays non-mutating — it never creates or logically modifies the Agent Trace DB (its schema inspection uses the non-creating read-only open of D4) and never writes maintenance state or refs; `sce doctor --fix` reports its actual cleanup result, exposes it as a typed `ref_reconciliation_fix` JSON object (outcome, counts, failure kind, state-warning kind/phase) alongside the unchanged `fix_results`, distinguishes `NotApplied` from `DurabilityUncertain` persistence warnings, and recovers from every failure, stale, and invalid-state condition. + - Validate: T05 S9, S25, S26 and S27 pass: counts for deleted/retained/local-required; distinct output for completed, completed-with-state-persist-failure, failed (unavailable/invalid DB, malformed refs, missing pins), and busy; plain `sce doctor` writes no ref and no state; a recommendation is shown when stale, when the latest explicit pass failed (regardless of `last_advised`), or when state is invalid. The doctor recommendation is independent of whether a hook reminder was ever seen, which is why lossy hook delivery is acceptable. +- [x] AC10: Context matches the implementation, the "retained and unwired" status is gone, and context states that automatic reclamation is deferred to a follow-up with its prerequisites. + - Validate: `nix shell nixpkgs#ripgrep -c rg -i "retained and unwired|unwired" context/cli` returns no stale claims; `rg -i "automatic.*(reclam|reconcil)" context` hits only the deferral/follow-up statements; documented APIs match code. +- [x] AC11: Exactly one `WorktreeLock` acquisition per maintenance operation, with no self-deadlock. + - Validate: T02 and T05 S12; `rg "acquire_inner_async|acquire_async|acquire_inner" cli/src/services/mutation_trace/runtime` shows each entrypoint acquires once and `reconcile_with_held_lock` never acquires. +- [x] AC12: The hook completion path contains no unbounded maintenance work, and the documented concurrency contract for explicit reconciliation holds. + - Validate: T05 S14, S15, S16 and S33 pass, as three distinct contention scenarios. S15 (direct advisory contention): with an explicit pass parked mid-inventory or mid-DB holding the lock, `advise_if_due` called directly returns `Busy` immediately and changes no state or refs. S16 (coordinator contention): with the lock held, `coordinate()` keeps its existing bounded lock acquisition; a short injected timeout fails with the existing lock-timeout error before durable completion, the advisory does not run after the failed coordinate, and the same boundary succeeds once the pass is released. S33 (post-coordinate handoff race): after a successful `coordinate()` released its lock, a competing explicit pass takes the lock before the paused advisory resumes; the advisory returns `Busy` and the committed boundary still returns success. The advisory takes no Git/DB parameters. No timeout, budget, or size cap is claimed to bound hook latency. +- [x] AC13: Maintenance state is durable, crash-safe, and cannot postpone eligibility indefinitely. + - Two guarantee classes plus a distinct uncertain-write outcome (matching D3). **Read-only or unsuccessful operations** (plain `sce doctor`, lock-busy advisory/explicit pass, state that cannot be read or evaluated (`StateUnavailable`, which is not a failed write), a write that fails before the atomic rename (`NotApplied`), a write interrupted before atomic replacement) never mutate maintenance state, may observe invalid timestamps, evaluate them conservatively, never indefinitely postpone an advisory or explicit repair, and never report invalid state as normalized. **`NotApplied`** means the failure happened before the rename: the previous state remains intact and byte-identical, the advisory reports `StateWriteFailed`, and no successful advisory is claimed. **`DurabilityUncertain`** means the failure happened after the rename: the new state may already be visible, crash durability is unconfirmed, the advisory reports `AdvisedDurabilityUncertain` when the recommendation was being recorded (an actionable diagnostic remains available) and `StateWriteFailed` carrying `DurabilityUncertain` when only an anchor or normalization was being written (no recommendation is emitted), and a normalization written this way is never claimed to be durably persisted; a later invocation recovers safely in either case. Explicit reconciliation retains `CompletedStatePersistFailed` and the typed persistence-warning phase. **Successful state-writing operations** normalize invalid timestamps under `WorktreeLock`, persist atomically, and afterwards every stored timestamp satisfies `t ≤ now + FUTURE_SKEW_TOLERANCE`; valid near-future timestamps are preserved; repeated reads never move eligibility forward; later invocations observe the persisted normalization. + - Validate: T05 S19–S25 and S34–S39 pass: single time-based advice per 24 h window with persistence before emission (a newly recorded failure may be advised inside that window, once, per AC5); concurrent processes cannot double-advise; state-persistence failure causes no advice and no scan; interrupted atomic write preserves the previous state; process restart preserves eligibility; timestamps within `FUTURE_SKEW_TOLERANCE` are valid (age 0, preserved, never advise by themselves) while timestamps beyond it and clock rollback beyond it are conservatively eligible and, on a successful write, normalized once, persisted, and recovered (S24 cases 1–7); the exact post-condition `t ≤ now + FUTURE_SKEW_TOLERANCE` is asserted only after successful state-writing operations (S24, S36, S37), while S34, S35 and S36's pre-rename leg assert byte-identical preexisting state instead, and S36's post-rename leg asserts only that the new state may be visible with durability reported as uncertain; corrupt/missing state is recovered. Deterministic regression tests (injected clocks and deterministic synchronization, no timing-based sleeps): S34 read-only doctor observing an invalid future timestamp; S35 lock-busy advisory leaving invalid state byte-identical; S36 failed state write during normalization, with both a pre-rename leg (previous bytes unchanged, `StateWriteFailed`) and a post-rename leg (visibility may have changed, `AdvisedDurabilityUncertain`, no durable-normalization claim), a later invocation recovering safely in both; S37 successful normalization persisting valid timestamps; S38 process restart after normalization; S39 eventual advisory eligibility after time progresses. +- [x] AC14: Reconciliation opens only a verified existing DB through a non-creating read-only open that production callers cannot replace, and never initializes, repairs, or migrates it; failure cases assert logical database immutability (incidental WAL bookkeeping on existing databases is permitted and documented), and only the missing-DB case asserts no newly created database-related files or directories, including when the file is removed between a preliminary check and the open. Plain `sce doctor` shares the same non-creating read-only open. + - **Missing DB:** the non-creating read-only Turso open fails without creating the DB file, its parent directory, or database sidecars (WAL/shared-memory); no Git ref is listed or deleted. A separate `sce doctor --fix` repair (global-state bootstrap) may create the parent directory independently of reconciliation; reconciliation itself creates nothing. + - **Existing but invalid DB:** opening is permitted; incidental Turso WAL/shared-memory bookkeeping is permitted if unavoidable and documented; no schema migration, migration-metadata repair, repository-metadata initialization, or application-table INSERT/UPDATE/DELETE; no Git-ref deletion; verification fails with the matching typed error. Tests do not assert filesystem immutability for this case. + - **Existing valid DB:** repository identity and schema readiness are verified before reconciliation; the existing multiprocess-WAL configuration is preserved; only the access reconciliation requires is performed; no initialization, repair, or migration. + - Validate: T05 S28–S32 pass: missing DB (no new files or directories, non-creating open fails, refs not listed), incompatible schema, missing metadata, repository identity mismatch (refs untouched, logical DB contents byte-for-byte equivalent by table/row comparison, typed error), and normal existing-DB operation. +- [x] AC15: Deferred automatic reclamation is explicit and not implied. + - Validate: the plan's D2 follow-up prerequisites are copied into the synchronized context by T06; no code path or doc claims automatic reclamation; T05 S14 proves hook completion never reconciles. + +### Review-finding traceability + +| Finding | Design resolution | Implementing task | Regression scenarios | +| --- | --- | --- | --- | +| R1 P1 double lock | D1 | T02 (T04 consumes) | S6, S7, S12, S17 | +| R1 P1 latency → R2 P1 unbounded inline latency | D2 (advisory only; reconciliation via doctor; automatic deferred) | T01 (investigation), T02, T03, T04 | S14, S15, S16, S17, S18a, S18b, S33 | +| R1 P2 failure state | D3 (diagnostic streak, no stored deadlines) | T02, T04 | S11, S26, S27 | +| R2 P2 persistence failure / repeated scans | D2/D3 (no automatic full pass; advice only after successful persist; reservation is a follow-up prerequisite) | T02, T03 | S20, S22, S23 | +| R2 P2 far-future timestamps → R3 P3 future-skew contradiction → R4 P2 normalization scope | D3 (past-only fields, `FUTURE_SKEW_TOLERANCE` valid as age 0, normalize beyond it once under lock on successful writes; read-only/unsuccessful operations never mutate and evaluate conservatively) | T02 | S24 (cases 1–7), S25, S34–S39 | +| R4 P2 database side-effect assertions | D4/AC14 (per-DB-state guarantees: missing = no files; existing invalid = logical immutability, incidental WAL allowed) | T02 | S28–S32 | +| R3 P2 external-taint marker assertion | AC4 (advisory never touches the marker; `MarkerClearAfterCommit` preserved) | T03, T05 | S18a, S18b | +| R3 P2 advisory vs coordinator contention | AC12 (three scenarios: direct, coordinator, post-coordinate handoff) | T05 | S15, S16, S33 | +| R3 P2 Git-ref assertions | AC2, AC5 (advisory adds no ref delta; coordinator pins legitimate) | T05 | S14 | +| R1 P2 AC3 durability claim | AC3 invariants | T02 (no behavior change), T05, T06 | S2, S3, S4, S5, S13 | +| R1 P2 PR metadata | Stacking section | T01 (re-verify) | none (plan metadata) | +| R1 P2 DB opening side effects | D4 | T01 (audit), T02 | S28–S32 | +| Concurrency / duplicate passes | D1, D3 | T02, T03 | S12, S21 | + +Previous acceptance criteria that changed in round 2: AC1 (explicit caller only; no automatic reconciliation caller), AC2 (doctor deletes; hook never does), AC4 (advisory, not automatic maintenance), AC5 (hook path does no I/O beyond state), AC9 (adds recovery conditions), AC10 (adds deferral statement), AC12 (budget/deferral language removed; concurrency contract added), AC13 (backoff replaced by durable-diagnostic and timestamp-recovery guarantees). New: AC15. Round 3 changed AC2 (ref assertion scoped to the advisory), AC4 (marker semantics, `MarkerClearAfterCommit` preserved), AC5 (S14/S15 references), AC12 (three contention scenarios, new S33) and AC13 (exact skew-tolerance invariant, S24 cases 1–7); scenarios S15, S16, S18 (now S18a/S18b), S24 were rewritten and S33 was added. + +*Round 4 (two findings; plan-only corrections, D1–D4 architecture and the T01–T06 structure unchanged)* + +14. **Timestamp-normalization scope (P2).** The `t ≤ now + FUTURE_SKEW_TOLERANCE` post-condition was asserted after every operation, which read-only (plain `sce doctor`), lock-busy, failed-read/write, and interrupted operations cannot satisfy. D3, AC13, T02 and T05 S24–S25 now separate read-only/unsuccessful operations (never mutate, evaluate conservatively, never postpone repair, never report normalization) from successful state-writing operations (normalize under `WorktreeLock`, persist atomically, post-condition holds). Added S34–S39. +15. **Database side-effect assertions (P2).** AC14 and S28–S32 asserted that failed verification never creates files, contradicting D4's accepted incidental WAL bookkeeping for existing databases. D4, AC14 and S28–S32 now specify per-state guarantees: missing DB (stronger: no Turso call, no new files/directories, no ref listing or deletion); existing invalid DB (logical immutability, no migration/repair/initialization/application writes, incidental WAL bookkeeping permitted); existing valid DB (verify before reconciliation, preserve WAL configuration, no initialization/repair/migration). + +### Full validation + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` +- `nix flake check` +- `nix build .#ci-checks` +- `nix run .#pkl-check-generated` (only if generated config inputs are touched) +- `git diff --check` +- `git diff fix-logging...HEAD --stat` (confirms no `protocol.rs` / Quint changes and only intended files) +- `nix shell nixpkgs#ripgrep -c rg "dyn Future|BoxFuture|tokio::spawn|Runtime::new|block_on|tokio::time::timeout|JoinHandle|abort\(" ` (AC7) +- Focused runs with non-zero match counts, repeated for race scenarios: `… test --manifest-path cli/Cargo.toml ref_reconciliation`, `… maintenance`, `… mutation_scope`, `… doctor`, `… agent_trace_storage` + +Record actual outcomes in the validation report; compilation alone does not validate the plan. + +### Context sync + +- `context/cli/mutation-trace-ref-reconciliation.md` +- `context/cli/mutation-trace-runtime-coordinator.md` +- `context/cli/mutation-trace-snapshot-service.md` +- `context/cli/mutation-trace-store.md` (if its usage contract changed) +- `context/cli/mutation-scope-runtime.md` +- `context/cli/mutation-scope-hook-ingress.md` +- `context/context-map.md` +- Relevant doctor documentation (`context/sce/agent-trace-hook-doctor.md`, `context/sce/doctor-human-text-contract.md`) +- Agent Trace storage documentation covering the new verified-existing-DB opener (located by T06) + +## Task context synchronization lifecycle + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/mutation_trace/runtime/` (new maintenance module with `reconcile_explicit`, `advise_if_due`, and the state module; `ref_reconciliation.rs` split; `git_snapshot.rs` and `worktree_lock.rs` only for test seams as needed), `cli/src/services/agent_trace_storage/mod.rs` and `cli/src/services/agent_trace_db/repository.rs` (the dedicated verified-existing-DB opener and its read-only metadata check only), `cli/src/services/hooks/mutation_scope.rs` (advisory call only), `cli/src/services/doctor/`, the checkout-local state file `/sce/ref-maintenance.json`, tests, and the context files listed above. +- **Out of scope:** `protocol.rs`, `spec/mutation_cursor.qnt`, mutation-attribution rules, Agent Trace schema migrations, the existing `open_agent_trace_db_for_hook_runtime` behavior for other callers, per-harness (Claude/Codex/OpenCode/Pi) adapter changes, adapter recovery state, any automatic or out-of-band reconciliation execution. +- **Constraints:** + 1. Reuse async reconciliation logic, `GitSnapshotService`, `MutationTraceStore`, `WorktreeLock`; the lock-held implementation is the single shared algorithm. + 2. Static dispatch and existing generic `AsyncFn` patterns only; no `dyn Future`, `BoxFuture`, additional Tokio runtime. + 3. Never delete a ref whose tree is in any repository-wide durable-root set. + 4. Never run any maintenance operation while holding a lock from `coordinate()`; maintenance acquires its own lock after `coordinate()` returned. + 5. Never convert a successful mutation-boundary completion into failure because the advisory was busy, failed, errored, or hit invalid state. + 6. No detached cleanup task, background worker, daemon, fire-and-forget subprocess, or timeout-cancelled Git/DB work; no reconciliation on the hook path. + 7. Do not modify `protocol.rs`, the Quint state machine, or attribution rules. + 8. Never invoke `git gc` or remove Git objects directly; ref deletion goes only through the existing atomic SHA-conditional transaction (`delete_pins`). No independent ref-deletion implementation. + 9. Do not bootstrap, migrate, repair, or initialize an Agent Trace DB to run cleanup (D4). + 10. Exactly one `WorktreeLock` acquisition per maintenance operation; never reacquire inside the lock-held implementation. + 11. Maintenance failures never arm `ExternalTaintMarker`. + 12. The maintenance state file never stores a deadline; every successful state write leaves no timestamp more than `FUTURE_SKEW_TOLERANCE` ahead of the injected `now`; read-only and unsuccessful operations (including a `NotApplied` write failure) never mutate it; a post-rename `DurabilityUncertain` failure may leave the new state visible and is reported as uncertain, never as durably persisted; all writes are atomic and under `WorktreeLock`. + 13. No schema migration, new attribution state, or new protocol action. + 14. Follow repository rules: no code comments in touched code, Cargo only via Nix, `nix flake check` for verification. +- **Non-goal (automatic reclamation):** see D2. Follow-up PR with the listed prerequisites, including the durable next-attempt reservation requirement. +- **Non-goal (retired-worktree cleanup):** the reconciler only inventories the namespace of a currently resolvable worktree, so refs owned by removed linked worktrees survive. No global namespace sweep here. Record a follow-up for repository-scoped cleanup with active-worktree inventory, repository-wide durable-root retention, and protection against worktree removal/recreation races. +- **Non-goal (repairing other worktrees' pins):** reconciliation of worktree `W` does not recreate or verify pins that only another worktree requires. +- **Non-goal (evidence-based advice):** the advisory is time-based; detecting actual orphans requires the Git/DB scan kept off the hook path. +- **Non-goal (historical event retention):** trees referenced by retained mutation events are never deleted; bounding history needs a separate event-retention and snapshot-compaction policy. +- **Non-goal (protocol changes):** no new action, state field, attribution category, Quint transition, or schema migration. + +## Assumptions + +- Constants (T03 records final values): `RECONCILIATION_ADVISORY_AFTER` 24 h; `MAINTENANCE_STATE_MAX_BYTES` 4096; `FUTURE_SKEW_TOLERANCE` 5 min; explicit lock wait 10 s (existing). +- The state file is per-worktree under `/sce/`, matching the existing lock and taint-marker layout; absent or unparsable means default state. +- Worktree identity is already Git-topology-derived (`resolve_worktree_id`: `main` and `worktrees/`); the checkout-ID wording in the context docs is stale, and `SkippedNoCheckoutIdentity` appears unreachable in code (declared, never constructed). T01 confirms and T06 removes or justifies it. +- Doctor output additions are additive fields/rows only. + +## Task stack + +- [x] T01: `Audit reconciliation and mutation-scope lifecycle integration points` (status:done) + - Task ID: T01 + - Scope: In — read-only audit against PR #304/#305 code of `runtime/ref_reconciliation.rs`, `git_snapshot.rs`, `worktree_lock.rs`, `coordinator.rs`, `protected_worktree.rs`, `hooks/mutation_scope.rs`, `hooks/commit_hooks.rs`, `hooks/` harness lifecycle modules, `doctor/mod.rs`, `doctor/fixes.rs`, `doctor/inspect.rs`, `hooks/runtime.rs`, `agent_trace_storage/mod.rs`, `agent_trace_db/repository.rs`, `db/mod.rs`, and existing reconciliation tests; record findings in this plan under an `Audit findings` subsection. Required findings: + - production call graph for `Start`/`Advance`/`Close`/`Flush`/`Abandon`; which `CoordinateOutcome`/`CoordinateError` variants are durably completed (including `MarkerClearAfterCommit`); when each lock is released, including the `abandon_after_spawn_without_unlock` path; + - **lock ownership (D1):** confirm `reconcile_worktree_inner` acquires `WorktreeLock` at its start and the lock is non-reentrant; specify the exact wrapper / lock-held split and the parameter carrying lock ownership; confirm `delete_pins` moves the lease into the `spawn_blocking` worker so dropping the caller future cannot release the file lock early; confirm the 10 s coordinator lock timeout; + - **execution-point investigation (D2):** re-verify each rejected alternative (post-commit, per-harness session-end/stop, setup lifecycle, detached/daemon) with file references, confirm that no lifecycle-owned noncritical-path executor exists, and confirm runtime shutdown joins blocking workers; confirm the selected design (advisory-only hook trigger plus `sce doctor --fix`) or escalate if a suitable mechanism is found; + - **advisory Git-dir resolution (D2 decision rule):** determine whether the worktree Git dir can be obtained for the advisory without a new Git subprocess (filesystem derivation proven equal to `resolve_git_dir` for the main and a linked worktree, or reuse of a value `coordinate` already resolves without protocol changes) and record the outcome: advisory kept, or hook trigger dropped; + - **state (D3):** identify the existing atomic staging-and-swap pattern to reuse for the state file; confirm placement beside the lock and marker; confirm no in-memory registry is needed; confirm the existing clock source and how to inject a wall clock; + - **DB opening (D4):** inspect `open_agent_trace_db_for_hook_runtime` and its resolver; record exactly which steps can create directories/files, repair migration metadata, or initialize repository metadata; determine whether Turso supports a non-creating/read-only open; define the smallest dedicated verified-existing-DB opener and exact verification queries (schema readiness including the durable-roots migration, metadata row presence, `repository_id` match, valid `source_instance_id`); + - current worktree identity forms; stale checkout-ID docs and unreachable branches including `SkippedNoCheckoutIdentity`; + - confirm PR #304 head and PR #305 head/branch (`ref-recon`, base `fix-logging`) and record the SHAs. + Out — any code or context change. + - Dependencies: none + - Done when: the plan records verified call sites with file/line references, lock ownership and the wrapper/lock-held split, error classification, the D2 execution-point conclusion and advisory Git-dir decision, the DB-opening side effects and chosen opener contract, and flags any finding that invalidates a later task's scope. + - Verify: re-read cited lines to confirm each recorded fact; `gh pr view 304 --json headRefOid` and `gh pr view 305 --json headRefName,baseRefName,headRefOid` match the recorded values. + - Completed: 2026-10-09 + - Files changed: `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Added the `Audit findings` section. Confirmed lock ownership and the wrapper/lock-held split, the call graph and completion classes, the D2 conclusion (advisory plus `sce doctor --fix`), a filesystem Git-dir derivation that avoids a new Git subprocess (hook trigger kept), the D3 staging-and-swap pattern and clock source, and the D4 non-read-only opener findings and contract. Flags recorded: the `sync/auto_sync.rs` detached-spawn precedent, the `ReconcileError` "003" vs `004` migration wording, and the `doctor/inspect.rs:305` repair-seam caller. No finding invalidates a later task's scope. + - Verify: cited lines re-read during the audit and line references corrected; `gh pr view 304` → head `0be74aad5bea5070828846d3277041a4649c9842`, `gh pr view 305` → `ref-recon`/base `fix-logging`/head `fb7e78fbfa41f9f054803eef2dddb180f5d5289a` (both match the recorded values); passed. + - Context impact: none (plan-only audit; no behavior or durable context change). + - Context synchronization: synced + +- [x] T02: `Add the reusable maintenance entrypoints (explicit reconciliation and hook advisory)` (status:done) + - Task ID: T02 + - Scope: In — + - **Reconciler split (D1):** extract the body of `reconcile_worktree_inner` after lock acquisition into `reconcile_with_held_lock`, which borrows the held `WorktreeLock` and contains no acquire call; keep `reconcile_worktree` and `reconcile_worktree_inner` behavior and signatures, now acquiring once and delegating. + - **`reconcile_explicit(repository_root)`:** takes no database opener; resolves the worktree and Git dir, acquires the lock **once** (existing bounded 10 s wait), resolves the repository identity and canonical Agent Trace path itself and opens the DB only through the D4 verified read-only opener, runs `reconcile_with_held_lock`, then records the outcome in the state file while still holding the same lock, then drops the lock. Typed outcome `Completed(report) | CompletedStatePersistFailed { report, warning } | Failed { error, state_warning } | Skipped(Busy)` (a failed pass keeps both the reconciliation error and any state-persistence warning); lock contention leaves the state untouched and is not a failure; it never calls the lock-acquiring wrapper; it ignores all prior state (stale, failed streak, invalid or corrupt) except to rewrite it. + - **`advise_if_due`:** synchronous function with no DB, Git, store, or snapshot parameters; zero-wait single `try_lock`; reads, normalizes, and writes the state under that one acquisition; returns `Anchored | NoAction | Advised | AdvisedDurabilityUncertain { warning } | Busy | StateWriteFailed { warning } | StateUnavailable` (see the second remediation); persists `last_advised` before returning `Advised`; takes injected wall clock and Git dir. Subject to the T01 decision rule. + - **State module (D3):** `ref-maintenance.json` read (capped, default-on-corrupt), invalid-timestamp normalization under lock, atomic write, past-only fields, success/failure transitions, and a pure `evaluate_recommendation(state, now)` function shared by the advisory and read-only doctor. + - **D4 opener:** the dedicated verified-existing-DB opener and the read-only repository metadata verification method. + - **Test seams only as required:** a worker-entered seam threaded through `delete_pins` to `run_ref_mutation_inner`'s existing `on_worker_entered`; phase-boundary hooks (post-lock, post-DB-open, post-inventory) used to park a pass deterministically; injectable write-failure and rename-interruption hooks for the atomic writer; injected clock. + Out — hook and doctor wiring, any new deletion code, any automatic/reservation/backoff scheduling, protocol/store schema changes, changes to the existing hook DB opener, capped inventory or budget logic. + - Dependencies: T01 + - Done when: both entrypoints are callable from tests; the explicit pass and the advisory each complete without self-deadlock; state never contains a deadline; after every successful state write no stored timestamp exceeds `now + FUTURE_SKEW_TOLERANCE`; timestamps within the tolerance are accepted as age 0 and preserved; on a successful write invalid timestamps normalize once and persist; read-only, lock-busy, unreadable-state (`StateUnavailable`) and pre-rename-failed (`NotApplied`) operations never mutate state (a post-rename `DurabilityUncertain` failure may leave the new state visible and is reported as uncertain), evaluate invalid timestamps conservatively, and never postpone eligibility indefinitely; failures leave refs untouched; the DB opener fails a missing DB at its single non-creating read-only open without creating files or directories, and for an existing invalid DB performs no migration, repair, initialization, or application-table write (incidental WAL bookkeeping permitted and documented); no second deletion path exists. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml ref_reconciliation`; focused maintenance-state and `agent_trace_storage` unit tests (deterministic, no sleeps); `rg` confirms the only ref-deletion call is the existing `delete_pins`, each entrypoint acquires `WorktreeLock` once, `reconcile_with_held_lock` contains no acquire call, and the advisory module imports no Git/DB/store symbol. + - Completed: 2026-10-09 + - Files changed: `cli/src/services/agent_trace_db/repository.rs`, `cli/src/services/agent_trace_storage/mod.rs`, `cli/src/services/db/mod.rs`, `cli/src/services/mutation_trace/runtime/mod.rs`, `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`, `cli/src/services/mutation_trace/runtime/maintenance_state.rs`, `cli/src/services/mutation_trace/runtime/ref_advisory.rs`, `cli/src/services/mutation_trace/runtime/ref_maintenance.rs`, `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Split `reconcile_worktree_inner` into lock acquisition plus `reconcile_with_held_lock` (borrows `&WorktreeLock`, phase-hook seam `ReconcilePhase`); added `ref_maintenance::reconcile_explicit[_with]` (single lock acquisition, typed `Completed | CompletedStatePersistFailed | Failed | Skipped(Busy)`), `ref_advisory::advise_if_due[_with]` (sync, zero-wait `try_lock`, filesystem Git-dir derivation, no Git/DB/store imports), and `maintenance_state` (capped versioned `ref-maintenance.json`, past-only timestamps, tolerance-based normalization under lock, staging-and-swap writes with injectable rename, pure `evaluate_recommendation`). Added the D4 opener `RepositoryAgentTraceDb::open_verified_existing_at` with `ExistingRepositoryDbError` (`Missing | Unreadable | IncompatibleSchema | MissingMetadata | RepositoryMismatch`), read-only `verify_existing_repository_metadata`, `TursoDb::open_existing_without_migrations_at` (no parent-dir creation), and `resolve_existing_agent_trace_storage_for_maintenance`. New items carry dead-code allowances until T03/T04 wire them. The existing hook opener and its callers are unchanged; `delete_pins` remains the only deletion path. + - Verify: `cargo clippy --all-targets --all-features -D warnings` clean; `test … ref_reconciliation` 5 passed; `test … maintenance` 8 passed; `test … mutation_trace` 123 passed; `rg` shows `reconcile_with_held_lock` has no acquire call and `maintenance_state.rs` has no Git/DB/store symbol (`ref_advisory.rs` matches only `Command` inside its test module); `git diff --check` clean. The `agent_trace_storage` filter matches zero tests, so opener coverage lives in the `ref_reconciliation` tests. Original T02 verification used an existence pre-check; superseded by the remediation below. + - Context impact: important (new runtime maintenance entrypoints, state file, and verified-existing-DB opener; T06 owns the durable context update). + - Context synchronization: synced + - Remediation (2026-10-09, PR #305 review of `d699b26`; T02 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01 unchanged, T03–T06 untouched): + - P1 authoritative DB: the production entrypoint is now `reconcile_explicit(repository_root: &Path) -> ExplicitOutcome`. It resolves the repository identity and canonical path itself (`resolve_agent_trace_storage_runtime_config` → `resolve_existing_agent_trace_storage_for_maintenance`) and opens only through `open_verified_existing_at`, which checks schema/all migrations, metadata presence, stored-vs-resolved repository identity, and `source_instance_id` before any pin inventory. `reconcile_explicit_with` (the opener-injection seam) is private to `ref_maintenance` and used only by its child test module; T04 must call `reconcile_explicit` and must not add an opener. The single lock-held algorithm and `reconcile_worktree` wrapper are unchanged. Typed `ExistingRepositoryDbError` stays downcastable from `ReconcileError::AgentTraceDbUnavailable`. + - P2 persistence diagnostics: `ExplicitOutcome::Failed { error, state_warning }` retains the reconciliation error and an optional `StatePersistWarning` (`PreviousStateUnreadable | WriteFailed(PersistFailure)`); `CompletedStatePersistFailed` carries the same warning type. Lock contention returns `Skipped(Busy)` before any state read or write. A completed ref deletion is never rolled back. + - P2 durability: `write_state_atomically` returns `PersistFailure::NotApplied { phase, source, staging_cleanup }` (previous state intact, staging removed, cleanup failure reported) for failures up to and including rename, and `PersistFailure::DurabilityUncertain { phase, source }` for parent-directory open/sync failures after rename (new state may already be visible; previous state is not claimed preserved). The filesystem seam is the statically dispatched `StateFilesystem` trait with `RealStateFilesystem`; `FaultInjectingFilesystem` is test-only. `advise_if_due` maps any `PersistFailure` to `StateUnavailable`, so `Advised` is never reported without a confirmed write. + - P2 existing-DB race: Turso 0.8.1 has no create-disabled builder flag, but `Builder::read_only(true)` opens the main file with `OpenFlags::ReadOnly`; the Unix/generic IO layers then open it without `create` and without write access, so existence and open are one OS call (missing → not-found, no file or directory created). `open_existing_without_migrations_at` now uses it with `experimental_multiprocess_wal(true)` unchanged; the `is_file()` pre-check is removed and a failed open is classified `Missing` only by a post-failure `symlink_metadata` NotFound. All verification queries run on the connection that open produced. + - Limitations (not claimed solved): (1) Turso opens the `-wal`/`-tshm` sidecars by path, so a replacement that swaps only the main file leaves a torn main+stale-WAL pairing whose contents are whatever Turso assembles; only a coherent replacement (main file and sidecars) is meaningful, and the check proves the connection's data matches the resolved repository, not that the path was not swapped later. (2) Turso's process-wide registry is keyed by file identity: if the same process already holds that DB open, the open shares that instance and the read-only flag is not enforced for it (the file necessarily exists, so nothing is created). (3) A DB replaced after verification but before the delete transaction is not detected; the guard is that roots are read from the verified connection under the held `WorktreeLock`. (4) Cross-process multiprocess-WAL coexistence is covered only by the builder configuration and an in-process live-writer test, not by a second OS process. + - Evidence: `test … ref_reconciliation` 20 passed (all in `ref_maintenance/tests.rs`, including: signature-only production API, other-repository DB cannot authorize deletion, mismatched/missing metadata and missing/incompatible schema fail before inventory (`phases` empty, pins retained), missing DB creates nothing, removed/replaced-before-open, removed-after-open never recreated, read-only connection, live-writer coexistence, lock-held path never reacquires, wrapper acquires once and releases, failure+persist-failure keeps both errors for missing DB and missing required pin, durability-uncertain completion keeps deletion, lock contention never increments the streak); `test … maintenance` 38 passed (`maintenance_state` tests cover staging create/write/sync, rename, parent open/sync failures, success, interrupted pre-rename recovery, post-rename visibility vs durability); `test … mutation_trace` 148 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; no new `allow`, `dyn`, `let _ =`, or comments; `acquire_inner_async` appears once in `ref_maintenance.rs` and once in `reconcile_worktree_inner`. + - Remediation 2 (2026-10-09, follow-up review of `de055d70`; T02 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01 unchanged and still done/synced, T03–T06 untouched). Architecture unchanged: reconciliation runs only through `reconcile_explicit`; the advisory stays lightweight (no Git, DB, or reconciliation), takes one zero-wait lock acquisition, uses static dispatch, and no mutation protocol, attribution, Quint, or ref-deletion code changed. + - P2 failure-based recommendation: `evaluate_recommendation` now treats an outstanding explicit-pass failure (`last_attempt_outcome == Failed` or `consecutive_failures > 0`) as an independent reason to recommend repair, in addition to staleness and invalid timestamps. A first-ever failed pass (no anchor, no `last_success`) is therefore recommended immediately. A never-attempted default state is not a failure. Multiple consecutive failures stay recommended; `record_success` clears the streak, `last_failure`, and the recommendation; a failure after a recent success is recommended again. Emitting an advisory never resets `consecutive_failures` or `last_failure`. + - Recommendation status vs reminder cadence: `Recommendation.recommended` answers "should `sce doctor` tell the operator to run `sce doctor --fix`" and ignores `last_advised`. `Evaluation.advise` answers "may the hook emit another reminder": staleness reminders are suppressed for 24 h after `last_advised`; a failure reminder is suppressed only when `last_advised` is within the window and not older than `last_attempt`, so a newly recorded failure can be advised immediately even if an earlier staleness reminder is recent, while a reminder issued after the failure is not repeated inside the window. A failure reminder repeats every 24 h until a successful explicit pass. Clock-skew normalization and the existing cadence are unchanged. + - P2 persistence outcomes: `AdvisoryOutcome` is now `Anchored | NoAction | Advised | AdvisedDurabilityUncertain { warning: PersistFailure } | Busy | StateWriteFailed { warning: PersistFailure } | StateUnavailable` (no longer `Copy`/`Eq` because it carries `PersistFailure`). `Advised` means the reminder state was persisted with confirmed durability. `PersistFailure::NotApplied` (up to and including rename) on an advisory yields `StateWriteFailed`: the previous state is intact, nothing is claimed persisted, `recommendation()` is `None`, and the next eligible call advises again. `PersistFailure::DurabilityUncertain` (parent-directory open or sync after rename) yields `AdvisedDurabilityUncertain`: the rename completed, `last_advised` may be visible, and the caller must still surface the recommendation. No rollback, second lock acquisition, or automatic reconciliation. A write failure while only anchoring or normalizing yields `StateWriteFailed` carrying whichever `PersistFailure` variant occurred; there is no recommendation to surface. With `NotApplied` the previous state is intact; with `DurabilityUncertain` the rename completed, the anchor or normalized state may be visible, and only crash durability is unconfirmed. The diagnostic text follows the variant ("could not be written" vs "durability could not be confirmed"). + - Caller contract for T03: `AdvisoryOutcome::recommendation()` returns `Reconciliation is recommended. Run sce doctor --fix.` for `Advised` and `AdvisedDurabilityUncertain`, and `None` otherwise; `persistence_warning()` returns the `PersistFailure` separately for logging. Outcomes are diagnostic values, not errors: T03 must log them and never convert them into a mutation-boundary failure. Suggested levels: `Advised` and `AdvisedDurabilityUncertain` (with the persistence warning as a distinct field) at warn; `StateWriteFailed` and `StateUnavailable` at warn; `Anchored`/`NoAction`/`Busy` at debug. + - Delivery semantics and limitations: a durable state update is not the same as an operator-visible advisory. The state is written before the caller surfaces the message, so a process crash between the write and the log leaves `last_advised` set and the reminder unseen for up to 24 h; exactly-once delivery is not promised. `sce doctor` is the backstop because it evaluates the recommendation from the failure/staleness fields independently of `last_advised`, so an unresolved failure or stale state is never hidden from plain `sce doctor`. After `AdvisedDurabilityUncertain`, a crash can still roll the directory entry back, in which case the next eligible call simply advises again. + - Regression scenarios and acceptance criteria (all deterministic, injected clocks and `FaultInjectingFilesystem`, no sleeps): first-ever failed pass with missing DB is recommended (unit and end-to-end through `reconcile_explicit`); failure advisory emits once, persists `last_advised`, keeps the failure diagnostics, and a second advisory inside the window is `NoAction`; plain evaluation stays recommended despite the recent reminder; the reminder repeats after the window; repeated failures stay recommended; success clears; a failure after a recent success and an older reminder is recommended and advised; rename failure preserves prior state and the next call advises; parent-open and parent-sync failures yield `AdvisedDurabilityUncertain` with the new state visible, the recommendation surfaced, a following call `NoAction`, and `evaluate_recommendation` still true; an anchor write failure is `StateWriteFailed`; confirmed advisories keep the ordinary duplicate-suppression tests. + - Evidence: `test … maintenance` 48 passed; `test … ref_reconciliation` 21 passed; `test … mutation_trace` 158 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; no added comments, `dyn`, `let _ =`, or new `allow`; no new `acquire` call sites. + +- [x] T03: `Trigger the advisory check from the shared mutation-scope ingress` (status:done) + - Task ID: T03 + - Scope: In — in `hooks/mutation_scope.rs`, after `classify_coordinate` yields a durably completed `Close` or `Flush` (including the `MarkerClearAfterCommit` case) and `coordinate_boundary(...)` has fully returned, call `advise_if_due` through an injected seam; the outcome is logged as structured events and, for actionable outcomes, written to an injectable diagnostic writer (stderr in production) and is never returned as a boundary error (`Anchored`/`NoAction`/`Busy` debug, no diagnostic; `Advised`/`AdvisedDurabilityUncertain`/`StateWriteFailed`/`StateUnavailable` warn plus diagnostic); no trigger from `Start`, `Advance`, `Abandon`, or failed coordinates; a seam error or unexpected condition is contained and logged; the hook module gains no reconciliation, Git, or DB dependency. Apply the T01 decision rule: if the advisory cannot avoid a new Git subprocess, implement no hook call and document that outcome. Record the final constants in the plan and document the deferred follow-up (D2) in the plan's audit findings. Out — per-harness adapter changes, reconciliation on the hook path, detached tasks, background workers, timeouts. + - Dependencies: T02 + - Done when: a completed `Close`/`Flush` can run the advisory and no other path does; hook output and exit status are identical whether the advisory returns any outcome or errors; the advisory runs only after the coordinator's lock has been released; `MarkerClearAfterCommit` handling is unchanged and the advisory never creates, clears, or modifies the external-taint marker; no reconciliation, ref inventory, or DB I/O is reachable from the hook module's new call. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml mutation_scope`; trigger-policy unit tests over every boundary and outcome with a fake advisory seam; a seam that tries to take the real `WorktreeLock` and asserts it is free at invocation; `rg` over the hook module shows no reconciliation/DB-opener/snapshot import added. + - Completed: 2026-10-09 + - Files changed: `cli/src/services/hooks/mutation_scope.rs`, `cli/src/services/mutation_trace/runtime/mod.rs`, `cli/src/services/mutation_trace/runtime/ref_advisory.rs`, `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: `drive_mutation_scope` takes an injected advisory seam and calls it once, only for `Close`/`Flush` whose `coordinate_boundary` result is `Ok` or `MarkerClearAfterCommit`, after the await (coordinator lock already released) and after `classify_coordinate`; the report is only logged (`sce.hooks.mutation_scope.ref_reconciliation_advisory`; warn for `advised*`/`state_write_failed`/`state_unavailable`, debug for `anchored`/`no_action`/`busy`) and never changes the hook result. Production passes `advise_after_completed_boundary` (system clock, filesystem Git-dir derivation, no Git subprocess), which maps `AdvisoryOutcome` to the new `AdvisoryReport`/`AdvisorySeverity`. The hook gains no reconciliation, snapshot, store, or DB-opener import for this purpose. Final constants unchanged from the plan assumptions (`RECONCILIATION_ADVISORY_AFTER` 24 h, `MAINTENANCE_STATE_MAX_BYTES` 4096, `FUTURE_SKEW_TOLERANCE` 5 min). Deferred automatic reclamation (D2 prerequisites) is already recorded in this plan; T06 carries it into context. + - Verify: `test … mutation_scope` 18 passed (5 new `advisory_trigger_tests`: runs after completed Close/Flush, runs after `MarkerClearAfterCommit` with unchanged success result, never for Start/Advance, never after a failed coordinate, never for Abandon); `test … hooks` 27 passed; `test … mutation_trace` 158 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; `rg` shows no new reconciliation/DB/snapshot import in the hook module and no added comments. Not covered here (T05 S17/S18/S33): a real-lock "acquirable at invocation" assertion, because the seam tests use a fake coordinate that never holds the lock; ordering after `coordinate_boundary` returns is structural. + - Context impact: important (hook ingress now emits the advisory; T06 owns the durable context update). + - Context synchronization: synced + - Remediation (2026-10-09, PR #305 review of `91e9ac2`; T03 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01/T02 unchanged, T04–T06 untouched). Architecture unchanged: advisory only on durably completed `Close`/`Flush`, no reconciliation/Git/DB work, single lock design, static dispatch. + - Delivery (P2): `Logger::warn` is not an operator channel (observability routes only ERROR to stderr, and only with file logging off), and `last_advised` is persisted before logging, so a reminder could be suppressed for 24 h without ever being seen. `drive_mutation_scope` now takes a generic `E: Write` diagnostic sink (production: `std::io::stderr()`, created in `run_mutation_scope_from_payload`; tests: `Vec`/failing writer). `AdvisoryReport.diagnostic` carries the text; `Advised` → `SCE: Reconciliation is recommended. Run sce doctor --fix.`; `AdvisedDurabilityUncertain` → that line plus `SCE: Advisory-state durability could not be confirmed: .`; `StateWriteFailed`/`StateUnavailable` → a state-write-failed / state-unavailable line plus `Run sce doctor to check reconciliation status.` (never claiming advice was persisted); `Anchored`/`NoAction`/`Busy` → none. Structured logging (event ID, `outcome`, `persistence_warning`) is retained; the diagnostic write is independent of the logger (works with `None`), ignores write/flush errors, and never alters the hook result. `Logger::warn` and observability defaults are unchanged. Stdout stays empty. + - Harness reality: Claude Code and Codex run the hook as a command, so stderr is available to the harness (shown only in the harness's verbose/hook output, not guaranteed in the transcript). The OpenCode plugin (`spawnSync`, captured output) and Pi extension (`stdio` `ignore`/captured and dropped) discard the child's stderr, so for those harnesses the stderr line is not operator-visible; the structured log file (default `log_to_file`) and `sce doctor` (which reports the stale/failed recommendation independent of hook delivery) are the supported surfaces. Plugin changes are out of T03 scope. + - Docs: AC4, AC5, AC9, AC13, the T03 scope and S20 now state the time-based vs failure-triggered cadence and the full seven-outcome enum. + - Tests: `mutation_scope` advisory tests (recording logger + injected sink) cover all seven report shapes (severity, event ID, outcome, persistence warning, diagnostic), missing logger, failing writer (result stays `Ok("")`), `MarkerClearAfterCommit`, failed coordinate (no advisory, no output), and single coordinate/advise invocation; `ref_advisory` adds `maintenance_report_maps_every_outcome_to_severity_and_diagnostic` over real outcomes. Existing `ref_advisory` tests already cover the 24 h cadence and an in-window new-failure reminder. Removed the now-unneeded file-level `#![allow(dead_code)]` from `ref_advisory.rs`. + - Verify: `test … mutation_scope` 25 passed; `test … hooks` 34 passed; `test … maintenance` 49 passed; `test … mutation_trace` 159 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean. + +- [x] T04: `Run explicit reconciliation from sce doctor --fix and report recommendation` (status:done) + - Task ID: T04 + - Scope: In — extend the existing doctor `--fix` flow to call `reconcile_explicit` (one lock acquisition inside the entrypoint; doctor never holds the worktree lock itself); additive report fields/rows for retained, deleted, and locally-required counts; distinct rows for completed, completed-with-state-persist-warning, failed (unavailable/invalid DB with the D4 failure kind, malformed refs, missing required pins, durable-root errors, delete-transaction error), and skipped-busy; plain `sce doctor` stays read-only but reads the state file (never writes it or any ref) and reports whether reconciliation is recommended (last success age, last attempt and failure streak, last advice, invalid-timestamp note) via the pure `evaluate_recommendation`; skipped or failed cleanup is never reported as success; `--fix` runs regardless of recommendation, failure streak, or timestamp validity; no DB bootstrap/migration/repair; no change to adapter recovery state; existing output contracts backward compatible; the documented concurrency note (a mutation boundary may wait up to 10 s while `--fix` reconciles) appears in help or fix output wording only if existing output contracts allow it additively. Out — new doctor subcommands, changes to mutation-scope repair logic. + - Dependencies: T02 + - Done when: `sce doctor --fix` always runs a pass and reports its true typed result; a successful pass resets the failure streak and sets `last_success`; `sce doctor` performs no ref or state mutation; existing doctor tests still pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml doctor` + - Completed: 2026-10-09 + - Files changed: `cli/src/services/doctor/mod.rs`, `cli/src/services/doctor/render.rs`, `cli/src/services/mutation_trace/runtime/mod.rs`, `cli/src/services/mutation_trace/runtime/ref_doctor.rs`, `cli/src/services/mutation_trace/runtime/ref_advisory.rs` (rustfmt-only: removed a leading blank line), `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Added the `pub(crate)` facade `runtime/ref_doctor.rs` (`run_explicit_reconciliation`, `inspect_reconciliation_recommendation`, plain-data `ReconciliationFix` / `ReconciliationCounts` / `ReconciliationRecommendation`) over `reconcile_explicit` and `evaluate_recommendation`. `sce doctor --fix` now always runs one explicit pass after the existing fixes and appends a fix-result row (category `mutation_scope_health`): completed → `fixed` with deleted/retained/locally-required counts; completed with state-persist failure → `fixed` plus a diagnostics warning; failed → `failed` with a typed kind (`agent_trace_db_missing`, `agent_trace_db_incompatible_schema`, `agent_trace_db_missing_metadata`, `agent_trace_db_repository_mismatch`, `malformed_ref`, `missing_required_pins`, `durable_roots_unavailable`, `delete_transaction_failed`, …) and any state warning; busy → `skipped`. Plain `sce doctor` reads the state file through a filesystem-derived Git dir (no lock, no write, no ref access) and, only when reconciliation is recommended or the state is unusable, adds a `Snapshot ref reconciliation` text section and a JSON `ref_reconciliation` object (`null` otherwise). Doctor receives the two calls as injected seams. No DB bootstrap/migration/repair, no adapter recovery state, and no mutation-scope repair logic changed. + - Verify: `test … doctor` 9 passed (4 new fix-result mapping tests, 4 new `ref_doctor` tests covering absent/recent state, failure recommendation with byte-identical state, unusable/invalid-timestamp state left unrewritten, outcome mapping); `test … mutation_trace` 163 passed; full `cargo test` 437 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; no comments added and the pre-existing comment in the edited `execute_doctor_with_lifecycle_providers` was removed per instruction. + - Context impact: important (doctor now runs explicit reconciliation and reports the recommendation; T06 owns the durable context update). + - Remediation (2026-10-09, PR #305 review of `bab36fe`; T04 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01–T03 unchanged, T05–T06 untouched). Architecture unchanged: single `WorktreeLock`, static dispatch, one reconciliation pass per `--fix`. + - Plain doctor DB inspection now opens through `RepositoryAgentTraceDb::open_existing_schema_ready_at` (non-creating read-only Turso open, multiprocess WAL preserved, no migrations, no metadata requirement); `Missing` yields no extra problem, `IncompatibleSchema`/`Unreadable` keep their existing diagnostics. The write-capable hook opener is untouched. + - `ReconciliationFix` keeps a typed `StateWarning` (`NotApplied` | `DurabilityUncertain` | `PreviousStateUnreadable`, with kind, phase, cause) instead of a string; human text reads "Maintenance-state update was not applied." / "Maintenance-state durability could not be confirmed.". + - JSON adds the optional top-level `ref_reconciliation_fix` object (`outcome` ∈ `completed | completed_state_persist_failed | failed | busy`, `counts`, `failure {kind,message}`, `state_warning {kind,phase,message,cause}`); `null` for plain doctor; `fix_results` unchanged. + - Human recommendation text adds "Last reconciliation attempt Nm ago." when an attempt timestamp exists. + - D3/D4/AC9/AC14 corrected above. + - Context synchronization: synced + +- [x] T05: `Add production-path and concurrency regression tests` (status:done; reopened `in_progress` 2026-10-09 for the verification-gap remediation below, returned to `done`/`synced` after it) + - Task ID: T05 + - Scope: In — real Git repos, linked worktrees, and repository Agent Trace DBs. Every race, ordering and timing scenario uses deterministic coordination (barriers, channels, existing `on_lock_contention`/`on_worker_entered`-style seams, parked phase hooks, injected clock); **no timing-based sleeps as correctness proofs**. T02 and T03 add unit-level coverage for their own code; T05 owns the production-path matrix below and each scenario must fail when the guarded behavior is deliberately broken. + - *Reconciliation core (explicit path):* (S1) `sce doctor --fix` reclaims an orphan pin left by an interrupted coordinate; (S2) `A → B → C → D` retains every historical tree; (S3) worktree A's pin is retained while worktree B needs the tree (**preservation**); (S4) missing required local pin fails closed with zero deletion; (S5) malformed/symbolic refs cause zero deletion; (S6) concurrent `coordinate()` and reconcile cannot race pin deletion — a coordinate attempt parked on the lock via barrier proceeds only after the pass releases it, and a pin it creates is never deleted by the in-flight pass; (S7) dropping the explicit-pass future while the deletion worker is parked at the worker-entered seam leaves the lock held until the worker is released, then free; (S8) idempotent reconciliation; (S9) plain `sce doctor` mutates neither refs nor state; `--fix` reports the true result for every outcome kind (completed, completed-with-persist-warning, failed kinds, busy); (S10) explicit pass against an external holder returns `Skipped(Busy)`, leaves state byte-identical, and does not count as a failure; (S11) explicit pass with DB failure leaves refs untouched, records a failure streak, never arms the external-taint marker; (S12) the explicit pass and the advisory each complete with one lock acquisition and no self-deadlock, and each returns busy (never deadlocks) against an externally held lock; (S13) with worktree B's required pin deliberately missing, reconciling A deletes only A's eligible pins, does not create or repair B's pin, does not fail, and never deletes any pin protecting B's tree. + - *Hook path and latency (P1):* (S14) the advisory is invoked only after a durably completed `Close`/`Flush` (including `MarkerClearAfterCommit`), never from `Start`/`Advance`/`Abandon`/failed coordinates, and never reaches reconciliation, ref inventory, or DB open — its signature takes none (originally proposed as spy/panic seams on the reconciler, inventory and opener; replaced, because such seams would add production dependencies to a deliberately path-only interface, by two complementary forms of evidence: (A) static inspection of the complete production call graph reachable from `advise_after_completed_boundary()` showing no reconciliation, `GitSnapshotService`, ref listing/mutation, Agent Trace DB access, Git subprocess or unbounded work, and (B) the real-hook behavioral tests named in the coverage map); `coordinate()` keeps all of its existing snapshot and pinning semantics (required pins are still created and intact — protecting pinning and durable evidence is not weakened), and the advisory itself performs no Git-ref creation, update, or deletion: the test compares the ref set produced by a full hook run against an equivalent coordinate-only baseline (or injects a fake advisory seam and asserts the real advisory adds no ref delta), so any ref change from the coordinator is legitimate and is never attributed to maintenance; (S15) direct advisory contention (scenario A): an explicit pass parked at the post-inventory seam (and, in a second parameterization, at the post-DB-open seam) holds `WorktreeLock`; `advise_if_due` is invoked directly, returns `Busy` through a channel handshake without waiting for the pass, and leaves the state file byte-identical and the ref set unchanged; (S16) coordinator contention (scenario B): with the explicit pass parked holding the lock, a `coordinate()` Close/Flush with a short injected lock timeout fails with the existing lock-timeout error before durable completion and mutates nothing, the existing bounded-acquisition semantics are unchanged, the failed coordinate is a legitimate pre-commit failure, the advisory seam is never invoked after it, and once the pass is released the same boundary succeeds (documents the D1 concurrency contract); (S17) the advisory runs only after the coordinator's lock is released — a seam asserts the worktree lock is acquirable at invocation; (S18a) normal completion: every advisory outcome, a state-write failure, a corrupt state file, and a seam error leave the hook result and `Ok` status identical to the coordinate-only baseline; the marker state is snapshotted immediately before the advisory runs and asserted byte-identical (existence) afterwards, so the advisory never creates, clears, or modifies it, and the normal coordinator-cleared state is unchanged; (S18b) `MarkerClearAfterCommit`: reusing the coordinator's existing marker-clear failure fixture (no change to `coordinator.rs`, `ProtectedWorktree`, or recovery semantics; if unreachable from the hook layer, a fake coordinate seam returns that variant while a real armed marker file exists), the boundary is classified as durably completed, the hook returns the same successful empty-stdout result with the advisory allowed to run, the marker the failed clear left behind is asserted still armed and unmodified after the advisory, and a subsequent real `coordinate()` invocation observes the inherited taint and takes the existing conservative recovery path; (S33) post-coordinate handoff race (scenario C): `coordinate()` completes successfully and releases its lock; a seam pauses the hook immediately before the advisory call; a competing explicit reconciliation acquires `WorktreeLock` through a barrier; the advisory resumes and returns `Busy`; the already committed boundary still returns success, and the maintenance state and refs are untouched by the advisory. + - *State, crashes, invalid timestamps (P2):* (S19) absent state → anchor written once, no advice; a second call writes nothing (file byte-identical); (S20) due state → exactly one time-based advice per window with `last_advised` persisted *before* the advice is returned; repeated calls inside the window do not write or advise; after the injected clock passes the window the advice recurs once; a failure recorded after `last_advised` is advised once inside the window, then suppressed, then repeats on the normal cadence until a successful explicit pass clears it; (S21) concurrent processes: one advisory parked inside its locked section, a second returns `Busy`, a third after release returns `NoAction` — exactly one advice, and an explicit pass concurrent with an advisory serializes (no duplicate explicit passes, second returns busy/waits per policy); (S22) state-persistence failure (injected write error): the advisory emits no advice and never scans; an explicit pass whose final write fails returns `CompletedStatePersistFailed` — distinct from `Completed` and `Failed` — with its deletions intact; (S23) crash durability: an interrupted atomic write (temp file left, rename not performed) leaves the previous state readable and eligibility preserved, a stale temp file is ignored/overwritten, and a fresh call (simulating process restart) re-reads state and advises at the correct time; (S24) clock-skew tolerance, with an injected clock for every case (`FUTURE_SKEW_TOLERANCE` = 5 min): (1) stored `last_success` equal to `now` — valid, age 0, no advice, file byte-identical; (2) one minute ahead of `now` — valid, age 0, preserved, no advice, no rewrite; (3) exactly five minutes ahead — valid (boundary is inclusive), age 0, preserved, no advice, no rewrite; (4) more than five minutes ahead (just beyond, and far-future) for `last_success`, `last_advised` and `anchor` — invalid, normalized once under the lock and persisted per D3 (invalid fields cleared; with `last_success` and `anchor` both unusable the advisory fires immediately once, persists `last_advised = now` and a fresh `anchor = now`); (5) clock rollback beyond tolerance (stored values valid at write time, injected `now` moved back more than five minutes) — treated as case 4; a rollback within tolerance is case 2/3 and never makes an advisory eligible by itself; (6) repeated reads after normalization — the file is byte-identical across repeated calls, no rewrite occurs, and eligibility is neither postponed nor slid; (7) process restart (fresh call re-reading the file) followed by clock advance past `RECONCILIATION_ADVISORY_AFTER` — advice recurs exactly once, proving eventual eligibility. After every **successful state-writing** operation an assertion checks the exact invariant `t ≤ now + FUTURE_SKEW_TOLERANCE` for every stored timestamp (not "no timestamp is in the future"); after read-only, lock-busy, or failed operations the assertion is instead that the state file is byte-identical to its prior content, with no normalization claimed; (S25) missing, unparsable, wrong-version, and oversized state files are recovered without failing any operation and are rewritten valid by the next successful recorded operation (until then, read-only and failed operations leave them untouched and treat them as default state). + - *Timestamp-normalization regressions (P2; injected clocks and deterministic synchronization, no sleeps):* (S34) read-only doctor observing an invalid future timestamp — plain `sce doctor` reports "recommended" with the invalid-timestamp note, the state file is byte-identical afterwards, and it is not reported as normalized; (S35) lock-busy advisory — with the lock held through a barrier and an invalid future timestamp stored, `advise_if_due` returns `Busy` and the state file is byte-identical; the same holds for a lock-busy explicit pass (`Skipped(Busy)`); (S36) failed state write during a normalizing operation, in two legs. *Pre-rename leg* (`NotApplied`, e.g. a rename failure): the previous state is byte-identical and readable, the advisory reports `StateWriteFailed` with no advice and no claimed normalization, the explicit pass reports `CompletedStatePersistFailed`, and a later successful operation still normalizes it. *Post-rename leg* (`DurabilityUncertain`, e.g. a parent-directory sync failure): the new state may already be visible, so byte identity is not asserted; the advisory reports `AdvisedDurabilityUncertain` with an actionable diagnostic, the normalization is not claimed durable, and a later invocation recovers safely (re-reads the visible state, performs no rewrite). `StateUnavailable` is reserved for state that cannot be read or evaluated and is not a write outcome; (S37) successful normalization — a normalizing advisory and a normalizing explicit pass each persist state in which every timestamp satisfies `t ≤ now + FUTURE_SKEW_TOLERANCE`, valid near-future values (within tolerance) are preserved byte-for-byte, and a repeated read performs no rewrite and does not move eligibility forward; (S38) process restart after normalization — a fresh call re-reading the persisted file observes the normalized timestamps, performs no rewrite, and does not re-fire the immediate advice; (S39) eventual eligibility — with invalid timestamps unnormalized (S34/S35/S36 states) repeated read-only evaluations never push eligibility out, and after the injected clock advances past `RECONCILIATION_ADVISORY_AFTER` a successful advisory fires exactly once and persists normalized state. + - *Explicit recovery:* (S26) `sce doctor --fix` runs and reports its true result from each of: failed streak, stale advice, invalid timestamps, corrupt state, unwritable state (→ `CompletedStatePersistFailed`); (S27) explicit success resets the failure streak and sets `last_success`; explicit failure increments the streak without touching `last_success`; lock-busy never changes state. + - *DB opening (D4):* (S28) missing DB — fails with the `Missing` typed error from the single non-creating read-only open (no existence pre-check); no database file, parent directory, WAL, or shared-memory file is created (directory listing before and after is identical, including when the parent directory is absent); Git refs are neither listed nor deleted (spy on `list_pins`/`delete_pins` never called); (S29) existing DB with incompatible schema — opening is permitted; fails with `IncompatibleSchema`; no migration or repair row written (migration table rows identical before and after); no ref deleted; (S30) existing DB with missing repository metadata — fails with `MissingMetadata`; no `repository_metadata` row inserted; (S31) existing DB with repository identity mismatch — fails with `RepositoryMismatch`; no row changed; (S32) existing valid DB — identity and schema readiness are verified before any ref listing (ordering asserted via seam), the multiprocess-WAL configuration is preserved, the pass completes, and no initialization, repair, or migration runs. For S29–S31 the assertions are logical: a before/after dump of every table's rows is equal (no migration, repair, initialization, or application-table INSERT/UPDATE/DELETE), refs are unchanged, and the external-taint marker is unarmed. Filesystem immutability is **not** asserted for existing-DB failure cases, because incidental Turso WAL/shared-memory bookkeeping is permitted when unavoidable; only S28 asserts that no database-related file or directory is created. + Out — production behavior changes beyond test seams strictly required. + - Dependencies: T03, T04 + - Done when: all scenarios pass reliably, each fails when the guarded behavior is deliberately broken, no test depends on timing or sleeps, and every review finding in the traceability table maps to at least one passing scenario. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` focused on the new tests, repeated runs for the race cases; zero-match filters are not accepted as passes; `rg "sleep"` over new test code returns no hits. + - Reopened: 2026-10-09 (the earlier `done`/`synced` marking was unsupported: S6, S7, S16, S17, S18b and S33 were listed as "not added" or covered only through fake closures, helper tests or enum mappings). + - Files changed (reopening pass): `cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs` and new `ref_maintenance/tests/{support,coordination,worktrees,cross_process,fail_closed,state_recovery}.rs`; `cli/src/app/doctor_reconciliation_cli_tests.rs` (new, registered in `cli/src/app.rs`); `ref_advisory.rs` (tests); `git_snapshot.rs` (deletion-cancellation test); production test seams only — `coordinator.rs` (`coordinate_inner` takes `lock_timeout`; `coordinate` passes `WORKTREE_LOCK_TIMEOUT`, so production behaviour is unchanged), `protected_worktree.rs` (`acquire_with_timeout` is `pub(super)`), `git_snapshot.rs` (`delete_pins_inner` forwards `on_worker_entered`; the dead `run_ref_mutation` wrapper was removed), `hooks/mutation_scope.rs` (`drive_mutation_scope` is `pub(crate)`). No protocol, Quint, dispatch, runtime or timeout changes. + - Findings that shaped the tests: (1) a parked pass must run on its own thread and runtime — parking inside `tokio::spawn` blocks a worker and starves the process/IO driver the coordinator needs, which made the coordinator stall before it ever tried the lock; (2) `sce doctor --fix` runs a separate global-state repair that creates the Agent Trace DB parent directory before reconciliation, so the CLI-level missing-DB guarantee is "no DB file or sidecars", not "no parent directory"; (3) doctor failure kinds are `agent_trace_db_missing` and `agent_trace_db_repository_mismatch`, not a generic `agent_trace_db_unavailable`. + - Test-suite simplification (2026-10-09, PR #305 audit; no production behavior changed; scenario statuses unchanged): 19 registrations removed (498 → 479). Deleted: `ref_reconciliation_production_entrypoint_accepts_only_a_repository_root` (signature-only), `ref_reconciliation_worktree_wrapper_acquires_once_and_releases` (with the then-unused `#[cfg(test)]` `reconcile_worktree`, `reconcile_worktree_inner` and `RECONCILIATION_LOCK_TIMEOUT`), `ref_reconciliation_advisory_is_busy_while_an_explicit_pass_holds_the_lock` (superseded by the two independent-task S15 tests), `advisory_runs_after_marker_clear_after_commit_with_success_result` (subset of `marker_clear_after_commit_logs_taint_warning_and_still_advises_once`). Consolidated into tables: five `maintenance_state` pre-rename tests → `maintenance_state_pre_rename_failures_preserve_previous_state`; two post-rename tests → `maintenance_state_post_rename_failures_report_uncertain_durability`; two timestamp tests → `maintenance_advisory_accepts_valid_timestamp_offsets_without_rewrite`; two unusable-state tests → `maintenance_advisory_reanchors_unusable_state_variants`; four doctor JSON tests → `json_ref_reconciliation_fix_results_match_contract` + `json_completed_state_persist_failed_results_match_contract` (whole-object equality); six doctor text tests → `reconciliation_fix_results_render_expected_text`. The duplicate `unreadable` leg was dropped from the S26 table. Close/Flush S16 tests share `assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary)` and keep their exact names. Strengthened: malformed ref compares the full `refs/sce/` inventory (name, object id, symref) and the `MalformedPin` ref name; unreadable previous state asserts `AgentTraceDbUnavailable` → typed `ExistingRepositoryDbError::Missing`, untouched state directory and pins; the verified opener test proves the statement valid on a writable connection first and asserts `database is readonly`; `Sandbox::run` now asserts the real `ExitCode` (typed `==`) for every doctor invocation; the non-database lifecycle test asserts `AgentTraceDbConnectionFailed`; the pre-rename table also checks byte identity, absent-file and recovery. The guard-removal records below remain historical: test names they cite that no longer exist are covered by the consolidated equivalents named in the map above. Guard re-check: forcing `read_only(false)` in `db/mod.rs` makes `ref_reconciliation_verified_open_is_read_only` fail (restored). Verification: full CLI suite 479 passed, 0 failed; focused `maintenance_state` 6, `ref_advisory` 24, `ref_reconciliation` 48, `doctor` 19, `mutation_scope` 24, `lifecycle` 9 (all non-zero); 8 consecutive passes of the 62 `doctor_reconciliation_cli`/`ref_reconciliation`/cancellation/lock-timeout tests; clippy `-D warnings` clean; `nix flake check` passed; `git diff --check` clean. + - Verification, remediation pass (2026-10-09): full CLI suite 495 passed, 0 failed (two new doctor CLI tests); focused `doctor_reconciliation_cli` 7, `ref_reconciliation` 50, `mutation_scope` 25, `maintenance` 85 (all non-zero); new doctor CLI tests 5 consecutive passes; every `ref_maintenance::tests` test and the cancellation tests 20 consecutive passes; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; `cargo fmt` made no change; `nix flake check` passed (clippy, fmt, CLI tests, Quint connect). + - Verification (earlier pass, 2026-10-09): full CLI suite 493 passed, 0 failed; focused `ref_reconciliation` 50, `mutation_scope` 25, `doctor` 25, `maintenance` 83 (all non-zero); `clippy --all-targets --all-features -D warnings` clean; `nix flake check` passed (clippy, fmt, CLI tests, Quint connect); `git diff --check` clean; 20 consecutive passes of every `ref_maintenance::tests::*` and cancellation test, 5 consecutive passes of the doctor CLI tests; no added comments, sleeps, `allow(dead_code)`, `dyn`, `BoxFuture`, detached tasks or extra production runtimes. + - Guard-removal checks actually performed: repository-wide roots replaced by worktree-local roots → S3 and S13 tests fail; deletion worker drops its lease → the new deletion-cancellation test and the existing pin-cancellation test fail; advisory ignores lock contention → S15 (both phases) and S33 fail. The remaining guards (S6, S16, S17, S18b, S21) were checked in the remediation pass; see the results table below. + - Remediation (2026-10-09, PR #305 review of `982976f4`, advisory persistence semantics; T01–T06 statuses unchanged): `AdvisoryOutcome::diagnostic()` now follows the carried `PersistFailure`. `StateWriteFailed` + `NotApplied` keeps `SCE: Reconciliation advisory state could not be written: . Run sce doctor to check reconciliation status.`; `StateWriteFailed` + `DurabilityUncertain` (post-rename anchor or normalization write) reads `SCE: Reconciliation advisory-state durability could not be confirmed: . Run sce doctor to check reconciliation status.` and never claims the update was not written or that advice was emitted. Outcome mapping, scheduling, locking, and Git-ref reconciliation are unchanged; no new variants or seams. Added tests in `ref_advisory::tests`: `maintenance_anchor_post_rename_failure_is_state_write_failed_with_visible_anchor` (absent state, `SyncParentDirectory` fault, anchor visible, next call `NoAction` with byte-identical state), `maintenance_no_action_normalization_post_rename_failure_is_state_write_failed` (fresh anchor plus far-future `last_advised`, no advice due; invalid timestamp removed; next call `NoAction`, no rewrite); extended `maintenance_report_maps_every_outcome_to_severity_and_diagnostic` (uncertain `StateWriteFailed`), `maintenance_advisory_rename_failure_preserves_state_and_next_call_can_advise` (pre-rename wording) and `assert_durability_uncertain_advisory` (recommendation retained, durability wording). Verification: see the final verification record for this remediation. + - Remediation (2026-10-09, PR #305 review of `39f78941`; no gap was waived and no production code changed): (1) S9 persistence-warning outcome and S26 recovery table now run through the real `sce doctor --fix` (`doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable`, `doctor_fix_recovers_from_every_initial_maintenance_state_condition`), using a naturally unreadable state (a directory at `.git/sce/ref-maintenance.json`) rather than a fault seam; `NotApplied` and `DurabilityUncertain` stay at the persistence layer (`ref_advisory::tests`, `rm::state_recovery`), with no CLI fault seam. (2) The doctor CLI child now captures the `ExitCode` returned by `run_with_dependency_check_and_streams`, writes `success`/`failure` to an `exit` file beside `stdout`/`stderr`, `CliRun` exposes it, and every doctor invocation asserts it. The CLI contract (unchanged): `render_report` always returns `Ok`, so plain doctor and `--fix` exit successfully for completed, failed-fix, busy and persistence-warning outcomes; an individual failed fix is reported in the payload, not in the exit status. (3) S14 is verified by static call-graph inspection plus real-hook ref-delta assertions (see the S14 row); no spy/panic seams were added. (4) The five remaining guard-removal checks were performed in a throwaway `git worktree` at `39f78941` (shared `CARGO_TARGET_DIR`, removed afterwards); results are in the table below. (5) D2 now distinguishes `StateUnavailable`, `StateWriteFailed`, `AdvisedDurabilityUncertain`, `Advised`, `NoAction` and `Busy`, and documents the single immediate advisory for a newly recorded failure; D3, AC5, AC13, S20 and S36 already agreed and were not changed. + - Guard-removal results for the five remaining guards (actual outcomes; every mutation lived only in the throwaway worktree and none is in the diff): + + | Scenario | Guard temporarily changed | Targeted test | Expected failure observed | Restored test passes | + |---|---|---|---|---| + | S6 | `ProtectedWorktree::acquire_with_timeout` took its lock from a diverted Git dir (`git_dir.join("bypass")`), so the coordinator no longer contends with the pass | `ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_db_open` and `…_after_pin_inventory` | Yes, both: the coordinator ran to completion without ever signalling lock contention (`signal sender dropped before signalling`) | Yes (2 passed) | + | S16 | `classify_coordinate` in `hooks/mutation_scope.rs` swallowed `CoordinateError::LockAcquisition` as a successful empty result | `ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries` and `ref_reconciliation_flush_lock_timeout_under_maintenance_commits_nothing_and_retries` | Yes, both: `expect_err` on the contended boundary failed (`the lock cannot be acquired while the pass holds it: ""`), i.e. the lock timeout was no longer reported as a pre-commit failure | Yes (2 passed) | + | S17 | `Close` leaked the coordinator's `WorktreeLock` past completion (flagged in `coordinate_inner`, `std::mem::forget` of the lock in `complete_blocking`), the observable equivalent of running the advisory before the release | `ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock` | Yes: the zero-wait acquisition inside the advisory callback failed (`the coordinator lock is released before the advisory runs: TimedOut { timeout: 0ns }`). A first attempt that leaked the lock on every boundary failed during test setup and was discarded as not evidence | Yes (1 passed) | + | S18b | `coordinate_inner` passed `false` instead of `protected.inherited_external_taint()` to `coordinate_protected` | `ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate` | Yes: `inherited taint conservatively abandons the live scope` failed with `left: Some(Active)` | Yes (1 passed) | + | S21 | `advise_if_due_with` acquired the lock and dropped it immediately, so no lock covered the state-writing section | `ref_reconciliation_advisory_lock_serializes_independent_processes` | Yes: the second independent process returned `Advised` where `Busy` was required (duplicate advice). A first attempt failed to compile (unused import) and was discarded as not evidence | Yes (1 passed) | + + No sleeps were used to force interleavings; the tests use their existing barriers, parked-phase seams and process rendezvous. + - Context impact: none (tests and test seams only). + - Context synchronization: synced + + - **Coverage map (S1–S39; 40 rows because S18 is split into S18a/S18b; all 40 Verified after the remediation pass).** Status is `Verified`, `Partially verified` or `Not verified`; a related unit test never upgrades a production-path scenario. Test names are exact; paths are relative to `cli/src/services/mutation_trace/runtime/` unless they start with `app::` or `services::`. `rm::` = `ref_maintenance::tests::`. + + | Scenario | Exact test | Test level | Verified invariant | Status | + |---|---|---|---|---| + | S1 | `app::doctor_reconciliation_cli_tests::doctor_inspection_is_read_only_and_fix_reconciles_the_orphan_pin_once`; `rm::ref_reconciliation_verified_database_reconciles_orphan_pin_and_records_success` | CLI end-to-end; pass-level | `sce doctor --fix` deletes an orphan SCE pin and records success (the orphan is created directly with `git update-ref`, not by an interrupted coordinate) | Verified | + | S2 | `rm::worktrees::ref_reconciliation_historical_roots_of_a_real_coordinator_sequence_stay_protected` | Production integration (real coordinator, real Git) | A→B→C→D from real `Start`/`Flush` boundaries: all four trees stay pinned although only D is the cursor; one orphan is reclaimed | Verified | + | S3 | `rm::worktrees::ref_reconciliation_pin_of_one_real_worktree_is_retained_while_another_needs_the_tree`; `rm::ref_reconciliation_retains_every_repository_wide_durable_root_and_never_repairs_other_worktrees` | Production integration (real linked worktree); synthetic roots | Main's pin of a tree that only the linked worktree durably requires is retained | Verified | + | S4 | `rm::fail_closed::ref_reconciliation_missing_local_pin_fails_closed_while_other_orphans_exist` | Pass-level | Missing local required pin returns `MissingRequiredPins` and deletes nothing, even with an eligible orphan present | Verified | + | S5 | `rm::ref_reconciliation_malformed_namespace_ref_deletes_nothing_and_records_failure` | Pass-level | Symbolic ref in the namespace yields `MalformedPin` naming that ref; the full `refs/sce/` inventory (ref name, object id, symref target) is identical before and after, so a delete-and-replace cannot pass; failure recorded | Verified | + | S6 | `rm::coordination::ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_db_open`; `…_after_pin_inventory` | Production integration (real `coordinate_inner`, real lock) | Coordinator hits contention, opens no DB and arms no marker while the pass holds the lock; the pin created after release survives; only the orphan is deleted; new pin precedes the durable-root commit; lock free afterwards | Verified | + | S7 | `services::mutation_trace::runtime::git_snapshot::tests::cancelled_delete_keeps_worktree_lock_until_git_ref_deletion_completes` | Cancellation integration (real `delete_pins`) | Dropping the caller and outer lock owner leaves the lock held by the parked worker; after release the pin is gone, no `.lock` files remain, a new pin proceeds. Existing `cancelled_pin_keeps_worktree_lock_until_git_ref_mutation_completes` covers the pin side | Verified | + | S8 | `rm::ref_reconciliation_retains_every_repository_wide_durable_root_and_never_repairs_other_worktrees` (second pass); CLI test second `--fix` run | Pass-level; CLI | Second pass deletes nothing | Verified | + | S9 | `app::doctor_reconciliation_cli_tests::doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable` (persist-warning); `…::doctor_inspection_is_read_only_and_fix_reconciles_the_orphan_pin_once` (plain doctor, completed); `…::doctor_fix_reports_missing_database_without_creating_it_or_deleting_refs` and `…::doctor_fix_reports_repository_mismatch_without_touching_the_database_or_refs` (failed); `…::doctor_fix_reports_busy_when_the_worktree_lock_is_held_and_keeps_other_repairs` (busy); `services::doctor::render::reconciliation_render_tests::{json_ref_reconciliation_fix_results_match_contract (cases completed, failed without warning, failed with warning, busy), json_completed_state_persist_failed_results_match_contract (cases not_applied, durability_uncertain, previous_state_unreadable)}` (whole-object JSON equality per case) and `services::doctor::reconciliation_fix_tests::reconciliation_fix_results_render_expected_text` (table: completed, pre-rename failure, post-rename failure, unreadable previous state, failed, busy; required and forbidden text per case) (mapping tests for `NotApplied`/`DurabilityUncertain`); `rm::ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome`; `ref_advisory::tests::{maintenance_normalization_pre_rename_failure_keeps_previous_bytes_and_recovers_later, maintenance_normalization_post_rename_failure_is_visible_but_durability_is_uncertain}` | CLI end-to-end for completed, failed, busy and `completed_state_persist_failed` (kind `previous_state_unreadable`, via a directory at the state path: orphan deleted, counts 1/0/0, `failure` null, detail reports the warning without "failed", directory untouched, exit success); persistence layer and render/mapping unit tests for `NotApplied` and `DurabilityUncertain` | Plain doctor mutates neither refs nor state; `--fix` reports the true result and the CLI exit status is asserted. At CLI level only `previous_state_unreadable` is exercised for the persist-warning outcome; `not_applied` and `durability_uncertain` are covered at the persistence and rendering layers by design (no CLI fault seam) | Verified | + | S10 | `rm::ref_reconciliation_failure_streak_counts_but_lock_contention_never_does`; `rm::fail_closed::ref_reconciliation_second_explicit_pass_is_busy_and_records_exactly_one_pass`; CLI busy test | Pass-level; CLI | Lock contention returns `Skipped(Busy)`, leaves state byte-identical and is not a failure | Verified | + | S11 | `rm::fail_closed::ref_reconciliation_database_failure_never_arms_the_external_taint_marker`; `rm::ref_reconciliation_missing_database_creates_nothing_and_deletes_nothing` | Pass-level | DB failure leaves refs untouched, records a streak, never arms the marker | Verified | + | S12 | `rm::ref_reconciliation_lock_held_implementation_never_reacquires_the_lock`; S6/S15 tests (the removed test-only `reconcile_worktree` wrapper test and synchronous advisory-busy test are superseded by these and by the production `reconcile_explicit` lock-once path exercised in S6/S15/S16/S21) | Pass-level; production integration | Single acquisition per operation, shown by bounded completion under the existing self-contending lock (no acquisition counter) | Verified | + | S13 | `rm::worktrees::ref_reconciliation_missing_pin_in_another_real_worktree_neither_fails_nor_is_repaired` | Production integration (real linked worktree) | Reconciling main reclaims its orphan, keeps the pin protecting the linked tree, never recreates the linked worktree's missing pin, does not fail; the linked worktree's own pass fails closed | Verified | + | S14 | Behavioral: `services::hooks::mutation_scope::advisory_trigger_tests::{advisory_runs_once_after_completed_close_and_flush, marker_clear_after_commit_logs_taint_warning_and_still_advises_once, advisory_never_runs_for_start_or_advance, advisory_never_runs_after_failed_coordinate, advisory_never_runs_for_abandon}` (when it runs); `rm::coordination::{ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries, ref_reconciliation_flush_lock_timeout_under_maintenance_commits_nothing_and_retries}` (real failed coordinate: advisory count 0); `rm::coordination::ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock` (real `Close`: refs identical before/after the advisory, marker snapshot identical, commit unchanged, zero-wait lock acquirable); `rm::coordination::ref_reconciliation_advisory_failures_after_a_real_commit_never_change_the_hook_result` (errors do not change the result); `rm::coordination::ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate` (marker fingerprint identical across the advisory); `rm::coordination::ref_reconciliation_advisory_after_a_real_commit_is_busy_when_a_pass_wins_the_handoff` (refs and state untouched by the advisory). Structural: call graph below | Hook policy (fake coordinate); production integration (real coordinator, real Git) for ref delta; static inspection | Advisory fires only after a durably completed `Close`/`Flush` and never after a failed coordinate, `Start`, `Advance` or `Abandon`; it adds no ref delta, never modifies the external-taint marker, its errors never change a committed result, and the coordinator's lock is already released. Structural, from `advise_after_completed_boundary(&Path)`: it calls only `advise_if_due(repository_root, system_unix_time_ms)` → `derive_git_dir` (filesystem walk, `std::fs`, no Git subprocess), `read_state`/`plan_advisory`/`normalize`/`evaluate` (pure or `std::fs`/`std::io`, capped read), the synchronous `worktree_lock::acquire_inner` with `Duration::ZERO` (one `try_lock`; returns `TimedOut` before any sleep), `write_state_atomically` (staged file, rename, directory sync) and `outcome_after_write` (pure). Production code in `ref_advisory.rs` and `maintenance_state.rs` has no `crate::` import, no `async`/`await`/`tokio`/`spawn_blocking`, no `Command`, no Git invocation (the only `git` token is the `.git` path component in `derive_git_dir`), and no reference to `reconcile_explicit`, `reconcile_with_held_lock`, `GitSnapshotService`, `list_pins`, `delete_pins`, `RepositoryAgentTraceDb` or a store. The `async`/`spawn_blocking` hits in `worktree_lock.rs` are only `acquire_async`/`acquire_inner_async`, which the advisory does not call. The only production caller is the injected seam in `hooks/mutation_scope.rs`. Verified through static production call-graph inspection and real-hook ref-delta assertions. No spy/panic seams added. | Verified | + | S15 | `rm::coordination::ref_reconciliation_independent_advisory_task_is_busy_while_a_pass_is_parked_after_db_open`; `…_after_pin_inventory`; | Concurrency integration (independent blocking task) | Advisory returns `Busy` without waiting, state and refs unchanged, pass continues, later advisory follows the resulting state (`NoAction`, then `Advised` after the window). The earlier synchronous in-callback test was deleted: it asserted nothing these two tests do not (`Busy`, no state file while parked, refs intact, `NoAction` afterwards) and was not independent concurrency | Verified | + | S16 | `rm::coordination::ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries` (`Close`); `…_flush_lock_timeout_under_maintenance_commits_nothing_and_retries` (`Flush`); both are thin `#[tokio::test]` wrappers over `assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary)`, so each stays separately runnable | Production integration (real coordinator, injected 300 ms timeout) | Existing lock-timeout error; no durable transition, no pins, no marker, no advisory after the failure; the same boundary succeeds after release | Verified | + | S17 | `rm::coordination::ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock` | Hook/coordinator integration | Zero-wait `WorktreeLock` acquisition succeeds inside the advisory callback; the commit is already durable and unchanged afterwards; refs unchanged; empty stdout | Verified | + | S18a | `rm::coordination::ref_reconciliation_advisory_failures_after_a_real_commit_never_change_the_hook_result`; `advisory_trigger_tests::{diagnostic_write_failure_does_not_change_mutation_result, state_failures_log_warn_and_write_diagnostic_without_claiming_advice, quiet_outcomes_log_debug_and_emit_no_diagnostic}` | Hook/coordinator integration; hook policy | Write-failed, corrupt and unreadable-state advisories leave the hook result, committed state, refs and marker unchanged | Verified | + | S18b | `rm::coordination::ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate` | Recovery integration (real coordinator; unlink fault injected inside the protected section) | Failed marker clear is a durable success; marker fingerprint identical across the advisory; after the fault is lifted the next real `coordinate()` takes the inherited-taint path (live scope `Abandoned`, health restored, revision advanced ≥2, marker cleared) whereas an untainted control flush left it `Active` | Verified | + | S19 | `ref_advisory::tests::maintenance_advisory_anchors_once_then_leaves_state_untouched` | Unit (advisory) | Anchor written once, second call byte-identical | Verified | + | S20 | `ref_advisory::tests::{maintenance_advisory_advises_once_per_window_and_persists_before_returning, maintenance_failure_advisory_emits_once_and_doctor_still_recommends_repair, maintenance_failure_advisory_repeats_after_the_cadence_window, maintenance_failure_after_recent_success_is_recommended_and_advised_despite_older_reminder}` | Unit (advisory) | One time-based advice per window, persisted first; failure advice once then cadence | Verified | + | S21 | `rm::cross_process::ref_reconciliation_advisory_lock_serializes_independent_processes`; `rm::fail_closed::ref_reconciliation_second_explicit_pass_is_busy_and_records_exactly_one_pass` | Independent OS processes (self re-exec, stdin/stdout rendezvous); in-process | Parked advisory in one process, `Busy` in a second, `NoAction` in a third; exactly one advice; no state write while parked. Explicit-vs-explicit is in-process only | Verified | + | S22 | `ref_advisory::tests::maintenance_advisory_anchor_write_failure_reports_state_write_failed_without_advice`; `rm::ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome` | Unit; pass-level | Persistence failure: no advice; explicit pass reports `CompletedStatePersistFailed` with deletions intact | Verified | + | S23 | `maintenance_state::tests::maintenance_state_recovers_after_interrupted_pre_rename_write`; `maintenance_state::tests::maintenance_state_pre_rename_failures_preserve_previous_state` (table: `CreateStaging`/`WriteStaging`/`SyncStaging`/`Rename` with previous state, `Rename` without; byte-identical or absent, no staging leftovers, later write recovers); `ref_advisory::tests::maintenance_advisory_rename_failure_preserves_state_and_next_call_can_advise` | Unit | Interrupted write preserves previous state; fresh call advises at the right time | Verified | + | S24 | `ref_advisory::tests::{maintenance_advisory_accepts_valid_timestamp_offsets_without_rewrite (table: offsets `0`, `+60 s`, `FUTURE_SKEW_TOLERANCE_MS`; `NoAction`, byte-identical state, no recommendation), maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately, maintenance_advisory_clock_rollback_beyond_tolerance_normalizes_while_within_it_does_not}`; `rm::state_recovery::ref_reconciliation_invalid_last_advised_does_not_suppress_a_due_advisory`; `maintenance_advisory_advises_once_per_window_and_persists_before_returning` | Unit (injected clocks) | Cases 1–7: equal and near-future valid; exactly +5 min valid; beyond tolerance (anchor, last_success, last_advised) normalized once; rollback beyond vs within tolerance; byte-identical repeats; window recurrence | Verified | + | S25 | `ref_advisory::tests::{maintenance_advisory_anchors_once_then_leaves_state_untouched, maintenance_advisory_reanchors_unusable_state_variants (table: wrong schema version, oversized file, corrupt JSON; `StateRead::Unusable`, `Anchored`, valid state with `anchor == NOW`, no recommendation, repeat call does not rewrite)}`; `rm::state_recovery::ref_reconciliation_explicit_pass_runs_and_rewrites_valid_state_from_every_stored_condition` | Unit; pass-level | Missing, corrupt, wrong-version and oversized state recover without failure and are rewritten valid | Verified | + | S26 | `app::doctor_reconciliation_cli_tests::doctor_fix_recovers_from_every_initial_maintenance_state_condition` (failed streak, expired advisory, invalid far-future timestamps, corrupt JSON, oversized file → real `sce doctor --fix`); `app::doctor_reconciliation_cli_tests::doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable` (the table no longer repeats the unreadable case; this dedicated test is the single end-to-end check for `CompletedStatePersistFailed` / `previous_state_unreadable`); pass-level: `rm::state_recovery::ref_reconciliation_explicit_pass_runs_and_rewrites_valid_state_from_every_stored_condition`, `…::ref_reconciliation_explicit_success_resets_the_streak_and_failure_keeps_last_success`, `rm::ref_reconciliation_unreadable_previous_state_is_reported_without_overwriting` (primary error `AgentTraceDbUnavailable` with typed `ExistingRepositoryDbError::Missing`, `PreviousStateUnreadable` warning, state directory and pin inventory unchanged), `rm::ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome` | CLI end-to-end (real repo, real Agent Trace DB, real `sce doctor --fix --format json`); pass-level | From each readable condition the explicit pass runs regardless of advisory eligibility, reclaims the orphan pin, reports `completed` (deleted 1), clears the failure streak and failure fields, sets `last_success`, normalizes far-future timestamps (all `≤ now + FUTURE_SKEW_TOLERANCE`), leaves a state a second `--fix` reads as valid, and exits successfully. From an unreadable state path (a directory) the pass still deletes the orphan, reports `completed_state_persist_failed` with `previous_state_unreadable`, makes no false persistence claim, leaves the directory untouched, and exits successfully | Verified | + | S27 | `rm::state_recovery::ref_reconciliation_explicit_success_resets_the_streak_and_failure_keeps_last_success`; `rm::ref_reconciliation_failure_streak_counts_but_lock_contention_never_does` | Pass-level | Success resets streak and sets `last_success`; failure increments without touching `last_success`; busy changes nothing | Verified | + | S28 | `rm::ref_reconciliation_missing_database_creates_nothing_and_deletes_nothing`; `rm::ref_reconciliation_verified_open_never_creates_missing_database_or_directories`; `rm::ref_reconciliation_verified_open_rejects_database_removed_before_open`; `app::doctor_reconciliation_cli_tests::doctor_fix_reports_missing_database_without_creating_it_or_deleting_refs` | Pass-level; opener; CLI | Typed `Missing`; no DB file, directory or sidecar created by the open; no refs listed or deleted. The CLI shows the parent directory comes from the separate global-state repair | Verified | + | S29 | `rm::state_recovery::ref_reconciliation_verification_failures_leave_the_logical_database_unchanged` (case `incompatible`); `rm::ref_reconciliation_missing_metadata_or_schema_fails_before_pin_inventory` | Pass-level | `IncompatibleSchema`; full-row dump of every table equal before and after; no pin inventory; refs and marker untouched | Verified | + | S30 | same test (case `missing_metadata`) | Pass-level | `MissingMetadata`; no `repository_metadata` row inserted (full-row dump equal) | Verified | + | S31 | same test (case `mismatch`); `app::doctor_reconciliation_cli_tests::doctor_fix_reports_repository_mismatch_without_touching_the_database_or_refs` | Pass-level; CLI | `RepositoryMismatch`; foreign metadata row intact; refs untouched | Verified | + | S32 | `rm::ref_reconciliation_verified_database_reconciles_orphan_pin_and_records_success` (phase order `DbOpened` → `PinsInventoried`); `rm::ref_reconciliation_verified_open_coexists_with_a_live_multiprocess_wal_writer`; `rm::ref_reconciliation_verified_open_is_read_only` | Pass-level; opener | Identity and schema verified before any ref listing; multiprocess-WAL coexistence; the opened handle rejects a statement first proven valid on a writable connection (rolled-back transaction) with `database is readonly`, and the rows stay empty. Breaking the guard (`read_only(false)` in `db/mod.rs`) makes this test fail | Verified | + | S33 | `rm::coordination::ref_reconciliation_advisory_after_a_real_commit_is_busy_when_a_pass_wins_the_handoff` | Concurrency integration (real coordinator; hook parked between coordinate and advisory; pass on its own thread) | Pass takes the lock in the handoff window; advisory returns `Busy`; state and refs unchanged by the advisory; committed `Close` still reports success; only the orphan is deleted by the pass | Verified | + | S34 | `ref_doctor::tests::recommendation_flags_unusable_and_invalid_state_without_rewriting` | Unit (doctor inspection) | Read-only doctor flags invalid timestamps and leaves bytes untouched | Verified | + | S35 | `ref_advisory::tests::maintenance_advisory_reports_busy_and_leaves_invalid_state_byte_identical`; `rm::state_recovery::ref_reconciliation_busy_explicit_pass_leaves_invalid_state_byte_identical` | Unit; pass-level | Lock-busy advisory and explicit pass leave invalid state byte-identical | Verified | + | S36 | `ref_advisory::tests::{maintenance_normalization_pre_rename_failure_keeps_previous_bytes_and_recovers_later, maintenance_normalization_post_rename_failure_is_visible_but_durability_is_uncertain}`; `maintenance_state::tests::{maintenance_state_pre_rename_failures_preserve_previous_state, maintenance_state_post_rename_failures_report_uncertain_durability (table: `OpenParentDirectory`, `SyncParentDirectory`; exact phase, `DurabilityUncertain`, new state visible, no `previous state preserved` claim, later write recovers)}`; `rm::state_recovery::ref_reconciliation_failed_state_write_during_normalization_has_two_distinct_legs` | Unit; pass-level | Pre-rename: previous bytes identical, `StateWriteFailed`/`NotApplied`. Post-rename: new state visible, `AdvisedDurabilityUncertain`/`DurabilityUncertain` when the normalization also recorded advice, or `StateWriteFailed`/`DurabilityUncertain` with no recommendation for an anchor or `NoAction` normalization (`ref_advisory::tests::{maintenance_anchor_post_rename_failure_is_state_write_failed_with_visible_anchor, maintenance_no_action_normalization_post_rename_failure_is_state_write_failed}`); no durable-normalization claim; later invocation recovers | Verified | + | S37 | `ref_advisory::tests::maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately`; `rm::state_recovery::ref_reconciliation_successful_pass_normalizes_invalid_state_and_a_later_read_does_not_rewrite` | Unit; pass-level | Normalizing advisory and explicit pass persist `t ≤ now + tolerance`; near-future value preserved; repeated read does not rewrite | Verified | + | S38 | `ref_advisory::tests::maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately` (fresh call re-reads the file); `rm::cross_process::ref_reconciliation_advisory_lock_serializes_independent_processes` | Unit; independent processes | A fresh call observes normalized state, performs no rewrite and does not re-fire | Verified | + | S39 | `rm::state_recovery::ref_reconciliation_unnormalized_invalid_state_stays_eligible_until_a_successful_advisory` | Pass-level (injected clocks, held lock) | Repeated read-only evaluation and busy advisories never move eligibility or bytes; the next successful advisory fires exactly once and persists normalized state | Verified | + +- [x] T06: `Remove dead-code exemptions and synchronize context` (status:done) + - Task ID: T06 + - Scope: In — remove declaration-level `#[allow(dead_code)]` from reconciliation declarations (`ref_reconciliation.rs`, `list_pins`/`delete_pins`/`PinnedRef`/`PinInventoryError` in `git_snapshot.rs`, durable-root readers in `store.rs`) that now have production consumers; separately review genuinely unreachable states (e.g. `SkippedNoCheckoutIdentity`, stale checkout-ID wording) and remove or justify each; update the context files in Context sync with the real async APIs, the wrapper/lock-held split and single-acquisition rule, the documented D1 concurrency contract, Git-derived worktree identities, the two maintenance entrypoints, the advisory trigger and why reconciliation is kept off the hook path, the maintenance state file (past-only fields, normalization, atomic writes), the distinct explicit outcomes, the verified-existing-DB opener contract, doctor reporting, and the corrected two-invariant durability statement (no claim that all repository-wide roots are pinned). Record in context, and not as implemented behavior: the deferred automatic out-of-band reclamation follow-up with its binding prerequisites from D2 (lifecycle-owned execution point, durable next-attempt reservation before expensive work, distinct outcomes, lock-hold analysis, timestamp normalization), the retired-worktree cleanup follow-up, and known storage limitations (historical retention, no repair of other worktrees' pins). Out — behavior changes. + - Dependencies: T05 + - Done when: no reconciliation-related `allow(dead_code)` remains without a stated reason; clippy with `-D warnings` is clean; context no longer says "retained and unwired", describes checkout-ID identity, or claims automatic reclamation; `context/context-map.md` entries match. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`; `nix shell nixpkgs#ripgrep -c rg -n "allow\(dead_code" cli/src/services/mutation_trace`; `rg -i "retained and unwired" context`; `rg -i "automatic.*(reclam|reconcil)" context` hits only deferral statements. + - Completed: 2026-10-09 + - Files changed: `cli/src/services/mutation_trace/runtime/{git_snapshot,maintenance_state,ref_maintenance,ref_reconciliation}.rs`, `cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs`, `cli/src/services/mutation_trace/store.rs`, `context/cli/{agent-trace-storage,async-persistence-guarantees,mutation-scope-runtime,mutation-trace-protocol,mutation-trace-ref-reconciliation,mutation-trace-runtime-coordinator,mutation-trace-snapshot-service,mutation-trace-store}.md`, `context/{context-map,overview}.md`, `context/sce/{agent-trace-hook-doctor,doctor-human-text-contract}.md`, `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Removed every "retained and unwired" and "wired by later tasks" dead-code allow from the reconciliation cluster (`ref_reconciliation.rs`, `git_snapshot.rs`, `store.rs` durable-root readers, `ref_maintenance.rs`, `maintenance_state.rs`). The lock-acquiring `reconcile_worktree`/`reconcile_worktree_inner`, `RECONCILIATION_LOCK_TIMEOUT` and their imports are now `#[cfg(test)]` because production uses `reconcile_explicit`. The single-event loaders in `store.rs` (`load_mutation_event`, `load_mutation_event_active_scopes`, `decode_boundary_kind`, their SQL constants and row mappers) have no production reader even in tests; they keep declaration-level allows with an accurate reason (attribution history uses `load_mutation_event_page`), not deleted to avoid scope expansion. Removed the unreachable `ReconciliationOutcome::SkippedNoCheckoutIdentity` and its `ref_maintenance` match arm; `ReconciliationOutcome` now has the single variant `Reconciled`. Removed comments in the touched and adjacent code per the user's instruction. Rewrote `context/cli/mutation-trace-ref-reconciliation.md` and updated the other context files listed in Context sync (no checkout-identity wording, no "unwired" claims, automatic reclamation stated only as a deferred follow-up with its D2 prerequisites, retired-worktree cleanup and storage limitations recorded). + - Verify: `clippy --all-targets --all-features -D warnings` clean; `nix flake check` passed (clippy, fmt, CLI tests, Quint connect); `git diff --check` clean; `rg "retained and unwired|unwired" context` (excluding plans) leaves no stale claim; `rg -i "automatic.*(reclam|reconcil)" context` hits only deferral statements; remaining `dead_code` allows in `mutation_trace` are the single-event loaders, coordinator/scope-runtime outcome payloads and test-model items, each with a stated reason. + - Context impact: important (this task is the context synchronization for the whole plan). + - Context synchronization: synced + +## Audit findings + +Recorded by T01 (read-only audit, 2026-10-09). Line numbers refer to the tree at `fb7e78fb`. + +**PR metadata (confirmed via `gh pr view`).** PR #304: head branch `fix-logging`, head `0be74aad5bea5070828846d3277041a4649c9842`, base `dead-code-blues`. PR #305: head branch `ref-recon`, base `fix-logging`, head `fb7e78fbfa41f9f054803eef2dddb180f5d5289a`. No rebase needed. + +**Lock ownership (D1) — confirmed.** +- `ref_reconciliation.rs:159-243` `reconcile_worktree_inner` resolves `git_dir` (`:168`), then acquires `WorktreeLock` via `acquire_inner_async(&git_dir, RECONCILIATION_LOCK_TIMEOUT /*10 s, :19*/, on_lock_contention)` (`:172`), then `resolve_worktree_id` (`:176`), `open_db` (`:180`), `GitSnapshotService::new` (`:184`), `list_pins` (`:190`), `load_tree_roots` (`:207`), `load_all_tree_roots` (`:222`), `delete_pins(lock.lease(), &stale)` (`:233`). `lock` drops on return. +- `worktree_lock.rs`: non-reentrant (`File::try_lock`, `:114`); `acquire_inner` loops `try_lock` and, on `WouldBlock`, checks `now >= deadline` before sleeping (`:123-135`), so `Duration::ZERO` is exactly one attempt then `TimedOut`. `WorktreeLockLease` is an `Arc` clone (`:147-151`); the file unlocks only when the last clone drops (`WorktreeLockInner::drop`, `:158-164`). `acquire_inner` (sync, `pub(super)`, `:79`) is available for the synchronous advisory; `acquire_inner_async` wraps it in `spawn_blocking` (`:65-77`). +- `git_snapshot.rs:281-298` `run_ref_mutation_inner` moves the lease into `spawn_blocking` (`let _lease = lease;`, `:292`) and calls `on_worker_entered()` (`:293`); `run_ref_mutation` passes `|| {}` (`:278`). `delete_pins` is `:214`, `delete_pins_inner` `:231`. Dropping the caller future therefore cannot release the file lock while the worker runs. +- `protected_worktree.rs:14` `WORKTREE_LOCK_TIMEOUT` = 10 s; `ProtectedWorktree::acquire_with_timeout` (`:109-138`) = `resolve_git_dir` → lock → `arm_marker_with_lock` → `resolve_worktree_id`. The lock is released when `coordinate_inner` (`coordinator.rs:215-252`) returns (`protected.complete()` consumes `self`; `complete_blocking` clears the marker, then the `lock` field drops). `abandon_after_spawn_without_unlock` (`:182-185`) calls `close_without_unlock` (`worktree_lock.rs:153`) for the external-guard child path only. +- **Split**: `reconcile_with_held_lock

(repository_root, &WorktreeLock, worktree_id/open_db…)` takes everything from `:176` onward; the lock is passed as `&WorktreeLock` (so it can call `lock.lease()` at the single `delete_pins` call). `reconcile_worktree_inner` becomes: resolve git dir, acquire, delegate. `reconcile_explicit` resolves git dir itself and acquires once; it never calls `reconcile_worktree[_inner]`. +- Plan-text correction: `ReconcileError::DurableRoots` doc says "migration `003`"; the durable-roots/tree-root queries belong to the mutation-trace protocol migration `004_mutation_trace_protocol.sql` (current set `001`–`006` in `cli/migrations/agent-trace-repository/`; `003` is `claude_model_state`). T02 should not hard-code a single migration id; D4 readiness must use the existing "all expected migrations applied" check (`db/mod.rs:1189 migration_metadata_problems`, used by `ensure_schema_ready` `:1251`). + +**Production call graph and completion classes.** +- Every harness adapter funnels into `hooks::mutation_scope::run_mutation_scope_from_payload` (`hooks/{claude,codex,opencode,pi}_mutation_scope/lifecycle.rs`; `hooks/mod.rs:108` direct subcommand; `doctor/inspect.rs:305` repair seam). → `run_mutation_scope_from_payload_with` (`mutation_scope.rs:269`) → `drive_mutation_scope` (`:303`): `Start`/`Advance`/`Close`/`Flush` → `coordinate_boundary(...).await` → `classify_coordinate` (`:362-376`); `Abandon` → `classify_abandon` (`:378-392`). `external_mutation_guard` is a separate subcommand (`hooks/mod.rs:123`) using `coordinate_on_held_worktree`. +- Durably completed: `Ok(_)` and `Err(CoordinateError::MarkerClearAfterCommit)` (`:366-371`); all other `CoordinateError` variants (`SnapshotFailure`, `CasConflictExhausted`, `RevisionExhausted`, `ExternalTaintMarker`, `AgentTraceDbUnavailable`, `LockAcquisition`, `ScopeIdentityConflict`, `ScopeProvenanceRegistration`, `Other`) are pre-completion failures. +- Hook-ingress seam: `drive_mutation_scope` takes injected `coordinate_boundary`/`abandon` closures; `classify_coordinate(coordinate_boundary(...).await, logger)` at `:356-359` is the exact insertion point (after the await, so the coordinator lock is already dropped). Requires `boundary` to be `Close`/`Flush`, and the result to be `Ok` or `MarkerClearAfterCommit`. `run_mutation_scope_from_payload_with` also gets `doctor/inspect.rs:305`'s repair path, so an advisory placed in `drive_mutation_scope` fires there too; harmless (it still never reconciles) but T03 should test it is unaffected. + +**Execution-point investigation (D2) — conclusion confirmed, with one flag.** +- No lifecycle-owned, noncritical-path, cross-harness executor exists. `post-commit` (`hooks/commit_hooks.rs:114`) is synchronous inside `git commit`; per-harness stop/session-end handlers (for example `hooks/codex/stop.rs`) are separate, per-harness, synchronous hook processes that persist their own DB rows and are not part of the shared mutation-scope ingress; `sce setup` is install-time; `doctor/mod.rs` is the operator-owned foreground path. +- **Flag (does not invalidate the plan):** `post-commit` already launches a detached child via `sync/auto_sync.rs:39` `launch` (spawns the current executable with null stdio and drops the `Child`). That is a fire-and-forget subprocess precedent. D2/constraint 6 prohibit fire-and-forget subprocesses for reconciliation, and a short-lived hook process cannot supervise one, so the selected design stands; the follow-up PR may cite this precedent when evaluating an out-of-band executor. +- Tokio shutdown joining blocking workers and the unbounded cost of `git for-each-ref`/durable-root queries are as stated in the authoring-time observations; nothing found contradicts them. +- Decision: advisory-only hook trigger plus `sce doctor --fix`. + +**Advisory Git-dir resolution (D2 decision rule) — advisory kept, filesystem derivation.** +- `coordinate` resolves the Git dir internally (`resolve_git_dir`, a `git rev-parse --absolute-git-dir` subprocess, `git_snapshot.rs:602-613`, via `run_rev_parse` `:699`) and does not return it (`CoordinateOutcome` carries `worktree_id`, `observed_tree`, …, not the Git dir), so reuse would need a protocol/API change. Not chosen. +- Filesystem derivation: the hook's `repository_root` is `std::env::current_dir()` (`hooks/mod.rs:63`), which may be a subdirectory. Starting there, walk ancestors to the nearest entry named `.git`; if it is a directory it is the Git dir; if it is a file containing `gitdir: `, the Git dir is that path (resolved relative to the file's parent when relative). Verified empirically in a scratch repo: from a subdirectory of the main worktree `git rev-parse --absolute-git-dir` equals `/.git`; in a linked worktree the `.git` file contains `gitdir: /.git/worktrees/`, which equals `--absolute-git-dir`. The lock file and state file are addressed by inode, so path spelling differences (symlinks) do not change behavior, but T02 should canonicalize both anyway and treat any mismatch as `StateUnavailable`. +- Guardrails T02 must implement: if `GIT_DIR`, `GIT_COMMON_DIR`, or `GIT_WORK_TREE` is set, or no `.git` is found, or the `gitdir:` target does not exist, return `StateUnavailable` (no advice, no Git subprocess). Because `coordinate` just succeeded from the same directory, the nearest-`.git` walk cannot find a different repository than Git did. +- Outcome: **hook trigger kept; no new Git subprocess.** The derivation function takes only `&Path`, performs only local `std::fs` reads, and lives in the advisory/state module (no `resolve_git_dir` import). + +**State (D3) — confirmed.** +- Placement: `/sce/` beside `mutation-cursor.lock` (`worktree_lock.rs:9-11`) and `mutation-cursor-tainted` (`external_taint.rs:26-29`). No in-memory registry is needed. +- No atomic-write helper exists in the runtime module: `ExternalTaintMarker::persist` (`external_taint.rs:82-112`) is create+`sync_data` of an empty file with best-effort parent-dir `sync_all` (`:141-145`). The repository's staging-and-swap pattern is `setup/install.rs` (`create_*_staging_path` with `{epoch_nanos}-{pid}-{attempt}` names at `:224-256`, `fs::write` to staging, `rename_fn` injected for test interruption at `:40,185`, `cleanup_path_if_exists` on failure). T02 should mirror this with an injected rename/write seam rather than invent a new pattern, plus the marker's best-effort parent-dir sync. +- Clock: there is no injectable wall clock in the runtime; `hooks/runtime.rs` has `current_unix_time_ms()` (`SystemTime`/`UNIX_EPOCH`, i64 ms). T02 takes `now: impl Fn() -> i64` and production passes `current_unix_time_ms`-equivalent logic. `sync/sync.rs:78` defines a `now()` trait for another subsystem; not reused. + +**DB opening (D4) — confirmed not read-only; opener contract.** +- `open_agent_trace_db_for_hook_runtime` (`hooks/runtime.rs:53-69`) resolves config → `resolve_agent_trace_storage_for_hook_runtime` → `open_storage_for_hook_runtime` → `open_repository_db_for_hook_runtime` (`agent_trace_storage/mod.rs:195-207`): (a) `TursoDb::open_without_migrations_at` calls `ensure_db_parent_dir` and `turso::Builder::new_local(path).experimental_multiprocess_wal(true).build()` (`db/mod.rs:856-880`), which can create parent directories and an empty DB file; (b) on `ensure_schema_ready_for_hooks` failure it calls `repair_missing_repository_schema_migration_metadata` (`agent_trace_db/repository.rs:136-150`), which inserts the baseline migration row; (c) `verify_or_initialize_repository_metadata` (`repository.rs:171-230`) inserts the `repository_metadata` row and claims `source_instance_id` when absent/invalid. +- Turso: crate `turso = 0.8.1` (`cli/Cargo.toml:32`); no read-only open or open flag is used anywhere in `cli/src`. T02 first confirms whether the 0.8.1 builder exposes one; if not, accept the documented limitation (existence pre-check, `SELECT`-only afterward, tests assert logical contents unchanged). The multiprocess-WAL builder option must be kept so the open coordinates with live hook writers. +- Smallest opener: a new function beside `open_repository_db_for_hook_runtime` that (1) resolves identity and path exactly as the hook runtime does (`resolve_identity`, `agent_trace_db_path_for_repository`), (2) returns `Missing` if the path is not an existing file before touching Turso, (3) opens without creating directories (a new `open_existing_without_migrations_at` on `TursoDb` that skips `ensure_db_parent_dir`, or an equivalent guard), (4) runs `ensure_schema_ready_for_hooks` (all expected migrations `001`–`006` applied) and on failure returns `IncompatibleSchema` without repair, (5) calls a new read-only `verify_existing_repository_metadata(repository_id)` built on `select_repository_metadata_row` (`repository.rs:232`) returning `MissingMetadata` when the row is absent or `source_instance_id` fails `is_valid_source_instance_id` (`:74`), and `RepositoryMismatch` via the existing `ensure_repository_id_matches`. The hook opener and its callers are unchanged. + +**Identity and stale-doc findings.** +- `resolve_worktree_id` (`git_snapshot.rs:666-685`): `git_dir == git_common_dir` → `main`; otherwise `worktrees/`. It is Git-topology-derived and failure is `Err`. +- `ReconciliationOutcome::SkippedNoCheckoutIdentity` (`ref_reconciliation.rs:59`) is declared and documented (`:36`, `:50-51`) but never constructed anywhere in `cli/src`; confirmed unreachable. `context/cli/mutation-trace-ref-reconciliation.md` still describes `read_checkout_id → Ok(None)` and "checkout identity" namespaces (stale). T06 removes the variant (callers must then match one variant) and rewrites the docs. +- `#[allow(dead_code, reason = "ref reconciliation retained and unwired; …")]` appears on `ref_reconciliation.rs` declarations, `git_snapshot.rs` (`run_ref_mutation`, `assert_pins_are_unchanged_direct_refs`, others) and ten places in `store.rs` (`:199, :225, :249, :259, :272, :713, :844, :874, :1022, :1195`); T06 removes only those that gain production consumers. + +**Findings affecting later tasks.** None invalidates a task's scope. T02 must: use `acquire_inner` (sync, zero timeout) for the advisory; implement filesystem Git-dir derivation with the stated guardrails; add the `Missing`/`IncompatibleSchema`/`MissingMetadata`/`RepositoryMismatch` opener and confirm Turso read-only support; reuse the `setup/install.rs` staging pattern. T03 must also account for the `doctor/inspect.rs:305` caller. T06 must also fix the stale "003" wording in the `ReconcileError` docs. + +## Open questions + +- Is a 24-hour `RECONCILIATION_ADVISORY_AFTER` the right window for the time-based recommendation? It is only a starting value recorded by T03. +- If T01 finds the advisory cannot get the Git dir without a new Git subprocess, the plan drops the hook trigger and relies on `sce doctor` alone (decision rule in D2, no further confirmation needed). Confirm you are happy with that outcome rather than accepting one extra `git rev-parse` on the `Close`/`Flush` path. +- Do you want a follow-up PR scoped now for automatic out-of-band reclamation (D2 prerequisites), or only the documented deferral? +- `SkippedNoCheckoutIdentity` looks unreachable in the current code (the identity is Git-topology-derived and its failure is an `Err`). The plan assumes T06 removes it; confirm you do not want it kept for compatibility. +- D4 adds a dedicated opener and one read-only verification method in `agent_trace_storage` / `agent_trace_db`. T01 may find that Turso cannot open an existing DB without incidental WAL bookkeeping; the plan treats that as an accepted limitation (logical contents unchanged) unless you want a stricter guarantee. + +## Validation Report + +**Status:** validated + +**Date:** 2026-10-09 + +### Commands run + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (498 passed, 0 failed before the test-suite simplification; 479 passed, 0 failed after it) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` -> exit 0 (clean) +- `nix flake check` -> exit 0 (all checks passed; non-host systems omitted) +- `nix build .#ci-checks` -> exit 0 +- `nix run .#pkl-check-generated` -> not run (no generated config inputs touched) +- `git diff --check` -> exit 0 +- `git diff fix-logging...HEAD --stat` -> exit 0 (no `protocol.rs` or `.qnt` change) +- AC7 `rg` over changed files -> every added hit is in test code; no production hit +- focused `ref_reconciliation` 50, `maintenance` 88, `mutation_scope` 25, `doctor` 27, `agent_trace_storage` 1 passed (all non-zero) + +### Success-criteria verification + +- [x] AC1: only caller chain is `doctor/mod.rs` -> `ref_doctor` -> `reconcile_explicit`; the hook module references only `advise_after_completed_boundary` +- [x] AC2: doctor fix and advisory ref tests pass +- [x] AC3: worktree and fail-closed tests pass +- [x] AC4: advisory outcome and marker tests pass (`mutation_scope` 25 passed) +- [x] AC5: `ref_advisory.rs` non-test code imports no Git, store or DB symbol +- [x] AC6: coordination tests pass in the full suite +- [x] AC7: no production-code hits for the forbidden patterns +- [x] AC8: no `protocol.rs` or `spec/mutation_cursor.qnt` change; `mbt` tests pass +- [x] AC9: doctor tests pass +- [x] AC10: no "unwired" claims in `context/cli`; automatic-reclamation hits are deferral statements +- [x] AC11: each entrypoint acquires `WorktreeLock` once; `reconcile_with_held_lock` does not acquire +- [x] AC12: contention and handoff tests pass +- [x] AC13: maintenance-state and recovery tests pass +- [x] AC14: verified-open and logical-immutability tests pass, including the new `agent_trace_storage` missing-DB test +- [x] AC15: deferral and prerequisites recorded in `context/cli/mutation-trace-ref-reconciliation.md` + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Cross-process multiprocess-WAL coexistence is covered only in-process (T02 limitation 4). +- The new `agent_trace_storage` test is uncommitted in the working tree at validation time. diff --git a/context/plans/observability-log-lock-striping.md b/context/plans/observability-log-lock-striping.md new file mode 100644 index 000000000..cfaefa010 --- /dev/null +++ b/context/plans/observability-log-lock-striping.md @@ -0,0 +1,177 @@ +# Plan: observability-log-lock-striping + +## Change summary + +`cli/src/services/observability.rs::file_log_lock` keeps a process-global `OnceLock>>>>`. Every distinct log path ever written adds an `Arc>` that is never evicted, so synchronization memory grows with the number of historical paths. This plan replaces the registry with a fixed `static LOG_LOCKS: [Mutex<()>; LOG_LOCK_STRIPES]` (64 stripes) selected by a deterministic hash of the path modulo the stripe count. It is a replacement of the lock-selection mechanism only; append, fallback, retention, permission, redaction, and formatting behavior are preserved. Same-process serialization only: no cross-process guarantee exists today and none is added. + +Investigation findings (recorded from the code before selecting the design; T01 re-confirms them while implementing): + +- Production callers: `append_log_line_with_cleanup` is reached only through `append_log_line`, which is reached only from `Logger::write_log_line` (`observability.rs`). `append_log_line_once_with_cleanup` is reached only from the v2 fallback closure inside `append_log_line_with_cleanup`. `file_log_lock` has exactly those two call sites. +- Ownership: `Logger` is a `Clone + Debug` plain value holding `ObservabilityConfig`, `log_dir`, and retention limit; it owns no shared state. App startup builds one in `app.rs` (`Logger::from_resolved_config`) and stores it in the app context, but `Clone` and `pub` construction let independent instances target the same path, so per-instance mutexes (Option A) would be unsound and would need new shared ownership. Option B (static striping) is selected as the smallest correct design. +- Concurrency: logging is synchronous `std::fs` I/O invoked from the main thread, `tokio::task::spawn_blocking` paths, and plain `std::thread` users, so the critical section stays a synchronous `std::sync::Mutex`; no async mutex. +- Deadlock hazard: today the v2 fallback runs while the primary path's guard is still held, and takes the v2 path's own lock. With striping the primary and v2 paths can hash to the same stripe, and `std::sync::Mutex` is not reentrant, so the fallback must run after the primary guard is released. This is the main correctness risk of the change. A regression here deadlocks instead of failing, so the same-stripe test must run in a bounded child process (T02), never in-process and never via a detached thread plus `recv_timeout` (the deadlocked thread would outlive the failed test). +- Poisoning failure domain: `lock()` is called only at the two sites in `append_log_line_with_cleanup` / `append_log_line_once_with_cleanup`. The guarded section is `persist_log_line` (returns `Result`, uses `?`, no panicking calls expected) plus the retention `cleanup` callback, which is the only caller-supplied code run under the guard and therefore the only realistic poisoning source (a panic inside cleanup). T01 re-confirms this by inspection. Today a poisoned lock affects one path; with 64 stripes it affects every path hashing to that stripe, including unrelated ones. This enlarged failure domain is a documented, accepted consequence; existing poisoned-lock handling is preserved unchanged (no `into_inner` recovery, no `clear_poison`, no new fallback). +- Test-process infrastructure: `cli/src/app.rs::app_output_boundary_uses_isolated_process` already re-executes `std::env::current_exe()` with `--exact --nocapture` and an env marker to run an isolated child. T02 reuses this pattern (std only, no new dependency). +- Behavior to preserve: lock-acquisition failure (poisoned stripe) is returned as `failed to lock log file '': ...` and does not trigger the fallback (the fallback is only reached from the `persist_log_line` error arm, after the lock was acquired); fallback is a single non-recursive `-v2` attempt whose combined error text is unchanged; retention cleanup runs only after a write that created the file and runs inside the guard of the path that was written; files are opened with owner-only `0o600` on Unix; redaction happens before `append_log_line`. +- Cross-process: no filesystem lock or cross-process guarantee exists or is documented; `context/sce/cli-observability-contract.md` only says "serializes writes independently per path", which becomes inaccurate once paths can share a stripe. + +## Acceptance criteria + +- [x] AC1: No path-keyed lock registry remains in `observability.rs`; synchronization storage is a fixed-size `[Mutex<()>; N]` static independent of the number of distinct log paths, with no per-path `Arc`/`Mutex` allocation, no `dyn`, no boxed callbacks, no `unsafe`, and no new dependency. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n "BTreeMap|OnceLock|Arc': ...` and does not attempt the fallback. + - Validate: code inspection of the guard scope in `append_log_line_with_cleanup` (guard dropped before `attempt_v2_log_fallback` is called, no guard live across it); the same-stripe fallback child test (steps 3-5); the existing combined-error test (step 6); the poisoned-lock test (step 7). +- [x] AC4b: Same-stripe fallback regression is deterministic and bounded: the test builds a primary path whose `v2_log_path` maps to the same stripe using the production `log_lock_stripe` selection, forces primary failure, and runs in an isolated child process with a finite parent-enforced timeout; the child is killed and reaped on timeout; timeout, unexpected termination, or wrong output fails the test; success requires the fallback file to exist with the expected content. + - Validate: the test passes under `observability`; temporarily holding the primary guard across the fallback makes it fail by timeout (not hang) and leaves no orphan child. +- [x] AC4c: Poisoning semantics are preserved and documented: a poisoned stripe still yields `failed to lock log file '': ...` with no fallback and no silent recovery; no new recovery or fallback code is introduced; the enlarged (per-stripe rather than per-path) failure domain is stated in the plan assumptions and durable context. + - Validate: focused poisoned-stripe test (if practical, see T02); `rg -n "into_inner|clear_poison|PoisonError" cli/src/services/observability.rs` shows no new recovery logic; inspection of `context/sce/cli-observability-contract.md`. +- [x] AC5: Retention-after-creation, owner-only Unix permissions, and redaction-before-persistence behave as before under the new locking. + - Validate: `observability` tests covering retention, `0o600` permissions, and redaction pass; no existing test is removed or weakened (`git diff main` shows only additions to the test module). +- [x] AC6: Durable context states the ownership model, constant memory bound, same-process guarantee, absence of cross-process coordination, and stripe-collision contention trade-off. + - Validate: inspection of `context/sce/cli-observability-contract.md` (and `context/overview.md`/`context/glossary.md` only if they already describe per-path locking). +- [x] AC7: The concurrency tests are not flaky. + - Validate: the targeted logging concurrency tests pass on 20 consecutive runs. + +### Full validation + +- All existing observability tests remain unchanged and pass (`git diff main` shows only additions to the test module). +- The same-stripe fallback regression executes in a subprocess with a bounded timeout (AC4b); the targeted concurrency tests are repeated 20 times (AC7). +- Review the diff for new dynamic dispatch, heap allocations on the lock-selection path, `unsafe`, dependencies, or unnecessary abstractions: `nix shell nixpkgs#ripgrep -c rg -n "dyn |Box<|Arc<|unsafe|Vec<" ` over the changed lock-selection code, and `git diff main -- cli/Cargo.toml cli/Cargo.lock` empty. +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` +- `nix flake check` +- `nix build .#ci-checks` +- `git diff --check` + +### Context sync + +- `context/sce/cli-observability-contract.md` must replace "serializes writes independently per path" with the striped same-process model and the explicit no-cross-process statement. +- `context/context-map.md` annotation for that file is updated only if its summary text becomes inaccurate. + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/observability.rs` (lock selection, the two append functions, inline tests) and the durable context named under Context sync. +- **Out of scope:** OpenTelemetry integration, other observability refactors, log formats/filenames/schema, retention policy, public configuration, logging queues, background workers, new runtimes, filesystem/cross-process locks. +- **Constraints:** std-only; static dispatch; keep generic `F: FnOnce(&Path) -> Result<()>` cleanup callbacks; no `dyn`, boxed callbacks, `unsafe`, new dependencies, or lint suppressions; the hash must be deterministic and independent of mutable global state (e.g. `std::hash::DefaultHasher::new()` with its fixed keys, not `RandomState`); `Cargo` runs go through `nix develop -c ./scripts/run-cli-cargo.sh`; tests stay a minimal focused set using the real implementation and no sleeps or timing assertions. +- **Non-goal:** guaranteeing cross-process serialization, per-instance lock ownership, or eviction caches. + +## Assumptions + +- 64 stripes (`LOG_LOCK_STRIPES`) is adequate; the exact count only affects collision contention, not correctness. +- Poisoned-lock behavior stays as today (error `failed to lock log file '': ...`, no fallback, no recovery). The failure domain changes from one path to every path on the poisoned stripe (~1/64 of paths per poisoned stripe). Poisoning requires a panic while the guard is held; the only caller-supplied code under the guard is the retention `cleanup` callback, so poisoning is not expected in normal logging. This is accepted, not mitigated, in this plan. +- Releasing the primary guard before the v2 fallback is acceptable: the fallback writes a different file, and the only previously-held-lock effect was ordering against other primary writers, which the fallback never touched. The fallback attempt is no longer atomic with the primary failure with respect to other writers; this is not observable beyond line ordering across two different files. +- Two stripes are never held at once by one call, so there is no lock-order inversion between stripes. +- The child-process timeout is bounded (a few seconds, generous for CI) and is a failure bound only, never a correctness assertion on timing. +- The existing contract doc, not a new ADR, is the right durable home for the model; no decision record is required. + +## Task stack + +- [x] T01: `Replace the path registry with fixed lock striping and release the guard before fallback` (status:done) + - Completed: 2026-10-08 + - Files changed: `cli/src/services/observability.rs` + - Result: Replaced `file_log_lock` registry with `LOG_LOCK_STRIPES = 64` / `static LOG_LOCKS` and `log_lock_stripe_index`/`log_lock_stripe` (deterministic `DefaultHasher::new()`); `append_log_line_with_cleanup` scopes the primary guard in an inner block that returns on success and yields the persist error, so `attempt_v2_log_fallback` runs only after the guard is dropped and never after lock-acquisition failure; `append_log_line_once_with_cleanup` uses the stripe directly. Poisoned-lock error text unchanged; only the retention `cleanup` callback runs caller code under the guard. + - Verify: `observability` tests passed (8/8); `rg "BTreeMap|OnceLock|Arc; LOG_LOCK_STRIPES] = [const { Mutex::new(()) }; LOG_LOCK_STRIPES]`; a small `log_lock_stripe(path) -> &'static Mutex<()>` using a deterministic std hash modulo the stripe count; remove `file_log_lock`, the `BTreeMap`/`Arc`/`OnceLock` imports, and the registry; update `append_log_line_with_cleanup` so the primary guard is dropped before the v2 fallback is attempted on both success and failure paths (scope the guard in an inner block/helper covering lock + persist + retention, returning the persist result; call `attempt_v2_log_fallback` only after that scope ends, and never when lock acquisition itself failed) and `append_log_line_once_with_cleanup` takes the stripe directly; keep error strings, retention-inside-guard, and lock-failure-skips-fallback behavior; keep existing poisoned-lock error handling unchanged (no recovery, no new fallback) and confirm by inspection that only the cleanup callback can panic under the guard; add a doc comment stating same-process-only synchronization, no cross-process guarantee, that stripe collisions only add contention, and that a poisoned stripe now affects all paths on that stripe. Out — tests beyond keeping existing ones green, context docs, any other observability change. + - Dependencies: none + - Done when: the registry is gone, both append functions use the static stripes, the fallback cannot nest a lock on the same stripe (AC4a ordering holds), poisoned-lock handling is unchanged, no `dyn`/`Box`/`unsafe`/new dependency is introduced, and all existing `observability` tests pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml observability`; `nix shell nixpkgs#ripgrep -c rg -n "BTreeMap|OnceLock|Arc': ...`, no v2 file, no recovery), retention-callback-runs-once under concurrent creation, `0o600` on Unix, redaction before persistence. No existing test changed. Also fixed a pedantic-clippy `cast_possible_truncation` in T01's `log_lock_stripe_index` (`usize::try_from(...).unwrap_or_default()`, same behavior on 64-bit). + - Verify: `observability` tests passed (20/20); 20 consecutive runs all passed; mutation holding the primary guard across the fallback made the same-stripe child test fail by 30s bounded timeout with the child killed and reaped (no orphan), then reverted; `clippy --all-targets --all-features -D warnings` passed; `git diff --check` clean. + - Context impact: none beyond T03 (tests only; the small stripe-index cast fix does not change documented behavior). + - Deviation: poisoned-stripe case runs in an isolated child so the shared static cannot be poisoned for other tests; the T01 cast fix was required for the clippy gate and is behavior-neutral. + - Task ID: T02 + - Scope: In — inline tests in `observability.rs` that call the real `append_log_line_with_cleanup`/`Logger` paths: (a) same-path multi-thread writes with exact complete-line count and per-line integrity; (b) multiple independent `Logger` clones/instances writing one path; (c) many (thousands of) distinct paths asserting stripe selection is stable, always in range, and the stripe array length is the constant; (d) fallback coverage: (d1) in-process, primary failure forcing the v2 fallback (e.g. a directory occupying the primary file name) with paths on different stripes, plus the combined error text when both fail; (d2) **same-stripe fallback regression in an isolated child process** (see below); (d2a) a focused poisoned-stripe test, if practical: poison the stripe of a temp path by panicking inside the `cleanup` callback of a write that creates the file (caught with `std::panic::catch_unwind` or a joined thread), then assert the next write to that path returns `failed to lock log file '': ...`, no `-v2` file is created, and nothing recovers the lock; use a path/stripe unique to the test so other tests are unaffected, and skip with a recorded rationale in the plan only if it cannot be isolated from the shared static; (e) retention under concurrent creation and `0o600` permissions on Unix (`#[cfg(unix)]`); (f) a secret in a logged record is redacted in the persisted file. Out — production behavior changes, correctness assertions on elapsed time, test-only copies of the locking logic, new dependencies, detached-thread-plus-`recv_timeout` harnesses, background workers. + - Same-stripe child-process test (d2) requirements: + 1. Build a primary path in a temp dir whose `v2_log_path(primary)` maps to the same stripe, using the production `log_lock_stripe` (compare by index or `std::ptr::eq` on the returned stripe); search deterministically over candidate file names (bounded loop) until one collides, and assert the collision before running. + 2. Force primary persistence to fail (e.g. a directory occupying the primary file name) so the fallback is triggered. + 3. Run the scenario in a child: parent re-executes `std::env::current_exe()` with `--exact --nocapture` and an env marker, mirroring `cli/src/app.rs::app_output_boundary_uses_isolated_process`; without the marker the child test is a no-op that passes. The child performs the real `append_log_line_with_cleanup` call, then prints a fixed success sentinel. + 4. Parent uses `Command::spawn` + a bounded `try_wait` polling loop against a finite deadline (std only; sleeps are only poll intervals, never assertions). + 5. On deadline expiry the parent calls `kill()` then `wait()` so the child is reaped, then fails the test with captured output. + 6. Timeout, non-success exit status or signal termination, or missing/incorrect sentinel output fails the test. + 7. On success the parent asserts the `-v2` fallback file exists with the exact expected line content and that the primary path is still the blocking directory. + 8. Child state (temp dir, env such as `HOME`/`XDG_*`) is isolated like the existing pattern; stdout/stderr are captured via pipes and drained so a full pipe cannot itself cause a hang. + - Dependencies: T01 + - Done when: each listed case has a deterministic test against the real implementation, all pass, the same-stripe child test fails by bounded timeout (not by hanging) if the guard were held across the fallback, no orphan child remains, and no existing test is removed or weakened. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml observability`; repeat the targeted concurrency and fallback tests 20 times in a loop to detect flakiness; temporarily mutate T01 to hold the primary guard across the fallback and confirm the child test fails by timeout, then revert. + - Context synchronization: synced + +- [x] T03: `Document the striped log-lock ownership model` (status:done) + - Completed: 2026-10-08 + - Files changed: `context/sce/cli-observability-contract.md` + - Result: Replaced "serializes writes independently per path" with a pointer to a new lock-model bullet (64 static `Mutex<()>` stripes, deterministic hash, constant memory, same-process serialization across threads and `Logger` instances, collision contention only, one stripe held at a time with the primary guard released before the v2 fallback, enlarged poisoning domain with unchanged error text/no fallback/no recovery, no cross-process coordination); v2 sentence now says "striped serialization". `context/context-map.md` annotation does not mention per-path locking, so left unchanged. + - Verify: inspected changed lines; `git diff --check` clean. + - Context impact: none beyond this task (this task is the context update). + - Task ID: T03 + - Scope: In — update `context/sce/cli-observability-contract.md` (and its `context/context-map.md` annotation only if inaccurate) to state: static 64-stripe `Mutex<()>` ownership, constant memory independent of path count, same-process serialization of identical paths across threads and independent `Logger` instances, stripe-collision contention trade-off, guard released before the v2 fallback (lock-ordering contract), the enlarged poisoning failure domain (a poisoned stripe fails every path on it, unchanged error text, no recovery), and the explicit absence of cross-process coordination. Out — broad documentation refactoring, other context files. + - Dependencies: T01 + - Done when: the contract no longer says writes are serialized "independently per path" and contains the accurate model, the bound, the guarantee, the non-guarantee, and the trade-off, in current-state wording. + - Verify: inspect the changed lines in `context/sce/cli-observability-contract.md`; `git diff --check`. + - Context synchronization: synced + +## Open questions + +None. The request fully specifies scope, design, and constraints; the one design hazard (same-stripe fallback deadlock) is handled inside T01 and proven by the bounded child-process test in T02. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-08 + +### Commands run + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml observability` -> exit 0 (20/20 passed, incl. stripe selection, many-paths, same-stripe child, poisoned-stripe child, retention, 0o600, redaction) +- `observability` test run repeated 20 consecutive times -> 0 failures +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (368 passed, 0 failed) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` -> exit 0 +- `nix flake check` -> exit 0 (all checks passed) +- `nix build .#ci-checks` -> exit 0 +- `git diff --check` -> exit 0 +- `rg -n "BTreeMap|OnceLock|Arc no hits +- `git diff d90415e0 -- cli/Cargo.toml cli/Cargo.lock` -> empty (plan commits only touch observability.rs and context) + +### Success-criteria verification + +- [x] AC1 -> registry removed; `static LOG_LOCKS: [Mutex<()>; 64]`; rg shows no BTreeMap/OnceLock/Arc/dyn/unsafe/Box hits; Cargo files unchanged by plan commits +- [x] AC2 -> concurrent same-path and independent-Logger tests pass +- [x] AC3 -> stripe-selection test (5000 paths, constant array length, stable index) passes +- [x] AC4 -> same-stripe bounded child test, different-stripe fallback, and combined-error tests pass +- [x] AC4a -> inspection of `append_log_line_with_cleanup` (observability.rs:343-363): guard scoped in an inner block, dropped before `attempt_v2_log_fallback`; lock failure returns via `?` without fallback; combined error text unchanged +- [x] AC4b -> child test with finite deadline and kill+wait passes; T02 recorded the mutation check (guard held across fallback fails by bounded timeout, no orphan); not re-run in this validation +- [x] AC4c -> poisoned-stripe child test passes; no into_inner/clear_poison/PoisonError in file; contract doc states the enlarged failure domain +- [x] AC5 -> retention, 0o600, and redaction tests pass; diff removals contain no test code +- [x] AC6 -> `context/sce/cli-observability-contract.md` lines 36-38 state ownership, constant bound, same-process guarantee, no cross-process coordination, collision contention +- [x] AC7 -> 20 consecutive runs, 0 failures + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- A poisoned stripe fails all paths hashing to it (accepted and documented). +- The guard-held-across-fallback mutation check relies on T02's recorded evidence rather than a fresh run. diff --git a/context/plans/sce-opentelemetry-tracing.md b/context/plans/sce-opentelemetry-tracing.md new file mode 100644 index 000000000..dd8f7570e --- /dev/null +++ b/context/plans/sce-opentelemetry-tracing.md @@ -0,0 +1,298 @@ +# Plan: sce-opentelemetry-tracing + +## Change summary + +Reintroduce opt-in OpenTelemetry tracing to the Rust CLI in five sequenced PR-sized phases, after first making the existing tracing path safe to attach a subscriber to. Today `Logger::log_forced` (`cli/src/services/observability.rs`) calls `emit_tracing_event` with the raw `message` and serialized `fields` before `redact_sensitive_text` (`cli/src/services/security.rs`) runs, and `log_cli_error` puts `error_source` (the full technical error chain) into those fields. Two further raw `tracing::warn!` sites carry free-form error text on the `sce` target: `sce.resilience.retry` (`error = %last_error`, `cli/src/services/resilience.rs`) and `sce.agent_trace_db.contention_exhausted` (`last_error = %last_error`, `cli/src/services/db/mod.rs`). `redact_sensitive_text` is key-name based and does not redact paths, so even a redacted line is not a safe export format. The logger API also accepts `event_id: &str` from arbitrary callers (including dynamically formatted identifiers such as `sce.error.{code}`), and the raw events carry caller-supplied strings such as `operation` and `db_name`; none of these are safe merely because they are short. Any subscriber attached to the process, not only an OTel exporter, can read this content. + +Phase 0 therefore establishes two independent security boundaries (A: existing events never carry free-form content to any subscriber; B: OTel export accepts only a dedicated, typed, allowlisted instrumentation surface). Phases 1-2 add a standalone OTLP/HTTP runtime (`sce.command` root span, bounded shutdown) and W3C `TRACEPARENT`/`TRACESTATE` propagation. Phase 3 adds managed Control Plane tracing with a pinned collector and a telemetry-only credential. Phase 4 adds operation spans. Phase 5 (Codex/Pi/OpenCode propagation, metrics, sampling controls, local forwarder) is deferred. + +Why reintroduction is justified despite `remove-cli-otel`: that plan removed an exporter that had no trustworthy sink and no consumer, and it explicitly scoped "do not introduce replacement telemetry/export libraries" to itself. This plan supersedes that constraint for the following reasons: a Control Plane collector and Dash0 pipeline now exist, Claude Code emits W3C trace context SCE can join, and the leak that made export unsafe is fixed first (T01) rather than inherited. The repository-level `otel` config key stays removed (see Configuration ownership). + +### Architecture decisions + +**D1 — Event surfaces.** Boundary A is allowlist-by-construction, not redaction. Existing `sce`-target logger/raw events remain for local logging semantics, but a tracing event may carry only values that are static, enumerated, typed, or demonstrably safe: + +- `event_id`: only members of a closed, static registry of known identifiers (`TracingEventId`, a sorted constant list validated by a unit test for uniqueness and an `[a-z0-9_.]{1,64}` shape) reach tracing. `Logger` keeps accepting `&str`; a classification function maps it to a registry member, and any unknown, dynamically constructed, or test-only identifier becomes the fixed classification `sce.unclassified`, never its raw string. The dynamic `sce.error.{code}` identifier is mapped from the typed error class, not from the formatted string. `redact_sensitive_text` is not relied on for any tracing value. +- Other preserved attributes: `level` (typed), numeric counters and durations (`u64`), `cause` (existing constant), and string-like attributes such as `operation` and `db_name` only after classification into enumerated types (`OperationClass`, `DbName`) with an `unclassified` fallback. Message arguments of every tracing macro on these paths are static literals. No other `&str`/`Display` value is recorded. +- Never reaching tracing: `event_message`, free-form `fields` JSON, `error_source`, `error`, `last_error`, paths, hook payloads. +- Local file/stderr logging keeps its existing format and raw event identifiers unchanged; Boundary A changes only what is handed to `tracing`. + +The OTel exporter reads only target `sce::otel` through a typed API (enumerated span/event names, enumerated attribute keys, typed/bounded values) and drops everything else (Boundary B). Exception messages and stack traces are never recorded on spans; only an enumerated error category is. + +**D2 — Subscriber scoping strategy.** The OTel runtime builds a private `tracing::Dispatch` (`tracing_subscriber::Registry` + OTel layer, never installed globally) and attaches it to the command future with `tracing::instrument::WithSubscriber::with_subscriber`, so the dispatcher is re-entered on every poll regardless of Tokio worker migration. This matches the contract already recorded in `context/sce/cli-observability-contract.md` (thread-local `with_default`/`set_default` guards held across `.await` are insufficient on multi-thread Tokio). Bauk's global-init pattern (`davidabram/bauk/src/telemetry/mod.rs`) is reused only for crate versions, provider and exporter construction, not for installation. Spawned tasks do not inherit the dispatcher: every `tokio::spawn` that must be traced in the audited call paths uses `.with_current_subscriber()` plus `.in_current_span()`; untraced spawns are listed and justified. + +**D3 — Configuration ownership.** + +- *Standalone/unmanaged mode:* explicit opt-in through an SCE-specific environment switch (`SCE_TELEMETRY=standalone`); standard `OTEL_EXPORTER_OTLP_*` variables configure a local or user-selected receiver. No managed token is ever attached in this mode, and the mode has no repository-level enablement. +- *Managed mode:* enabled only by a foreground setup action (`sce auth telemetry`, final subcommand wording confirmed in T09 against `cli-command-surface.md`) that persists enablement and credential in the user-owned encrypted auth DB under SCE's platform state/config roots (`cli/src/services/default_paths.rs`). Repository-local `.sce/config.json` cannot enable managed export or override its destination. The managed collector endpoint is a trusted, pinned compile-time constant (HTTPS only), not configurable by env or file outside test builds. +- *Precedence:* when managed mode is active, `OTEL_EXPORTER_OTLP_ENDPOINT`, `*_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, `*_TRACES_HEADERS`, and protocol/certificate overrides are ignored (not merged), and `sce doctor` reports that they were ignored. Managed mode wins over `SCE_TELEMETRY=standalone`. The managed exporter uses a dedicated HTTP client with redirects disabled so credentials are never forwarded. +- *Config keys:* no repository-level or global `otel` key is reintroduced; `remove-cli-otel` made it an unknown key and this plan keeps that. +- *Hook-export gate (applies to every production mode):* before the T05 performance gate passes, an ordinary SCE hook invocation never exports telemetry remotely, in standalone or managed mode alike. Inheriting `SCE_TELEMETRY=standalone` and `OTEL_EXPORTER_OTLP_*` from a parent process cannot enable hook export, and managed enablement plus a valid cached credential cannot either. The gate is applied during telemetry-mode resolution, before any exporter, credential read, or auth-DB access: a hook invocation resolves to `NoopTelemetry`, and `sce doctor` reports `hook export: gated`. Hook classification is a property of SCE's existing typed command/dispatch model (the parsed command variant that routes to the `hooks` command family), never string matching on raw CLI arguments, `argv[0]`, or environment values. The only pre-approval exception is the D6 compile-time feature-gated loopback receiver. After T05, production hook export may be enabled only through an explicit, documented, user-owned policy (stored under SCE's user state/config roots, never repository-local, never inherited from the environment) and only when the budget T05 selected passes `telemetry_shutdown_subprocess` and `telemetry_hook_export_gated`; implementing that policy is a separate change outside this plan. + +**D4 — Bounded shutdown and exporter concurrency.** Telemetry failures never change exit codes, stdout, hook permission decisions, durable state, lock ordering, or cancellation behavior. The SDK `BatchSpanProcessor` is kept; no custom buffer is built to force post-command-only networking. Its contract is explicit: + +- *Span end never performs network I/O.* It enqueues into a bounded in-memory queue; a full queue drops spans and never blocks or applies backpressure to the caller. +- *Background batch export may run concurrently with command processing*, including while mutation, Git, adapter-state, or DB locks are held, because the batch timer or batch-size threshold can fire at any time. This is permitted precisely because exporter activity is isolated: it runs on the SDK's own background thread (T03 confirms this against the pinned SDK version and ensures it never occupies the command runtime's workers), shares no lock with any protected critical section, and nothing in a critical section waits on it. +- *Must not run while those locks are held:* credential issuance, exporter initialization that needs network access, explicit `force_flush()`, and shutdown. Exporter construction is lazy or offline; flush and shutdown happen only after the command future has completed and locks are released. +- *No correctness-sensitive code waits for successful export or reads exporter state.* Export failure, delay, or blockage cannot alter results or durable state. +- Final flush/shutdown runs on a dedicated thread; the main thread waits at most the flush budget on a channel, then proceeds to process exit without joining, so a hung connection cannot extend process lifetime. HTTP connect and total timeouts are set below the budget so the thread normally finishes on its own. A timeout around a blocking call is not relied on; the process-lifetime bound is the contract. +- Budgets: foreground commands default 1000 ms, configurable via `SCE_TELEMETRY_FLUSH_TIMEOUT_MS`; hook invocations have **no remote export in any production mode** (standalone or managed; see the D3 hook-export gate) until T05 selects a budget from benchmark evidence, the regression tests pass, and a user-owned policy enables it. Telemetry failures never change hook permission decisions, stdout, exit codes, mutation durability, or lock ordering, whether or not export is gated. Strict post-command-only export is not required and not claimed; if it later becomes a requirement it needs a different architecture and a new plan. + +**D5 — Production-verification decoupling.** T01 precedes any exporter activation. T03/T04/T06/T07/T08 use a local OTLP receiver and an offline Control Plane fixture and never need production access. T02 (public collector and Dash0 evidence) and the T09 combined-trace evidence gate only managed-mode production acceptance. + +**D6 — Controlled local-receiver test mechanism.** Production hook export stays disabled until the T05 gate passes, yet T05 and T07 need real exporting hooks. T04 therefore delivers a test-only implementation boundary, defined here so no implementer invents one: + +- *Compile-time gate:* Cargo feature `telemetry-test-receiver`, off by default and excluded from `.#default`, release, crates.io, and Flatpak builds; production hook telemetry never depends on it. Without the feature, the code path does not exist and the environment variables below are ignored (proved by a `cfg(not(feature))` test `telemetry_test_receiver_inert` run in the default-feature pass, plus a packaged-binary check that `.#default` with the D6 variables set sends nothing to a loopback listener). T04 wires the feature into the test and benchmark builds only. Two test configurations are run explicitly and separately, in local verification and CI: the default-feature pass, and the feature-enabled pass using `--features telemetry-test-receiver`. A filtered Cargo run that matches zero tests is never a pass: every filtered run goes through a count check (`scripts/check-cargo-test-count.sh`, delivered by T04) that fails unless the reported `N passed` is at least the minimum recorded for that filter and configuration. +- *Runtime opt-in (both required):* `SCE_TELEMETRY=test-receiver` and `SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT=http://:`. The endpoint must parse as plain HTTP to a loopback IP literal (`127.0.0.0/8` or `::1`); anything else (hostname, non-loopback, HTTPS) is rejected and telemetry stays disabled. +- *Isolation:* resolution is a pure function over an injected environment source returning a `TelemetryConfig` variant `TestReceiver { endpoint, flush_budget }` that has no credential field. That variant never opens the auth DB, never calls the credential API, ignores `OTEL_*` endpoint/header overrides, and cannot construct the managed exporter or its pinned endpoint. It is exclusive: it wins over standalone and is rejected rather than merged when managed enablement is present. +- *Hook behavior:* in this mode hook-style invocations export only to the validated loopback receiver, regardless of the production hook-export gate (the single pre-approval exception to D3). Hook stdout, exit code, permission decisions, and mutation semantics are unchanged (asserted by comparison with a telemetry-disabled run). +- *Lifecycle markers:* with the feature, `SCE_TELEMETRY_TEST_LIFECYCLE_FILE=` makes the process append monotonic-clock lines `command_complete`, `shutdown_begin`, `shutdown_end` (and `process_exit_requested`) to that file only, never stdout/stderr. This is the observable boundary for shutdown measurement (AC5). +- Ordinary execution (no feature, or feature without both variables) is byte-for-byte the disabled baseline. + +## Acceptance criteria + +How this plan is proven complete. Each criterion is observable and names the check that proves it. `/validate` runs these checks; no task in the stack performs final validation. + +- [ ] AC1 (Boundary A — existing events): With an arbitrary capturing `tracing` subscriber (not only the OTel exporter) attached, no `Logger` record, `log_cli_error` record, `sce.resilience.retry`, or `sce.agent_trace_db.contention_exhausted` event exposes any free-form message, field value, event identifier outside the static registry, filesystem path, error chain, credential, or hook payload; unknown or dynamic event IDs and string-valued attributes appear only as fixed safe classifications; local file/stderr log output and its event identifiers remain the existing output. (T01; `cli/src/services/observability.rs`, `observability/traits.rs`, `security.rs`, `resilience.rs`, `db/mod.rs`) + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml tracing_boundary_a` with injected (1) a secret-bearing event ID, (2) an event ID containing a filesystem path, (3) an unknown event ID, (4) a valid event ID with a malicious string-valued attribute (`operation`/`db_name`/field value), plus secret-bearing messages, fields, paths, and multi-level error chains; the test fails if any injected sensitive value, or `event_message`, `fields`, `error_source`, `error`, `last_error`, appears in any captured event, and asserts file/stderr output is unchanged. +- [ ] AC2 (Boundary B — OTel export): The exporter layer passes only target `sce::otel`, only enumerated span/event names and attribute keys, only typed bounded values, and never exception messages or stack traces; general `sce` target events are excluded. (T01 policy, T03 enforcement) + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml tracing_boundary_b` (capturing exporter fed disallowed names, keys, oversized strings, and `sce`-target events; asserts all dropped or truncated) +- [ ] AC3 (disabled and offline behavior): With telemetry disabled, command stdout, stderr, exit codes, and timing are the baseline; with telemetry enabled but the collector unreachable, stdout and exit codes are still unchanged and only bounded overhead (see AC5) is added. + - Validate: `nix flake check` (existing CLI tests unchanged) plus the actual-subprocess tests `telemetry_disabled_baseline` (T03) and `telemetry_offline_parity` (T04), both default features. Hook invocations are covered by AC17: they stay on the disabled baseline in every production mode. Overhead bounds are those of AC5. +- [ ] AC4 (async semantics): The telemetry subscriber is active on every poll of the command future, across multiple yields, Tokio worker-thread migration, nested spans, and concurrent command executions, with correct parent-child relationships, and without global subscriber initialization or `Box`. (T03; `cli/src/services/observability/traits.rs`, `cli/src/app.rs`) + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_async_scope` (multi-yield, multi-thread, nested span, concurrent-command, and spawned-task cases) and the existing `telemetry_scopes_every_poll_of_yielding_work` test unchanged. +- [ ] AC5 (bounded shutdown and exporter isolation): Overhead is defined as `telemetry overhead = enabled process duration - disabled process duration` for the same command, and two bounds are required. (a) Post-command: using the D6 lifecycle markers, `process_exit_requested - command_complete` is at most flush budget + slack, so ordinary long-running commands are never required to finish within the budget. (b) End-to-end: the delta against the telemetry-disabled baseline stays within the recorded bounds below. Protocol (T04): workload `sce version` and `sce doctor` (command names confirmed against `cli-command-surface.md`); 5 warmup runs then 30 measured runs per receiver mode, interleaved with disabled runs on the same build; modes reachable, connection-refused, non-routable address, and accept-but-never-respond; flush budget 1000 ms; slack 250 ms for (a). For (b), blackhole and non-routable modes must satisfy p50/p95 overhead ≤ budget + 250 ms and max ≤ budget + 500 ms (budget-derived); reachable and connection-refused p50/p95/max overhead bounds are measured in T04, recorded in the T04 note, and fixed as constants in the test before T04 completes. Every subprocess runs under a 5 s hard timeout whose expiry fails the test, proving no hang after command completion. Stdout and exit code are unchanged. Separately, no credential issuance, exporter initialization needing network access, `force_flush()`, or shutdown occurs while a mutation, Git, adapter-state, or DB lock is held; background batch export may run concurrently with lock holders but must not block them (see `telemetry_exporter_cannot_block_locks`). Hook budget gate: production hook export remains disabled in standalone and managed modes (AC17) until T05 records a budget chosen from its benchmark, `telemetry_shutdown_subprocess` passes configured with it, and a user-owned policy enables it. + - Validate (feature-enabled pass, because the D6 lifecycle markers require it): `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml --features telemetry-test-receiver telemetry_shutdown_subprocess` (real subprocess, D6 markers, protocol above); `... --features telemetry-test-receiver telemetry_exporter_cannot_block_locks` (negative regression: exporter pointed at a blackhole with batch size/delay forced low so export starts mid-command, exporter additionally delayed/blocked; a real mutation-scope/worktree-lock operation still releases its locks, completes, and a second contender acquires the lock within a bound independent of the exporter timeout; command result and durable state identical to the telemetry-disabled run); inspection of the T05 benchmark record. +- [ ] AC6 (standalone trace): With `SCE_TELEMETRY=standalone` and a local OTLP/HTTP protobuf receiver, `sce doctor` emits one root `sce.command` trace with a typed outcome and no managed credential header. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml sce_command_span` +- [ ] AC7 (trace context): Valid `TRACEPARENT`/`TRACESTATE` set the remote parent of `sce.command` with exact trace ID, parent span ID, and sampled flag; malformed or absent context yields an independent root; unsampled parents are honored. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_context` +- [ ] AC8 (Claude Code connectivity): A real Claude Code smoke test records, for Bash, PowerShell, and lifecycle-hook subprocesses separately, whether `TRACEPARENT` reached SCE, the exact trace ID and remote parent span ID observed on both sides, span hierarchy, and sampling behavior; SCE configuration works when Claude does not propagate `OTEL_*`. Lifecycle-hook parenting is claimed only if demonstrated; otherwise independent-root behavior is documented. Hook subprocess runs use the same `telemetry-test-receiver` feature-enabled binary recorded by T05 and export through the D6 local-receiver mechanism, with no managed credential and no production access; production hook export is not exercised or implied. + - Validate: inspect the T07 evidence note under `context/sce/` (Claude Code version, flags verbatim, captured IDs, per-subprocess table). +- [ ] AC9 (credential identity and lifecycle): `/me` stable `user.id` and `workspace.id` are decoded; the telemetry credential record (`token`, `accessor`, `expiresAt`, binding) is stored encrypted in a record separate from WorkOS tokens; mismatched, expired, or stale credentials are rejected; logout clears the cache; concurrent provisioning does not corrupt the cache; hooks never mint credentials; tokens, authorization headers, and raw credential responses are never logged. Documentation states that local deletion does not revoke an already issued OpenBao token. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_credential` +- [ ] AC10 (managed exfiltration safety): A malicious repository-local `.sce/config.json` (and malicious `OTEL_*`/header/endpoint env) cannot enable managed export, change its destination, or cause the managed token to be sent anywhere except the pinned endpoint; redirects carrying the token are not followed. A hook invocation with managed enablement and a valid cached credential makes no exporter or credential-API request (AC17). + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_managed_exfiltration` (default features; local listeners record any request received; assert none receives the token, redirect target receives no request) +- [ ] AC11 (production ingestion evidence): The T02 note records, each with timestamp, environment, and evidence source: public receiver authentication test, public trace acceptance test, workspace dataset routing, actual Dash0 ingestion, and the internal exporter failure investigation with disposition. Unavailable checks are marked blocked with exact reason and retry condition; no evidence is fabricated. + - Validate: inspect the T02 context note and its `context-map.md` link. +- [ ] AC12 (doctor readiness): `sce doctor` reports telemetry mode and readiness (credential presence, binding validity, expiry, ignored override variables, hook-export gate state) in stable text and JSON, never printing credentials. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml doctor_telemetry` +- [ ] AC13 (operation spans): Traces explain where mutation and reconciliation operations spend time and where they fail, exporting only duration, operation type, typed outcome, error category, and safe counts; existing mutation-scope, reconciliation, and DB tests pass unchanged, proving durability and lock ordering are unchanged. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml operation_span` plus the existing `mutation_scope`, `reconciliation`, and DB test selections unchanged. +- [ ] AC14 (dependency impact recorded): Dependency count, release binary size, and clean build time deltas introduced by the OTel crates are measured in T03 and recorded. + - Validate: inspect the T03 note for before/after `cargo tree` crate counts, release binary size, and clean-build time with methodology. +- [ ] AC15 (combined production trace): For managed-mode rollout acceptance, one distributed trace is demonstrated end to end in the intended Dash0 workspace dataset: (1) Claude Code emits an upstream span with a known trace ID; (2) SCE receives valid W3C context from an actual supported Claude Code subprocess (per T07); (3) SCE emits a child span with the identical trace ID and the correct parent span ID; (4) both export successfully through their own configured authentication paths; (5) both spans arrive in the same assigned Dash0 dataset; (6) a Dash0 trace query returns both spans and their parent-child relationship; (7) neither is routed to the internal collector dataset or another workspace's dataset. The evidence note records trace ID, span IDs, dataset, Claude Code version, timestamps, and the query result, never credentials. If Claude Code's export cannot authenticate or route to the same dataset, this criterion is recorded as blocked (Blocker, Required action, Retry condition) and T09 production acceptance is not complete. Local tests stay independent of Dash0. + - Validate: inspect the T09 combined-trace evidence note and its `context-map.md` link. +- [ ] AC16 (controlled local-receiver mechanism): The D6 mechanism exports real hook traces only to a loopback receiver under explicit opt-in in feature-enabled builds, cannot load or send a managed credential, cannot activate the managed exporter, leaves hook output/permission decisions/mutation semantics unchanged, and is inert in default builds, release builds, and ordinary execution. (T04) + - Validate, default features (feature unavailable): `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_test_receiver_inert` (D6 environment variables have no effect when the feature is absent: no export, no output change), plus the packaged `.#default` binary check that D6 variables produce no request to a loopback listener and that the release feature set excludes `telemetry-test-receiver`. + - Validate, feature enabled: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml --features telemetry-test-receiver telemetry_test_receiver` (loopback-only endpoint acceptance and non-loopback/hostname/HTTPS rejection, no auth DB access, no managed credential loadable or exportable, no managed header, hook parity with disabled run). + - Both configurations run explicitly in CI and local verification, each through `scripts/check-cargo-test-count.sh`; zero matching tests fails the check, and the executed test counts are recorded in the T04 note. +- [ ] AC17 (hook-export gate in every production mode): Before the T05 gate passes, ordinary SCE hook invocations export nothing remotely and read no credential in standalone or managed mode; hook classification comes from the typed command/dispatch model; the only exception is the D6 feature-gated loopback receiver; telemetry failure never changes hook permission decisions, stdout, exit codes, mutation durability, or lock ordering. (T03 gate and standalone cases, T04 D6 and unreachable-receiver cases, T09 managed case) + - Validate, default features: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_hook_export_gated` covering (1) standalone enabled with a reachable OTLP receiver and inherited `OTEL_EXPORTER_OTLP_*`, hook export not approved: the receiver gets zero requests; (2) managed enabled with a valid cached credential, hook export not approved: zero exporter requests and no credential-API call (T09); (4) a non-hook standalone command still emits its normal trace; (5) an unreachable or blackhole receiver leaves hook stdout, exit code, and permission decision identical to the telemetry-disabled run. Each case asserts its listener count, so removing the gate fails the test. + - Validate, feature enabled: `... --features telemetry-test-receiver telemetry_hook_export_gated` adds (3) D6 receiver with explicit opt-in: hook export reaches only the loopback receiver, with no managed credential read. + - Both runs go through `scripts/check-cargo-test-count.sh`. + +### Full validation + +Repository-wide checks `/validate` runs after the last task, regardless of which criterion they map to. + +- `nix flake check` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml tracing_boundary` (focused observability/security) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_context` (propagation) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_credential` and `telemetry_managed_exfiltration` (credential boundary) +- Default-feature pass, each via `scripts/check-cargo-test-count.sh` (zero matching tests fails): `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_test_receiver_inert`, `telemetry_hook_export_gated`, `telemetry_offline_parity`, `telemetry_disabled_baseline` +- Feature-enabled pass, each via `scripts/check-cargo-test-count.sh`: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml --features telemetry-test-receiver telemetry_test_receiver`, `telemetry_hook_export_gated`, `telemetry_shutdown_subprocess`, and `telemetry_exporter_cannot_block_locks` (actual-process timeout, lock isolation, controlled receiver, hook gate) +- Release/default package check: the `.#default` build excludes `telemetry-test-receiver` and ignores the D6 environment variables +- Existing `mutation_scope` and `reconciliation` regression selections, unmodified +- `nix run .#pkl-check-generated` if any Pkl/schema/generated input changes + +Existing tests are not rebaselined or deleted; only tests that assert the removed tracing payload shape (`event_message`, `fields`) are updated, and each such change is listed in the owning task's completion record. + +### Context sync + +- `context/sce/cli-observability-contract.md` (two-boundary contract, event-ID registry, `sce::otel` surface, modes, bounded shutdown and background-export concurrency, D6 test mechanism, supersession of the "no production sink" statements) +- `context/overview.md`, `context/architecture.md`, `context/glossary.md` (OTel runtime, `sce.command`, Boundary A/B, standalone vs managed, trace-context propagation, telemetry credential) +- `context/cli/cli-command-surface.md`, `context/cli/config-precedence-contract.md`, `context/cli/default-path-catalog.md` (setup action, env switches, managed state location) +- `context/sce/auth-db.md` (telemetry credential record, separation from WorkOS tokens, logout clearing) +- Doctor context file for telemetry readiness +- `context/context-map.md` entries for new context notes (T02, T05, T07, T09 combined-trace evidence) +- Supersession note relative to `remove-cli-otel` + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. A completed task must be `synced` before another task can start or the plan can finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** beside the status. Never infer `synced` from conversation history; write every lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/observability*`, `cli/src/services/security.rs`, `cli/src/services/resilience.rs` and `cli/src/services/db/mod.rs` (tracing call sites only), `cli/src/app.rs` telemetry wiring, `cli/Cargo.toml`/`Cargo.lock`, `cli/src/services/agent_trace_sync/control_plane.rs` (`/me` decoding and credential client), auth DB credential storage (`cli/src/services/auth_db/`), `cli/src/services/auth_command/mod.rs` (logout), `sce doctor`, the foreground setup action, instrumentation call sites listed in Phase 4, and matching context files. +- **Out of scope:** Phase 5 items; changes to durability, lock ordering, or mutation-trace semantics; changes to local logging formats and file routing; Control Plane server code; server-side token revocation. +- **Constraints:** ordinary hooks never export remotely before the T05 gate passes, in any production mode, with classification from the typed command model and the D6 feature-gated loopback receiver as the only exception; production hook telemetry never depends on `telemetry-test-receiver`; static dispatch through the generic `Telemetry` trait with `NoopTelemetry` default and no `Box`; no global subscriber initialization; OTLP/HTTP protobuf only; crate versions pinned exactly and matching Bauk's; no comments in code; export is allowlist-only; managed credentials are never minted from a hook or while locks are held; exporter activity never blocks or is awaited by protected critical sections, and telemetry errors never change mutation outcomes; run Cargo and tests via Nix; local tests never require production Dash0 access. +- **Non-goal:** exporting free-form log messages, command arguments, paths, hook payloads, SQL text, or error text as span attributes. +- **Non-goal:** claiming trace parenting for any harness or subprocess type that was not demonstrated. +- **Non-goal:** guaranteeing that all network traffic occurs after command completion; the batch exporter may export concurrently. +- **Non-goal:** a production bypass flag or environment-inherited switch for hook export before T05; only the feature-gated D6 mechanism exists. The post-T05 user-owned enablement policy is implemented by a separate change. +- **Non-goal:** promising revocation of issued telemetry tokens. + +## Assumptions + +- Bauk reference is `davidabram/bauk/src/telemetry/mod.rs` and its `Cargo.toml`; the repository is not checked out on this machine, so T03 reads the exact crate versions from it before pinning them and records them in the task. +- Control Plane contract is fixed by `control-plane/context/api/telemetry-credential.md`: `POST /auth/telemetry/credential` takes no payload, is protected by the WorkOS bearer, returns exactly `{ token, accessor, expiresAt }` (non-renewable ~90-day OpenBao token), and returns 403 without a workspace, 503 when not configured/unavailable, and 500 on issuance failure. Control Plane `GET /me` returns stable `user.id` and `workspace.id` (nullable workspace) that SCE's current `MeUser`/`MeWorkspace` types do not yet decode. +- Control Plane does not persist or cache issued credentials and its issuer cannot revoke; SCE therefore documents local-cache deletion as non-revoking. +- Standalone opt-in uses an SCE-specific switch rather than a bare `OTEL_*` variable, because `OTEL_*` may be inherited from an unrelated process environment. +- Claude Code propagation behavior (which subprocess types receive `TRACEPARENT`) is unknown until T07 measures it. +- Phase 4 spans are grouped into three tasks by subsystem to keep each commit atomic. + +## Task stack + +- [x] T01: `Enforce Boundary A and define the Boundary B export policy` (status:done) + - Task ID: T01 + - Scope: In — stop `Logger::log_forced`/`emit_tracing_event` and the `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted` raw events from carrying `event_message`, serialized `fields`, `error_source`, `error`, or `last_error` to tracing; introduce the closed `TracingEventId` registry and classification (unknown or dynamic IDs, including test-only and `sce.error.{code}` string forms, become `sce.unclassified`; `sce.error.*` derives from the typed error class); classify `operation` and `db_name` into enumerated types with an `unclassified` fallback; keep only level, typed numerics, and existing constants; inventory every `tracing::` macro site under `cli/src` with a test that fails on a new site recording an unclassified field; define the `sce::otel` target constants, enumerated span/event names, enumerated attribute keys, and typed bounded value types with no exception-message/stack-trace recording; arbitrary-subscriber capture tests for both boundaries. Out — OTel crates, exporter wiring, local file/stderr log rendering (format and raw event IDs unchanged), changing `redact_sensitive_text` behavior. + - Dependencies: none + - Done when: with an arbitrary capturing subscriber, the four injected cases of AC1 (secret-bearing event ID, path-bearing event ID, unknown event ID, valid event with malicious string attribute) plus secret-bearing messages, fields, paths, and error chains never reach any captured `sce`-target event; the `sce::otel` policy types reject disallowed names/keys and truncate or reject unbounded strings; file/stderr log output is byte-identical to before; existing tests asserting the old tracing payload shape are updated and listed. Boundary A is fully enforced here, before any OTel exporter exists. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml tracing_boundary_a`; `... tracing_boundary_b`; `... observability`; `... resilience` + - Completed: 2026-10-09 + - Files changed: `cli/src/services/observability.rs`, `cli/src/services/observability/tracing_boundary.rs` (new), `cli/src/services/observability/otel_policy.rs` (new), `cli/src/services/resilience.rs`, `cli/src/services/db/mod.rs`, `cli/src/services/db/tracing_boundary_tests.rs` (new) + - Result: `Logger` now hands `tracing` only a registry-classified `event_id` (closed sorted `TRACING_EVENT_IDS`, unknown/dynamic/test IDs become `sce.unclassified`, `log_cli_error` maps `sce.error.{parse,validation,runtime,dependency}` from the typed failure class) plus a typed `log_level`, with a static message; `event_message`, serialized `fields`, and `error_source` are no longer emitted. `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted` no longer record `error`/`last_error`, and their `operation`/`db_name` pass through `OperationClass`/`DbName` enums with an `unclassified` fallback. A source-scanning inventory test fails on any new `tracing::` macro site with an unlisted field, `%`/`?` capture, or non-literal message. Boundary B policy (`sce::otel` target, `OtelName`, `OtelAttributeKey`, bounded `OtelLabel`/`OtelValue`, `admit_otel_record`) lives in `otel_policy.rs` with scoped `allow(dead_code)` until T03 consumes it. Local file/stderr rendering is unchanged. No pre-existing test asserted the old tracing payload shape, so none were updated. + - Verify results: `tracing_boundary_a` passed (9 tests); `tracing_boundary_b` passed (3 tests); `observability` passed (30 tests); `resilience` passed (1 test); `nix flake check` passed (clippy, fmt, tests, quint-connect). + - Hardening (PR #306 review, two P1 findings; completed 2026-10-09): (1) Boundary B string attributes were a character-class check (`[a-z0-9_.-]{1,64}`) and so admitted secrets such as `hunter2`, `password123`, `secret_token_123`; `OtelLabel`/`OtelValue::Label`/`OtelAttributeKey` are replaced by the closed `OtelAttribute` enum over `CommandName`, `Outcome`, `ErrorCategory`, and `OperationType` (= `OperationClass`) plus numeric variants, with raw input admitted only by exact enum lookup (`OtelAttribute::admit`). (2) The text-scanning macro inventory could be bypassed (`tracing::warn!("token={secret}")`, `operation = untrusted_value`); all `sce`-target emission now goes through `emit_logger_event`, `emit_retry_event`, and `emit_contention_event` in `tracing_boundary.rs` (typed enums, registry-backed `EventId` newtype, numerics, static messages), `observability.rs`/`resilience.rs`/`db/mod.rs` contain no `tracing` macros, and a test-only `syn` audit (`tracing_boundary/audit.rs`; `syn` and `proc-macro2` dev-dependencies) rejects any tracing path/import/alias/macro outside the boundary and any interpolation or unclassified field value inside it. The retry event's target is now `sce` (was the incidental module path). Hardening files: `cli/Cargo.toml`, `cli/Cargo.lock`, `cli/src/services/observability/{tracing_boundary.rs,otel_policy.rs,tracing_boundary/audit.rs}`, `observability.rs`, `resilience.rs`, `db/mod.rs`, `context/sce/cli-observability-contract.md`. + - Hardening verify results: `tracing_boundary_a` passed (22 tests, includes the syn audit regression cases and the repo-wide scan `tracing_boundary_a_every_tracing_macro_site_records_only_classified_fields`); `tracing_boundary_b` passed (12 tests); `observability` passed (52 tests); `resilience` passed (1 test); `nix flake check` passed (clippy pedantic, fmt, tests, quint-connect). Compile-fail (trybuild) tests were not added because `sce` is a bin-only crate; `tracing_boundary_a_emission_functions_accept_only_closed_types` pins the emitter signatures at compile time instead. + - Context impact: important — `context/sce/cli-observability-contract.md` needs the Boundary A event-ID registry/classification and the Boundary B `sce::otel` policy; glossary terms Boundary A/B. + - Context synchronization: synced + +- [ ] T02: `Record public collector and Dash0 ingestion verification` (status:todo) + - Task ID: T02 + - Scope: In — independently run and record, each with timestamp, environment, and evidence source: public receiver authentication test, public trace acceptance test, workspace dataset routing, actual Dash0 ingestion, and the internal exporter failure investigation with disposition; write the note under `context/sce/` and link it from `context/context-map.md`. Checks that cannot be run are recorded as blocked with exact reason and retry condition; evidence is never fabricated. Out — Control Plane code changes, SCE runtime changes. + - Dependencies: none + - Done when: the note exists, every listed check has a result or an explicit blocked entry with reason and retry condition, and internal exporter failures are tracked separately from public pipeline readiness. + - Verify: inspect the note and its `context-map.md` link; no SCE tests required. + - Context synchronization: pending + +- [x] T03: `Add opt-in standalone OpenTelemetry runtime with async-correct subscriber scoping` (status:done) + - Task ID: T03 + - Scope: In — the D3 hook-export gate, delivered together with (and activated no later than) the first production exporter: hook classification from the typed command/dispatch model (never raw-argument string matching) applied during mode resolution so hook invocations resolve to `NoopTelemetry` in standalone mode regardless of inherited `SCE_TELEMETRY`/`OTEL_*`, plus `sce doctor` gate reporting hooks (text/JSON detail owned by T10); the default-feature cases of `telemetry_hook_export_gated` (standalone with reachable receiver gets zero hook requests, non-hook standalone command still traces, unreachable receiver leaves hook output/exit code unchanged); exact-pinned OTel crates matching Bauk (`davidabram/bauk/src/telemetry/mod.rs` and its `Cargo.toml`), provider and OTLP/HTTP protobuf exporter, a statically dispatched `Telemetry` implementation that builds a private `Dispatch` and attaches it with `WithSubscriber::with_subscriber` (D2), no global init, exporter layer enforcing Boundary B (target `sce::otel`, name/key allowlist), standalone-mode configuration (`SCE_TELEMETRY=standalone`, `OTEL_EXPORTER_OTLP_*` for a local receiver, no managed token), `.with_current_subscriber()` on audited spawn sites, and the AC14 dependency-impact measurement. Out — `sce.command` span content, flush policy, trace-context extraction, managed mode. + - Dependencies: T01 + - Done when: hooks export nothing in standalone mode and `telemetry_hook_export_gated` (default features) fails if the gate is removed; telemetry unset uses `NoopTelemetry` with identical behavior; the runtime installs without global subscriber state; tests prove the subscriber is active on every poll across multiple yields, worker-thread migration, nested spans with correct parents, concurrent commands, and spawned tasks; repository-local config cannot enable standalone mode; the exporter layer enforces Boundary B independently of T01's Boundary A work; the batch export thread is confirmed not to run on command runtime workers; dependency/binary-size/build-time deltas are recorded. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_async_scope`; `... telemetry_disabled_baseline`; `... telemetry_hook_export_gated` (default features, via `scripts/check-cargo-test-count.sh` once T04 delivers it; until then the executed count is recorded manually and must be non-zero); existing `telemetry_scopes_every_poll_of_yielding_work` unchanged; `nix flake check` + - Completed: 2026-10-09 + - Files changed: `cli/Cargo.toml`, `cli/Cargo.lock`, `cli/src/app.rs`, `cli/src/app/telemetry_tests.rs` (new), `cli/src/services/command_registry.rs`, `cli/src/services/agent_trace_sync/control_plane.rs`, `cli/src/services/observability.rs`, `cli/src/services/observability/otel_runtime.rs` (new), `cli/src/services/observability/otel_runtime/tests.rs` (new), `cli/src/services/observability/otel_policy.rs`, `cli/src/services/observability/tracing_boundary.rs`, `cli/src/services/observability/tracing_boundary/audit.rs` + - Result: Crate versions were read from `davidabram/bauk` (`Cargo.toml` and `Cargo.lock`, fetched with `gh api` because the repo is not checked out) and pinned exactly: `opentelemetry` 0.33.0, `opentelemetry_sdk` 0.33.0, `opentelemetry-otlp` 0.33.0 (`trace`, `http-proto`, `reqwest-blocking-client`; Bauk uses `grpc-tonic`, this plan requires HTTP protobuf), `tracing-opentelemetry` 0.34.0, `tracing-subscriber` 0.3.23 (`registry`, `std`). `RuntimeCommand::is_hook_invocation` classifies from the typed command variant (exhaustive `match`, no wildcard). `resolve_telemetry_mode` reads only `SCE_TELEMETRY` from an injected environment source, and consults the invocation only when `standalone` was requested, so hook invocations (and unparseable arguments, fail-closed) resolve to `NoopTelemetry` before any exporter is built. `OtelTelemetry` builds a private `Registry` + `tracing-opentelemetry` layer behind a `filter_fn` (spans only, target `sce::otel`, enumerated `OtelName`) and wraps each action future with `ScopedDispatch::scope` (`WithSubscriber::with_subscriber`), with no global subscriber. `BoundaryBExporter` wraps the OTLP exporter and independently drops non-enumerated span names, re-admits every attribute through `OtelAttribute::admit` (numeric-looking strings are re-admitted only for numeric keys), clears events and links, and blanks error status descriptions. `AppRuntime.telemetry` is the two-variant `RuntimeTelemetry` enum (static dispatch, no `Box`). Standalone export uses OTLP/HTTP protobuf with a 750 ms connect/total timeout, redirects disabled, and a tracked fallback to an empty-root TLS client when the platform certificate store cannot be loaded (found in the Nix sandbox: plaintext local receivers still work). Repository config cannot enable the mode because resolution never reads it. Spawn audit: the only production `tokio::spawn` (`control_plane.rs` credential refresh) now goes through `spawn_in_current_scope` (`.with_current_subscriber()` + `.in_current_span()`). Untraced and justified: the `spawn_blocking` sites in `worktree_lock.rs`, `protected_worktree.rs`, `git_snapshot.rs`, `mutation_scope_lock.rs`, and `external_mutation_guard.rs`, and the `std::thread::spawn` in `codex_hook_policy.rs`, because no `sce::otel` span exists under them until T11-T13, which will wrap them. The batch export thread is the SDK's `OpenTelemetry.Traces.BatchProcessor` std thread, asserted distinct from the command runtime threads. No `sce.command` span, flush policy, trace-context, or managed mode was added (T04/T06/T09). + - Verify results: `telemetry_async_scope` passed (5 tests); `telemetry_disabled_baseline` passed (2); `telemetry_hook_export_gated` passed (5, default features; executed count recorded manually because `scripts/check-cargo-test-count.sh` arrives in T04); removing the hook classification made `telemetry_hook_export_gated_standalone_hook_invocations_export_nothing` fail, and restoring it passed; existing `telemetry_scopes_every_poll_of_yielding_work` passed unchanged; `tracing_boundary_a` passed (22, unchanged); `tracing_boundary_b` passed (15, 3 new exporter-layer tests); `nix flake check` passed (clippy pedantic, fmt, tests, quint-connect). + - Dependency impact (AC14), single run each, same host, clean `--release` build in an empty target directory, baseline = `HEAD` (2703775e) in a temporary worktree: normal-edge crates (`cargo tree -e normal --prefix none | sort -u | wc -l`) 526 -> 537 (+11); release binary 54,603,600 -> 56,500,856 bytes (+1,897,256, +3.5%); clean release build 102 s -> 102 s (no measurable delta at one-second resolution; one sample, not a benchmark). `reqwest` stayed a single version (0.13.4) because the OTLP blocking client reuses it. Methodology script lived in the session scratchpad and is not committed. + - Deviations and assumptions: `sce policy` is classified as a hook invocation together with `sce hooks` (it is the editor-hook policy entrypoint and is latency-sensitive), which is stricter than the plan's "hooks command family" wording. The audit allow-list `BOUNDARY_ALLOWED_USES` gained `tracing::Dispatch`, `tracing::instrument::Instrument`, and `tracing::instrument::WithSubscriber` so the trusted boundary file can host the scoping helpers; the blanket `allow(dead_code)` on `otel_policy.rs` is narrowed to the three items T04 first consumes. No existing test was rebaselined. New files were registered with `git add -N` so the Nix source filter sees them; nothing was committed. + - Context impact: important — `context/sce/cli-observability-contract.md` (OTel runtime, `ScopedDispatch`, `BoundaryBExporter`, mode resolution, hook gate, spawn audit), `context/architecture.md`, `context/glossary.md` (standalone mode, hook-export gate), `context/cli/config-precedence-contract.md` (`SCE_TELEMETRY`), and the AC14 dependency-impact record. + - Context synchronization: synced + +- [x] T04: `Emit sce.command root span with bounded best-effort shutdown and the controlled test receiver` (status:done) + - Task ID: T04 + - Scope: In — `scripts/check-cargo-test-count.sh` (fails on zero or below-minimum executed tests) and CI wiring that runs the default-feature and `--features telemetry-test-receiver` passes explicitly as separate flake checks with the recorded minimum test counts; a packaged-binary check that `.#default` ignores the D6 variables and its feature set excludes `telemetry-test-receiver`; the feature-enabled cases of `telemetry_hook_export_gated` (D6 receiver is the only hook export, loopback only, no credential read) and `telemetry_test_receiver_inert` (D6 variables inert without the feature); `sce.command` span with command name and typed outcome/error category only; SDK batch processor with a bounded drop-on-full queue so span end performs no I/O and never blocks (D4); background export explicitly permitted to run concurrently with command processing and lock holders; `force_flush()`/shutdown only after the command future completes and locks are released, on a dedicated thread with the main thread waiting at most the flush budget (default 1000 ms, `SCE_TELEMETRY_FLUSH_TIMEOUT_MS`) and exiting without joining; HTTP connect/total timeouts below the budget; the D6 `telemetry-test-receiver` feature, loopback validation, injected-env config resolution, and lifecycle markers; real-subprocess tests per the AC5 protocol (workload, warmup, sample count, four receiver modes, bounds, 5 s hard timeout) and the `telemetry_exporter_cannot_block_locks` negative regression. Out — production hook export enablement, hook benchmark methodology and budget selection, context propagation. + - Dependencies: T03 + - Done when: both test configurations (default features and `--features telemetry-test-receiver`) run explicitly in CI through the count check with recorded non-zero minimums, and the T04 note records the executed counts; `telemetry_hook_export_gated` passes in both configurations; `.#default` provably excludes the feature; `sce doctor` against a local receiver emits one root `sce.command` trace with typed outcome; the AC5 (a) and (b) bounds hold across all four receiver modes with unchanged stdout and exit code; the lock-isolation regression passes with a delayed/blocked exporter; no credential issuance, network-requiring exporter initialization, flush, or shutdown happens before the command future completes (background batch export may); D6 isolation tests pass (AC16); reachable/refused bounds are recorded in the T04 note and fixed in the tests. + - Verify (default features): `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml sce_command_span`; `... telemetry_offline_parity`; `... telemetry_test_receiver_inert`; `... telemetry_hook_export_gated`. Verify (feature enabled, each adding `--features telemetry-test-receiver`): `telemetry_shutdown_subprocess`; `telemetry_exporter_cannot_block_locks`; `telemetry_test_receiver`; `telemetry_hook_export_gated`. All runs go through `scripts/check-cargo-test-count.sh`. + - Completed: 2026-10-09 + - Files changed: `cli/Cargo.toml` (feature `telemetry-test-receiver`), `cli/src/app.rs`, `cli/src/app/telemetry_tests.rs`, `cli/src/services/hooks/mod.rs` (`mutation_scope_lock` now `pub(crate)` so the lock-isolation test can use a real adapter lock), `cli/src/services/observability/otel_runtime.rs`, `cli/src/services/observability/otel_runtime/{lifecycle_markers,test_receiver,test_support,tests}.rs`, `cli/src/services/observability/tracing_boundary.rs` (`CommandSpan`, test helpers), `cli/src/services/observability/tracing_boundary/audit.rs`, `cli/tests/{support/mod,telemetry_default,telemetry_test_receiver,telemetry_shutdown}.rs`, `scripts/check-cargo-test-count.sh`, `scripts/test-check-cargo-test-count.sh`, `flake.nix` (checks `cli-telemetry-default-features`, `cli-telemetry-test-receiver`, `cargo-test-count-guard`, `packaged-binary-excludes-test-receiver`), `nix/flatpak/cargo-sources.nix` and `packaging/flatpak/cargo-sources.json` (PR #306 CI fix, committed separately as 498e33d5), `context/sce/cli-observability-contract.md`, `context/cli/config-precedence-contract.md`. + - Result: `OtelTelemetry` owns a `Teardown` (provider plus scoped dispatch). After the command future completes and stdout/stderr are rendered and flushed, `RuntimeTelemetry::shutdown(self)` moves the teardown to a dedicated `sce-otel-shutdown` thread that calls `SdkTracerProvider::shutdown_with_timeout(budget)` once and drops the provider and dispatch there; the caller awaits a oneshot under `tokio::time::timeout(budget)` and never joins. Dropping an unshut-down handle hands the same teardown to a detached thread. SDK 0.33 facts that drove this (read from source): the provider shuts down on last-reference drop unless `is_shutdown` is set; an explicit shutdown sets it, so later clone drops cannot block; `with_batch_exporter` and `BatchConfigBuilder::default()` read `OTEL_BSP_*`, so `BatchSettings::PRODUCTION` sets queue, batch size, and delay explicitly; the provider stringifies processor errors, so `ShutdownOutcome` is `Completed`/`Incomplete`/`Unavailable`. The OTLP client now uses `RetryPolicy::disabled()` (the SDK default added about 450 ms on connection refused). `CommandSpan` emits one `sce.command` span per successfully parsed command with only command name, outcome, and error category. Test receiver: pure `test_receiver::resolve`, loopback IP literal only, fails closed when `OTEL_EXPORTER_OTLP_(TRACES_)HEADERS` is set (the SDK merges them unconditionally), wins over standalone; lifecycle markers via `SCE_TELEMETRY_TEST_LIFECYCLE_FILE`. + - Verify results (default features): `tracing_boundary_a` 23 passed; `tracing_boundary_b` 17; `telemetry_async_scope` 5 (includes the deterministic cross-thread `telemetry_async_scope_survives_worker_thread_migration`); `telemetry_hook_export_gated` 5; `telemetry_shutdown` 4 (`telemetry_shutdown_exactly_once*`, `telemetry_shutdown_stalled_exporter_does_not_hold_tokio_worker`); `telemetry_command_span` 4; `telemetry_exporter_cannot_block_locks` 1; `telemetry_test_receiver_inert` 1; `telemetry_offline_parity` 4 (real `sce` subprocesses); `telemetry_disabled_baseline` 2; `sce_command_span` 2. Feature-enabled (`--features telemetry-test-receiver`): `telemetry_test_receiver` 8; `telemetry_hook_export_gated` 6; `telemetry_shutdown_subprocess` 5; `telemetry_exporter_cannot_block_locks` 1. Counts above are the minimums enforced through `scripts/check-cargo-test-count.sh` by the two flake checks, both of which passed in the Nix sandbox. Mutation evidence: inserting a blocking `thread::sleep` into `shutdown` makes `telemetry_shutdown_stalled_exporter_does_not_hold_tokio_worker` fail ("worker was starved for 300.441ms"); the first version of that test passed the same mutation and was strengthened. `cargo clippy --all-targets --all-features` and fmt clean. `nix flake check` passed on x86_64-linux (all checks including `cli-tests`, `cli-clippy`, `cli-fmt`, `cli-telemetry-default-features`, `cli-telemetry-test-receiver`, `cargo-test-count-guard`, `packaged-binary-excludes-test-receiver`, `cargo-sources-parity`, `mutation-trace-quint-connect`); darwin and aarch64 systems were not evaluated. No Magic Nix Cache or substituter failures occurred locally; the CI cache throttling seen on PR #306 is infrastructure, separate from the Flatpak hash mismatch fixed in 498e33d5. + - Measured AC5 results (single host, debug test build, 5 warmup + 30 measured interleaved runs, flush budget 1000 ms, workloads `version` and `doctor`): reachable overhead p50 43-47 ms, p95 49-53 ms, max 49-62 ms, post-command max 6.5 ms; connection refused overhead p50 43-45 ms, p95 47-51 ms, max 47-53 ms, post-command max 1.7 ms; accept-but-never-respond and non-routable overhead p50 about 793 ms, p95 about 796-799 ms, max about 796-803 ms, post-command about 751 ms (the 750 ms client timeout, inside the 1000 ms budget). With `SCE_TELEMETRY_FLUSH_TIMEOUT_MS=200` against a blackhole the post-command wait is 200.5 ms, so the budget (not the exporter) bounds exit. Fixed bounds in `tests/telemetry_shutdown.rs`: reachable/refused p50 250 ms, p95 400 ms, max 750 ms; blackhole/non-routable p50 and p95 budget + 250 ms, max budget + 500 ms; post-command budget + 250 ms. + - Deviations and assumptions: (1) Isolated multi-second stalls (4.2 s, simultaneous across concurrent subprocess tests) appeared on this host while `/proc/pressure/io` reported `full avg10` around 90%; an independent heartbeat showed no gaps in later passing runs, so these are recorded as host IO pressure, not exporter behavior. The protocol therefore retries a hard-timeout run once per measurement (second stall fails), applies the max bound to the second-worst run, and bounds the single worst run only by the 5 s hard timeout; p50/p95 stay strict. (2) `telemetry_exporter_cannot_block_locks` is an in-process test with a real adapter-state advisory lock (`AdapterLockSpec`), a stalled exporter forced to export mid-command (batch size 1, 10 ms delay), a contender acquiring within 2 s, and state identical to a telemetry-disabled run; it is not a separate subprocess. (3) The plan's `sce_command_span` wire check counts the protobuf `sce.command` span-name field and key strings in the captured OTLP body (no protobuf decoder dependency was added); root-ness and attributes are asserted on decoded `SpanData` in `telemetry_command_span_*`. (4) D6 "ignores `OTEL_*` header overrides" is implemented as fail-closed rejection because the SDK reads them unconditionally. (5) Non-routable mode uses standalone resolution (the D6 receiver is loopback-only); the lifecycle markers work in any mode. (6) `RetryPolicy::disabled()` (single export attempt) was added beyond the original T04 wording to keep shutdown latency bounded. The PR #306 review items (Boundary B target contract, deterministic migration test) were folded into T03 hardening records of this task; T01 and T03 regressions pass unchanged. + - Hardening (PR #306 review, two P2 findings; completed 2026-10-09): (1) Shutdown failure paths could still run blocking SDK teardown on the caller: `spawn_teardown` fell back to `shutdown_with_timeout(Duration::ZERO)`, which is not guaranteed nonblocking, and dropping the last provider reference also runs SDK shutdown. Now `spawn_teardown` takes a statically dispatched worker-spawn closure (production: `spawn_shutdown_thread`, named `sce-otel-shutdown`); when thread creation or the handoff send fails the recovered `Teardown` is `abandon`ed with `std::mem::forget`, which skips the provider and dispatch destructors so no SDK shutdown code runs on the caller, nothing is exported or written, and the memory stays allocated until process exit (accepted leak, exceptional path only, never used on the successful path). Outcome is `Unavailable`. New tests `telemetry_shutdown_thread_spawn_failure_is_nonblocking`, `telemetry_shutdown_handoff_failure_is_nonblocking`, `telemetry_drop_without_shutdown_spawn_failure_is_nonblocking` use a blocking test exporter and assert the exporter shutdown is never entered, no second shutdown occurs, and the single Tokio worker keeps polling. (2) `tests/telemetry_shutdown.rs` no longer retries a hard-timeout subprocess or exempts the worst sample: any hard timeout panics with workload, mode, iteration, flush budget, loadavg, and thread dump (`require_completed`), and `assert_strict_max` bounds the actual maximum sample for overhead and post-command latency. Timing measurements are serialized within the test binary (`TIMING_LOCK`); p50/p95, 5 warmup and 30 measured runs, and the 5 s hard timeout are unchanged. `telemetry_shutdown_measurement_rejects_hard_timeout` covers the helpers directly. This supersedes deviation (1) below regarding the retry and second-worst allowance. + - Context impact: important — `context/sce/cli-observability-contract.md` (Boundary B responsibilities, `sce.command`, shutdown lifecycle, test receiver) and `context/cli/config-precedence-contract.md` are updated; `context/overview.md`, `architecture.md`, `glossary.md` not changed. + - Context synchronization: synced + +- [x] T05: `Benchmark hook latency and record the hook export decision gate` (status:done) + - Task ID: T05 + - Scope: In — benchmark real hook-style `sce` subprocess invocations, built with the D6 `telemetry-test-receiver` feature and exporting to the loopback receiver (the only way a hook exports before the gate passes; record the build command and that the feature adds only a config branch), with `nix shell nixpkgs#hyperfine -c hyperfine` under four modes: disabled, enabled with reachable local receiver, enabled with blackhole listener, enabled with connection refused; fixed warmup and at least 30 runs each; record p50, p95, and max end-to-end overhead (`enabled - disabled`) and the post-command shutdown duration from the D6 lifecycle markers, host and build profile; select and record the hook flush budget from this evidence (none is pre-chosen) or the decision to keep hook export disabled; if no budget is justified, hooks stay export-disabled and a local forwarder is recorded as a separate follow-up. Out — implementing a forwarder, enabling production hook export. + - Dependencies: T04 (provides D6 mechanism, shutdown tests, the hook-export gate tests, and the count check); T03 (hook-export gate) + - Done when: the benchmark binary is built with `--features telemetry-test-receiver`, the exact build command is recorded, and that same feature-enabled binary is the one T07 reuses; the note records methodology, raw results, selected budget (or disable decision), and states that production hook export stays blocked in standalone and managed modes (the T03 gate remains in force) until the recorded budget is met by `telemetry_shutdown_subprocess` configured with it, `telemetry_hook_export_gated` still passes, and a separate change adds an explicit documented user-owned enablement policy (not repository-local, not environment-inherited). + - Verify: confirm the recorded build command includes `--features telemetry-test-receiver` and that the released `.#default` build does not; run `... --features telemetry-test-receiver telemetry_hook_export_gated` through the count check; inspect the recorded benchmark note; re-run the recorded hyperfine command to confirm reproducibility. + - Completed: 2026-10-09 + - Files changed: `scripts/bench-hook-telemetry.sh` (new), `context/sce/hook-telemetry-benchmark.md` (new), `context/context-map.md`, `context/plans/sce-opentelemetry-tracing.md` + - Result: Release binary built with `nix develop -c ./scripts/run-cli-cargo.sh build --release --features telemetry-test-receiver --manifest-path cli/Cargo.toml` and benchmarked with `scripts/bench-hook-telemetry.sh` (hyperfine, `sce hooks pre-commit`, 5 warmup + 40 runs per mode, 40 lifecycle-marker runs per enabled mode) in disabled, reachable, blackhole, and refused modes at budgets 1000 ms and 100 ms. Reachable and refused add about 4-5 ms end to end (p95 4.7-5.0 ms) with under 1 ms post-command; blackhole adds about 755 ms at 1000 ms (bounded by the 750 ms client timeout) and about 105 ms at 100 ms. Candidate hook flush budget recorded as 100 ms; production hook export stays disabled (T03 gate unchanged) pending `telemetry_shutdown_subprocess` at that budget, a passing `telemetry_hook_export_gated`, and a separate user-owned enablement policy; a local forwarder is recorded as a separate follow-up. Note: `context/sce/hook-telemetry-benchmark.md`, linked from `context/context-map.md`. + - Verify results: build command includes `--features telemetry-test-receiver` and the flake check `packaged-binary-excludes-test-receiver` (`.#default`) passed; `--features telemetry-test-receiver telemetry_hook_export_gated` passed through `scripts/check-cargo-test-count.sh` (6 executed, minimum 6); benchmark script run twice end to end with consistent reachable/refused p50 (8.57 vs 8.76 ms, 8.61 vs 8.49 ms); `shellcheck` clean. `nix flake check` was not run (no Rust, Nix, or generated-input change). + - Deviations and assumptions: workload is `sce hooks pre-commit` (typed hook command, no-op runtime); loopback only, single host, so remote-collector latency is not measured; the 100 ms budget is a candidate derived from the evidence, not a gate pass. The script runs the markers as separate runs because hyperfine cannot collect them. + - Context impact: minor — new context note plus `context-map.md` link; no behavior or contract change. + - Context synchronization: synced + +- [x] T06: `Extract W3C trace context as parent of sce.command` (status:done) + - Task ID: T06 + - Scope: In — parse `TRACEPARENT`/`TRACESTATE` from the environment, set the remote parent on `sce.command`, harness-neutral fixtures (direct invocation, malformed context, sampled, unsampled, absent) reusable for future harnesses. Out — real Claude Code verification, other harnesses, proxy-specific propagation. + - Dependencies: T04 + - Done when: valid context yields a child with the exact trace ID, parent span ID, and sampled flag; malformed or absent context yields an independent root; unsampled parents are honored; no parent relationship is invented. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_context` + - Completed: 2026-10-09 + - Files changed: `cli/src/app.rs`, `cli/src/app/telemetry_tests.rs`, `cli/src/services/observability.rs`, `cli/src/services/observability/trace_context.rs` (new), `cli/src/services/observability/trace_context/tests.rs` (new), `cli/src/services/observability/tracing_boundary.rs` + - Result: `extract_remote_parent` (pure, injected env source) feeds `TRACEPARENT`/`TRACESTATE` to the SDK `TraceContextPropagator` and returns a context only when the span context is valid and remote. `CommandSpan::start` now takes `Option` and applies it with `OpenTelemetrySpanExt::set_parent`; `app.rs` passes `extract_remote_parent(&process_env)`. Valid context yields a child with the exact trace ID, parent span ID, sampled flag and `tracestate`; malformed or absent context yields an independent root (no parent, fresh trace ID, empty trace state); an unsampled parent is honored by the provider's parent-based sampler (nothing exported). No attributes were added, so Boundary B is unchanged, and hook gating is untouched. + - Verify results: `trace_context` passed (6 tests); `tracing_boundary` passed (40); `sce_command_span` passed (2); `telemetry_command_span` passed (4) with and without an ambient `TRACEPARENT`; `nix flake check` passed (first attempt was OOM-killed, exit 137; rerun with `--max-jobs 1` passed). + - Deviations and assumptions: `telemetry_command_span_success_records_one_root_with_typed_attributes` read the process environment through the real lifecycle and failed when the shell exported `TRACEPARENT` (Claude Code does); it now expects the parent derived from the same ambient environment instead of a hard-coded invalid parent. `tracestate` is propagated on the span context per W3C but never recorded as an attribute. + - Context impact: important — `context/sce/cli-observability-contract.md` (trace-context extraction and `CommandSpan::start` remote parent), `context/glossary.md` (trace-context propagation), `context/cli/config-precedence-contract.md` (`TRACEPARENT`/`TRACESTATE` read from the process environment). + - Context synchronization: synced + +- [ ] T07: `Verify Claude Code to SCE trace propagation per subprocess type` (status:todo) + - Task ID: T07 + - Scope: In — real Claude Code smoke test with the relevant enhanced-telemetry flags recorded verbatim for the tested version, covering standalone SCE execution, Bash, PowerShell, and lifecycle-hook subprocesses; verify exact trace ID, exact remote parent span ID, span hierarchy, sampling behavior, whether `TRACEPARENT` reaches each subprocess type, that SCE configuration works when Claude does not propagate `OTEL_*`, and any proxy-specific traceparent setting before it is required anywhere; record independent-root behavior where propagation is absent. Hook-subprocess runs export through the D6 loopback receiver using the exact feature-enabled (`--features telemetry-test-receiver`) binary T05 recorded, never a default or release build, so the test needs no managed credential and no production hook export. Out — Codex/Pi/OpenCode, production Dash0 access. + - Dependencies: T04, T05 (feature-enabled binary and build command), T06 + - Done when: the evidence note has one row per subprocess type with observed propagation, IDs on both sides, and a conclusion; lifecycle-hook parenting is claimed only if demonstrated. + - Verify: confirm the note records the feature-enabled build command and that the hook runs reached only the loopback receiver; inspect the note against the local receiver's captured traces. + - Context synchronization: pending + +- [ ] T08: `Add identity-bound telemetry credential client and encrypted storage` (status:todo) + - Task ID: T08 + - Scope: In — extend `MeUser`/`MeWorkspace` in `control_plane.rs` to decode stable `user.id` and `workspace.id`; client for `POST /auth/telemetry/credential` returning `{ token, accessor, expiresAt }` with secrets redacted from `Debug` and errors; a separate encrypted auth-DB record (new migration through the existing auth DB mechanism in `cli/src/services/auth_db/`) storing `token`, `accessor`, `expiresAt`, bound `user_id` and `workspace_id`, distinct from WorkOS tokens; reject expired, stale, and mismatched records; clear the record on logout (`auth_command/mod.rs`); safe concurrent provisioning (single-flight via DB transaction or cross-process lock, no cache corruption); tests use a local API fixture, no production access. Out — setup flow, exporter wiring, doctor, server-side revocation. + - Dependencies: none + - Done when: credentials are readable only under a matching user and workspace; workspace or account switch yields "no credential" and requires foreground re-provisioning; expired credentials are rejected; logout deletes the record; two concurrent provisioning attempts leave one valid record; failures never log tokens, authorization headers, or raw responses; docs state that local deletion does not revoke already issued tokens. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_credential` (identity mismatch, expiry, logout, concurrent provisioning, safe failure, WorkOS record untouched) + - Context synchronization: pending + +- [ ] T09: `Wire managed mode with pinned collector and exfiltration guards` (status:todo) + - Task ID: T09 + - Scope: In — foreground setup action that provisions via T08 and persists managed enablement in the auth DB; managed exporter using the pinned HTTPS collector constant, a dedicated HTTP client with redirects disabled, token attached only by SCE and only to the pinned endpoint; ignore `OTEL_EXPORTER_OTLP_*` endpoint, header, protocol, and certificate overrides in managed mode; managed wins over standalone; the managed-mode case of `telemetry_hook_export_gated` (managed enabled with a valid cached credential, hook export not approved: zero exporter requests, no credential-API call, no auth-DB credential read; the T03 hook classification runs before credential access); hooks never contact the credential API or mint tokens; no provisioning while locks are held; tests with local listeners including a malicious repository-local `.sce/config.json`, malicious env endpoints/headers, and a redirecting server; credential provisioning never occurs under a lock; production acceptance including the AC15 combined Claude Code + SCE trace verified in the assigned Dash0 dataset, with the evidence note (trace ID, span IDs, dataset, Claude Code version, timestamps, query result, no credentials) linked from `context-map.md`. Out — doctor check, hook export if the T05 gate is unmet. + - Dependencies: T02, T03, T04, T05, T07, T08 + - Done when: a configured installation exports to the user's assigned Dash0 dataset without manual exporter-token handling per the T02 evidence; the exfiltration tests prove no listener receives the managed token and the redirect target receives no request; hook export stays disabled in managed mode unless the T05 gate passed and a user-owned enablement policy exists, and `telemetry_hook_export_gated` fails if the managed gate is removed; if T02 recorded blocked checks, or the AC15 combined trace cannot be produced in the same dataset, production acceptance is recorded as blocked (Blocker, Required action, Retry condition) rather than passed. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_managed_exfiltration`; `... telemetry_managed`; `... telemetry_hook_export_gated` (default features, managed case; count-checked); manual production check recorded per T02 method; combined-trace evidence note (AC15). + - Context synchronization: pending + +- [ ] T10: `Add telemetry readiness check to sce doctor` (status:todo) + - Task ID: T10 + - Scope: In — doctor diagnostic for mode, hook-export gate state (`gated` until T05 and a user-owned policy), credential presence, identity/workspace binding validity, expiry, pinned endpoint configuration, and a note when `OTEL_*` overrides are being ignored, with stable text and JSON output. Out — remediation automation, network calls to the Control Plane. + - Dependencies: T09 + - Done when: `sce doctor` reports ready/not-ready with a concrete reason in text and JSON and never prints credentials; output is deterministic. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml doctor_telemetry` + - Context synchronization: pending + +- [ ] T11: `Instrument mutation scope coordination and worktree lock spans` (status:todo) + - Task ID: T11 + - Scope: In — `sce.mutation_scope.coordinate` and `sce.worktree.lock` spans via the `sce::otel` typed API exporting duration, operation type, typed outcome, error category, and safe counts only; spans end without I/O and nothing in a critical section waits on the exporter; re-run `telemetry_exporter_cannot_block_locks` (feature-enabled pass, `--features telemetry-test-receiver`, count-checked) with the real spans. Out — any change to lock ordering, durability, or scope semantics; new attributes beyond the allowlist. + - Dependencies: T04 + - Done when: both spans appear under `sce.command` with allowlisted attributes only; existing mutation-scope tests pass unchanged; the lock-isolation regression still passes with a blocked exporter. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml operation_span`; existing `mutation_scope` selection unchanged + - Context synchronization: pending + +- [ ] T12: `Instrument reconciliation and git snapshot spans` (status:todo) + - Task ID: T12 + - Scope: In — `sce.reconciliation` and `sce.git.snapshot` spans with allowlisted attributes, ending without I/O or exporter waits. Out — reconciliation or snapshot behavior changes. + - Dependencies: T11 + - Done when: spans show duration and typed outcome, including failure category; existing reconciliation tests pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml operation_span`; existing `reconciliation` selection unchanged + - Context synchronization: pending + +- [ ] T13: `Instrument database operation and sync spans` (status:todo) + - Task ID: T13 + - Scope: In — `sce.db.operation` and `sce.sync` spans with allowlisted attributes; no SQL text, row content, or paths exported; no exporter waits inside DB transactions or lock sections. Out — persistence semantics changes. + - Dependencies: T12 + - Done when: spans carry operation type, outcome, and safe counts only; capture tests prove no SQL or payload content is exported; existing DB tests pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml operation_span` + - Context synchronization: pending + +## Open questions + +Resolved and recorded above, so no longer open: Bauk reference, Control Plane credential contract, configuration ownership and precedence, justification relative to `remove-cli-otel`, and measurement of dependency growth (T03). + +Genuinely unresolved, each needing new experimental evidence: + +- Which Claude Code subprocess types (Bash, PowerShell, lifecycle hooks) actually receive `TRACEPARENT`, and whether a proxy-specific traceparent setting exists and is required? Answered by T07 for the tested version. +- What numeric hook flush budget is justified, or must hooks remain export-disabled? Answered by T05 from D6-mode benchmarks; production hook export stays blocked until then. +- Can Claude Code's own OTLP export authenticate and route to the same Dash0 dataset as SCE's managed export (AC15)? Answered by T09; if not, combined-trace acceptance is blocked. +- Reachable and connection-refused overhead bounds (AC5b) are set from T04 measurements, not guessed here. +- Does the public collector accept authenticated traces and route them to the correct Dash0 dataset, and what is the disposition of the observed internal exporter failures? Answered by T02; gates T09 production acceptance only. diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index c18442627..c2d431dc2 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -23,12 +23,10 @@ - `MessageRole` enum: `User` / `Assistant` — maps to `messages.role` DB constraint. - `InsertMessageInsert`: owned payload struct with insertable parent `messages` columns (`session_id`, `message_id`, `role`, `generated_at_unix_ms`); message body text belongs to `parts.text`, not the parent message row. - `INSERT_MESSAGE_SQL`: parameterized single-row SQL using `INSERT ... ON CONFLICT (session_id, message_id) DO NOTHING` — leverages the unique index `idx_messages_session_message` so duplicate parent-message events remain non-failing without mutating the existing row. -- `insert_message(input)`: typed single-row helper that executes the duplicate-ignore parent-message insert; retained as part of the adapter surface. - `insert_messages(inputs)`: typed batch helper that generates and executes one parameterized multi-row `messages` insert for valid conversation-trace `message` batches while preserving duplicate-ignore semantics. - `PartType` enum: `Text` / `Reasoning` / `Patch` / `Question` — serializes known conversation part kinds for typed inserts. `parts.type` is stored as `TEXT NOT NULL` without a database-level enum `CHECK` constraint. - `InsertPartInsert`: owned payload struct with `part_type`, `text`, `session_id`, `message_id`, and `generated_at_unix_ms`. - `INSERT_PART_SQL`: parameterized single-row append-only INSERT into `parts` (no upsert; multiple rows per `(session_id, message_id)` allowed). -- `insert_part(input)`: typed single-row helper that inserts a part row without requiring a matching `messages` row (supports out-of-order writes); retained as part of the adapter surface. - `insert_parts(inputs)`: typed batch helper that generates and executes one parameterized multi-row append-only `parts` insert for valid conversation-trace `message.part` batches. - `insert_conversation_text_event(message, part)`: atomic one-message-plus-one-part write for exactly-once conversation text events (currently used only by `sce hooks codex`'s `UserPromptSubmit`/`Stop` arms; see [codex-integration-runtime.md](codex-integration-runtime.md)). Delegates to `TursoDb::execute_transactional_insert_pair_if_absent` (`cli/src/services/db/mod.rs`), a generic, schema-agnostic primitive: inside one `BEGIN IMMEDIATE` transaction it checks whether `(session_id, message_id)` already exists and, only if absent, inserts the message row then the part row and commits; an existing row causes no insert statements to run, and the no-write transaction commits, returning `Ok(false)`. `BEGIN IMMEDIATE` serializes concurrent callers against the same database file, so a replayed or concurrent duplicate delivery leaves exactly one message row and one part row — stronger than `insert_messages`/`insert_parts`' own guarantee, which dedups only the parent message row via `ON CONFLICT DO NOTHING` and leaves `parts` unguarded. `insert_messages`/`insert_parts` remain unchanged and are still what OpenCode/Claude/Pi conversation-trace intake uses. - `lifecycle.rs`: service lifecycle provider for setup/doctor integration. @@ -45,7 +43,7 @@ Repository persistence, readiness checks, transactions, and migration operations This adapter has no canonical `DbSpec::db_path()`; callers must resolve `/sce/repos//agent-trace.db` first and use explicit-path `TursoDb` constructors. Its migration list is `generated_migrations::AGENT_TRACE_REPOSITORY_MIGRATIONS`: the fresh multi-statement baseline `cli/migrations/agent-trace-repository/001_repository_schema.sql` plus the additive `002_repository_source_instance_id.sql` (adds `repository_metadata.source_instance_id`), `003_claude_model_state.sql` (adds the non-exported Claude model-state register), `004_mutation_trace_protocol.sql`, and `005_mutation_scope_provenance.sql` (the mutation-cursor protocol tables and the scope-provenance table; see [mutation-trace store](../cli/mutation-trace-store.md) and [mutation-scope provenance](../cli/mutation-scope-provenance.md)). The baseline schema includes `repository_metadata` plus the existing repository-level Agent Trace tables, indexes, and triggers, and intentionally has no `checkout_id` columns on trace tables. `RepositoryAgentTraceDb::verify_or_initialize_repository_metadata(repository_id) -> Result` inserts the singleton metadata row on first initialization, errors if an existing DB stores a different repository ID, and atomically claims `source_instance_id` for this physical database via `UPDATE ... WHERE source_instance_id = ''` (a losing racer's generated candidate is discarded and an already-valid stored value is never overwritten), returning the typed `RepositoryMetadata { repository_id, source_instance_id }`. `source_instance_id` is generated by application code (`generate_source_instance_id()`, UUID v4) and validated with `is_valid_source_instance_id()` (non-empty once trimmed); it is never derived from `repository_id`, remote URL, checkout ID, filesystem path, hostname, or user/workspace identity, and stays stable across reopen and repeated `sce setup` runs. `RepositoryAgentTraceDb::repair_missing_repository_schema_migration_metadata()` is a narrow concurrent-first-open repair seam: it never creates trace tables, but if every required repository schema table already exists and only the one-file baseline migration record is missing, it records `001_repository_schema` and rechecks readiness. -`RepositoryAgentTraceDb` exposes repository-level write helpers for the current row families by delegating to the same typed insert payloads and parameterized SQL used by the checkout-scoped adapter: `insert_diff_trace`, `insert_post_commit_patch_intersection`, `insert_agent_trace`, `insert_message`, `insert_messages`, `insert_part`, `insert_parts`, and `insert_conversation_text_event`. It also exposes `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` by delegating to the shared recent diff-trace query/parser helper, so repository-scoped attribution reads use the same chronological inclusive window semantics without a checkout filter. These methods preserve the existing row shapes and do not add checkout provenance columns or checkout-scoped write/query APIs. +`RepositoryAgentTraceDb` exposes repository-level write helpers for the current row families by delegating to the same typed insert payloads and parameterized SQL used by the checkout-scoped adapter: `insert_diff_trace`, `insert_post_commit_patch_intersection`, `insert_agent_trace`, `insert_messages`, `insert_parts`, and `insert_conversation_text_event`. It also exposes `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` by delegating to the shared recent diff-trace query/parser helper, so repository-scoped attribution reads use the same chronological inclusive window semantics without a checkout filter. These methods preserve the existing row shapes and do not add checkout provenance columns or checkout-scoped write/query APIs. The repository-scoped adapter is consumed by `agent_trace_storage`, active hook runtime opening, Agent Trace setup/doctor lifecycle, and `sce sync`. Hook writers/readers resolve the current repository storage context before using `RepositoryAgentTraceDb`. The migration-running `new_at(path)` constructor is used by setup/lifecycle; hook runtime uses the no-migration constructor and fails open with `Run 'sce setup'.` guidance when schema readiness is not met. There is no longer a checkout-scoped adapter or trace database inspection service. @@ -57,7 +55,7 @@ to `diff_traces` or the direct intersection table. ## Non-goals -- No read/query helper for loading messages with their joined parts exists in the current runtime; the typed write helpers (`insert_message`, `insert_messages`, `insert_part`, `insert_parts`, `insert_conversation_text_event`) are the only exposed message/part API surface. Message/part query helpers are deferred to a future task. +- No read/query helper for loading messages with their joined parts exists in the current runtime; the typed write helpers (`insert_messages`, `insert_parts`, `insert_conversation_text_event`) are the only exposed message/part API surface. Message/part query helpers are deferred to a future task. - No part upsert/deduplication; `parts` uses only the internal integer `id` for row identity (append-only per the `INSERT_PART_SQL` contract). ## Database path @@ -205,7 +203,7 @@ Both triggers compare `OLD.*` vs `NEW.*` for all mutable columns (excluding `upd - AgentTraceDb open failures are logged at error level through `sce.hooks.diff_trace.agent_trace_db_open_failed`; later conversion/insert failures retain `sce.hooks.diff_trace.agent_trace_db_write_failed`. Both failure classes preserve deterministic failed-persistence success text and create no artifact fallback. Open failures use the producer-native unprefixed session and do not also emit the write-failure event. - Existing artifact files are not backfilled into the database. -Post-commit intersection rows are written by the active `post-commit` hook flow through repository-scoped Agent Trace DB access, and the same flow inserts built Agent Trace payloads into `agent_traces` via `RepositoryAgentTraceDb::insert_agent_trace()` (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). The persisted `trace_json` is the schema-validated `build_agent_trace(...)` output and includes top-level `metadata.sce.version` from the compiled `sce` CLI package version, always-emitted `metadata.sce.line_changes` touched-line attribution counts, plus `content_hash` on every emitted range. Range `content_hash` values are computed from the touched-line kind/content of the post-commit hunk that produced the persisted range, not from DB IDs, paths, line positions, or runtime metadata. +Post-commit intersection rows are written by the active `post-commit` hook flow through repository-scoped Agent Trace DB access, and the same flow inserts built Agent Trace payloads into `agent_traces` via `RepositoryAgentTraceDb::insert_agent_trace()` (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). The persisted `trace_json` is the schema-validated `build_agent_trace_from_evidence(...)` output and includes top-level `metadata.sce.version` from the compiled `sce` CLI package version, always-emitted `metadata.sce.line_changes` touched-line attribution counts, plus `content_hash` on every emitted range. Range `content_hash` values are computed from the touched-line kind/content of the post-commit hunk that produced the persisted range, not from DB IDs, paths, line positions, or runtime metadata. `sce hooks conversation-trace` is the current runtime writer for `messages` and `parts`. diff --git a/context/sce/agent-trace-embedded-schema-validation.md b/context/sce/agent-trace-embedded-schema-validation.md index 81cbfa6d8..516d993e8 100644 --- a/context/sce/agent-trace-embedded-schema-validation.md +++ b/context/sce/agent-trace-embedded-schema-validation.md @@ -13,7 +13,6 @@ Current internal validation seam for Agent Trace JSON in the Rust CLI. - The canonical schema remains at `config/schema/agent-trace.schema.json`. For repository builds, `cli/build.rs` copies it to `OUT_DIR/static/schema/agent-trace.schema.json`, and the CLI embeds that staged file at compile time via `include_str!`. Validation does not read the schema from disk at runtime; channel-specific package fallback preparation remains a distribution concern. - `agent_trace_schema_validator()` compiles the embedded schema once and caches the `jsonschema::Validator` in a `OnceLock`. - `validate_agent_trace_value(&serde_json::Value)` validates already-parsed JSON values against the embedded schema. -- `validate_agent_trace_json(&str)` parses a JSON string and then validates it against the embedded schema. - Top-level `version` must match strict numeric `x.y.z` (`^[0-9]+\.[0-9]+\.[0-9]+$`); two-part values like `x.y` are rejected. - Top-level `vcs` is optional at schema level; payloads without `vcs` validate, and payloads that include `vcs` must still provide both `type` and `revision`. - Validation failures use `AgentTraceValidationError` to distinguish: @@ -24,7 +23,7 @@ Current internal validation seam for Agent Trace JSON in the Rust CLI. ## Boundaries - This seam is internal-only; no user-facing `sce` command currently exposes Agent Trace validation. -- The existing minimal `build_agent_trace(...)` output remains unchanged by this validation seam. +- The `build_agent_trace_from_evidence(...)` output remains unchanged by this validation seam. - File-path loading and validation entrypoints are not part of the current implemented slice. ## Verification diff --git a/context/sce/agent-trace-export-readers.md b/context/sce/agent-trace-export-readers.md index e3bc47264..72fb87c96 100644 --- a/context/sce/agent-trace-export-readers.md +++ b/context/sce/agent-trace-export-readers.md @@ -19,7 +19,7 @@ flowchart LR `AgentTraceExportReader::new(&db)` takes a `&RepositoryAgentTraceDb` directly. The reader does not resolve storage, open a database, or generate identity itself. The existing storage resolver composes cleanly with it: ```rust -let storage = resolve_agent_trace_storage_at_state_root(&context, &state_root).await?; +let storage = resolve_agent_trace_storage(&context).await?; let reader = AgentTraceExportReader::new(&storage.db); let rows = reader.read_messages_after(cursor, limit).await?; ``` @@ -29,8 +29,6 @@ let rows = reader.read_messages_after(cursor, limit).await?; - `repository_id` / `source_instance_id` identify the *source* (the physical database). - Each stream's `table.id` (returned as `sourceRowId`) is the per-stream *progress marker* — the caller's cursor. It has no relationship to `source_instance_id` and is never used to derive or validate it. -Test coverage: `cli/src/services/agent_trace_export/mod.rs::tests::source_instance_integration` resolves storage through `resolve_agent_trace_storage_at_state_root`, asserts both metadata fields are populated, and reads one stream through a reader built from `storage.db`. - ## Reader contract Four async methods, one per capture stream, sharing one shape: `(cursor: i64, limit: usize) -> Result>`, awaiting the DB query and row materialization, running diff --git a/context/sce/agent-trace-hook-doctor.md b/context/sce/agent-trace-hook-doctor.md index 26fa399b5..ad2be4b91 100644 --- a/context/sce/agent-trace-hook-doctor.md +++ b/context/sce/agent-trace-hook-doctor.md @@ -132,6 +132,10 @@ The broadened contract for `sce doctor` must cover the following problem invento - Agent Trace DB file exists but cannot be opened (connection failure) or has incomplete schema (missing/unapplied migrations) — reported as `AgentTraceDbConnectionFailed` / `AgentTraceDbSchemaNotReady` with manual-only remediation directing to `sce setup` - Repository-scoped DB path/health remains in the complete report and JSON; human text summarizes it under `Environment` → `Repository identity` without exposing healthy identity metadata. Repository DB records include repository ID, identity source, safe canonical identity, configured remote name, and never raw remote URLs. Checkout identity was removed by the `remove-checkout-id` plan and is no longer reported (see `context/cli/checkout-identity.md`) +### Snapshot-ref reconciliation + +- `sce doctor --fix` always runs one explicit snapshot-ref reconciliation pass and reports its true typed result; plain `sce doctor` stays read-only (no ref or maintenance-state write) and reports a recommendation only when one is due. Details in [../cli/doctor-ref-reconciliation.md](../cli/doctor-ref-reconciliation.md). Reconciliation is never automatic. + ### Repository targeting and git readiness - repo-scoped checks are required but `sce doctor` is run outside a git repository diff --git a/context/sce/agent-trace-minimal-generator.md b/context/sce/agent-trace-minimal-generator.md index 8f6631781..5c5928010 100644 --- a/context/sce/agent-trace-minimal-generator.md +++ b/context/sce/agent-trace-minimal-generator.md @@ -11,21 +11,20 @@ Given a `constructed_patch` (AI candidate) and a `post_commit_patch` (canonical - **`ai`** — every touched line in the `post_commit_patch` hunk is covered by direct evidence, mutation-derived AI evidence, or both. - **`mixed`** — a non-empty proper subset of the hunk's touched lines is covered. - **`unknown`** — no touched line in the hunk is covered. -3. With no mutation evidence this is equivalent to the direct-only slot rule, since a direct-intersection hunk always holds an ordered sub-multiset of its `post_commit_patch` hunk's touched lines: `ai` when the `intersection_patch` hunk has identical touched lines (same count, kind, `line_number`, content, order), `mixed` when that hunk exists but is a proper subset, `unknown` when no `intersection_patch` hunk shares the `old_start`. The direct-only path `build_agent_trace(...)` produces exactly this classification. +3. With no mutation evidence this is equivalent to the direct-only slot rule, since a direct-intersection hunk always holds an ordered sub-multiset of its `post_commit_patch` hunk's touched lines: `ai` when the `intersection_patch` hunk has identical touched lines (same count, kind, `line_number`, content, order), `mixed` when that hunk exists but is a proper subset, `unknown` when no `intersection_patch` hunk shares the `old_start`. `build_agent_trace_from_evidence(...)` with an empty `mutation_ai_patch` produces exactly this classification. 4. Map `Conversation.contributor.model_id` from the matched direct and mutation hunks when contributor type is `ai` or `mixed`: an absent evidence source does not count as unknown, one present model passes through, and both present models must be known and equal. Omit `model_id` when both sources are present but either model is missing or they disagree. A hunk classified `ai`/`mixed` purely through mutation-derived coverage may therefore emit the mutation model. 5. For each emitted conversation, derive optional `conversation.related` entries from non-empty `session_id` values on touched lines in both matched evidence hunks; emit related entries as `{ "type": "session", "url": "https://sce.crocoder.dev/sessions/" }`, deduplicated by session ID with deterministic ordering, and omit `related` when no included lines provide `session_id`. Structured diff-trace reconstruction supplies the persisted canonical `cc_...` row session on direct lines, while mutation reconstruction supplies scope provenance on mutation-attributed lines. 6. Emit one `Conversation` per `post_commit_patch` hunk, each carrying the trace lookup `url`, one `TraceFile` per `post_commit_patch` file, and one range per hunk with a deterministic `content_hash` computed from that hunk's touched-line kind/content. ## Separated direct/mutation evidence -`build_agent_trace_from_evidence(evidence, post_commit_patch, metadata)` is the seam that classifies from two independent AI-evidence sources: +`build_agent_trace_from_evidence(evidence, post_commit_patch, metadata)` is the only supported generator entrypoint. It handles direct-only evidence (empty `mutation_ai_patch`), mutation-only evidence, and combined evidence through one path, classifying from two independent AI-evidence sources: - `evidence.direct_patch` — the reconstructed direct patch, intersected against `post_commit_patch` internally exactly as before. It supplies direct model/session provenance and remains the sole source of the top-level `tool` object (still omitted when the direct intersection is empty). - `evidence.mutation_ai_patch` — a target-shaped set of committed touched lines with resolved scope provenance that causal mutation-lineage replay attributed to AI (an AI event's line that survived every later observed tree transition into the committed tree), produced by [../cli/mutation-trace-agent-attribution.md](../cli/mutation-trace-agent-attribution.md). It widens combined AI coverage for hunk classification and supplies mutation model/session metadata when available. Per hunk, a `post_commit_patch` touched line is covered when it pairs one-to-one on `(kind, line_number, content)` with a line in the direct intersection hunk or the mutation-AI hunk at the same `old_start`; the covered fraction drives `ai` / `mixed` / `unknown` as in the Contract above. `line_changes` buckets follow that combined classification. -`build_agent_trace(constructed_patch, post_commit_patch, metadata)` is retained unchanged and delegates to `build_agent_trace_from_evidence` with an empty `mutation_ai_patch`. Wiring the mutation-AI patch in at post-commit is owned by [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md). ## Domain types @@ -58,7 +57,7 @@ the boundary that mutation protocol attribution is not itself scope provenance. Current output includes top-level metadata fields with this contract: - `version` is fixed to `"0.1.0"` and remains the Agent Trace payload/schema version -- `id` is generated per `build_agent_trace(...)` call as a UUIDv7 string derived from the same commit-time moment used for `timestamp` +- `id` is generated per `build_agent_trace_from_evidence(...)` call as a UUIDv7 string derived from the same commit-time moment used for `timestamp` - `timestamp` is sourced from explicit commit metadata input (`AgentTraceMetadataInput.commit_timestamp`) and must be RFC 3339 - `vcs` is emitted only when explicit commit metadata input includes `AgentTraceMetadataInput.vcs_type` - when `vcs` is emitted, `vcs.type` is sourced from the schema-aligned enum (`git | jj | hg | svn`) and `vcs.revision` is sourced from `AgentTraceMetadataInput.commit_revision` @@ -109,15 +108,13 @@ Current output includes top-level metadata fields with this contract: ## Public API -- `classify_hunk(post_commit_hunk, intersection_hunks) -> HunkContributor` — the direct-only slot rule, retained as a primitive; the builder itself now classifies through the internal combined direct+mutation line-coverage rule. - `range_content_hash(hunk) -> String` — internal helper that computes the serialized range-level `murmur3:` content fingerprint from `PatchHunk.lines` using versioned, length-delimited touched-line serialization in patch order. The hash input includes touched-line kind and content, and excludes hunk positions, line numbers, file paths, trace metadata, contributor/model metadata, VCS metadata, tool metadata, and database IDs. -- `build_agent_trace(constructed_patch, post_commit_patch, metadata) -> Result` — direct-only entrypoint, retained unchanged; delegates to `build_agent_trace_from_evidence` with an empty `mutation_ai_patch`. It validates `metadata.commit_timestamp` as RFC 3339, uses it as top-level `timestamp`, derives a UUIDv7 `id` from that same commit-time moment, derives one conversation URL from that `id`, conditionally emits `vcs` only when `metadata.vcs_type` is present (mapping `vcs.type` from metadata and `vcs.revision` from `metadata.commit_revision`), carries optional tool metadata inputs (`metadata.tool_name`, `metadata.tool_version`) for top-level `tool` mapping, and always emits `metadata.sce.version` from the compiled package version. When the direct `intersection_patch.files` is empty, `tool` is always `None` regardless of metadata values. - `build_agent_trace_from_evidence(evidence: AgentTraceEvidence, post_commit_patch, metadata) -> Result` — separated-evidence entrypoint (see [Separated direct/mutation evidence](#separated-directmutation-evidence)): identical top-level metadata behavior, classifies each hunk from the union of direct and mutation-derived AI coverage, unions direct/mutation related sessions, and emits a model only when the present evidence sources agree under the combined rule; top-level `tool` remains bound to the direct intersection. ## Test fixture contract - Golden fixtures under `cli/src/services/agent_trace/fixtures/**/golden.json` pin deterministic literal values for top-level `id`, `timestamp`, optional `vcs`, `metadata.sce.version`, `metadata.sce.line_changes`, per-conversation `url`, range-level `content_hash`, and expected file/conversation shapes. -- Reconstruction fixtures pair `incremental_*.patch` inputs with a `post_commit.patch` and drive `build_agent_trace`. Evidence fixtures (`direct_only`, `exclusive_without_direct`, `direct_plus_mutation`, `partial_combined`, `newer_nonexclusive_blocks`, `mutation_only_no_provenance`) instead pair `direct.patch` + `mutation_ai.patch` + `post_commit.patch` and drive `build_agent_trace_from_evidence`, pinning mutation-only coverage with and without resolved provenance, the combined model-agreement and session-union rules, the absence of fabricated metadata when provenance is unknown, and the empty-mutation path's byte identity with `build_agent_trace`. +- Evidence fixtures (`direct_only`, `exclusive_without_direct`, `direct_plus_mutation`, `partial_combined`, `newer_nonexclusive_blocks`, `mutation_only_no_provenance`) pair `direct.patch` + `mutation_ai.patch` + `post_commit.patch` and drive `build_agent_trace_from_evidence`, pinning mutation-only coverage with and without resolved provenance, the combined model-agreement and session-union rules, the absence of fabricated metadata when provenance is unknown, and the direct-only case (empty `mutation_ai_patch`), whose classification and provenance match the direct-intersection rule. - Tests validate golden fixtures and built payloads against the embedded schema, assert core runtime metadata directly (`version`, `timestamp`, optional `vcs`, and `metadata.sce.version`), and compare `vcs`, optional `tool`, `metadata.sce.line_changes`, and normalized `files` against fixture truth. Expected fixture URLs are normalized to the runtime `AgentTrace.id` before the existing file-shape comparison because UUIDv7 generation includes non-deterministic bits. ## Relationship to existing patch service diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index ddbcbbc82..8bc73b87f 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -33,8 +33,9 @@ Runtime observability consumes the shared resolved observability config from `cl - Sessionless file logs route to `/sce-.log`; session-aware file logs route to `/sce--.log`. - Session filename sanitization preserves ASCII letters, digits, `-`, and `_`; percent-encodes every other UTF-8 byte as uppercase `%HH`; and represents an explicitly empty `Some("")` session ID with the reserved `%EMPTY` token. - `sce hooks diff-trace` and `conversation-trace` pass producer-native session context into this existing routing argument when available. Diff-trace logging never uses the AgentTraceDb-only `oc_`/`cc_`/`pi_` prefix; skipped conversation items use their own session; batch-wide conversation insert failures use the first valid insert's session. Agent Trace DB open failures use hook-specific error events (`sce.hooks.diff_trace.agent_trace_db_open_failed` and `sce.hooks.conversation_trace.agent_trace_db_open_failed`) and do not also emit their broader write/intake events for the same failure. Session IDs remain absent from rendered record fields unless separately supplied as fields. -- File routing creates the configured directory when needed, uses owner-only create permissions on Unix, and serializes writes independently per path. -- If file logging is enabled and the selected primary file cannot be opened, appended to, or flushed, the logger retries the complete rendered record exactly once at a sibling path with `-v2` inserted before `.log`: `sce--v2.log` or `sce---v2.log`. Existing v2 files use the same create-or-append and per-path serialization behavior. +- File routing creates the configured directory when needed, uses owner-only create permissions on Unix, and serializes writes through a fixed set of in-process lock stripes (see the next bullet). +- Log-file write synchronization is a static array of 64 `std::sync::Mutex<()>` stripes in `cli/src/services/observability.rs`; a path selects its stripe via a deterministic `DefaultHasher::new()` hash modulo the stripe count. Synchronization memory is constant and independent of the number of distinct log paths (no per-path allocation or eviction). Writes to the same path are serialized within one process across threads and independent `Logger` instances; distinct paths that hash to the same stripe only contend, they never fail. No cross-process coordination exists or is guaranteed. A call holds at most one stripe at a time: the primary path's guard covers persistence and retention cleanup and is released on success and failure before the v2 fallback acquires its own stripe, so primary and v2 paths sharing a stripe cannot deadlock. A poisoned stripe (a panic under the guard, realistically only inside the retention cleanup callback) fails every path on that stripe with the unchanged `failed to lock log file '': ...` error, does not trigger the fallback, and is never recovered. +- If file logging is enabled and the selected primary file cannot be opened, appended to, or flushed, the logger retries the complete rendered record exactly once at a sibling path with `-v2` inserted before `.log`: `sce--v2.log` or `sce---v2.log`. Existing v2 files use the same create-or-append and striped serialization behavior. - Successful v2 persistence suppresses the terminal `Failed to write SCE log file` diagnostic and leaves the CLI command result unchanged. If both persistence attempts fail, the logger emits one redacted terminal file-write diagnostic to stderr and continues fail-open; a partial primary append may therefore coexist with the complete fallback record. - Directory creation, primary lock acquisition, and retention cleanup failures do not trigger alternate-name generation. The fallback attempt is non-recursive: no v3, timestamped, random, or unbounded variants are tried. - After a successful write to a newly created primary or v2 SCE log file, the logger runs one best-effort retention pass over direct regular `*.log` children of `log_dir`. Existing-file appends do not scan or delete files. Cleanup keeps the resolved `log_file_retention_limit` newest files (default `10`). Files are ordered newest-first by filesystem modification time with path/name ordering as the deterministic tie-break, and older `.log` files are removed regardless of whether their names are SCE-owned. @@ -49,12 +50,50 @@ Runtime observability consumes the shared resolved observability config from `cl - `sce.command.dispatch_start` (debug level - logged before dispatch) - `sce.command.dispatch_end` (debug level - logged after successful dispatch) - `sce.command.completed` -- Error logging uses the pattern `sce.error.{code}` where `{code}` is the classified error code (e.g., `sce.error.SCE-ERR-RUNTIME`). +- Error logging uses the pattern `sce.error.{code}` where `{code}` is the classified error code (e.g., `sce.error.SCE-ERR-RUNTIME`) for the local file/stderr record only; the tracing event for the same record carries the typed `sce.error.{parse,validation,runtime,dependency}` identifier instead (see Tracing boundaries). - All `CliError` instances are logged via `Logger::log_cli_error()` before user-facing stderr diagnostics are written; observability retains full technical detail independently of what is rendered to the terminal, and never writes a second competing stderr diagnostic for the same error. - Event records include deterministic metadata keys used by automation (`command`, `failure_class`, `component` when applicable). - Error log records include `error_code` and `error_class` fields for structured observability, plus `error_surface` (`user` for `CliError::User`, `internal` for `CliError::Internal`), `user_error` (the catalog `UserError::key()`, present only for `CliError::User`), and `error_source` (the full technical source chain, present whenever one was preserved — always for `CliError::Internal`, and for `CliError::User` only when constructed with `CliError::user_with_source`). -- App runtime wraps parse/dispatch in `Telemetry::with_default_subscriber`. Production uses `NoopTelemetry`, which installs no tracing subscriber and no tracer provider. Raw `tracing` events emitted outside `Logger`, such as `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted`, are instrumentation points with no production sink. `Logger` records still follow the file/stderr routing below. -- Tracing event emission checks the `sce` target and requested tracing level before constructing serialized `fields` payloads; disabled or filtered tracing events return without building field JSON while enabled events preserve the same `event_id`, `event_message`, and `fields` payload shape. +- App runtime wraps parse/dispatch in `Telemetry::with_default_subscriber`. Production uses `RuntimeTelemetry`: `NoopTelemetry` (no tracing subscriber, no tracer provider) unless the opt-in standalone OTel runtime is selected (see Standalone OTel runtime). Raw `tracing` events emitted outside `Logger`, such as `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted`, have no production sink unless a subscriber is attached that way. `Logger` records still follow the file/stderr routing below. + +## Tracing boundaries + +Any `tracing` subscriber attached to the process can read what `sce` emits, so tracing content is allowlisted by construction rather than redacted. `cli/src/services/observability/tracing_boundary.rs` owns Boundary A and `cli/src/services/observability/otel_policy.rs` owns Boundary B. + +- Boundary A (existing `sce`-target events): all `sce`-target event emission lives in three typed functions in `tracing_boundary.rs`: `emit_logger_event(LogLevel, EventId)`, `emit_retry_event(OperationClass, u32, u32, u64, u64)`, and `emit_contention_event(DbName, OperationClass, u32, u64, u64, u64)`. They accept only closed enums, the registry-backed `EventId` newtype (private field, built only by `EventId::classify`, `EventId::from_cli_error`, or its associated constants), and numerics, and use static messages. `Logger`, `resilience.rs`, and `db/mod.rs` call these and contain no `tracing` macro. The retry event now carries target `sce` (previously the incidental module path). Each enabled logger record emits one event with only `event_id` and `log_level`. `event_message`, serialized `fields`, `error_source`, `error`, `last_error`, paths, and hook payloads never reach `tracing`; local file/stderr rendering and its raw event identifiers are unchanged. +- `event_id` reaches tracing only if it is a member of the closed, sorted `TRACING_EVENT_IDS` registry (shape `[a-z0-9_.]{1,64}`, uniqueness and ordering unit-tested); any unknown, dynamically built, or test-only identifier becomes `sce.unclassified`. `log_cli_error` derives `sce.error.{parse,validation,runtime,dependency}` from the typed `FailureClass`, never from the formatted string. New static identifiers must be added to the registry. +- `sce.resilience.retry` and `sce.agent_trace_db.contention_exhausted` record only `event_id`, enumerated `operation` (`OperationClass`) and `db_name` (`DbName`) with an `unclassified` fallback, numeric counters/durations, and the constant `cause`. +- A syn-based test-only audit (`tracing_boundary/audit.rs`, `syn` and `proc-macro2` dev-dependencies) parses every `cli/src` file as Rust, so comments and string literals are never mistaken for code. Outside `tracing_boundary.rs` it rejects any `tracing`-rooted path, import (including aliases, globs, and `extern crate`), `#[tracing::...]` attribute, macro named `error`/`warn`/`info`/`debug`/`trace`/`event`, and macro token stream mentioning `tracing::`. Inside the boundary (non-test code) it requires `tracing::`-qualified `error`/`warn`/`info`/`debug`/`enabled` only, `target: SCE_TRACING_TARGET`, a single brace-free string-literal message, allowlisted field names, and values that are same-named bindings of numeric parameters or `as_str()` results of closed-type parameters (or approved constants). No per-file counts are maintained. Span macros and `#[tracing::instrument]` are also rejected outside the boundary, so T03 span creation must go through a typed boundary function. +- Boundary B (OTel export surface, enforced by the standalone runtime's exporter): the `sce::otel` target, enumerated span names (`OtelName`), and the closed `OtelAttribute` enum (`CommandName`, `Outcome`, `ErrorCategory`, `OperationType` = `OperationClass`, plus numeric `DurationMs`/`Count`/`Attempts`). Every string value is a static `as_str()` of a closed enum, so there is no string-carrying attribute type. `OtelAttribute::admit` and `admit_otel_record` map raw `(key, value)` input back only by exact lookup against those enums and drop other targets, unknown names/keys, unknown or mismatched values, and anything resembling exception messages or stack traces. Character-class validation is no longer used. +- Capture tests (`tracing_boundary_a*`) use a dependency-free capturing `tracing::Subscriber` rather than the OTel exporter; `tracing_boundary_b*` additionally drive the real exporter layer with an in-memory exporter. + +## Standalone OTel runtime + +`cli/src/services/observability/otel_runtime.rs` is the opt-in, statically dispatched OTel runtime. It is a private subscriber, never a global one, and exports only the Boundary B surface. + +- Enablement: only the environment switch `SCE_TELEMETRY=standalone` (exact value). Resolution (`resolve_telemetry_mode`) reads that single key from an injected environment source; repository `.sce/config.json` cannot enable it and no `otel` config key exists. No managed token is ever attached. `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (used as-is) or `OTEL_EXPORTER_OTLP_ENDPOINT` (`/v1/traces` appended) selects the receiver; otherwise the SDK default applies. +- Hook-export gate: when standalone is requested, the parsed command is classified through the typed `RuntimeCommand::is_hook_invocation` (exhaustive match; `hooks` and `policy` variants are hook invocations). Hook invocations, and arguments that fail to parse, resolve to `NoopTelemetry` before any exporter is built, regardless of inherited `SCE_TELEMETRY`/`OTEL_*`. Initialization failure also falls back to `NoopTelemetry`; telemetry failure never changes stdout, exit codes, or hook decisions. The gate stays in force after the T05 benchmark; its candidate 100 ms hook flush budget and the conditions for any future user-owned enablement are recorded in [hook telemetry benchmark](hook-telemetry-benchmark.md). +- Scoping: `OtelTelemetry` builds a `Registry` plus `tracing-opentelemetry` layer filtered to spans on target `sce::otel` with enumerated `OtelName`s, and `ScopedDispatch::scope` (in `tracing_boundary.rs`) attaches it to the command future with `WithSubscriber::with_subscriber`, so it is active on every poll across Tokio worker migration. Spawned tasks do not inherit it: production `tokio::spawn` that must be traced goes through `spawn_in_current_scope` (`.with_current_subscriber()` + `.in_current_span()`); today only the credential-refresh spawn in `agent_trace_sync/control_plane.rs` does. `spawn_blocking`/`std::thread` sites are untraced until the operation-span tasks wrap them. +- Export: OTLP/HTTP protobuf through the SDK `BatchSpanProcessor` (its own `OpenTelemetry.Traces.BatchProcessor` thread, bounded drop-on-full queue). The HTTP client has 750 ms connect/total timeouts and redirects disabled, and is built on a separate init thread (the blocking `reqwest` client must not be built or dropped on a Tokio runtime); if the platform certificate store cannot be loaded it falls back to an empty-root client so plaintext local receivers still work. +- Boundary B enforcement is split by what each layer can actually observe: + - Subscriber layer (`scoped_dispatch` `filter_fn`, the only place the originating `tracing` target is visible): accepts spans only when the target is exactly `sce::otel` and the name is an enumerated `OtelName`; ordinary `sce` logger events are never spans and are rejected, and a span named like an allowed `OtelName` is still rejected when emitted from target `sce` or any other module target. + - Exporter wrapper (`BoundaryBExporter`, wraps the OTLP exporter): `SpanData` carries no originating `tracing` target, so the wrapper never claims target filtering and no target attribute is fabricated. It independently drops spans whose name is not an `OtelName`, re-admits every attribute through `OtelAttribute::admit` (a canonical decimal string is accepted only for numeric keys, because `tracing-opentelemetry` records `u64` fields as strings), drops unclassified attributes, clears events and links, and blanks error status descriptions. Tests: `tracing_boundary_b_production_subscriber_rejects_wrong_target` (subscriber) and `tracing_boundary_b_exporter_wrapper_enforces_policy_without_layer_filter` / `tracing_boundary_b_exporter_wrapper_cannot_observe_the_tracing_target` (wrapper, explicitly limited to name, attribute, event, link, and status guarantees). + +## `sce.command` span and bounded shutdown + +- `CommandSpan` (in `tracing_boundary.rs`, the only production `tracing::span!` site, accepted by the audit only with `target: OTEL_TARGET`, `Level::INFO`, an enumerated name, and allowlisted fields recorded as `Empty`) is created in `run_command_lifecycle_with_context` after the arguments parse successfully, wraps `execute_command_phase`, and records only `sce.command.name` (`CommandName`), `sce.outcome` (`success`/`failure`) and, on failure, `sce.error.category` (from `FailureClass`) plus an error status with an empty description. Parse failures and hook invocations in ordinary standalone mode produce no span. No message, argument, path, or stack trace is recorded. The span closes when the command future completes, before any flush. +- Trace-context propagation: `trace_context::extract_remote_parent` (pure over an injected environment source; `app.rs` passes the process environment) feeds `TRACEPARENT`/`TRACESTATE` to the SDK `TraceContextPropagator` and yields a context only when the resulting span context is valid and remote. `CommandSpan::start(command, remote_parent)` applies it with `OpenTelemetrySpanExt::set_parent`, so a valid header gives a child with the exact trace ID, parent span ID, sampled flag, and trace state. Malformed or absent headers give an independent root and no parent is invented. An unsampled parent is honored by the provider's parent-based sampler, so nothing is exported. `tracestate` rides on the span context only and is never recorded as an attribute. Extraction runs only inside the OTel dispatch, so disabled and hook-gated invocations ignore both variables. Which harness subprocesses actually receive `TRACEPARENT` is not claimed here; it is measured separately. +- `OtelTelemetry` holds its tracer provider and scoped dispatch in a single `Teardown`. `run_with_dependency_check_and_streams` renders stdout/stderr first, flushes them, then calls `RuntimeTelemetry::shutdown(self)`, which consumes the telemetry: the `Teardown` moves to a dedicated `sce-otel-shutdown` OS thread that calls `SdkTracerProvider::shutdown_with_timeout(budget)` exactly once (the SDK batch worker exports the remaining queue, then shuts the exporter down) and drops the provider and dispatch there; the caller awaits a oneshot with `tokio::time::timeout(budget)` and never joins. No lock is held at that point and no Tokio worker blocks. After an explicit shutdown the SDK `is_shutdown` flag makes any later final drop of a retained provider/dispatch clone non-blocking. If an `OtelTelemetry` is dropped without `shutdown`, `Drop` hands the same teardown to a detached thread instead of blocking the caller. `ShutdownOutcome` is `Completed`, `Incomplete` (deadline or SDK error), or `Unavailable` (shutdown thread creation or handoff failure). On that failure path the `Teardown` is leaked with `std::mem::forget` so neither `shutdown_with_timeout` nor the final provider drop (which runs SDK shutdown) executes on the caller; memory is not reclaimed before process exit, which is accepted for this short-lived CLI. Worker creation is injected through a generic closure for tests (`shutdown_with`, `release_with`). +- Bounded-shutdown subprocess tests treat any hard timeout as a failure and enforce bounds against the maximum sample (no retry, no outlier allowance). +- Budget: `SCE_TELEMETRY_FLUSH_TIMEOUT_MS`, default 1000 ms, accepted range 1 to 5000 ms (anything else uses the default). The OTLP client uses one attempt (`RetryPolicy::disabled()`; the SDK default of 3 retries added about 450 ms when the connection was refused) with 750 ms connect/total timeouts. `BatchSettings::PRODUCTION` fixes the batch processor explicitly (queue 256, batch 64, delay 2 s, drop on full) and overrides every `OTEL_BSP_*` variable the SDK would otherwise read. +- Telemetry failure never changes stdout, stderr, exit code, hook decisions, or durable state (`telemetry_offline_parity_*`, real `sce` subprocesses against reachable, refused, accept-but-never-respond, and non-routable receivers). + +## Controlled test receiver (`telemetry-test-receiver`) + +- Cargo feature `telemetry-test-receiver` (off by default; not used by `.#default`, release, crates.io, or Flatpak). Without it the code, the environment variable names, and the lifecycle markers do not exist (`telemetry_test_receiver_inert_without_the_feature` plus the `packaged-binary-excludes-test-receiver` flake check, which greps the `.#default` binary for the variable names and proves a loopback listener receives nothing). +- With the feature, `SCE_TELEMETRY=test-receiver` plus `SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT=http://:` exports to that endpoint only. Resolution (`test_receiver::resolve`) is pure over an injected environment, has no credential field, never touches the auth DB, ignores `OTEL_*` endpoint/timeout/protocol, and wins over standalone. The SDK exporter merges `OTEL_EXPORTER_OTLP_HEADERS` and `OTEL_EXPORTER_OTLP_TRACES_HEADERS` unconditionally, and that cannot be disabled without a new dependency, so resolution fails closed (telemetry stays disabled) when either header variable is non-empty. Hostnames, HTTPS, non-loopback, paths, userinfo, and port 0 are rejected. Hook-style invocations export only to this receiver (the single exception to the hook-export gate). +- `SCE_TELEMETRY_TEST_LIFECYCLE_FILE` makes the process append `command_complete`, `shutdown_begin`, `shutdown_end`, and `process_exit_requested` lines (monotonic nanoseconds since the first marker) to that file only, in any telemetry mode, for shutdown measurement. +- `scripts/check-cargo-test-count.sh ` fails when the command fails or the summed `N passed` is below the minimum, so a filter matching zero tests is never a pass; `scripts/test-check-cargo-test-count.sh` tests it. Flake checks `cli-telemetry-default-features` and `cli-telemetry-test-receiver` run the default-feature and feature-enabled passes separately through it with recorded minimums; `cli-clippy` already uses `--all-features`. ## Format contract @@ -71,8 +110,8 @@ Runtime observability consumes the shared resolved observability config from `cl - The concrete `services::observability::Logger` implements the trait while retaining the existing inherent methods and behavior. - `NoopLogger` is available from the same traits module for tests and future dependency-injected services that need a logger without side effects. - The same traits module exposes deliberately non-object-safe `services::observability::traits::Telemetry`: async `with_default_subscriber` accepts generic `FnMut() -> Fut` actions and awaits command-lifecycle execution through static associated capability dispatch. -- Production currently uses `NoopTelemetry` from the same traits module. Its `with_default_subscriber` implementation awaits the action once and installs no tracing subscriber or tracer provider. Future statically dispatched telemetry/OTEL must attach the subscriber to the command future (for example `WithSubscriber::with_subscriber`) or deliberately install it globally; holding a thread-local `with_default`/`set_default` guard across awaits does not scope every poll on multi-thread Tokio. -- `cli/src/app.rs` stores the concrete production `Logger` and `NoopTelemetry` as `AppRuntime` fields, creates borrowed `AppContext` views for command execution, and exposes logger/telemetry access through associated-type context accessors instead of owned `Arc` fields or object-erased accessor return values. +- `NoopTelemetry` (same traits module) awaits the action once and installs no tracing subscriber or tracer provider; production selects it unless the standalone runtime is enabled. Statically dispatched telemetry attaches its subscriber to the command future with `WithSubscriber::with_subscriber`; holding a thread-local `with_default`/`set_default` guard across awaits does not scope every poll on multi-thread Tokio. +- `cli/src/app.rs` stores the concrete production `Logger` and the two-variant `RuntimeTelemetry` enum (`Noop` or `Otel`, chosen after runtime init by `select_runtime_telemetry`) as `AppRuntime` fields, creates borrowed `AppContext` views for command execution, and exposes logger/telemetry access through associated-type context accessors instead of owned `Arc` fields or object-erased accessor return values. - Final stream rendering uses `RunOutcome` in `cli/src/services/app_support.rs`, so classified-error and stdout-write-failure logging depends on the logger trait boundary rather than the concrete production logger type. - `run_command_lifecycle` expects the telemetry subscriber action to execute command dispatch at most once; arguments and the mutable stderr reference are taken from single-use options before constructing the action future; if a telemetry implementation invokes the action again, the app returns a `SCE-ERR-RUNTIME` classified error rather than panicking or reparsing consumed arguments. @@ -87,6 +126,6 @@ Runtime observability consumes the shared resolved observability config from `cl ## Ownership and verification - `cli/src/services/config/resolver.rs` owns shared observability value resolution, including independent config-file/default `log_to_file` and `log_dir` resolution, config-file discovery/merge, env-over-config/default precedence for supported runtime inputs, default `log_dir` resolution through `default_paths::observability_log_dir()`, and config-file/default-only `log_file_retention_limit` resolution. -- `cli/src/services/observability.rs` owns runtime logger construction from resolved values, storage and application of `log_file_retention_limit`, `log_dir` non-empty validation, level filtering, tracing-event enablement checks, record rendering, local-date/session file-name selection, session filename sanitization, primary append plus one-time v2 fallback persistence, and best-effort `.log` retention; `cli/src/services/observability/traits.rs` owns the logger and telemetry trait boundaries plus the no-op logger implementation. +- `cli/src/services/observability.rs` owns runtime logger construction from resolved values, storage and application of `log_file_retention_limit`, `log_dir` non-empty validation, level filtering, tracing-event enablement checks, record rendering, local-date/session file-name selection, session filename sanitization, primary append plus one-time v2 fallback persistence, and best-effort `.log` retention; `cli/src/services/observability/traits.rs` owns the logger and telemetry trait boundaries plus the no-op logger implementation; `tracing_boundary.rs` and `otel_policy.rs` own the tracing allowlists described above. - `cli/src/app.rs` owns lifecycle event emission around parse/dispatch success and failure paths, resolves observability config before command dispatch, emits startup invalid-config warning events for skipped discovered config files, wraps dispatch inside the observability subscriber context, and guards the single-use command-dispatch action against repeated telemetry invocation with a runtime-classified error. `cli/src/services/app_support.rs` owns final stdout/stderr rendering and generic logger-backed classified-error logging. - Retention-specific validation uses packaged CLI smoke checks for config/schema behavior and direct review of the primary/v2 logger cleanup plumbing. The root flake check suite validates the build, lint, formatting, generated parity, and remaining repository tests; no retention-specific Rust test module is currently kept in `observability.rs`, `config/resolver.rs`, or `config/schema.rs`. diff --git a/context/sce/cli-shared-output-format-contract.md b/context/sce/cli-shared-output-format-contract.md index aa7b5ec3a..a6d13dccb 100644 --- a/context/sce/cli-shared-output-format-contract.md +++ b/context/sce/cli-shared-output-format-contract.md @@ -6,7 +6,7 @@ - Canonical type lives at `cli/src/services/output_format.rs` as `OutputFormat`. - Allowed values are `text` and `json`. -- Parsing is command-context aware via `OutputFormat::parse(raw, help_command)` so invalid values include command-specific help guidance. +- `OutputFormat` derives clap `ValueEnum` for parsing; the former `OutputFormat::parse(raw, help_command)` helper was removed as unused. ## Current command integration diff --git a/context/sce/doctor-human-text-contract.md b/context/sce/doctor-human-text-contract.md index bf00c59d4..e4116634e 100644 --- a/context/sce/doctor-human-text-contract.md +++ b/context/sce/doctor-human-text-contract.md @@ -16,6 +16,12 @@ The `Environment` domain contains `State`, `Configuration`, and `Repository identity`. The `Repository` domain contains `Git repository`, `Post-commit Agent Trace auto-sync`, and `Git hooks`. +When snapshot-ref reconciliation is recommended (or its maintenance state is +unusable), plain `sce doctor` additionally renders an additive `Snapshot ref +reconciliation` section; it is omitted otherwise. `sce doctor --fix` appends one +`mutation_scope_health` fix-result row for its reconciliation pass. Details in +[../cli/doctor-ref-reconciliation.md](../cli/doctor-ref-reconciliation.md). + ## Header and status vocabulary - Diagnose mode renders `SCE doctor`. diff --git a/context/sce/hook-telemetry-benchmark.md b/context/sce/hook-telemetry-benchmark.md new file mode 100644 index 000000000..0f23a5cc2 --- /dev/null +++ b/context/sce/hook-telemetry-benchmark.md @@ -0,0 +1,56 @@ +# Hook telemetry benchmark and export decision gate (T05) + +Evidence for the `sce-opentelemetry-tracing` plan, task T05. It benchmarks a hook-style `sce` subprocess exporting through the D6 `telemetry-test-receiver` loopback mechanism and records the hook flush-budget decision. + +## Decision + +- Production hook export stays disabled in standalone and managed modes. The D3 hook-export gate (`RuntimeCommand::is_hook_invocation`, `telemetry_hook_export_gated`) remains in force; this note does not enable anything. +- Candidate hook flush budget: **100 ms**. Rationale: a healthy or refused receiver adds about 4-5 ms end to end and under 1 ms after the command completes, while a blackhole receiver costs the full budget (about 100 ms at a 100 ms budget, about 755 ms at the 1000 ms foreground default, where the 750 ms client timeout bounds it). A 100 ms ceiling keeps the worst case a small fraction of typical hook latency budgets without cutting off the healthy case. +- The budget is not yet a gate pass. Enabling production hook export additionally needs `telemetry_shutdown_subprocess` configured with this budget (hook workload), `telemetry_hook_export_gated` still passing, and a separate change adding an explicit, documented, user-owned enablement policy (never repository-local, never environment-inherited). +- A remote collector is not measured here (loopback only). A local forwarder, which would keep hook export off the network path entirely, is recorded as a separate follow-up and is not implemented by this plan. + +## Build command + +``` +nix develop -c ./scripts/run-cli-cargo.sh build --release --features telemetry-test-receiver --manifest-path cli/Cargo.toml +``` + +Binary: `cli/target/release/sce` (release profile, 52,154,152 bytes). The feature adds only the D6 resolution branch and lifecycle markers. The packaged `.#default` build excludes it, verified by the flake check `packaged-binary-excludes-test-receiver`. T07 reuses this same feature-enabled binary. + +## Methodology + +- Script: `scripts/bench-hook-telemetry.sh `, run as `nix shell nixpkgs#hyperfine nixpkgs#python3 -c scripts/bench-hook-telemetry.sh cli/target/release/sce `; `SCE_TELEMETRY_FLUSH_TIMEOUT_MS` selects the budget. +- Workload: `sce hooks pre-commit` (typed hook command, exits 0 outside a repository), `env -i` with a throwaway `HOME`/`XDG_*`, stdin null. +- Modes: `disabled`; `reachable` (loopback listener answering 200); `blackhole` (loopback listener that accepts and never responds); `refused` (loopback port with no listener). Enabled modes set `SCE_TELEMETRY=test-receiver` and `SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT=http://127.0.0.1:`. +- hyperfine: 5 warmup and 40 measured runs per mode, one mode at a time. Overhead is `enabled - disabled p50` per sample. Hyperfine warns the disabled command is under 5 ms, so its shell-start calibration is imprecise; the same shell wrapper is applied to every mode and the figures are relative. +- Post-command duration: 40 additional runs per enabled mode with `SCE_TELEMETRY_TEST_LIFECYCLE_FILE`; post-command is `process_exit_requested - command_complete`, shutdown is `shutdown_end - shutdown_begin` (process-relative monotonic nanoseconds). +- Host: AMD Ryzen 9 5900X (24 threads), Linux 6.18.37 NixOS, rustc 1.95.0, single host, repository HEAD 77462524 plus the uncommitted T05 working tree. + +## Results (milliseconds) + +Flush budget 1000 ms (run 1): + +| Mode | e2e p50 | e2e p95 | e2e max | overhead p50 | overhead p95 | overhead max | post-command p50 / p95 / max | +| --- | --- | --- | --- | --- | --- | --- | --- | +| disabled | 4.56 | 5.05 | 5.26 | - | - | - | - | +| reachable | 8.57 | 9.23 | 9.44 | 4.01 | 4.67 | 4.88 | 0.61 / 0.72 / 0.79 | +| blackhole | 759.30 | 759.76 | 759.82 | 754.74 | 755.20 | 755.26 | 750.60 / 750.89 / 751.50 | +| refused | 8.61 | 9.22 | 9.24 | 4.05 | 4.66 | 4.67 | 0.39 / 0.46 / 0.53 | + +Flush budget 100 ms (run 2, candidate hook budget; its own disabled baseline was re-measured): + +| Mode | e2e p50 | e2e p95 | e2e max | overhead p50 | overhead p95 | overhead max | post-command p50 / p95 / max | +| --- | --- | --- | --- | --- | --- | --- | --- | +| reachable | 8.76 | 9.43 | 9.51 | 4.32 | 4.99 | 5.07 | 0.63 / 0.74 / 0.93 | +| blackhole | 109.19 | 109.49 | 109.65 | 104.75 | 105.05 | 105.21 | 100.37 / 100.64 / 101.48 | +| refused | 8.49 | 9.15 | 9.25 | 4.05 | 4.71 | 4.81 | 0.39 / 0.49 / 0.51 | + +Overhead is measured against the disabled p50 of the same run (run 1 disabled p50 4.56 ms; run 2 about 4.44 ms), so overhead p95 and max include the enabled mode's own spread. + +Reading: the reachable and refused overhead of about 4-5 ms is dominated by exporter and runtime construction, not by export waiting (post-command under 1 ms). The blackhole post-command wait equals the budget plus about 1 ms, so the flush budget, not the exporter, bounds process lifetime at both budgets. + +## Verification record + +- Build command includes `--features telemetry-test-receiver`; `.#default` check `packaged-binary-excludes-test-receiver` passed. +- `... --features telemetry-test-receiver telemetry_hook_export_gated` through `scripts/check-cargo-test-count.sh`: 6 executed (minimum 6). +- The benchmark script was run twice end to end (budgets 1000 ms and 100 ms) with consistent reachable/refused figures (8.57 vs 8.76 ms and 8.61 vs 8.49 ms p50). diff --git a/context/sce/mutation-scope-health-status.md b/context/sce/mutation-scope-health-status.md index 12e4021e6..6b657ee25 100644 --- a/context/sce/mutation-scope-health-status.md +++ b/context/sce/mutation-scope-health-status.md @@ -62,7 +62,7 @@ four-way status each adapter module already computed. Doctor is a read-only snapshot: a currently executing hook process can, in a narrow window, hold a state shape mid-transition (for example Claude's `recovery_pending = true` with non-empty `attempts` between -`mark_recovery_pending()` and `remove_attempt()` inside one still-running +`mark_recovery_pending_and_pending_abandon()` and `remove_attempt()` inside one still-running hook call). Health classifies the durable state observed at inspection time against whether normal future adapter lifecycle events can recover it, assuming the operation that produced it has stopped progressing; it is not a diff --git a/context/sce/setup-githooks-hook-asset-packaging.md b/context/sce/setup-githooks-hook-asset-packaging.md index 8a5a8e3e5..a7d12ea67 100644 --- a/context/sce/setup-githooks-hook-asset-packaging.md +++ b/context/sce/setup-githooks-hook-asset-packaging.md @@ -24,16 +24,7 @@ Available-CLI behavior remains hook-specific: ## Setup-service accessor surface -`cli/src/services/setup/mod.rs` exposes hook-template access through: - -- `iter_required_hook_assets()` for deterministic full-set iteration -- `get_required_hook_asset(RequiredHookAsset)` for stable per-hook lookup - -`RequiredHookAsset` is the canonical hook mapping enum for this packaging layer: - -- `PreCommit` -- `CommitMsg` -- `PostCommit` +`cli/src/services/setup/mod.rs` exposes hook-template access through `iter_required_hook_assets()` for deterministic full-set iteration; the per-hook lookup helper and `RequiredHookAsset` enum were removed as unused. ## Determinism and validation diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 99ea18372..8b5d07143 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -16,21 +16,21 @@ - Agent Trace write-contention deadline, resolved by `resolve_contention_deadline::()` from `policies.database_retry.agent_trace_db.contention_deadline_ms` with fallback to `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS = 2_250` (zero for every other database). It is the cutoff for starting another outer write-contention retry, not a hard operation timeout - Agent Trace write-contention retry (`write_contention_policy::()`, `run_with_write_contention_retry`), returned only for `agent_trace_db` and applied only to write units whose whole replay is safe: the two transactional primitives below (each attempt is a complete `BEGIN IMMEDIATE` → `COMMIT` unit) and `execute_idempotent_write(sql, params)`, the opt-in single-statement entrypoint used only by the repository-metadata `INSERT … ON CONFLICT DO NOTHING`, the guarded source-instance claim `UPDATE … WHERE source_instance_id = ''`, and the mutation-trace worktree/scope/scope-provenance `…_IF_ABSENT` inserts. It retries only typed `Busy`/`BusySnapshot` (classified before conversion to `anyhow`), allows at most `2` attempts, sleeps a full-jitter backoff drawn from `0..=100ms`, and admits the second attempt at two points: before the backoff sleep it requires `remaining contention deadline >= backoff + busy_timeout`, and after the sleep actually returns it re-checks `remaining >= busy_timeout`, so scheduler oversleep can never start an attempt outside the admission contract (time is measured on a monotonic clock from the operation start, so Turso's busy wait counts; nothing starts after the deadline expires). Deterministic failures return after one attempt without sleeping. On every other database these entrypoints keep the generic query retry, and `execute_idempotent_write` is plain `execute()` - config-driven connection-open retry around only the `build().await.connect()` block using async `run_with_retry` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) - - config-driven operation retry for `execute()`, `query()`, and `query_map()` using async `run_with_retry` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). This is the outer retry for every operation outside the Agent Trace write-contention set, including Agent Trace reads (`query`/`query_values`/`query_map`), `passive_checkpoint`, and generic `execute()`; migration execution is deliberately outside every retry. Its retry-policy budget is not a wall-clock bound; see the retry-budget note below + - config-driven operation retry for `execute()` and `query_map()` using async `run_with_retry` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). This is the outer retry for every operation outside the Agent Trace write-contention set, including Agent Trace reads (`query_map`), `passive_checkpoint`, and generic `execute()`; migration execution is deliberately outside every retry. Its retry-policy budget is not a wall-clock bound; see the retry-budget note below - parent-directory creation - - retry-backed async `execute()`, `query()`, raw-value `query_values()`, and row-mapping `query_map()` operations via the public adapter methods, with config-driven query retry resolved from `policies.database_retry..query` + - retry-backed async `execute()` and row-mapping `query_map()` operations via the public adapter methods, with config-driven query retry resolved from `policies.database_retry..query` - migration-running initialization through `new()` and generic embedded migration execution through `run_migrations()` using shared migration helpers with per-database `__sce_migrations` metadata - explicit-path migration-running initialization through `new_at(path)`, preserving the same service-specific retry/migration behavior while letting callers supply a database path outside `DbSpec::db_path()` - no-migration opening through `open_without_migrations()`, which preserves parent-directory creation and connection-open retry but does not create `__sce_migrations` or apply embedded migrations - explicit-path no-migration opening through `open_without_migrations_at(path)` for path-resolved hot runtime callers - `migration_metadata_problems(&self) -> Result>`: non-mutating readiness check that queries `__sce_migrations` metadata and compares applied migration IDs against `M::migrations()`; returns a list of problems (missing metadata table, incomplete applied migrations, unexpected extra migrations) or an empty list when the schema is ready - `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>`: non-mutating hook-readiness gate that calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found; returns `Ok(())` when the schema is ready - - `passive_checkpoint(&self) -> Result<()>`: runs `PRAGMA wal_checkpoint(PASSIVE)` through the same async query/retry path as `execute()`/`query()` (config-driven query retry), draining the checkpoint result row without exposing its busy/log/checkpointed statistics. PASSIVE checkpoints only what is currently safe to move from the WAL into the main database file and never blocks on active readers or writers, so it does not guarantee WAL truncation; safe to call repeatedly; not a durability boundary on its own. Routine maintenance only, not exposed on `EncryptedTursoDb`. `sce hooks post-commit` is the sole current caller: it runs `RepositoryAgentTraceDb::passive_checkpoint()` exactly once after post-commit Agent Trace persistence succeeds; a failing checkpoint is logged as a warning and never fails the hook or affects already-persisted data (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). + - `passive_checkpoint(&self) -> Result<()>`: runs `PRAGMA wal_checkpoint(PASSIVE)` through the same async query/retry path as `execute()` (config-driven query retry), draining the checkpoint result row without exposing its busy/log/checkpointed statistics. PASSIVE checkpoints only what is currently safe to move from the WAL into the main database file and never blocks on active readers or writers, so it does not guarantee WAL truncation; safe to call repeatedly; not a durability boundary on its own. Routine maintenance only, not exposed on `EncryptedTursoDb`. `sce hooks post-commit` is the sole current caller: it runs `RepositoryAgentTraceDb::passive_checkpoint()` exactly once after post-commit Agent Trace persistence succeeds; a failing checkpoint is logged as a warning and never fails the hook or affects already-persisted data (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). - `count_write_contention(body) -> (T, WriteContentionCounts)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports the write-contention `attempts`, `outer_retries`, and `exhaustions` it caused on the current thread. Test instrumentation only; not production telemetry and not shared across threads or processes. - `record_write_contention_timeline(body) -> (T, Vec<(Instant, WriteContentionTimelineEvent)>)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and returns the `Instant`-stamped attempt start/end, backoff requested and backoff slept events of the write-contention retry on the current thread. Test-only measurement instrumentation used by the lock-contention suite; never logged in production. -- `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query`/`query_values`/`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. +- `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. - `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. -- `EncryptedTursoDb` exposes async `execute()`, `query()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. +- `EncryptedTursoDb` exposes async `execute()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. - Both public adapters keep concrete Turso connections and marker types; there is no shared runtime-owning core or synchronous executor bridge. - Shared lifecycle helpers: - `collect_db_path_health()` emits common parent/path health problems for DB-backed services. @@ -43,7 +43,7 @@ Both adapters expose native async operations and own no Tokio runtime. Construct ## Transactional primitives `TursoDb` offers two generic multi-statement transaction primitives beyond -the single-statement `execute()`/`query()`/`query_map()` wrappers. Both are +the single-statement `execute()`/`query_map()` wrappers. Both are public on `TursoDb` only (not `EncryptedTursoDb`). Each attempt runs the whole `BEGIN IMMEDIATE` → `COMMIT` unit and classifies every `turso::Error` as retryable (`Busy`/`BusySnapshot`) or deterministic. On the Agent Trace DB @@ -133,7 +133,7 @@ All three database areas (local DB, auth DB, Agent Trace DB) have lifecycle prov Migrations are deliberately outside the connection-open retry block. The generic constructors retry only local Turso open/connect; schema changes are not generically retried because migration SQL must not be replayed after partial execution. Service-specific resolvers may add bounded recovery around their own initialization contracts when they can prove safety; the repository-scoped Agent Trace resolver does this only for its one-file fresh schema and only repairs missing migration metadata after all required repository schema tables already exist. -`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). `execute()`, `query()`, and `query_values()` convert caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic rendering by operator-facing services. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. +`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). `execute()` converts caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. Retry-policy budget versus wall-clock time: the generic query retry policy keeps its historical `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff configuration, and its earlier `<= 2_000ms` figure describes that configured retry/budget calculation, not a hard wall-clock operation timeout. `run_with_retry` preserves post-attempt diagnostic timeout semantics and does not cancel an operation merely because its configured attempt budget elapsed: it only compares an attempt's elapsed time with `timeout_ms` after the attempt returns, to label the error. Actual failure latency is the sum of each attempt's real duration plus backoff. For the Agent Trace write-contention set, each attempt can include up to `busy_timeout_ms` (default 1000 ms) of Turso busy waiting and at most one outer retry starts, so a write blocked by a long-held lock fails after about 2–2.1 s with the defaults (about 1–1.1 s under the earlier 500 ms / 1250 ms defaults). None of `query.timeout_ms`, the `<= 2_000ms` policy figure, or `contention_deadline_ms` is a hard wall-clock deadline. diff --git a/flake.nix b/flake.nix index dde943a68..f918b463d 100644 --- a/flake.nix +++ b/flake.nix @@ -390,6 +390,110 @@ cargoArtifacts = craneLib.buildDepsOnly cargoDepsArgs; + telemetryCountScript = ./scripts/check-cargo-test-count.sh; + + mkCountedCargoTestPass = + { + pname, + featureArgs, + passes, + }: + craneLib.mkCargoDerivation ( + commonCargoArgs + // { + inherit pname cargoArtifacts; + pnameSuffix = ""; + doCheck = true; + buildPhaseCargoCommand = ""; + nativeCheckInputs = [ pkgs.git ]; + checkPhaseCargoCommand = '' + export CARGO_PROFILE="''${CARGO_PROFILE:-}" + export -f cargoWithProfile + '' + + pkgs.lib.concatMapStringsSep "\n" ( + pass: + "bash ${telemetryCountScript} ${toString pass.minimum} cargoWithProfile test --locked ${featureArgs} ${pass.filter}" + ) passes; + } + ); + + telemetryDefaultFeaturePasses = [ + { filter = "tracing_boundary_a"; minimum = 23; } + { filter = "tracing_boundary_b"; minimum = 17; } + { filter = "telemetry_async_scope"; minimum = 5; } + { filter = "telemetry_hook_export_gated"; minimum = 5; } + { filter = "telemetry_shutdown"; minimum = 4; } + { filter = "telemetry_command_span"; minimum = 4; } + { filter = "telemetry_exporter_cannot_block_locks"; minimum = 1; } + { filter = "telemetry_test_receiver_inert"; minimum = 1; } + { filter = "telemetry_offline_parity"; minimum = 4; } + { filter = "telemetry_disabled_baseline"; minimum = 2; } + { filter = "sce_command_span"; minimum = 2; } + ]; + + telemetryTestReceiverPasses = [ + { filter = "telemetry_test_receiver"; minimum = 8; } + { filter = "telemetry_hook_export_gated"; minimum = 6; } + { filter = "telemetry_shutdown_subprocess"; minimum = 5; } + { filter = "telemetry_exporter_cannot_block_locks"; minimum = 1; } + ]; + + cargoTestCountGuardCheck = pkgs.runCommand "cargo-test-count-guard" { } '' + set -euo pipefail + cp ${./scripts/check-cargo-test-count.sh} ./check-cargo-test-count.sh + cp ${./scripts/test-check-cargo-test-count.sh} ./test-check-cargo-test-count.sh + chmod +x ./check-cargo-test-count.sh ./test-check-cargo-test-count.sh + ${pkgs.bash}/bin/bash ./test-check-cargo-test-count.sh + mkdir -p "$out" + ''; + + packagedBinaryTestReceiverExclusionCheck = + pkgs.runCommand "packaged-binary-excludes-test-receiver" + { + nativeBuildInputs = [ pkgs.python3 pkgs.gnugrep ]; + } + '' + set -euo pipefail + binary=${scePackage}/bin/sce + + for marker in \ + SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT \ + SCE_TELEMETRY_TEST_LIFECYCLE_FILE; do + if grep -aq "$marker" "$binary"; then + echo "packaged-binary-excludes-test-receiver: $marker is compiled into .#default" >&2 + exit 1 + fi + done + + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + python3 - "$binary" <<'PY' + import os, socket, subprocess, sys + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(8) + listener.settimeout(0.5) + port = listener.getsockname()[1] + env = { + "PATH": os.environ["PATH"], + "HOME": os.environ["HOME"], + "SCE_TELEMETRY": "test-receiver", + "SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT": f"http://127.0.0.1:{port}", + "SCE_TELEMETRY_TEST_LIFECYCLE_FILE": os.path.join(os.environ["HOME"], "lifecycle"), + } + result = subprocess.run([sys.argv[1], "version"], env=env, capture_output=True, timeout=30) + assert result.returncode == 0, result.stderr + try: + listener.accept() + except socket.timeout: + pass + else: + raise SystemExit("packaged .#default connected to the D6 loopback receiver") + assert not os.path.exists(env["SCE_TELEMETRY_TEST_LIFECYCLE_FILE"]), "lifecycle markers written" + PY + mkdir -p "$out" + ''; + scePackage = craneLib.buildPackage ( commonCargoArgs // { @@ -1574,6 +1678,21 @@ } ); + cli-telemetry-default-features = mkCountedCargoTestPass { + pname = "sce-cli-telemetry-default-features"; + featureArgs = ""; + passes = telemetryDefaultFeaturePasses; + }; + + cli-telemetry-test-receiver = mkCountedCargoTestPass { + pname = "sce-cli-telemetry-test-receiver"; + featureArgs = "--features telemetry-test-receiver"; + passes = telemetryTestReceiverPasses; + }; + + cargo-test-count-guard = cargoTestCountGuardCheck; + packaged-binary-excludes-test-receiver = packagedBinaryTestReceiverExclusionCheck; + cli-clippy = craneLib.cargoClippy ( commonCargoArgs // { diff --git a/nix/flatpak/cargo-sources.nix b/nix/flatpak/cargo-sources.nix index 258103e90..937c8b990 100644 --- a/nix/flatpak/cargo-sources.nix +++ b/nix/flatpak/cargo-sources.nix @@ -33,7 +33,7 @@ let outputHashMode = "flat"; outputHashAlgo = "sha256"; - outputHash = "sha256-ZLcI8Basvo3MrOan7XnIUqq8UUFQkijzLGw/So3RFi0="; + outputHash = "sha256-Nuby7b45BrNnGAzFSCLB041p8XIJvxSFxnz4jvyFfKk="; }; regenerateApp = pkgs.writeShellApplication { diff --git a/packaging/flatpak/cargo-sources.json b/packaging/flatpak/cargo-sources.json index c6bd0fcd2..61aa31bb9 100644 --- a/packaging/flatpak/cargo-sources.json +++ b/packaging/flatpak/cargo-sources.json @@ -1962,6 +1962,19 @@ "dest": "cargo/vendor/futures-core-0.3.32", "dest-filename": ".cargo-checksum.json" }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/futures-executor/futures-executor-0.3.32.crate", + "sha256": "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d", + "dest": "cargo/vendor/futures-executor-0.3.32" + }, + { + "type": "inline", + "contents": "{\"package\": \"baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d\", \"files\": {}}", + "dest": "cargo/vendor/futures-executor-0.3.32", + "dest-filename": ".cargo-checksum.json" + }, { "type": "archive", "archive-type": "tar-gzip", @@ -2378,6 +2391,19 @@ "dest": "cargo/vendor/httparse-1.10.1", "dest-filename": ".cargo-checksum.json" }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/httpdate/httpdate-1.0.3.crate", + "sha256": "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9", + "dest": "cargo/vendor/httpdate-1.0.3" + }, + { + "type": "inline", + "contents": "{\"package\": \"df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9\", \"files\": {}}", + "dest": "cargo/vendor/httpdate-1.0.3", + "dest-filename": ".cargo-checksum.json" + }, { "type": "archive", "archive-type": "tar-gzip", @@ -3691,6 +3717,71 @@ "dest": "cargo/vendor/openssl-probe-0.2.1", "dest-filename": ".cargo-checksum.json" }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/opentelemetry/opentelemetry-0.33.0.crate", + "sha256": "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5", + "dest": "cargo/vendor/opentelemetry-0.33.0" + }, + { + "type": "inline", + "contents": "{\"package\": \"6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5\", \"files\": {}}", + "dest": "cargo/vendor/opentelemetry-0.33.0", + "dest-filename": ".cargo-checksum.json" + }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/opentelemetry-http/opentelemetry-http-0.33.0.crate", + "sha256": "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300", + "dest": "cargo/vendor/opentelemetry-http-0.33.0" + }, + { + "type": "inline", + "contents": "{\"package\": \"ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300\", \"files\": {}}", + "dest": "cargo/vendor/opentelemetry-http-0.33.0", + "dest-filename": ".cargo-checksum.json" + }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/opentelemetry-otlp/opentelemetry-otlp-0.33.0.crate", + "sha256": "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344", + "dest": "cargo/vendor/opentelemetry-otlp-0.33.0" + }, + { + "type": "inline", + "contents": "{\"package\": \"699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344\", \"files\": {}}", + "dest": "cargo/vendor/opentelemetry-otlp-0.33.0", + "dest-filename": ".cargo-checksum.json" + }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/opentelemetry-proto/opentelemetry-proto-0.33.0.crate", + "sha256": "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d", + "dest": "cargo/vendor/opentelemetry-proto-0.33.0" + }, + { + "type": "inline", + "contents": "{\"package\": \"25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d\", \"files\": {}}", + "dest": "cargo/vendor/opentelemetry-proto-0.33.0", + "dest-filename": ".cargo-checksum.json" + }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/opentelemetry_sdk/opentelemetry_sdk-0.33.0.crate", + "sha256": "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520", + "dest": "cargo/vendor/opentelemetry_sdk-0.33.0" + }, + { + "type": "inline", + "contents": "{\"package\": \"cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520\", \"files\": {}}", + "dest": "cargo/vendor/opentelemetry_sdk-0.33.0", + "dest-filename": ".cargo-checksum.json" + }, { "type": "archive", "archive-type": "tar-gzip", @@ -5823,6 +5914,19 @@ "dest": "cargo/vendor/tracing-log-0.2.0", "dest-filename": ".cargo-checksum.json" }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/tracing-opentelemetry/tracing-opentelemetry-0.34.0.crate", + "sha256": "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe", + "dest": "cargo/vendor/tracing-opentelemetry-0.34.0" + }, + { + "type": "inline", + "contents": "{\"package\": \"0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe\", \"files\": {}}", + "dest": "cargo/vendor/tracing-opentelemetry-0.34.0", + "dest-filename": ".cargo-checksum.json" + }, { "type": "archive", "archive-type": "tar-gzip", diff --git a/scripts/bench-hook-telemetry.sh b/scripts/bench-hook-telemetry.sh new file mode 100755 index 000000000..0f30ac466 --- /dev/null +++ b/scripts/bench-hook-telemetry.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: scripts/bench-hook-telemetry.sh + +Benchmarks `sce hooks pre-commit` under four telemetry modes (disabled, reachable +loopback receiver, accept-but-never-respond blackhole, connection refused) with +hyperfine, then measures post-command shutdown from the D6 lifecycle markers. + +Build the binary first: + nix develop -c ./scripts/run-cli-cargo.sh build --release \ + --features telemetry-test-receiver --manifest-path cli/Cargo.toml + +Run: + nix shell nixpkgs#hyperfine nixpkgs#python3 -c \ + scripts/bench-hook-telemetry.sh cli/target/release/sce + +Environment: BENCH_WARMUP (default 5), BENCH_RUNS (default 40), +BENCH_MARKER_RUNS (default 40), SCE_TELEMETRY_FLUSH_TIMEOUT_MS (default 1000). +USAGE +} + +if [ "$#" -ne 2 ]; then + usage + exit 2 +fi + +for tool in hyperfine python3; do + if ! command -v "${tool}" >/dev/null 2>&1; then + printf 'bench-hook-telemetry: %s not found; run via nix shell nixpkgs#hyperfine nixpkgs#python3 -c\n' "${tool}" >&2 + exit 2 + fi +done + +sce_binary="$(realpath "$1")" +output_dir="$(mkdir -p "$2" && realpath "$2")" +warmup="${BENCH_WARMUP:-5}" +runs="${BENCH_RUNS:-40}" +marker_runs="${BENCH_MARKER_RUNS:-40}" +flush_ms="${SCE_TELEMETRY_FLUSH_TIMEOUT_MS:-1000}" + +if [ ! -x "${sce_binary}" ]; then + printf 'bench-hook-telemetry: %s is not an executable file\n' "${sce_binary}" >&2 + exit 2 +fi + +sandbox="$(mktemp -d "${TMPDIR:-/tmp}/bench-hook-telemetry.XXXXXX")" +listener_pids=() +cleanup() { + for pid in "${listener_pids[@]:-}"; do + [ -n "${pid}" ] && kill "${pid}" 2>/dev/null || true + done + rm -rf "${sandbox}" +} +trap cleanup EXIT + +listener="${sandbox}/listener.py" +cat >"${listener}" <<'PY' +import socket +import sys +import threading + +mode = sys.argv[1] +port_file = sys.argv[2] +server = socket.socket() +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(("127.0.0.1", 0)) +server.listen(256) +with open(port_file, "w") as handle: + handle.write(str(server.getsockname()[1])) +held = [] + + +def serve(conn): + data = b"" + while b"\r\n\r\n" not in data: + chunk = conn.recv(65536) + if not chunk: + conn.close() + return + data += chunk + head, _, body = data.partition(b"\r\n\r\n") + length = 0 + for line in head.split(b"\r\n"): + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + while len(body) < length: + chunk = conn.recv(65536) + if not chunk: + break + body += chunk + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + conn.close() + + +while True: + conn, _ = server.accept() + if mode == "blackhole": + held.append(conn) + else: + threading.Thread(target=serve, args=(conn,), daemon=True).start() +PY + +start_listener() { + local mode="$1" port_file="${sandbox}/${1}.port" + python3 "${listener}" "${mode}" "${port_file}" >/dev/null 2>&1 & + listener_pids+=("$!") + for _ in $(seq 1 100); do + [ -s "${port_file}" ] && break + sleep 0.05 + done +} + +closed_port() { + python3 - <<'PY' +import socket +s = socket.socket() +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close() +PY +} + +start_listener reachable +start_listener blackhole +reachable_port="$(cat "${sandbox}/reachable.port")" +blackhole_port="$(cat "${sandbox}/blackhole.port")" +refused_port="$(closed_port)" + +base_env=( + "PATH=${PATH}" + "HOME=${sandbox}" + "XDG_CONFIG_HOME=${sandbox}/config" + "XDG_STATE_HOME=${sandbox}/state" + "XDG_CACHE_HOME=${sandbox}/cache" + "NO_COLOR=1" +) + +receiver_env() { + printf 'SCE_TELEMETRY=test-receiver SCE_TELEMETRY_TEST_RECEIVER_ENDPOINT=http://127.0.0.1:%s SCE_TELEMETRY_FLUSH_TIMEOUT_MS=%s' "$1" "${flush_ms}" +} + +mode_command() { + local mode="$1" extra="" + case "${mode}" in + disabled) extra="" ;; + reachable) extra="$(receiver_env "${reachable_port}")" ;; + blackhole) extra="$(receiver_env "${blackhole_port}")" ;; + refused) extra="$(receiver_env "${refused_port}")" ;; + esac + printf 'env -i %s %s %s hooks pre-commit' "${base_env[*]}" "${extra}" "${sce_binary}" +} + +modes=(disabled reachable blackhole refused) +mkdir -p "${sandbox}/work" +cd "${sandbox}/work" + +for mode in "${modes[@]}"; do + hyperfine --warmup "${warmup}" --runs "${runs}" --input null --style basic \ + --export-json "${output_dir}/hyperfine-${mode}.json" \ + -n "${mode}" "$(mode_command "${mode}")" \ + | tee "${output_dir}/hyperfine-${mode}.txt" +done + +for mode in reachable blackhole refused; do + marker_file="${output_dir}/markers-${mode}.txt" + : >"${marker_file}" + for i in $(seq 1 "${marker_runs}"); do + run_file="${sandbox}/run.markers" + rm -f "${run_file}" + marker_command="$(mode_command "${mode}")" + marker_command="${marker_command/env -i /env -i SCE_TELEMETRY_TEST_LIFECYCLE_FILE=${run_file} }" + bash -c "${marker_command}" /dev/null 2>&1 || true + awk -v run="${i}" '{ t[$1] = $2 } END { printf "%d %d %d %d\n", run, t["command_complete"], t["shutdown_end"] - t["shutdown_begin"], t["process_exit_requested"] - t["command_complete"] }' "${run_file}" >>"${marker_file}" + done +done + +python3 - "${output_dir}" "${flush_ms}" <<'PY' +import json +import statistics +import sys + +out, flush_ms = sys.argv[1], int(sys.argv[2]) + + +def pct(values, q): + values = sorted(values) + index = (len(values) - 1) * q + low = int(index) + high = min(low + 1, len(values) - 1) + return values[low] + (values[high] - values[low]) * (index - low) + + +def load(mode): + with open(f"{out}/hyperfine-{mode}.json") as handle: + return [t * 1000 for t in json.load(handle)["results"][0]["times"]] + + +disabled = load("disabled") +summary = {"flush_budget_ms": flush_ms, "disabled_ms": {"p50": pct(disabled, 0.5), "p95": pct(disabled, 0.95), "max": max(disabled)}, "modes": {}} +d50, d95 = pct(disabled, 0.5), pct(disabled, 0.95) +for mode in ("reachable", "blackhole", "refused"): + times = load(mode) + overhead = sorted(t - d50 for t in times) + paired = [t - statistics.median(disabled) for t in times] + rows = [list(map(int, line.split())) for line in open(f"{out}/markers-{mode}.txt") if line.strip()] + post = [r[3] / 1e6 for r in rows] + shutdown = [r[2] / 1e6 for r in rows] + summary["modes"][mode] = { + "runs": len(times), + "end_to_end_ms": {"p50": pct(times, 0.5), "p95": pct(times, 0.95), "max": max(times)}, + "overhead_ms_vs_disabled_p50": {"p50": pct(paired, 0.5), "p95": pct(paired, 0.95), "max": max(paired)}, + "post_command_ms": {"p50": pct(post, 0.5), "p95": pct(post, 0.95), "max": max(post)} if post else None, + "shutdown_ms": {"p50": pct(shutdown, 0.5), "p95": pct(shutdown, 0.95), "max": max(shutdown)} if shutdown else None, + "marker_runs": len(rows), + } +with open(f"{out}/summary.json", "w") as handle: + json.dump(summary, handle, indent=2) +print(json.dumps(summary, indent=2)) +PY diff --git a/scripts/check-cargo-test-count.sh b/scripts/check-cargo-test-count.sh new file mode 100755 index 000000000..60ac0e89c --- /dev/null +++ b/scripts/check-cargo-test-count.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: scripts/check-cargo-test-count.sh [args...] + +Runs (normally a `cargo test` invocation), then fails unless it exits +successfully and the summed `N passed` counts across every `test result:` line +are at least . A filtered run that matches zero tests +exits 0 under Cargo, so it is never accepted as a pass here. + +Example: + scripts/check-cargo-test-count.sh 5 nix develop -c ./scripts/run-cli-cargo.sh \ + test --manifest-path cli/Cargo.toml telemetry_hook_export_gated +USAGE +} + +if [ "$#" -lt 2 ]; then + usage + exit 2 +fi + +minimum="$1" +shift + +if ! [[ "${minimum}" =~ ^[1-9][0-9]*$ ]]; then + printf 'check-cargo-test-count: minimum must be a positive integer, got %q\n' "${minimum}" >&2 + exit 2 +fi + +output_file="$(mktemp "${TMPDIR:-/tmp}/check-cargo-test-count.XXXXXX")" +cleanup() { + rm -f "${output_file}" +} +trap cleanup EXIT + +command_status=0 +"$@" 2>&1 | tee "${output_file}" || command_status="${PIPESTATUS[0]}" + +if [ "${command_status}" -ne 0 ]; then + printf 'check-cargo-test-count: command failed with exit status %s\n' "${command_status}" >&2 + exit "${command_status}" +fi + +executed=0 +result_pattern='^test result: [A-Za-z]+\. ([0-9]+) passed;' +while IFS= read -r line; do + if [[ "${line}" =~ ${result_pattern} ]]; then + executed=$((executed + BASH_REMATCH[1])) + fi +done < "${output_file}" + +if [ "${executed}" -lt "${minimum}" ]; then + printf 'check-cargo-test-count: executed %s test(s), expected at least %s; a filter matching zero tests is not a pass.\n' \ + "${executed}" "${minimum}" >&2 + exit 1 +fi + +printf 'check-cargo-test-count: executed %s test(s) (minimum %s)\n' "${executed}" "${minimum}" >&2 diff --git a/scripts/test-check-cargo-test-count.sh b/scripts/test-check-cargo-test-count.sh new file mode 100755 index 000000000..60ede0989 --- /dev/null +++ b/scripts/test-check-cargo-test-count.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +checker="${script_dir}/check-cargo-test-count.sh" +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/test-check-cargo-test-count.XXXXXX")" +trap 'rm -rf "${work_dir}"' EXIT + +fake_cargo() { + local name="$1" + shift + local path="${work_dir}/${name}" + { + printf '#!%s\n' "${BASH}" + printf 'cat <<"OUT"\n' + printf '%s\n' "$@" + printf 'OUT\n' + printf 'exit "${FAKE_EXIT:-0}"\n' + } > "${path}" + chmod +x "${path}" + printf '%s' "${path}" +} + +expect_status() { + local expected="$1" + local label="$2" + shift 2 + local status=0 + bash "${checker}" "$@" > /dev/null 2>&1 || status=$? + if [ "${status}" -ne "${expected}" ]; then + printf 'FAIL %s: expected exit %s, got %s\n' "${label}" "${expected}" "${status}" >&2 + exit 1 + fi + printf 'ok %s\n' "${label}" >&2 +} + +zero="$(fake_cargo zero \ + 'running 0 tests' \ + 'test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 540 filtered out; finished in 0.00s' \ + 'running 0 tests' \ + 'test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 12 filtered out; finished in 0.00s')" +three="$(fake_cargo three \ + 'test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 540 filtered out; finished in 0.10s' \ + 'test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 12 filtered out; finished in 0.10s')" + +expect_status 1 "zero executed tests is rejected" 1 "${zero}" +expect_status 1 "below the minimum is rejected" 4 "${three}" +expect_status 0 "sum across binaries meets the minimum" 3 "${three}" +expect_status 2 "missing command is a usage error" 3 +expect_status 2 "non-numeric minimum is a usage error" many "${three}" +expect_status 2 "zero minimum is a usage error" 0 "${three}" +FAKE_EXIT=101 expect_status 101 "failing command propagates its status" 1 "${three}" +printf 'all check-cargo-test-count checks passed\n' >&2