From 79c9d43b65ceb0a4312e22dab2fb23a506818d17 Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:35:47 +0000 Subject: [PATCH 01/12] lens: let a summary say what its record spent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record can record what it cost, and two rows want the total: the group row over a folded run, and the status bar over the document. Both are in the record seam, because a lens may not decide a row. Summary::tokens is the exact count a dialect read, and lens::tokens is the one spelling of it — floored to at most four columns, so a row that says 18k promises at least 18,000 and never overstates. A group row gains a third clause, omitted at zero, so the two shipped dialects paint exactly as before. --- src/lens/agent.rs | 13 ++++++- src/lens/atif.rs | 5 ++- src/lens/mod.rs | 15 +++++++++ src/lens/num.rs | 54 ++++++++++++++++++++++++++++++ src/lens/tests.rs | 54 ++++++++++++++++++++++++++++++ src/source/record/lensrow.rs | 50 ++++++++++++++++++--------- src/source/record/lensrow_tests.rs | 33 ++++++++++++++++++ src/source/record/plan_fixture.rs | 1 + 8 files changed, 208 insertions(+), 17 deletions(-) create mode 100644 src/lens/num.rs create mode 100644 src/source/record/lensrow_tests.rs diff --git a/src/lens/agent.rs b/src/lens/agent.rs index 7080e59..6363ad6 100644 --- a/src/lens/agent.rs +++ b/src/lens/agent.rs @@ -179,7 +179,17 @@ impl Agent { }; // A step's row stays one line — what it *did* — and whatever it was // thinking goes under it. Reasoning is text, so it is shown. - Summary { class: Class::Step, who, actor, time: None, what, calls: seen.calls.len(), body } + Summary { + class: Class::Step, + who, + actor, + time: None, + what, + calls: seen.calls.len(), + // What a turn *spent* is `--lens usage`'s question, not this one's. + tokens: 0, + body, + } } /// Read the blocks of one message into the four things a row can say. @@ -277,6 +287,7 @@ fn said(who: Who, actor: String, what: String, body: Option) -> Summary { time: None, what, calls: 0, + tokens: 0, body, } } diff --git a/src/lens/atif.rs b/src/lens/atif.rs index 5f5c413..381fdc0 100644 --- a/src/lens/atif.rs +++ b/src/lens/atif.rs @@ -155,6 +155,7 @@ fn session(v: &Value) -> Option { time: None, what: parts.join(" \u{b7} "), calls: 0, + tokens: 0, // The envelope is a headline over a tree, not something anyone said: // its keys are the row, and `Enter` opens them. body: None, @@ -194,7 +195,8 @@ fn spoken(source: &str, said: &str, thinking: bool, calls: usize) -> Summary { what.push_str(&format!(" \u{b7} {}", call_count(calls))); } let body = Body::new(said, vec![Step::Key("message")]); - Summary { class: Class::Message, who, actor, time: None, what, calls, body: Some(body) } + // What a step *spent* is `--lens usage-atif`'s question, not this one's. + Summary { class: Class::Message, who, actor, time: None, what, calls, tokens: 0, body: Some(body) } } /// A step that only did things: the thought and the calls it made, each with @@ -233,6 +235,7 @@ fn worked(source: &str, v: &Value, thinking: bool, calls: Vec) -> Summary time: None, what: collapse(parts).join(" \u{b7} "), calls: n, + tokens: 0, // Reasoning is text, so it is shown: a step that only thought puts the // thought under its row, clipped, the way a message puts what was said // there. The row keeps saying what the step *did* — the body goes diff --git a/src/lens/mod.rs b/src/lens/mod.rs index 4953c06..c1324c5 100644 --- a/src/lens/mod.rs +++ b/src/lens/mod.rs @@ -42,8 +42,10 @@ pub mod agent; pub mod atif; +pub mod num; pub mod part; +pub use num::tokens; pub use part::Part; #[cfg(test)] @@ -215,6 +217,18 @@ pub struct Summary { pub what: String, /// Tool calls this record makes, for the group row's `· 4 tool calls`. pub calls: usize, + /// Every token unit this record recorded, exact; `0` when it records none. + /// + /// Exact and not a spelling, because the two places a total is shown add + /// these up — the group row over a run ([`crate::source::record::lensrow`]) + /// and the status bar over the document + /// ([`crate::source::record::view`]) — and a sum of rounded numbers is not + /// the rounding of a sum. [`tokens`] is the *only* spelling of one, so the + /// row and the total cannot disagree about what `18k` means. + /// + /// A dialect that reads no counters leaves it `0`, and the two clauses that + /// would show it are then omitted entirely rather than saying `0 tokens`. + pub tokens: u64, /// The record's own text, under the row: what was said for a message, and /// what the model was thinking for a step that only thought. /// @@ -237,6 +251,7 @@ impl Summary { time: None, what: what.into(), calls: 0, + tokens: 0, body: None, } } diff --git a/src/lens/num.rs b/src/lens/num.rs new file mode 100644 index 0000000..08a9883 --- /dev/null +++ b/src/lens/num.rs @@ -0,0 +1,54 @@ +//! A token count as a row spells it. +//! +//! One function, in the seam and not in a dialect, because the same number +//! appears on a record's row, on the group row over a folded run and in the +//! status bar — and two spellings of it would drift, which is the argument that +//! put `jsonrow` under both JSON sources. Its own module so +//! [`super`] stays under the size limit. +#![deny(unsafe_code)] + +/// The scales a token count is spelled at, largest first. `u64::MAX` is about +/// 1.8×10¹⁹, so `E` is the last rung there can be and the widest answer this +/// can give is four columns. +const SCALES: [(u64, char); 6] = [ + (1_000_000_000_000_000_000, 'E'), + (1_000_000_000_000_000, 'P'), + (1_000_000_000_000, 'T'), + (1_000_000_000, 'G'), + (1_000_000, 'M'), + (1_000, 'k'), +]; + +/// A token count in at most **four display columns**: `380`, `1.2k`, `18k`, +/// `1.8M`. A number column is only a column if every number in it is the same +/// width, so the four is a promise and not an estimate. +/// +/// # Floored, never rounded +/// +/// `999` is `999` and not `1.0k`; `1999` is `1.9k` and not `2.0k`. A row that +/// says `18k` is therefore a promise of *at least* 18,000, which is the reading +/// a person makes of a truncated number and the only one that never overstates +/// what a session spent. The cost is that a bucket hides its magnitude — `18k` +/// is anything up to 18,999, and floored numbers will not add up by eye. The +/// exact integers are one `Enter` away in `detail()`, and the group row and the +/// status bar spell the *exact* sum through this function rather than summing +/// what the rows show. +/// +/// It lives in the seam and not in a dialect because a group row spells the +/// same number a record row does, and two spellings would drift — the argument +/// that put `jsonrow` under both JSON sources. +pub fn tokens(n: u64) -> String { + for (scale, suffix) in SCALES { + if n < scale { + continue; + } + let whole = n / scale; + // Two digits leave no room for a decimal, and `18k` is as much as four + // columns can honestly say. + if whole >= 10 { + return format!("{whole}{suffix}"); + } + return format!("{whole}.{}{suffix}", (n % scale) * 10 / scale); + } + format!("{n}") +} diff --git a/src/lens/tests.rs b/src/lens/tests.rs index b0662de..bcb88b2 100644 --- a/src/lens/tests.rs +++ b/src/lens/tests.rs @@ -139,6 +139,60 @@ fn a_timestamp_with_a_multibyte_character_is_refused_rather_than_split() { assert_eq!(clock("2026-08-05T21:28\u{20ac}").as_deref(), Some("21:28")); } +/// The number spelling, at every rung and at every boundary between them. +#[test] +fn a_token_count_is_spelled_at_the_scale_it_reaches() { + assert_eq!(tokens(0), "0"); + assert_eq!(tokens(380), "380"); + assert_eq!(tokens(999), "999", "999 is not a thousand of anything"); + assert_eq!(tokens(1_000), "1.0k"); + assert_eq!(tokens(1_200), "1.2k"); + assert_eq!(tokens(9_999), "9.9k"); + assert_eq!(tokens(10_000), "10k"); + assert_eq!(tokens(999_999), "999k"); + assert_eq!(tokens(1_000_000), "1.0M"); +} + +/// Floored, never rounded up: a row that says `18k` promises at least 18,000, +/// and a reader reconciling it against a bill must never find less than it said. +#[test] +fn a_token_count_floors() { + assert_eq!(tokens(1_999), "1.9k", "not 2.0k"); + assert_eq!(tokens(18_999), "18k", "not 19k"); + assert_eq!(tokens(1_899_999), "1.8M", "not 1.9M"); + assert_eq!(tokens(999_999_999), "999M"); +} + +/// The whole product of this lens family is a column of numbers, so the +/// spelling may never be wider than the column — at any rung, at any boundary, +/// and at the extremes a real session reaches. +#[test] +fn a_token_count_never_outgrows_its_column() { + let cases = [ + 0, + 1, + 999, + 1_000, + 9_999, + 10_000, + 999_999, + 1_000_000, + 20_385, + 709_541, + 802_174, + 1_823_744, + u64::MAX / 2, + u64::MAX, + ]; + for n in cases { + let text = tokens(n); + assert!( + crate::render::str_width(&text) <= 4, + "{n} spelled as {text}, wider than four columns" + ); + } +} + #[test] fn record_helpers_read_only_what_is_there() { let v = crate::json::parse(br#"{"type":"user","timestamp":"2026-08-05T21:28:58.659Z"}"#) diff --git a/src/source/record/lensrow.rs b/src/source/record/lensrow.rs index b913487..8fd4451 100644 --- a/src/source/record/lensrow.rs +++ b/src/source/record/lensrow.rs @@ -352,7 +352,7 @@ pub(crate) fn toc_line(plan: Option<&Plan>, record: usize) -> Option { /// It always carries a fold marker, because a group only exists when there /// is something inside it to open. pub(crate) fn group_row(plan: Option<&Plan>, item: usize) -> Line { - let (first, count, calls) = group_counts(plan, item); + let (first, count, calls, tokens) = group_counts(plan, item); let time = plan .and_then(|p| p.summary(first)) .and_then(|s| s.time.clone()) @@ -361,24 +361,30 @@ pub(crate) fn group_row(plan: Option<&Plan>, item: usize) -> Line { Span::new(pad("", ACTOR), theme::text()), Span::new(pad(&time, TIME), theme::lens_time()), Span::new(" ", theme::text()), - Span::new(group_text(count, calls), theme::lens_group()), + Span::new(group_text(count, calls, tokens), theme::lens_group()), ]; summary_line(marker(rest), first + 1) } -/// `(first record, records, tool calls)` of a group. -fn group_counts(plan: Option<&Plan>, item: usize) -> (usize, usize, usize) { +/// `(first record, records, tool calls, tokens)` of a group. +/// +/// The token total is the **exact** sum of the members' exact counts, spelled +/// once by [`crate::lens::tokens`] — not a sum of what the rows show, which are +/// floored and would not add up. +fn group_counts(plan: Option<&Plan>, item: usize) -> (usize, usize, usize, u64) { let Some(plan) = plan else { - return (0, 0, 0); + return (0, 0, 0, 0); }; let Some(it) = plan.item(item) else { - return (0, 0, 0); + return (0, 0, 0, 0); }; - let calls = (it.first..it.first + it.count) - .filter_map(|r| plan.summary(r)) - .map(|s| s.calls) - .sum(); - (it.first, it.count, calls) + let mut calls = 0; + let mut tokens = 0u64; + for sum in (it.first..it.first + it.count).filter_map(|r| plan.summary(r)) { + calls += sum.calls; + tokens = tokens.saturating_add(sum.tokens); + } + (it.first, it.count, calls, tokens) } /// One summary row: never wrapped, and a landmark for `Tab`. @@ -431,9 +437,14 @@ fn body_style(class: Class) -> crate::term::Style { } } -/// `⟨6 steps · 4 tool calls⟩`, and the singulars, and no call clause when the -/// run made no calls (a run of thoughts and bookkeeping). -fn group_text(steps: usize, calls: usize) -> String { +/// `⟨6 steps · 4 tool calls · 128k tokens⟩`, and the singulars, and no clause +/// for a count that is zero — a run of thoughts and bookkeeping made no calls, +/// and a lens that reads no counters spent nothing it knows of, so a row under +/// either says neither. +/// +/// One number rather than a breakdown because a group row has one column and no +/// room for four; the breakdown is one `Enter` away on the member rows. +fn group_text(steps: usize, calls: usize, tokens: u64) -> String { let steps = match steps { 1 => "1 step".to_string(), n => format!("{n} steps"), @@ -443,7 +454,12 @@ fn group_text(steps: usize, calls: usize) -> String { 1 => " \u{b7} 1 tool call".to_string(), n => format!(" \u{b7} {n} tool calls"), }; - format!("\u{27e8}{steps}{calls}\u{27e9}") + let tokens = match tokens { + 0 => String::new(), + 1 => " \u{b7} 1 token".to_string(), + n => format!(" \u{b7} {} tokens", crate::lens::tokens(n)), + }; + format!("\u{27e8}{steps}{calls}{tokens}\u{27e9}") } /// Pad to `cols` display columns — display columns, not bytes, because an @@ -457,3 +473,7 @@ fn pad(text: &str, cols: usize) -> String { s.push(' '); s } + +#[cfg(test)] +#[path = "lensrow_tests.rs"] +mod tests; diff --git a/src/source/record/lensrow_tests.rs b/src/source/record/lensrow_tests.rs new file mode 100644 index 0000000..2216aaf --- /dev/null +++ b/src/source/record/lensrow_tests.rs @@ -0,0 +1,33 @@ +//! The group row's text: what a folded run says it swallowed. +//! +//! One of the two places a total appears (the other is the status bar in +//! `view.rs`), and both are here in the seam rather than in a dialect, because a +//! lens may not decide a row. +#![deny(unsafe_code)] + +use super::*; + +/// A run that recorded no tokens reads exactly as it always has. This pins that +/// adding the clause moved no shipped row: `agent` and `atif` both leave +/// `Summary::tokens` at 0, so every one of their group rows is byte-identical. +#[test] +fn a_run_that_counted_nothing_says_nothing_about_tokens() { + assert_eq!(group_text(6, 4, 0), "\u{27e8}6 steps \u{b7} 4 tool calls\u{27e9}"); + assert_eq!(group_text(1, 0, 0), "\u{27e8}1 step\u{27e9}"); + assert_eq!(group_text(1, 1, 0), "\u{27e8}1 step \u{b7} 1 tool call\u{27e9}"); + assert_eq!(group_text(3, 0, 0), "\u{27e8}3 steps\u{27e9}"); +} + +/// The third clause, with the singular and plural the other two already have. +#[test] +fn a_run_that_counted_tokens_totals_them() { + assert_eq!( + group_text(15, 3, 128_000), + "\u{27e8}15 steps \u{b7} 3 tool calls \u{b7} 128k tokens\u{27e9}" + ); + assert_eq!(group_text(1, 0, 1), "\u{27e8}1 step \u{b7} 1 token\u{27e9}"); + assert_eq!(group_text(2, 0, 380), "\u{27e8}2 steps \u{b7} 380 tokens\u{27e9}"); + // Spelled by the one spelling there is, so the group row and a record row + // cannot disagree about what `1.9k` means. + assert_eq!(group_text(2, 0, 1_999), "\u{27e8}2 steps \u{b7} 1.9k tokens\u{27e9}"); +} diff --git a/src/source/record/plan_fixture.rs b/src/source/record/plan_fixture.rs index b1ed52b..2fdd2aa 100644 --- a/src/source/record/plan_fixture.rs +++ b/src/source/record/plan_fixture.rs @@ -36,6 +36,7 @@ impl crate::lens::Lens for Fake { time: None, what: String::new(), calls: 1, + tokens: 0, // A body is the source's to measure; these tests give one to an // item directly (`with_bodies`), which is the same number the // measurement would have produced and keeps this file file-free. From 961561b687d6cfb7eb9446600fcc97a53e1002af Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:42:03 +0000 Subject: [PATCH 02/12] lens: read a session log as what it spent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --lens usage over a Claude Code session log answers what each record cost rather than what it said: a fixed-width block of four token counters, then what the record did. Three different things a cell can say, and they are kept apart deliberately — a number means the format recorded that value (0 included), a dash means this record did not record a field its format has, and an absent column means the format has no such field. Numbers live at message.usage and nowhere else; usage.iterations is counted and never summed, because its elements repeat the outer counters. A human turn breaks the run and everything else is mechanics, so the group row over a run totals exactly one turn. A subagent is marked as ↳assistant, ten columns exactly, because a mark that cost a column would bend the numbers on more than half the rows of a real session. --- src/lens/mod.rs | 3 + src/lens/usage.rs | 206 +++++++++++++++++++++++++++++++ src/lens/usage_agent.rs | 225 ++++++++++++++++++++++++++++++++++ src/lens/usage_agent_tests.rs | 207 +++++++++++++++++++++++++++++++ src/lens/usage_tests.rs | 102 +++++++++++++++ 5 files changed, 743 insertions(+) create mode 100644 src/lens/usage.rs create mode 100644 src/lens/usage_agent.rs create mode 100644 src/lens/usage_agent_tests.rs create mode 100644 src/lens/usage_tests.rs diff --git a/src/lens/mod.rs b/src/lens/mod.rs index c1324c5..fa1d921 100644 --- a/src/lens/mod.rs +++ b/src/lens/mod.rs @@ -44,6 +44,8 @@ pub mod agent; pub mod atif; pub mod num; pub mod part; +pub mod usage; +pub mod usage_agent; pub use num::tokens; pub use part::Part; @@ -346,6 +348,7 @@ type Make = fn() -> Box; const LENSES: &[(&str, Make)] = &[ (agent::NAME, || Box::new(agent::Agent::default())), (atif::NAME, || Box::new(atif::Atif)), + (usage_agent::NAME, || Box::new(usage_agent::Usage)), ]; /// The lens called `name`, or `None`. diff --git a/src/lens/usage.rs b/src/lens/usage.rs new file mode 100644 index 0000000..cf982d7 --- /dev/null +++ b/src/lens/usage.rs @@ -0,0 +1,206 @@ +//! What a record *spent*, said without naming a format. +//! +//! The half the two usage dialects share: the counters a record can carry, the +//! exact total that becomes [`crate::lens::Summary::tokens`], and the one +//! spelling of the numeric block a row shows them in. `usage_agent.rs` reads a +//! Claude Code session log into it and `usage_atif.rs` reads an ATIF trajectory +//! into it; neither spells a number, and neither decides a row — that is still +//! `src/source/record/`'s. +//! +//! # Three different things a cell can say +//! +//! This is the whole design, and it is why every counter is an [`Option`] +//! rather than a `u64` that defaults to zero: +//! +//! * **a number** — the format recorded that value, and `0` means zero tokens +//! were spent, which is a real and observable thing; +//! * **`-`** — this *record* did not record a field its format has. The column +//! stays, because other records in the same file fill it; +//! * **no column at all** — the *format* has no such field. ATIF-v1.7 records no +//! cache-creation counter of any kind, so a `usage-atif` row has three fields +//! and never a fourth. A `0` there would read as "this agent wrote nothing to +//! cache" when the truth is "this format does not record cache writes", and a +//! `-` on every row of every file would be a column of nothing. +//! +//! The rule stated once and applied twice: a *format*-level absence removes the +//! column for the whole file, because alignment is what a number column is for +//! and alignment is per file; a *record*-level absence inside a format that has +//! the field prints `-`. +//! +//! # Why the block is a constant width +//! +//! A reader of this lens is scanning down one column. Four fields are +//! `4×8 + 3×2 = 38` columns and three are `3×8 + 2×2 = 28`, always, whatever +//! the numbers are — which is what [`crate::lens::tokens`] flooring to four +//! columns buys. The row is never wrapped (it scrolls), so a narrow terminal +//! pans across it rather than losing the action off the end. +#![deny(unsafe_code)] + +use crate::lens::tokens; + +/// Columns a field's label gets, left-justified. +const LABEL: usize = 4; + +/// Columns a field's number gets, right-aligned. [`crate::lens::tokens`] +/// promises never to exceed it. +const NUMBER: usize = 4; + +/// Columns one field occupies: its label then its number, no separator, because +/// the label is short and the number is right-aligned into the gap. +pub const FIELD: usize = LABEL + NUMBER; + +/// Columns between two fields. +const GAP: &str = " "; + +/// Between the numbers and what the record did. +const TURN: &str = " \u{b7} "; + +/// What a record with no numbers at all shows in the number cell of a field its +/// format does have. Never `0`: a recorded zero and an unrecorded field are +/// different claims, and this lens exists to keep them apart. +const ABSENT: &str = "-"; + +/// The counters a record recorded, exactly as the file recorded them. +/// +/// `None` is "this record did not say", not "zero" — see the module docs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Tokens { + /// Tokens sent to the model. + pub input: Option, + /// Tokens the model produced. + pub output: Option, + /// Tokens served from cache rather than re-sent. + pub cache_read: Option, + /// Tokens written *to* cache. No ATIF field maps to it. + pub cache_new: Option, +} + +impl Tokens { + /// Every unit this record recorded, added once. + /// + /// **The four counters and nothing else.** A record's reasoning count is a + /// *subset* of its output and a Claude Code `usage.iterations[]` list + /// repeats the outer counters per attempt — adding either would count the + /// same token twice, and a total that double-counts is worse than no total. + /// That mistake is the one a later reader of this file will be tempted into, + /// which is why it is written here and pinned by a test rather than left in + /// the documentation. + pub fn total(&self) -> u64 { + [self.input, self.output, self.cache_read, self.cache_new] + .into_iter() + .flatten() + .fold(0u64, |a, b| a.saturating_add(b)) + } + + /// Did this record record anything at all? A record that recorded nothing + /// gets no number columns — it shows what kind of record it is and stops. + pub fn any(&self) -> bool { + self.input.is_some() + || self.output.is_some() + || self.cache_read.is_some() + || self.cache_new.is_some() + } +} + +/// One column of the numeric block: which counter, under which label. +/// +/// A dialect names the set its format actually has, once, and a counter that is +/// not in the set never reaches a row — that is the "absent column" of the +/// three-way distinction above. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Field { + In, + Out, + Read, + New, +} + +impl Field { + /// The four a format that records cache writes shows. + pub const ALL: [Field; 4] = [Field::In, Field::Out, Field::Read, Field::New]; + + fn label(self) -> &'static str { + match self { + Field::In => "in", + Field::Out => "out", + Field::Read => "read", + Field::New => "new", + } + } + + fn of(self, t: &Tokens) -> Option { + match self { + Field::In => t.input, + Field::Out => t.output, + Field::Read => t.cache_read, + Field::New => t.cache_new, + } + } +} + +/// The row a usage-bearing record shows: the numbers, then what it did. +/// +/// Every field is [`FIELD`] columns whatever it holds, so the block is +/// `fields.len() * FIELD + (fields.len() - 1) * 2` columns for every record in +/// the file and the column reads straight down. +pub fn row_text(t: &Tokens, fields: &[Field], action: &str) -> String { + let mut out = String::new(); + for (i, f) in fields.iter().enumerate() { + if i > 0 { + out.push_str(GAP); + } + out.push_str(&cell(*f, t)); + } + if !action.is_empty() { + out.push_str(TURN); + out.push_str(action); + } + out +} + +/// One field: its label left-justified, its number right-aligned into what is +/// left. ASCII throughout — the labels are this module's own and +/// [`crate::lens::tokens`] emits digits and one suffix letter — so columns and +/// bytes agree here and the padding is exact. +fn cell(f: Field, t: &Tokens) -> String { + let number = match f.of(t) { + Some(n) => tokens(n), + None => ABSENT.to_string(), + }; + let label = f.label(); + let mut s = String::with_capacity(FIELD); + s.push_str(label); + for _ in label.len()..LABEL { + s.push(' '); + } + for _ in number.len()..NUMBER { + s.push(' '); + } + s.push_str(&number); + s +} + +/// Adjacent entries that read the same collapsed to `Read ×3`. +/// +/// A turn that read three files made one kind of move, and spelling it three +/// times pushes what happened next off the row. The same rule `atif` applies to +/// its own row, for the same reason. +pub fn collapse(items: Vec) -> Vec { + let mut out: Vec<(String, usize)> = Vec::new(); + for it in items { + match out.last_mut() { + Some((last, n)) if *last == it => *n += 1, + _ => out.push((it, 1)), + } + } + out.into_iter() + .map(|(text, n)| match n { + 1 => text, + n => format!("{text} \u{d7}{n}"), + }) + .collect() +} + +#[cfg(test)] +#[path = "usage_tests.rs"] +mod tests; diff --git a/src/lens/usage_agent.rs b/src/lens/usage_agent.rs new file mode 100644 index 0000000..4b4cf06 --- /dev/null +++ b/src/lens/usage_agent.rs @@ -0,0 +1,225 @@ +//! The `usage` lens: a Claude Code session log read as what it *spent*. +//! +//! Same file as `--lens agent`, a different question. `agent` asks what was +//! said; this asks what each record cost, and answers in a fixed-width numeric +//! block a reader can scan straight down: +//! +//! ```text +//! assistant 21:31 in 1.2k out 380 read 18k new 2.1k · Bash(cargo test) +//! ``` +//! +//! Named `usage` and not `cost`: no price table is compiled in and no money is +//! shown. A lens must not promise a currency it cannot compute. +//! +//! # Where the numbers are +//! +//! Exactly one path, `message.usage`, and nothing else is read for a number. In +//! the sessions this was written against it is present on every `assistant` +//! record and on no record of any other type, so a record without it is not a +//! record whose numbers are hidden somewhere else — it is a record that spent +//! nothing this file recorded, and it shows its `type` and stops. +//! +//! | key | column | +//! | --- | --- | +//! | `input_tokens` | `in` | +//! | `output_tokens` | `out` | +//! | `cache_read_input_tokens` | `read` | +//! | `cache_creation_input_tokens` | `new` | +//! +//! `output_tokens_details.thinking_tokens` is a *subset* of `output_tokens` and +//! gets no column. **`usage.iterations[]` is never summed**: it is a list whose +//! elements repeat the outer counters per attempt, so adding both counts every +//! token twice ([`usage::Tokens::total`] says so where the adding happens). +//! +//! # The one decision that shapes the document +//! +//! A **human** turn — a `user` record that is not carrying a tool result — is a +//! [`Class::Message`] and breaks the run. Everything else is a [`Class::Step`]. +//! So the run between two human turns is exactly one turn's mechanics, and the +//! group row over it totals what that turn cost. +//! +//! Drawing that line anywhere else would ruin the lens. A `user` record whose +//! content blocks are `tool_result` is mechanics — `agent.rs` already draws that +//! same line — and if it were conversation instead, tool results would shred +//! every run into pairs and no group row would total anything. +//! +//! # What this lens does not show +//! +//! Any message text at all. [`Summary::body`] is `None` on every row, because +//! "what was said" is the question `--lens agent` answers. It also means this +//! dialect allocates nothing per record but its own one-line `what`, so a log +//! whose longest line is most of a megabyte costs it a parse and no more. +#![deny(unsafe_code)] + +use super::usage::{self, Field, Tokens}; +use super::{record_clock, record_type, Class, Lens, Summary, Who}; +use crate::json::Value; + +pub const NAME: &str = "usage"; +pub const ABOUT: &str = "Claude Code session logs: what each turn spent, not what it said"; + +/// The subagent mark, with **no space after it**. +/// +/// `agent.rs` writes `↳ assistant`, which is eleven columns and pushes every +/// column on that row one to the right. More than half of the records in a real +/// session carry `isSidechain`, and on a lens whose whole product is a column of +/// numbers a shift on half the rows is fatal — so here it is `↳assistant`, +/// exactly the ten columns the actor field is wide. `Who` still carries the +/// colour, so the row still paints as an assistant. +const SIDE: char = '\u{21b3}'; + +#[derive(Default)] +pub struct Usage; + +impl Lens for Usage { + fn name(&self) -> &'static str { + NAME + } + + fn about(&self) -> &'static str { + ABOUT + } + + fn read(&mut self, v: &Value) -> Option { + // No string `type` is not this dialect's record: it falls to the + // generic tree with nothing hidden. + let kind = record_type(v)?; + let time = record_clock(v); + let (who, actor) = speaker(v, kind); + let calls = call_blocks(v).count(); + let sum = match read_usage(v) { + Some(t) => Summary { + class: class_of(v, kind), + who, + actor, + time: None, + what: usage::row_text(&t, &Field::ALL, &action(v, kind)), + calls, + tokens: t.total(), + body: None, + }, + // Nothing was spent here that the file recorded, so there is no + // number column at all — not a row of zeroes, and not a row of + // dashes. A `type` this dialect has never seen prints its own name + // rather than being swallowed. + None => Summary { + class: class_of(v, kind), + who, + actor, + time: None, + what: kind.to_string(), + calls, + tokens: 0, + body: None, + }, + }; + Some(sum.at(time)) + } +} + +/// `message.usage`, read into the four counters. `None` when the record carries +/// no usage object at all. +fn read_usage(v: &Value) -> Option { + let usage = v.get("message")?.get("usage")?; + let t = Tokens { + input: count(usage, "input_tokens"), + output: count(usage, "output_tokens"), + cache_read: count(usage, "cache_read_input_tokens"), + cache_new: count(usage, "cache_creation_input_tokens"), + }; + // A `usage` that is there but says none of the four is still a record with + // no numbers, and it reads better as its own kind than as four dashes. + match t.any() { + true => Some(t), + false => None, + } +} + +/// One counter, when it is there and is a non-negative integer. +/// +/// Exact, through the number's own literal text: these are added up into a +/// session total, and a count that went through an `f64` would stop being the +/// number the file wrote somewhere past 2⁵³. A negative or fractional value is +/// not a token count and is refused rather than clamped — the record then shows +/// `-` in that cell, which is true. +pub(super) fn count(usage: &Value, key: &str) -> Option { + u64::try_from(usage.get(key)?.as_number()?.as_i64()?).ok() +} + +/// Conversation or mechanics. Only a human turn is conversation; see the module +/// docs for why the line is drawn exactly there. +fn class_of(v: &Value, kind: &str) -> Class { + match kind == "user" && !carries_results(v) { + true => Class::Message, + false => Class::Step, + } +} + +/// Is this `user` record the tool answering rather than a person typing? +fn carries_results(v: &Value) -> bool { + blocks(v).is_some_and(|bs| bs.iter().any(|b| block_type(b) == "tool_result")) +} + +fn blocks(v: &Value) -> Option<&[Value]> { + v.get("message")?.get("content")?.as_array() +} + +fn block_type(b: &Value) -> &str { + b.get("type").and_then(|t| t.as_str()).unwrap_or("") +} + +fn call_blocks(v: &Value) -> impl Iterator { + blocks(v) + .unwrap_or(&[]) + .iter() + .filter(|b| block_type(b) == "tool_use") +} + +/// What the record did, after the numbers: its tool calls, or its own kind when +/// it made none. The numbers come first because they are the column being +/// scanned; the action is what tells one row of them from another. +fn action(v: &Value, kind: &str) -> String { + let calls: Vec = call_blocks(v).map(call_text).collect(); + match calls.is_empty() { + true => kind.to_string(), + false => usage::collapse(calls).join(" \u{b7} "), + } +} + +/// `Bash(cargo test)`, through the same argument budget every dialect uses. +fn call_text(block: &Value) -> String { + let name = block.get("name").and_then(|n| n.as_str()).unwrap_or("tool"); + match block.get("input").and_then(super::agent::tool_arg) { + Some(arg) => format!("{name}({arg})"), + None => name.to_string(), + } +} + +/// Who the row is attributed to, with a subagent's own conversation marked. +/// +/// Three names and no more — `user`, `assistant`, `system` — because the actor +/// field is ten columns and a record whose `type` is `file-history-snapshot` +/// would push the numbers of every row it sits between. The `type` is not lost: +/// on a record with no usage it *is* the row. +fn speaker(v: &Value, kind: &str) -> (Who, String) { + let role = v + .get("message") + .and_then(|m| m.get("role")) + .and_then(|r| r.as_str()) + .unwrap_or(kind); + let (who, name) = match role { + "user" => (Who::User, "user"), + "assistant" => (Who::Assistant, "assistant"), + _ => (Who::System, "system"), + }; + let side = v.get("isSidechain").and_then(|s| s.as_bool()).unwrap_or(false); + let name = match side { + true => format!("{SIDE}{name}"), + false => name.to_string(), + }; + (who, name) +} + +#[cfg(test)] +#[path = "usage_agent_tests.rs"] +mod tests; diff --git a/src/lens/usage_agent_tests.rs b/src/lens/usage_agent_tests.rs new file mode 100644 index 0000000..1e146c4 --- /dev/null +++ b/src/lens/usage_agent_tests.rs @@ -0,0 +1,207 @@ +//! The `usage` dialect over hand-written, synthetic session records. +//! +//! Nothing here is copied from a real session log; every fixture is written by +//! hand to the shape the format documents. +#![deny(unsafe_code)] + +use super::*; +use crate::render::str_width; + +/// Columns the row's actor field is padded to (`lensrow::ACTOR`). Named here +/// because a subagent mark that outgrew it would shift every number on the row, +/// and this dialect's whole product is that the numbers line up. +const ACTOR: usize = 10; + +fn read(json: &str) -> Option { + let v = crate::json::parse(json.as_bytes()).expect("fixture parses"); + Usage.read(&v) +} + +fn sum(json: &str) -> Summary { + read(json).expect("the dialect reads this record") +} + +/// An assistant record with all four counters: the block, aligned, and a total +/// that is the exact sum. +#[test] +fn an_assistant_record_shows_the_four_counters_and_totals_them() { + let s = sum( + r#"{"type":"assistant","timestamp":"2026-08-05T21:31:00.000Z", + "message":{"role":"assistant","content":[ + {"type":"tool_use","id":"a","name":"Bash","input":{"command":"cargo test"}}], + "usage":{"input_tokens":1200,"output_tokens":380, + "cache_read_input_tokens":18000, + "cache_creation_input_tokens":2100}}}"#, + ); + assert_eq!(s.what, "in 1.2k out 380 read 18k new 2.1k \u{b7} Bash(cargo test)"); + assert_eq!(s.class, Class::Step, "only a human turn is conversation"); + assert_eq!(s.actor, "assistant"); + assert_eq!(s.time.as_deref(), Some("21:31")); + assert_eq!(s.tokens, 1200 + 380 + 18_000 + 2_100); + assert_eq!(s.calls, 1); + assert!(s.body.is_none(), "this lens shows no message text at all"); +} + +/// A turn that really did spend zero prints zeroes, not dashes. +#[test] +fn a_recorded_zero_is_still_a_row_of_numbers() { + let s = sum( + r#"{"type":"assistant","message":{"role":"assistant","content":[], + "usage":{"input_tokens":0,"output_tokens":0}}}"#, + ); + assert!(s.what.starts_with("in 0 out 0"), "{}", s.what); + assert_eq!(s.tokens, 0); + // The two counters this record did not write are dashes, not zeroes. + assert!(s.what.contains("read -"), "{}", s.what); + assert!(s.what.contains("new -"), "{}", s.what); +} + +/// No usage at all: the record's kind, verbatim, and no number columns anywhere. +#[test] +fn a_record_with_no_usage_shows_its_kind_and_nothing_more() { + for kind in ["file-history-snapshot", "queue-operation", "bridge-session", "agent-color"] { + let s = sum(&format!(r#"{{"type":"{kind}","messageId":"x"}}"#)); + assert_eq!(s.what, kind, "the kind verbatim"); + assert_eq!(s.tokens, 0); + for label in ["in", "out", "read", "new"] { + assert!(!s.what.contains(&format!("{label} ")), "{kind} shows a number column"); + } + assert_eq!(s.actor, "system", "and the actor field still fits"); + } +} + +/// A type this dialect has never seen prints its own name rather than being +/// swallowed — the seam's rule that a lens never hides anything. +#[test] +fn an_unknown_type_prints_its_own_name() { + let s = sum(r#"{"type":"some-future-thing"}"#); + assert_eq!(s.what, "some-future-thing"); +} + +/// The load-bearing decision: a human turn breaks the run, and a tool result +/// does not. If the second were a `Message` too, every run would shred into +/// pairs and no group row would total a turn. +#[test] +fn only_a_human_turn_is_conversation() { + let typed = sum(r#"{"type":"user","message":{"role":"user","content":"run the tests"}}"#); + assert_eq!(typed.class, Class::Message); + assert_eq!(typed.actor, "user"); + assert_eq!(typed.what, "user", "no numbers on a turn that recorded none"); + + let result = sum( + r#"{"type":"user","message":{"role":"user","content":[ + {"type":"tool_result","tool_use_id":"a","content":"ok"}]}}"#, + ); + assert_eq!(result.class, Class::Step, "a tool result is mechanics"); + + // A record with blocks but no result is still a person typing. + let blocks = sum( + r#"{"type":"user","message":{"role":"user","content":[ + {"type":"text","text":"and again"}]}}"#, + ); + assert_eq!(blocks.class, Class::Message); + + // Everything else is mechanics whatever it holds. + assert_eq!(sum(r#"{"type":"system","subtype":"turn_duration"}"#).class, Class::Step); +} + +/// A run is consecutive steps, so this is the item list the plan would build +/// from a whole turn: one message, its mechanics, then the next message. +#[test] +fn a_turn_is_one_message_then_a_run_of_mechanics() { + let records = [ + r#"{"type":"user","message":{"role":"user","content":"go"}}"#, + r#"{"type":"assistant","message":{"role":"assistant","content":[], + "usage":{"input_tokens":10,"output_tokens":5}}}"#, + r#"{"type":"user","message":{"role":"user","content":[ + {"type":"tool_result","tool_use_id":"a","content":"ok"}]}}"#, + r#"{"type":"assistant","message":{"role":"assistant","content":[], + "usage":{"input_tokens":20,"output_tokens":1}}}"#, + r#"{"type":"user","message":{"role":"user","content":"thanks"}}"#, + ]; + let classes: Vec = records.iter().map(|r| sum(r).class).collect(); + assert_eq!( + classes, + vec![Class::Message, Class::Step, Class::Step, Class::Step, Class::Message], + "one run of three between two turns, not three runs of one" + ); +} + +/// The subagent mark costs no alignment: `↳assistant` is exactly the ten +/// columns the actor field is wide, and `↳ assistant` (which `agent` uses) +/// would be eleven. More than half the records in a real session carry the flag. +#[test] +fn a_subagent_is_marked_without_costing_a_column() { + let s = sum( + r#"{"type":"assistant","isSidechain":true,"message":{"role":"assistant","content":[], + "usage":{"input_tokens":1,"output_tokens":1}}}"#, + ); + assert_eq!(s.actor, "\u{21b3}assistant"); + assert_eq!(str_width(&s.actor), ACTOR, "the mark must not push the numbers right"); + assert_eq!(s.who, Who::Assistant, "and it still paints as an assistant"); + + let side_user = sum(r#"{"type":"user","isSidechain":true,"message":{"role":"user","content":"x"}}"#); + assert_eq!(side_user.actor, "\u{21b3}user"); + assert!(str_width(&side_user.actor) <= ACTOR); + assert!(str_width(&sum(r#"{"type":"assistant"}"#).actor) <= ACTOR); +} + +/// `iterations` is a list whose elements repeat the outer counter names. +/// Summing both double-counts every token on the records that carry it, which is +/// the mistake this pins against. +#[test] +fn iterations_are_never_added_to_the_total() { + let s = sum( + r#"{"type":"assistant","message":{"role":"assistant","content":[], + "usage":{"input_tokens":100,"output_tokens":50, + "iterations":[{"input_tokens":60,"output_tokens":30}, + {"input_tokens":40,"output_tokens":20}]}}}"#, + ); + assert_eq!(s.tokens, 150, "the outer four once, the iterations not at all"); + assert!(s.what.starts_with("in 100 out 50"), "{}", s.what); +} + +/// Several calls of one kind read as one move with a count. +#[test] +fn several_calls_of_one_kind_collapse() { + let s = sum( + r#"{"type":"assistant","message":{"role":"assistant","content":[ + {"type":"tool_use","id":"a","name":"Read","input":{"file_path":"a.rs"}}, + {"type":"tool_use","id":"b","name":"Read","input":{"file_path":"a.rs"}}, + {"type":"tool_use","id":"c","name":"Read","input":{"file_path":"a.rs"}}], + "usage":{"input_tokens":1}}}"#, + ); + assert!(s.what.ends_with("Read(a.rs) \u{d7}3"), "{}", s.what); + assert_eq!(s.calls, 3); +} + +/// Not this dialect's record: it falls back to the generic tree rather than +/// being summarised wrongly or hidden. +#[test] +fn a_record_that_is_not_an_object_is_not_read() { + assert!(read("[1,2,3]").is_none()); + assert!(read(r#""just a string""#).is_none()); + assert!(read(r#"{"no":"type"}"#).is_none()); + assert!(read(r#"{"type":7}"#).is_none(), "a non-string type is not a type"); +} + +/// A `usage` object that is present but says none of the four is a record with +/// no numbers, and reads as its kind rather than as four dashes. +#[test] +fn a_usage_object_with_no_counters_is_not_a_row_of_dashes() { + let s = sum(r#"{"type":"assistant","message":{"role":"assistant","usage":{"service_tier":"standard"}}}"#); + assert_eq!(s.what, "assistant"); + assert_eq!(s.tokens, 0); +} + +/// A counter that is not a non-negative integer is refused rather than clamped: +/// the cell then says `-`, which is true. +#[test] +fn a_counter_that_is_not_a_count_is_refused() { + let s = sum( + r#"{"type":"assistant","message":{"role":"assistant","content":[], + "usage":{"input_tokens":-5,"output_tokens":"lots","cache_read_input_tokens":7}}}"#, + ); + assert_eq!(s.tokens, 7); + assert!(s.what.starts_with("in - out - read 7"), "{}", s.what); +} diff --git a/src/lens/usage_tests.rs b/src/lens/usage_tests.rs new file mode 100644 index 0000000..36c64f7 --- /dev/null +++ b/src/lens/usage_tests.rs @@ -0,0 +1,102 @@ +//! The numeric block: its width, its alignment, and the three different things +//! one cell can say. +//! +//! Every fixture here is hand-written and synthetic. +#![deny(unsafe_code)] + +use super::*; +use crate::render::str_width; + +fn four(input: u64, output: u64, read: u64, new: u64) -> Tokens { + Tokens { + input: Some(input), + output: Some(output), + cache_read: Some(read), + cache_new: Some(new), + } +} + +/// The block is a constant width whatever the numbers are — that is the whole +/// product, and a field that grew by one column would bend the column below it. +#[test] +fn the_numeric_block_is_the_same_width_for_every_record() { + let wide = four(1_234_567, 999_999, 18_999, 2_100); + let narrow = four(0, 0, 0, 0); + let block = |t: &Tokens| row_text(t, &Field::ALL, ""); + assert_eq!(str_width(&block(&wide)), 4 * FIELD + 3 * 2); + assert_eq!(str_width(&block(&wide)), 38, "four fields are 38 columns"); + assert_eq!(str_width(&block(&narrow)), 38); + // And three fields, for a format with no cache-creation counter. + let three = row_text(&wide, &[Field::In, Field::Out, Field::Read], ""); + assert_eq!(str_width(&three), 3 * FIELD + 2 * 2); + assert_eq!(str_width(&three), 28, "three fields are 28 columns"); +} + +/// Labels left, numbers right, so the digits line up under each other. +#[test] +fn a_field_is_its_label_then_its_number_right_aligned() { + let t = four(1_200, 380, 18_000, 2_100); + let row = row_text(&t, &Field::ALL, "Bash(cargo test)"); + assert_eq!(row, "in 1.2k out 380 read 18k new 2.1k \u{b7} Bash(cargo test)"); + // The action begins exactly one separator past the block. + assert!(row.starts_with("in 1.2k")); + assert_eq!(&row[..38], "in 1.2k out 380 read 18k new 2.1k"); +} + +/// A recorded zero is a fact about the session and is printed as one. Fourteen +/// records in the corpus this was measured against record exactly that. +#[test] +fn a_recorded_zero_is_shown_as_zero() { + let t = four(0, 0, 0, 0); + let row = row_text(&t, &Field::ALL, ""); + assert!(row.contains("in 0"), "{row}"); + assert!(!row.contains('-'), "a zero is not an absence: {row}"); +} + +/// A field this record did not record, inside a format that has it: `-`, never +/// `0`. The two are different claims and keeping them apart is why every +/// counter is an `Option`. +#[test] +fn a_field_this_record_did_not_record_is_a_dash() { + let t = Tokens { input: Some(500), output: None, cache_read: None, cache_new: Some(0) }; + let row = row_text(&t, &Field::ALL, ""); + assert_eq!(row, "in 500 out - read - new 0"); + assert_eq!(str_width(&row), 38, "an absence costs no alignment"); +} + +/// A field the *format* does not have never reaches a row at all — no dash, no +/// zero, no column. The counter may even be set; the field list is the answer. +#[test] +fn a_field_the_format_does_not_have_has_no_column() { + let t = four(500, 100, 20, 7); + let row = row_text(&t, &[Field::In, Field::Out, Field::Read], "bash"); + assert!(!row.contains("new"), "no cache-creation column: {row}"); + assert!(row.contains("read"), "{row}"); +} + +/// The total is the four counters, added once each. +#[test] +fn a_total_adds_the_four_counters_once() { + assert_eq!(four(1, 2, 4, 8).total(), 15); + assert_eq!(Tokens::default().total(), 0); + let partial = Tokens { input: Some(10), output: None, cache_read: Some(5), cache_new: None }; + assert_eq!(partial.total(), 15, "an unrecorded field adds nothing"); + // Saturating, so a corrupt file cannot panic a release build either. + let huge = four(u64::MAX, u64::MAX, 0, 0); + assert_eq!(huge.total(), u64::MAX); +} + +/// A record that recorded nothing gets no columns at all. +#[test] +fn a_record_with_no_counters_records_nothing() { + assert!(!Tokens::default().any()); + assert!(four(0, 0, 0, 0).any(), "a recorded zero is a recording"); +} + +#[test] +fn adjacent_repeats_collapse() { + let items = vec!["Read".to_string(), "Read".to_string(), "Read".to_string()]; + assert_eq!(collapse(items), vec!["Read \u{d7}3".to_string()]); + let mixed = vec!["Read".to_string(), "Bash".to_string(), "Read".to_string()]; + assert_eq!(collapse(mixed.clone()), mixed); +} From cc08fed206b39168b4739551a00811624955fe0c Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:45:16 +0000 Subject: [PATCH 03/12] lens: read an atif trajectory as what it spent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --lens usage-atif is the document twin of usage, over the same numeric block and a different set of counters — and the difference is visible on the row. ATIF-v1.7 records no cache-creation counter of any kind, so this dialect has three columns and never a fourth: a format-level absence removes the column for the whole file, where a record-level absence inside a format that has the field prints a dash. Reasoning is a subset of the completion count and llm_call_count counts models rather than tools, so neither reaches the total. Two registry entries over one shared module because records_at() returns a single answer, and a lens declaring Lines cannot legally be pointed at a document. --lens list now keeps its gutter outside the padding, since usage-atif fills the name field exactly. --- src/lens/atif.rs | 2 +- src/lens/mod.rs | 8 +- src/lens/usage_atif.rs | 192 +++++++++++++++++++++++++++++++++++ src/lens/usage_atif_tests.rs | 128 +++++++++++++++++++++++ 4 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 src/lens/usage_atif.rs create mode 100644 src/lens/usage_atif_tests.rs diff --git a/src/lens/atif.rs b/src/lens/atif.rs index 381fdc0..3f08b6a 100644 --- a/src/lens/atif.rs +++ b/src/lens/atif.rs @@ -313,7 +313,7 @@ fn calls_of(v: &Value) -> Vec { /// JSON-encoded string the wire format this schema descends from emits. A /// value that is not a string (a `todos` list, a numeric `limit`) names /// nothing, which is why the tool then shows its name alone. -fn arg_of(call: &Value) -> Option { +pub(super) fn arg_of(call: &Value) -> Option { let raw = call.get("arguments")?; let decoded; let args = match raw.as_str() { diff --git a/src/lens/mod.rs b/src/lens/mod.rs index fa1d921..6a08ed7 100644 --- a/src/lens/mod.rs +++ b/src/lens/mod.rs @@ -46,6 +46,7 @@ pub mod num; pub mod part; pub mod usage; pub mod usage_agent; +pub mod usage_atif; pub use num::tokens; pub use part::Part; @@ -349,6 +350,7 @@ const LENSES: &[(&str, Make)] = &[ (agent::NAME, || Box::new(agent::Agent::default())), (atif::NAME, || Box::new(atif::Atif)), (usage_agent::NAME, || Box::new(usage_agent::Usage)), + (usage_atif::NAME, || Box::new(usage_atif::UsageAtif)), ]; /// The lens called `name`, or `None`. @@ -378,10 +380,14 @@ pub fn names() -> Vec<&'static str> { /// `--lens list`: the available lenses, one per line. The description comes /// from the lens itself, so a dialect says what it is in one place. +/// +/// The gutter is *after* the padding rather than inside it: `usage-atif` is +/// exactly ten columns, and a name that filled the field would otherwise run +/// straight into its own description. pub fn list_text() -> String { let mut s = String::from("lenses (--lens ):\n"); for (name, make) in LENSES { - s.push_str(&format!(" {name:<10}{}\n", make().about())); + s.push_str(&format!(" {name:<10} {}\n", make().about())); } s.push_str("\nWithout --lens, records render as the generic JSON tree.\n"); s diff --git a/src/lens/usage_atif.rs b/src/lens/usage_atif.rs new file mode 100644 index 0000000..06585c3 --- /dev/null +++ b/src/lens/usage_atif.rs @@ -0,0 +1,192 @@ +//! The `usage-atif` lens: an ATIF trajectory read as what it *spent*. +//! +//! The document twin of `usage`, over the same [`usage`] vocabulary. Same +//! question, same numeric block, a different file shape and a different set of +//! counters — and the difference in the counters is visible on the row, which is +//! the point. +//! +//! # Two entries over one module, and why +//! +//! [`Lens::records_at`] returns **one** answer, and `src/open/lens.rs` refuses a +//! record-per-line lens on a document and a document lens on a `.jsonl`. So one +//! name cannot legally cover both shapes: `usage` declares `Lines` and this +//! declares `Member("steps")`. Everything that is not that declaration is shared. +//! +//! # Where the numbers are +//! +//! `steps[].metrics`, not `message.usage`: +//! +//! | key | column | +//! | --- | --- | +//! | `prompt_tokens` | `in` | +//! | `completion_tokens` | `out` | +//! | `cached_tokens` | `read` | +//! +//! And **there is no fourth**. ATIF-v1.7 records no cache-*creation* counter of +//! any kind, so this dialect's field set is three and a `usage-atif` row never +//! shows a `new` column at all — not a `-`, which would say this record failed +//! to record something the format has, and above all not a `0`, which would say +//! the agent wrote nothing to cache when the truth is that the format does not +//! record cache writes. A format-level absence removes the column for the whole +//! file; a record-level absence inside a format that has the field prints `-`. +//! +//! `metrics.extra.reasoning_tokens` is a subset of `completion_tokens` and is +//! never added to the total; `llm_call_count` is a count of calls to a model and +//! not a count of tokens, so it is not a column either. Neither is lost — both +//! are on the open level. +//! +//! There is no `isSidechain` here and nothing that means it, so no row is +//! marked as a subagent. Inventing a mark would be a guess. +#![deny(unsafe_code)] + +use super::atif::STEPS; +use super::usage::{self, Field, Tokens}; +use super::usage_agent::count; +use super::{record_clock, Class, Lens, RecordsAt, Summary, Who}; +use crate::json::Value; + +pub const NAME: &str = "usage-atif"; +pub const ABOUT: &str = "ATIF agent trajectories: what each step spent, not what it said"; + +/// The three columns this format has. There is no cache-creation counter in +/// ATIF-v1.7, so there is no fourth column — see the module docs. +const FIELDS: [Field; 3] = [Field::In, Field::Out, Field::Read]; + +#[derive(Default)] +pub struct UsageAtif; + +impl Lens for UsageAtif { + fn name(&self) -> &'static str { + NAME + } + + fn about(&self) -> &'static str { + ABOUT + } + + fn records_at(&self) -> RecordsAt { + RecordsAt::Member(STEPS) + } + + fn read(&mut self, v: &Value) -> Option { + match v.get("step_id") { + Some(_) => step(v), + None => session(v), + } + } +} + +/// One step of the trajectory. +fn step(v: &Value) -> Option { + // There is no `type` field on an ATIF step; `source` is the discriminator, + // and it is also the turn boundary. + let source = v.get("source").and_then(|s| s.as_str()).unwrap_or("agent"); + let (who, actor, class) = match source { + "user" => (Who::User, "user", Class::Message), + _ => (Who::Assistant, "agent", Class::Step), + }; + let calls = calls_of(v); + let metrics = read_metrics(v); + let what = match &metrics { + Some(t) => usage::row_text(t, &FIELDS, &action(&calls, source)), + // No metrics: the kind, and nothing more. No dashes and no zeroes, + // because the step recorded nothing at all. + None => source.to_string(), + }; + let tokens = metrics.map(|t| t.total()).unwrap_or(0); + Some( + Summary { + class, + who, + actor: actor.to_string(), + time: None, + what, + calls: calls.len(), + tokens, + // What was said is `--lens atif`'s question. + body: None, + } + .at(record_clock(v)), + ) +} + +/// Record 0: the document's own keys, which the source keeps so this row can +/// exist. It names the agent and never a total — ATIF records no document-level +/// counters, and a total invented here would be one the file does not contain. +/// +/// A [`Class::Message`], so it never folds into the run that follows it. +fn session(v: &Value) -> Option { + let schema = text(v, "schema_version"); + let id = text(v, "session_id"); + if schema.is_none() && id.is_none() { + return None; + } + let mut parts: Vec = Vec::new(); + if let Some(a) = v.get("agent") { + parts.push(text(a, "name").unwrap_or_else(|| "agent".to_string())); + parts.extend(text(a, "model_name")); + } + parts.extend(schema); + parts.extend(id); + Some(Summary { + class: Class::Message, + who: Who::System, + actor: "session".to_string(), + time: None, + what: parts.join(" \u{b7} "), + calls: 0, + tokens: 0, + body: None, + }) +} + +/// `steps[].metrics`, read into the three counters this format has. +fn read_metrics(v: &Value) -> Option { + let m = v.get("metrics")?; + let t = Tokens { + input: count(m, "prompt_tokens"), + output: count(m, "completion_tokens"), + cache_read: count(m, "cached_tokens"), + // ATIF-v1.7 has no cache-creation counter, and `FIELDS` gives it no + // column, so this is never read. It is spelled out rather than left + // implicit so the next reader does not go looking for the field. + cache_new: None, + }; + match t.any() { + true => Some(t), + false => None, + } +} + +/// What the step did, after the numbers: its tool calls, or its own kind. +fn action(calls: &[String], source: &str) -> String { + match calls.is_empty() { + true => source.to_string(), + false => usage::collapse(calls.to_vec()).join(" \u{b7} "), + } +} + +/// The tool calls of one step, as the row names them. +fn calls_of(v: &Value) -> Vec { + let Some(items) = v.get("tool_calls").and_then(|c| c.as_array()) else { + return Vec::new(); + }; + items.iter().map(call_text).collect() +} + +/// `bash(cargo test)`, through the same argument budget every dialect uses. +fn call_text(call: &Value) -> String { + let name = text(call, "function_name").unwrap_or_else(|| "tool".to_string()); + match super::atif::arg_of(call) { + Some(arg) => format!("{name}({arg})"), + None => name, + } +} + +fn text(v: &Value, key: &str) -> Option { + Some(v.get(key)?.as_str()?.to_string()) +} + +#[cfg(test)] +#[path = "usage_atif_tests.rs"] +mod tests; diff --git a/src/lens/usage_atif_tests.rs b/src/lens/usage_atif_tests.rs new file mode 100644 index 0000000..58b4440 --- /dev/null +++ b/src/lens/usage_atif_tests.rs @@ -0,0 +1,128 @@ +//! The `usage-atif` dialect over hand-written, synthetic ATIF-shaped steps. +//! +//! Nothing here is copied from a real trajectory; every fixture is written by +//! hand to the shape the schema documents. +#![deny(unsafe_code)] + +use super::*; +use crate::render::str_width; + +fn read(json: &str) -> Option { + let v = crate::json::parse(json.as_bytes()).expect("fixture parses"); + UsageAtif.read(&v) +} + +fn sum(json: &str) -> Summary { + read(json).expect("the dialect reads this record") +} + +/// Three fields, twenty-eight columns, and no fourth. +#[test] +fn a_step_shows_the_three_counters_this_format_has() { + let s = sum( + r#"{"step_id":4,"source":"agent","timestamp":"2026-08-05T21:31:00+00:00", + "metrics":{"prompt_tokens":1200,"completion_tokens":380,"cached_tokens":18000}, + "tool_calls":[{"function_name":"bash","tool_call_id":"c1", + "arguments":{"command":"cargo test"}}]}"#, + ); + assert_eq!(s.what, "in 1.2k out 380 read 18k \u{b7} bash(cargo test)"); + assert_eq!(&s.what[..28], "in 1.2k out 380 read 18k"); + assert_eq!(str_width(&s.what[..28]), 28); + assert_eq!(s.class, Class::Step); + assert_eq!(s.actor, "agent"); + assert_eq!(s.time.as_deref(), Some("21:31")); + assert_eq!(s.tokens, 1200 + 380 + 18_000); + assert_eq!(s.calls, 1); + assert!(s.body.is_none()); +} + +/// The format has no cache-creation counter, so no row of this file ever shows +/// a `new` column — not a `-`, and above all not a `0`, which would claim the +/// agent wrote nothing to cache when the truth is that ATIF does not record it. +#[test] +fn no_step_ever_shows_a_cache_creation_column() { + for fixture in [ + r#"{"step_id":1,"source":"agent","metrics":{"prompt_tokens":10}}"#, + r#"{"step_id":2,"source":"agent","metrics":{"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0}}"#, + r#"{"step_id":3,"source":"agent","metrics":{"cached_tokens":1823744}}"#, + ] { + let s = sum(fixture); + assert!(!s.what.contains("new"), "{}", s.what); + } +} + +/// A step with no metrics at all: its kind, and nothing more. +#[test] +fn a_step_with_no_metrics_shows_its_kind_and_nothing_more() { + let s = sum(r#"{"step_id":7,"source":"agent","message":"","tool_calls":[]}"#); + assert_eq!(s.what, "agent"); + assert_eq!(s.tokens, 0); + // An empty `metrics` object is the same thing: nothing was recorded. + let empty = sum(r#"{"step_id":8,"source":"agent","metrics":{}}"#); + assert_eq!(empty.what, "agent"); +} + +/// `source` is the discriminator and the turn boundary: a user step breaks the +/// run, an agent step joins it. +#[test] +fn source_decides_whether_a_step_breaks_the_run() { + assert_eq!(sum(r#"{"step_id":0,"source":"user"}"#).class, Class::Message); + assert_eq!(sum(r#"{"step_id":1,"source":"agent"}"#).class, Class::Step); + // An unrecognised source reads as the agent rather than vanishing. + assert_eq!(sum(r#"{"step_id":2,"source":"tool"}"#).class, Class::Step); +} + +/// Record 0 names the agent and carries no numbers: ATIF records no +/// document-level total, and a total invented here would be one the file does +/// not contain. +#[test] +fn the_session_record_names_the_agent_and_totals_nothing() { + let s = sum( + r#"{"schema_version":"ATIF-v1.7","session_id":"s-1", + "agent":{"name":"someagent","model_name":"a-model-id"}}"#, + ); + assert_eq!(s.class, Class::Message, "the envelope never folds away"); + assert_eq!(s.actor, "session"); + assert_eq!(s.what, "someagent \u{b7} a-model-id \u{b7} ATIF-v1.7 \u{b7} s-1"); + assert_eq!(s.tokens, 0); + assert_eq!(s.calls, 0); + for label in ["in ", "out ", "read"] { + assert!(!s.what.contains(label), "{}", s.what); + } +} + +/// Not an ATIF envelope and not a step: the generic tree, with nothing hidden. +#[test] +fn a_record_that_is_neither_a_step_nor_an_envelope_is_not_read() { + assert!(read(r#"{"unrelated":true}"#).is_none()); + assert!(read("[1,2]").is_none()); + assert!(read("42").is_none()); +} + +/// The call count is the calls, and it is what the group row's `· N tool calls` +/// totals. `llm_call_count` is a count of model calls, not of tools, and is not +/// it. +#[test] +fn the_call_count_is_the_tool_calls() { + let s = sum( + r#"{"step_id":9,"source":"agent","llm_call_count":11, + "metrics":{"prompt_tokens":5}, + "tool_calls":[{"function_name":"read","arguments":{"filePath":"a.rs"}}, + {"function_name":"read","arguments":{"filePath":"a.rs"}}, + {"function_name":"grep","arguments":{"pattern":"fn"}}]}"#, + ); + assert_eq!(s.calls, 3); + assert!(s.what.ends_with("read(a.rs) \u{d7}2 \u{b7} grep(fn)"), "{}", s.what); +} + +/// Reasoning is a subset of `completion_tokens`; adding it would count the same +/// token twice. It is not lost — it is on the open level. +#[test] +fn reasoning_is_not_added_to_the_total() { + let s = sum( + r#"{"step_id":10,"source":"agent", + "metrics":{"prompt_tokens":100,"completion_tokens":50, + "extra":{"reasoning_tokens":40}}}"#, + ); + assert_eq!(s.tokens, 150, "not 190"); +} From 8b47ccb1359a1383b6a554e8db4be3ef2e2624fa Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:49:49 +0000 Subject: [PATCH 04/12] record: say in the status bar what has been counted so far MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A whole-file token total belongs where the seam allows it rather than where a dialect wishes: a lens may not decide a row, and the record seam has no document-level row to hang a footer on, so it goes in the one per-document field a format already owns. The plan keeps the total running as records are classified — one add per record, nothing per frame, and classification runs once per record, so opening a folded run cannot make the total jump. It carries the same lower-bound mark the record count beside it does, because classification only reaches as far as the reader has scrolled. The clause is omitted at zero, so no shipped lens's status bar moves. --- src/source/jsonl/mod.rs | 4 ++ src/source/jsonl/usage_tests.rs | 93 +++++++++++++++++++++++++++++++ src/source/record/lensstate.rs | 26 +++++++++ src/source/record/plan.rs | 22 ++++++++ src/source/record/plan_fixture.rs | 17 +++++- src/source/record/plan_tests.rs | 32 +++++++++++ src/source/record/view.rs | 4 +- 7 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 src/source/jsonl/usage_tests.rs diff --git a/src/source/jsonl/mod.rs b/src/source/jsonl/mod.rs index 7c42b28..63aabde 100644 --- a/src/source/jsonl/mod.rs +++ b/src/source/jsonl/mod.rs @@ -31,6 +31,10 @@ mod lensrow_tests; #[path = "block_tests.rs"] mod block_tests; +#[cfg(test)] +#[path = "usage_tests.rs"] +mod usage_tests; + use std::cell::RefCell; use std::io; use std::path::Path; diff --git a/src/source/jsonl/usage_tests.rs b/src/source/jsonl/usage_tests.rs new file mode 100644 index 0000000..a99a4c5 --- /dev/null +++ b/src/source/jsonl/usage_tests.rs @@ -0,0 +1,93 @@ +//! A record file read through a lens that counts, end to end: the status bar's +//! session total, and the rows the two usage dialects paint over a real source. +//! +//! Every fixture is hand-written and synthetic — a real session log is never +//! copied into this repository. +#![deny(unsafe_code)] + +use super::*; +use crate::source::Source; + +/// Two turns: a prompt, an answer that spent, a tool result, an answer that +/// spent, another prompt. +const SPEND: &str = concat!( + r#"{"type":"user","timestamp":"2026-08-05T14:01:00.000Z","message":{"role":"user","content":"go"}}"#, + "\n", + r#"{"type":"assistant","timestamp":"2026-08-05T14:02:00.000Z","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"cargo test"}}],"usage":{"input_tokens":1200,"output_tokens":380,"cache_read_input_tokens":18000,"cache_creation_input_tokens":2100}}}"#, + "\n", + r#"{"type":"user","timestamp":"2026-08-05T14:02:01.000Z","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"ok"}]}}"#, + "\n", + r#"{"type":"assistant","timestamp":"2026-08-05T14:03:00.000Z","message":{"role":"assistant","content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":800,"output_tokens":20}}}"#, + "\n", + r#"{"type":"user","timestamp":"2026-08-05T14:04:00.000Z","message":{"role":"user","content":"thanks"}}"#, + "\n", +); + +fn lensed(text: &str, lens: &str) -> JsonlSource { + let mut s = JsonlSource::from_bytes(text.as_bytes().to_vec()); + s.set_lens(crate::lens::find(lens).expect("a registered lens")); + s.set_width(200); + while s.extend() {} + s +} + +fn rows(s: &mut JsonlSource) -> Vec { + let n = s.len(); + s.lines(0..n).iter().map(|l| l.text().trim_end().to_string()).collect() +} + +/// The status bar carries what has been counted so far, and once the whole file +/// is classified it is the session total with no qualification. +#[test] +fn the_status_bar_carries_the_session_total() { + let mut s = lensed(SPEND, "usage"); + let _ = rows(&mut s); + let text = s.position_text(0).expect("a position"); + let total = 1200 + 380 + 18_000 + 2_100 + 800 + 20; + assert_eq!(crate::lens::tokens(total), "22k"); + assert!(text.contains(" \u{b7} 22k tokens"), "{text}"); + assert!(!text.contains('\u{2265}'), "the file is fully classified: {text}"); +} + +/// A lens that reads no counters says nothing about tokens, so the status line +/// of every shipped lens is exactly what it always was. +#[test] +fn a_lens_that_counts_nothing_says_nothing_about_tokens() { + let mut s = lensed(SPEND, "agent"); + let _ = rows(&mut s); + let text = s.position_text(0).expect("a position"); + assert!(!text.contains("tokens"), "{text}"); + assert!(text.starts_with("agent \u{b7} record 1/5"), "{text}"); +} + +/// The rows themselves, over a real source: the numbers line up under each +/// other, and the group row totals the turn the run belongs to. +#[test] +fn the_numbers_line_up_and_the_run_totals_the_turn() { + let mut s = lensed(SPEND, "usage"); + let got = rows(&mut s); + // A prompt, one folded run of three mechanics, and the next prompt. + assert_eq!(got.len(), 3, "{got:#?}"); + assert!(got[0].contains("user 14:01 user"), "{:?}", got[0]); + assert!( + got[1].contains("\u{27e8}3 steps \u{b7} 1 tool call \u{b7} 22k tokens\u{27e9}"), + "{:?}", + got[1] + ); + assert!(got[2].contains("user 14:04 user"), "{:?}", got[2]); +} + +/// The numeric block is at the same column on every row of the file, which is +/// the whole product of this lens. +#[test] +fn every_usage_row_puts_its_numbers_in_the_same_column() { + let mut s = lensed(SPEND, "usage"); + let _ = rows(&mut s); + // Open the run so its members get rows of their own. + let id = s.section_at(1).expect("the run's fold id"); + s.set_fold(id, false); + let got = rows(&mut s); + let at: Vec = got.iter().filter_map(|r| r.find("in ")).collect(); + assert_eq!(at.len(), 2, "two records recorded numbers: {got:#?}"); + assert_eq!(at[0], at[1], "the column bends: {got:#?}"); +} diff --git a/src/source/record/lensstate.rs b/src/source/record/lensstate.rs index 35f2b2e..758f095 100644 --- a/src/source/record/lensstate.rs +++ b/src/source/record/lensstate.rs @@ -183,6 +183,32 @@ impl RecordSource { format!(" \u{b7} block {}/{total}", i.saturating_add(1)) } + /// The status bar's token clause — ` · ≥1.2M tokens` — or nothing at all. + /// + /// A whole-file total belongs in the one per-document field a *format* owns + /// ([`crate::source::Source::position_text`]), because a lens may not decide + /// a row and the record seam has no document-level row to hang a footer on. + /// It is dialect-agnostic: any lens that fills [`crate::lens::Summary`]'s + /// token count gets it, and a lens that fills none gets an empty string, so + /// no shipped lens's status bar changes. + /// + /// The `≥` is not cosmetic. Classification runs only as far as the reader + /// has scrolled, so on a multi-megabyte log an unqualified total would be + /// wrong by most of the file — exactly the qualification the record count + /// beside it already carries. + pub(crate) fn tokens_text(&self) -> String { + let classified = self.plan.as_ref().map(|p| p.classified()).unwrap_or(0); + let total = self.plan.as_ref().map(|p| p.tokens_seen()).unwrap_or(0); + if total == 0 { + return String::new(); + } + let mark = match self.complete() && classified >= self.known() { + true => "", + false => "\u{2265}", + }; + format!(" \u{b7} {mark}{} tokens", crate::lens::tokens(total)) + } + /// `Y` on a group's row: every record the run holds. pub(crate) fn yank_group(&self, item: usize) -> Option { ops::yank_group(self, self.plan.as_ref(), item) diff --git a/src/source/record/plan.rs b/src/source/record/plan.rs index 9b094e2..010eb21 100644 --- a/src/source/record/plan.rs +++ b/src/source/record/plan.rs @@ -153,6 +153,18 @@ pub struct Plan { /// Every record is at the open level: what a dump is, and what `zR` leaves /// behind. all_full: bool, + /// Every token every *classified* record recorded, added as it was + /// classified. + /// + /// A running total and not a sweep: it costs one add per record and nothing + /// per frame, and `classify` runs exactly once per record — a record + /// swallowed into a closed group is counted there and then, and opening the + /// group later does not classify it again, so the total cannot double. + /// + /// It is over the records the reader has reached, which is not the file: + /// classification is lazy, so [`crate::source::record::view`] prints it with + /// the same `\u{2265}` the record count beside it already carries. + tokens: u64, } /// The rows one record shows under its own summary row. @@ -186,6 +198,7 @@ impl Plan { dirty: 0, width: 80, all_full: false, + tokens: 0, } } @@ -198,6 +211,12 @@ impl Plan { self.seen.len() } + /// Every token the classified records recorded. `0` under a lens that reads + /// no counters, and the status bar then says nothing about tokens at all. + pub fn tokens_seen(&self) -> u64 { + self.tokens + } + /// Read the next record — it must be record [`Plan::classified`] — and fold /// it into the item list. /// @@ -209,6 +228,9 @@ impl Plan { return; } let sum = value.and_then(|v| self.lens.read(v)); + if let Some(s) = &sum { + self.tokens = self.tokens.saturating_add(s.tokens); + } let step = matches!(sum, Some(Summary { class: crate::lens::Class::Step, .. })); self.seen.push(sum); self.under.push(Under::default()); diff --git a/src/source/record/plan_fixture.rs b/src/source/record/plan_fixture.rs index 2fdd2aa..045f1aa 100644 --- a/src/source/record/plan_fixture.rs +++ b/src/source/record/plan_fixture.rs @@ -36,7 +36,9 @@ impl crate::lens::Lens for Fake { time: None, what: String::new(), calls: 1, - tokens: 0, + // `"t"` is what this record claims to have spent, so the seam's + // running total can be exercised without a dialect. + tokens: v.get("t").and_then(|t| t.as_number()).and_then(|n| n.as_i64()).unwrap_or(0) as u64, // A body is the source's to measure; these tests give one to an // item directly (`with_bodies`), which is the same number the // measurement would have produced and keeps this file file-free. @@ -59,6 +61,19 @@ pub(super) fn plan_of(kinds: &str) -> (Plan, RowMap) { (plan, map) } +/// The same, with every record claiming to have spent `each` tokens. +pub(super) fn plan_of_spend(kinds: &str, each: u64) -> (Plan, RowMap) { + let mut plan = Plan::new(Box::new(Fake)); + let mut map = RowMap::default(); + for (i, ch) in kinds.chars().enumerate() { + let json = format!(r#"{{"k":"{ch}","t":{each}}}"#); + let value = crate::json::parse(json.as_bytes()).expect("fixture"); + plan.classify(i, Some(&value), &mut map); + } + plan.sync(); + (plan, map) +} + /// Every row of the document, as `(record, sub)` or `group N`. pub(super) fn walk(plan: &Plan, map: &RowMap, known: usize) -> Vec { (0..plan.rows(known, map)) diff --git a/src/source/record/plan_tests.rs b/src/source/record/plan_tests.rs index a31ff11..03b4ede 100644 --- a/src/source/record/plan_tests.rs +++ b/src/source/record/plan_tests.rs @@ -362,3 +362,35 @@ fn a_width_change_re_lays_every_body() { assert_eq!(plan.width(), 40); } +/// The running total the status bar prints: every classified record counted +/// exactly once, however the plan later reshapes the document around it. +/// +/// The failure this guards against is a double count. Records swallowed into a +/// closed run are classified there and then, and opening that run later must +/// not classify them again — if it did, a reader who opened a group would watch +/// the session total jump. +#[test] +fn the_token_total_counts_each_record_exactly_once() { + let (mut plan, mut map) = plan_of_spend("msssm", 100); + assert_eq!(plan.classified(), 5); + assert_eq!(plan.tokens_seen(), 500, "five records at a hundred each"); + // Three of those were swallowed into one group. Opening it changes the rows + // and must not change the total. + let group = plan + .items() + .iter() + .position(|it| it.is_group()) + .expect("a run of three steps"); + assert!(plan.set_open(group, true, &mut map)); + plan.sync(); + assert_eq!(plan.tokens_seen(), 500, "opening a run re-counts nothing"); +} + +/// A lens that reads no counters totals nothing, so the status bar says nothing +/// about tokens and no shipped lens's status line moves. +#[test] +fn a_plan_over_records_that_spent_nothing_totals_zero() { + let (plan, _) = plan_of("msssm"); + assert_eq!(plan.tokens_seen(), 0); +} + diff --git a/src/source/record/view.rs b/src/source/record/view.rs index e1208b0..5438a20 100644 --- a/src/source/record/view.rs +++ b/src/source/record/view.rs @@ -96,9 +96,9 @@ impl Source for RecordSource { }; let head = match self.lens_name() { Some(lens) => format!( - "{lens} \u{b7} record {}/{total}{}", + "{lens} \u{b7} record {}/{total}{}{}", record.saturating_add(1), - self.block_text(row) + self.block_text(row), self.tokens_text() ), None => format!("record {}/{total}", record.saturating_add(1)), }; From 154cfb9ff526d6f11d3ab1c983dfe47c94f2f6d7 Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:52:42 +0000 Subject: [PATCH 05/12] lens: open a usage row into the exact numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four floored columns are what a row can honestly carry, so the level under it is where they become the integers the file wrote, and where everything the row had no column for lands: the cache-creation breakdown, the thinking count that is a subset of the output, the model the numbers were spent on, and the build that wrote the record. Built from text parts alone — this lens is not re-telling the conversation. A service tier or a speed is shown only when it is not the standard one, because a column for the ordinary case would be noise and the exceptions are what a reader opened the row to find. An ATIF step opens into no cache-creation number under any circumstance, since there is none to show. --- src/lens/usage.rs | 43 +++++++++++++++- src/lens/usage_agent.rs | 94 +++++++++++++++++++++++++++++++++- src/lens/usage_agent_tests.rs | 95 +++++++++++++++++++++++++++++++++++ src/lens/usage_atif.rs | 38 +++++++++++++- src/lens/usage_atif_tests.rs | 57 +++++++++++++++++++++ 5 files changed, 324 insertions(+), 3 deletions(-) diff --git a/src/lens/usage.rs b/src/lens/usage.rs index cf982d7..4309c05 100644 --- a/src/lens/usage.rs +++ b/src/lens/usage.rs @@ -36,7 +36,8 @@ //! pans across it rather than losing the action off the end. #![deny(unsafe_code)] -use crate::lens::tokens; +use crate::json::Value; +use crate::lens::{tokens, Body, Part}; /// Columns a field's label gets, left-justified. const LABEL: usize = 4; @@ -201,6 +202,46 @@ pub fn collapse(items: Vec) -> Vec { .collect() } +// -- the open level ------------------------------------------------------------- +// +// Where the row's floored four columns become exact, and where everything the +// row had no column for lands. Nothing a record holds is unreachable, which is +// the seam's rule — and this level is built from `Part::Text` alone, because +// this lens is not re-telling the conversation. + +/// One `name value` line of an open level, aligned so a column of numbers +/// still reads as a column when it is exact. +pub fn line(name: &str, value: impl std::fmt::Display) -> String { + format!("{name:<28}{value}") +} + +/// A `name value` line for an integer field, when the record has one. +/// +/// The **exact** integer, not [`tokens`]: the row above is floored to four +/// columns and this is the rung a reader descends to when they need the real +/// number — `1999` on the row is `1.9k`, and here it is `1999`. +pub fn exact(v: &Value, key: &str) -> Option { + Some(line(key, v.get(key)?.as_number()?.as_i64()?)) +} + +/// A `name value` line for a string field, when the record has one. +pub fn named(v: &Value, key: &str) -> Option { + Some(line(key, v.get(key)?.as_str()?)) +} + +/// The lines gathered under `label` as one part, or nothing when there were +/// none — an empty part is a row that says a name and shows nothing under it. +/// +/// The body has an **empty path**: it is a summary this module composed, not a +/// string node of the record, so there is nothing to walk back to. Every byte +/// of it is short, and the record's own tree is one `r` away regardless. +pub fn part(label: &'static str, lines: Vec) -> Option { + match lines.is_empty() { + true => None, + false => Some(Part::Text { label, body: Body::new(&lines.join("\n"), Vec::new()) }), + } +} + #[cfg(test)] #[path = "usage_tests.rs"] mod tests; diff --git a/src/lens/usage_agent.rs b/src/lens/usage_agent.rs index 4b4cf06..f312c4c 100644 --- a/src/lens/usage_agent.rs +++ b/src/lens/usage_agent.rs @@ -52,7 +52,7 @@ #![deny(unsafe_code)] use super::usage::{self, Field, Tokens}; -use super::{record_clock, record_type, Class, Lens, Summary, Who}; +use super::{record_clock, record_type, Class, Lens, Part, Summary, Who}; use crate::json::Value; pub const NAME: &str = "usage"; @@ -115,6 +115,98 @@ impl Lens for Usage { }; Some(sum.at(time)) } + + /// The exact numbers, and everything the row had no column for. + /// + /// Four columns of floored numbers are what a row can honestly carry; this + /// is where they become the integers the file wrote, and where the fields a + /// column would have been 99.96% noise for finally appear — a `service_tier` + /// or a `speed` that is *not* `standard` is exactly the anomaly a reader + /// opened the row to find, and one that is `standard` says nothing and is + /// not shown. + /// + /// `Part::Text` only: this lens is not re-telling the conversation, so a + /// call gets no part of its own. `&self` and nothing stored — a `Vec` + /// is per keystroke. + fn detail(&self, v: &Value) -> Vec { + let mut out: Vec = Vec::new(); + if let Some(u) = v.get("message").and_then(|m| m.get("usage")) { + out.extend(usage::part("tokens", token_lines(u))); + out.extend(usage::part("request", request_lines(u))); + } + out.extend(usage::part("model", model_lines(v))); + out.extend(usage::part("envelope", envelope_lines(v))); + out + } +} + +/// Every counter the record wrote, exact — including the two the row has no +/// column for, and the cache-creation breakdown when the record splits it. +fn token_lines(u: &Value) -> Vec { + let mut lines: Vec = Vec::new(); + for key in [ + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + ] { + lines.extend(usage::exact(u, key)); + } + if let Some(c) = u.get("cache_creation") { + for key in ["ephemeral_5m_input_tokens", "ephemeral_1h_input_tokens"] { + lines.extend(usage::exact(c, key)); + } + } + // A subset of `output_tokens`, which is why it gets no column and is never + // added to the total. + if let Some(d) = u.get("output_tokens_details") { + lines.extend(usage::exact(d, "thinking_tokens")); + } + lines +} + +/// How the request itself went, when it went in a way worth saying. +fn request_lines(u: &Value) -> Vec { + let mut lines: Vec = Vec::new(); + for key in ["service_tier", "speed"] { + if let Some(text) = u.get(key).and_then(|t| t.as_str()) { + // Nearly every record says `standard`, so saying it again is noise; + // anything else is the reason a reader is here. + if text != "standard" { + lines.push(usage::line(key, text)); + } + } + } + // A list whose elements repeat the outer counters per attempt. Its *length* + // is the fact — the request was retried — and its contents are never summed + // (`usage::Tokens::total` says why). + if let Some(n) = u.get("iterations").and_then(|i| i.as_array()).map(|a| a.len()) { + if n > 1 { + lines.push(usage::line("iterations", n)); + } + } + lines +} + +/// Which model the numbers were spent on. Without it a count is a number with +/// no price attached to it. +fn model_lines(v: &Value) -> Vec { + let Some(m) = v.get("message") else { + return Vec::new(); + }; + ["model", "id", "stop_reason"] + .into_iter() + .filter_map(|key| usage::named(m, key)) + .collect() +} + +/// What wrote the record. `version` above all: it is the schema-drift signal, so +/// a row whose numbers look wrong can be checked against the build that wrote it. +fn envelope_lines(v: &Value) -> Vec { + ["requestId", "sessionId", "gitBranch", "version"] + .into_iter() + .filter_map(|key| usage::named(v, key)) + .collect() } /// `message.usage`, read into the four counters. `None` when the record carries diff --git a/src/lens/usage_agent_tests.rs b/src/lens/usage_agent_tests.rs index 1e146c4..ea4ed45 100644 --- a/src/lens/usage_agent_tests.rs +++ b/src/lens/usage_agent_tests.rs @@ -205,3 +205,98 @@ fn a_counter_that_is_not_a_count_is_refused() { assert_eq!(s.tokens, 7); assert!(s.what.starts_with("in - out - read 7"), "{}", s.what); } + +// -- the open level -------------------------------------------------------------- + +/// Every part's label and text, for asking what the open level says. +fn detail_text(json: &str) -> Vec<(&'static str, String)> { + let v = crate::json::parse(json.as_bytes()).expect("fixture parses"); + Usage + .detail(&v) + .into_iter() + .map(|p| match p { + Part::Text { label, body } => (label, body.head), + Part::Call { tool, .. } => panic!("this lens builds no call parts, got {tool}"), + }) + .collect() +} + +fn part_of<'a>(parts: &'a [(&'static str, String)], label: &str) -> &'a str { + &parts.iter().find(|(l, _)| *l == label).unwrap_or_else(|| panic!("no {label} part")).1 +} + +/// The row floors to four columns; the level under it is the integer the file +/// wrote. That is the whole bargain of flooring. +#[test] +fn the_open_level_is_where_the_numbers_are_exact() { + let record = r#"{"type":"assistant","message":{"role":"assistant","content":[], + "model":"a-model-id","usage":{"input_tokens":1999,"output_tokens":7, + "cache_creation":{"ephemeral_5m_input_tokens":11,"ephemeral_1h_input_tokens":2}, + "output_tokens_details":{"thinking_tokens":3}}}}"#; + assert!(sum(record).what.starts_with("in 1.9k"), "the row floors"); + let parts = detail_text(record); + let tokens = part_of(&parts, "tokens"); + for want in [ + "input_tokens 1999", + "output_tokens 7", + "ephemeral_5m_input_tokens 11", + "ephemeral_1h_input_tokens 2", + "thinking_tokens 3", + ] { + assert!(tokens.contains(want), "{want:?} not in {tokens:?}"); + } + assert!(part_of(&parts, "model").contains("a-model-id")); +} + +/// `standard` is what nearly every record says, so saying it again is noise; +/// anything else is the anomaly a reader opened the row to find. +#[test] +fn a_tier_is_shown_only_when_it_is_not_standard() { + let with = |tier: &str| { + format!( + r#"{{"type":"assistant","message":{{"role":"assistant", + "usage":{{"input_tokens":1,"service_tier":{tier}}}}}}}"# + ) + }; + let standard = detail_text(&with(r#""standard""#)); + assert!(!standard.iter().any(|(l, _)| *l == "request"), "{standard:?}"); + assert!(part_of(&detail_text(&with(r#""priority""#)), "request").contains("priority")); + // A tier that is null is not a string and says nothing either. + assert!(!detail_text(&with("null")).iter().any(|(l, _)| *l == "request")); +} + +/// One attempt is the ordinary case and says nothing; two is a retry, and the +/// *length* is the fact. The elements are never summed into the total. +#[test] +fn iterations_appear_only_when_there_was_more_than_one() { + let one = r#"{"type":"assistant","message":{"role":"assistant", + "usage":{"input_tokens":1,"iterations":[{"input_tokens":1}]}}}"#; + assert!(!detail_text(one).iter().any(|(l, _)| *l == "request")); + let two = r#"{"type":"assistant","message":{"role":"assistant", + "usage":{"input_tokens":1,"iterations":[{"input_tokens":1},{"input_tokens":1}]}}}"#; + let parts = detail_text(two); + let req = part_of(&parts, "request"); + assert!(req.contains("iterations"), "{req}"); + assert!(req.contains('2'), "{req}"); +} + +/// `version` is the schema-drift signal: a row whose numbers look wrong can be +/// checked against the build that wrote it. +#[test] +fn the_envelope_carries_what_wrote_the_record() { + let parts = detail_text( + r#"{"type":"assistant","requestId":"req-1","sessionId":"s-1","gitBranch":"main", + "version":"9.9.9","message":{"role":"assistant","usage":{"input_tokens":1}}}"#, + ); + let env = part_of(&parts, "envelope"); + for want in ["req-1", "s-1", "main", "9.9.9"] { + assert!(env.contains(want), "{env}"); + } +} + +/// A record with nothing to say at this level says nothing, rather than a row +/// with an empty part hanging under it. +#[test] +fn a_record_with_nothing_to_open_has_no_parts() { + assert!(detail_text(r#"{"type":"file-history-snapshot"}"#).is_empty()); +} diff --git a/src/lens/usage_atif.rs b/src/lens/usage_atif.rs index 06585c3..03f2712 100644 --- a/src/lens/usage_atif.rs +++ b/src/lens/usage_atif.rs @@ -42,7 +42,7 @@ use super::atif::STEPS; use super::usage::{self, Field, Tokens}; use super::usage_agent::count; -use super::{record_clock, Class, Lens, RecordsAt, Summary, Who}; +use super::{record_clock, Class, Lens, Part, RecordsAt, Summary, Who}; use crate::json::Value; pub const NAME: &str = "usage-atif"; @@ -74,6 +74,42 @@ impl Lens for UsageAtif { None => session(v), } } + + /// The exact numbers, and the two fields the row has no column for. + /// + /// Nothing is invented for the four counters this format does not have: + /// there is no cache-creation part here under any circumstance, because + /// there is no cache-creation number to show. + fn detail(&self, v: &Value) -> Vec { + let mut out: Vec = Vec::new(); + if let Some(m) = v.get("metrics") { + out.extend(usage::part("tokens", metric_lines(m))); + } + out.extend(usage::part("model", model_lines(v))); + out + } +} + +/// Every counter `metrics` wrote, exact, plus the two that get no column: +/// `reasoning_tokens`, which is a subset of the completion count, and +/// `llm_call_count`, which counts calls to a model rather than tokens. +fn metric_lines(m: &Value) -> Vec { + let mut lines: Vec = Vec::new(); + for key in ["prompt_tokens", "completion_tokens", "cached_tokens", "total_tokens"] { + lines.extend(usage::exact(m, key)); + } + if let Some(extra) = m.get("extra") { + lines.extend(usage::exact(extra, "reasoning_tokens")); + } + lines +} + +/// Which model the numbers were spent on, and how many times it was called. +fn model_lines(v: &Value) -> Vec { + let mut lines: Vec = Vec::new(); + lines.extend(usage::named(v, "model_name")); + lines.extend(usage::exact(v, "llm_call_count")); + lines } /// One step of the trajectory. diff --git a/src/lens/usage_atif_tests.rs b/src/lens/usage_atif_tests.rs index 58b4440..d04e09e 100644 --- a/src/lens/usage_atif_tests.rs +++ b/src/lens/usage_atif_tests.rs @@ -126,3 +126,60 @@ fn reasoning_is_not_added_to_the_total() { ); assert_eq!(s.tokens, 150, "not 190"); } + +// -- the open level -------------------------------------------------------------- + +fn detail_text(json: &str) -> Vec<(&'static str, String)> { + let v = crate::json::parse(json.as_bytes()).expect("fixture parses"); + UsageAtif + .detail(&v) + .into_iter() + .map(|p| match p { + Part::Text { label, body } => (label, body.head), + Part::Call { tool, .. } => panic!("this lens builds no call parts, got {tool}"), + }) + .collect() +} + +fn part_of<'a>(parts: &'a [(&'static str, String)], label: &str) -> &'a str { + &parts.iter().find(|(l, _)| *l == label).unwrap_or_else(|| panic!("no {label} part")).1 +} + +/// The row floors; the level under it is exact, and it carries the two numbers +/// the row has no column for. +#[test] +fn the_open_level_is_where_the_numbers_are_exact() { + let record = r#"{"step_id":1,"source":"agent","model_name":"a-model-id","llm_call_count":11, + "metrics":{"prompt_tokens":1999,"completion_tokens":50,"cached_tokens":3, + "extra":{"reasoning_tokens":40}}}"#; + assert!(sum(record).what.starts_with("in 1.9k"), "the row floors"); + let parts = detail_text(record); + let tokens = part_of(&parts, "tokens"); + for want in ["prompt_tokens 1999", "reasoning_tokens 40"] { + assert!(tokens.contains(want), "{want:?} not in {tokens:?}"); + } + let model = part_of(&parts, "model"); + assert!(model.contains("a-model-id"), "{model}"); + assert!(model.contains("llm_call_count"), "{model}"); +} + +/// There is no cache-creation number in this format, so no part ever mentions +/// one — nothing is invented for a counter the file does not have. +#[test] +fn no_step_ever_opens_into_a_cache_creation_number() { + for fixture in [ + r#"{"step_id":1,"source":"agent","metrics":{"prompt_tokens":1,"cached_tokens":2}}"#, + r#"{"step_id":2,"source":"user","metrics":{"completion_tokens":9}}"#, + ] { + for (_, text) in detail_text(fixture) { + assert!(!text.contains("cache_creation"), "{text}"); + assert!(!text.contains("ephemeral"), "{text}"); + } + } +} + +/// The envelope is a headline over a tree, and `Enter` opens the tree. +#[test] +fn the_session_record_has_no_parts() { + assert!(detail_text(r#"{"schema_version":"ATIF-v1.7","session_id":"s-1"}"#).is_empty()); +} From 65305174c93cd681fb268ff1897eb1db59ed8fb7 Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:56:04 +0000 Subject: [PATCH 06/12] docs: write down the usage dialects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The column table, the number rule and why it floors, the three-way distinction between a printed number, a dash and an absent column, and the two places a total lives — the group row and the status bar — with the reason neither is in a dialect. Also the note that records_at() is one answer, so a dialect that wants to read two file shapes is two entries over one shared module. --- CLAUDE.md | 2 +- SPEC.md | 12 ++++- docs/lenses.md | 137 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 147 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c2ff56c..ead2434 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,7 @@ corpus of linked documents. Crate `tread`, binary `tread`. | `src/code/` | the language grammars, pure like `md`/`csv`/`json`: a lexer and a declaration recogniser per language (`rust`, `ts`, `py`, `java`), over the shared line arithmetic in `decl.rs`. `docs/code.md` is the contract | | `src/source/code/` | code behind the seam: rows and colouring (`render.rs`, `paint.rs`), import resolution (`jump.rs`, `tsconfig.rs`, `workspace.rs`), the `Source` impl | | `src/source/fold.rs` | what folds and what folding hides. Prose infers a section from the next heading; code *states* where a region ends, because a block that ended at the next heading would swallow the statements after it | -| `src/lens/` | the `--lens` seam (SPEC.md §Lenses): `mod.rs` holds `Lens`, `Summary`, `RecordsAt` and the registry; one module per dialect (`agent.rs` — Claude Code session logs; `atif.rs` — ATIF trajectories, whose records live inside a document). A lens is a **transform over records**, never a `Source`: it says what one record *is* — including its `Summary::body`, which is a bounded head plus the path back to the text inside the record, never a copy of the message, because a summary is kept for every record — plus the one declaration `records_at()` — where its records live, which is what `src/open/lens.rs` turns into a format and a refusal — and nothing about rows, folding or search. A record it does not recognise falls back to the generic tree and is never hidden. `part.rs` is the row-free vocabulary a dialect answers `detail()` in — `Part::Text` and `Part::Call`, every stretch of text a bounded `Body`, plus `args_of`, the **one** reading of a call's argument list, which both dialects use so an `arguments` that is not the object the schema promises becomes one named row rather than nothing; `agent_parts.rs` is that dialect's half of it, split so both stay under the size limit. Adding a dialect is a module plus one line in `LENSES` — `docs/lenses.md` is the contract | +| `src/lens/` | the `--lens` seam (SPEC.md §Lenses): `mod.rs` holds `Lens`, `Summary`, `RecordsAt` and the registry; one module per dialect (`agent.rs` — Claude Code session logs; `atif.rs` — ATIF trajectories, whose records live inside a document; `usage_agent.rs` / `usage_atif.rs` — the same two files read as what each record *spent*, over the shared `usage.rs`, which owns the counters, the exact total and the one spelling of the fixed-width numeric block. Two entries rather than one because `records_at()` is a single answer and a `Lines` lens may not be pointed at a document). A lens is a **transform over records**, never a `Source`: it says what one record *is* — including its `Summary::body`, which is a bounded head plus the path back to the text inside the record, never a copy of the message, because a summary is kept for every record — plus the one declaration `records_at()` — where its records live, which is what `src/open/lens.rs` turns into a format and a refusal — and nothing about rows, folding or search. A record it does not recognise falls back to the generic tree and is never hidden. `part.rs` is the row-free vocabulary a dialect answers `detail()` in — `Part::Text` and `Part::Call`, every stretch of text a bounded `Body`, plus `args_of`, the **one** reading of a call's argument list, which both dialects use so an `arguments` that is not the object the schema promises becomes one named row rather than nothing; `agent_parts.rs` is that dialect's half of it, split so both stay under the size limit. `lens::tokens` is the **one** spelling of a token count — floored, never wider than four columns — because the group row and the status bar spell the same number a record row does and two spellings would drift. Adding a dialect is a module plus one line in `LENSES` — `docs/lenses.md` is the contract | | `src/open.rs` | building the `Box` behind an input; `open/input.rs` resolves *what* to open (the path or the pipe, the first bytes, the encoding refusal), and `open/lens.rs` is the one place a `--lens` meets a format — it turns a dialect's `RecordsAt` into a reader and into the refusal for pointing a record-per-line lens at a document, or a document lens at a `.jsonl` | | `src/render/` | AST + width to styled wrapped lines | | `src/theme.rs` | palette, heading styles, banner glyphs | diff --git a/SPEC.md b/SPEC.md index e7f8199..53c676e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -402,7 +402,17 @@ run is a conversation, and what a reader wants is the conversation with the mechanics folded away. Messages stay visible; consecutive tool calls and their results collapse into one summary row (`⟨6 steps · 4 tool calls⟩`) that opens. Two dialects read one today — `agent` for Claude Code session logs, `atif` for -ATIF trajectories — and a third is a module and a line. +ATIF trajectories — and another is a module and a line. + +A lens says what a record *is*, and there is more than one honest answer for the +same file. `usage` and `usage-atif` read those same two shapes as what each +record **spent**: a fixed-width block of token counters, then what the record +did, with a record that spent nothing showing its kind and no number columns at +all. A printed number means the format recorded that value, `-` means this +record did not record a field its format has, and an absent column means the +format has no such field. Counts are floored to four columns, so a row that says +`18k` promises at least 18,000; the exact integers are on the level under it, +and the exact sums are on the run's row and in the status bar. **A row is a headline; the record is under it.** A summary row is one line — who, when, what — and for a message that `what` **is the message's own first diff --git a/docs/lenses.md b/docs/lenses.md index a3b3f0d..26e82e5 100644 --- a/docs/lenses.md +++ b/docs/lenses.md @@ -366,6 +366,136 @@ thing that scrolls sideways. `--toc --lens atif` prints the whole run as 50 tab-separated lines, headlines only — a list is a list, and the levels are for a screen. +## The `usage` dialects + +The same two files, a different question: not what was said, but what each +record **spent**. `usage` reads a Claude Code session log +(`records_at() -> Lines`) and `usage-atif` reads an ATIF trajectory +(`records_at() -> Member("steps")`); everything that is not that one declaration +is shared, in `src/lens/usage.rs`. + +```text + user 14:01 user +▸ ⟨3 steps · 1 tool call · 22k tokens⟩ 14:02 + assistant 14:02 in 1.2k out 380 read 18k new 2.1k · Bash(cargo test) + ↳assistant 14:03 in 800 out 20 read - new - · assistant + system 14:03 file-history-snapshot +``` + +They are named `usage` and not `cost`: no price table is compiled in and no money +is shown. A lens must not promise a currency it cannot compute. + +**The columns.** The actor is 10 and the clock is 5, so `what` starts at column +21 — the same column every lens's `what` starts at. Then the numeric block, and +then ` · ` and what the record did: + +| Field | Columns | Made of | +| --- | --- | --- | +| one field | 8 | its label left-justified in 4, then `lens::tokens` right-aligned in 4 | +| the gap between two fields | 2 | | +| `usage`'s block (`in`, `out`, `read`, `new`) | **38** | 4×8 + 3×2 | +| `usage-atif`'s block (`in`, `out`, `read`) | **28** | 3×8 + 2×2 | + +The block is that width on **every** row of the file, whatever the numbers are, +which is what makes it a column a reader can scan down. The row never wraps — it +scrolls sideways like every other summary row — so a narrow terminal pans across +the action rather than losing it. + +**Three different things a cell can say**, and this is the whole design: + +| On the row | Means | +| --- | --- | +| a number, `0` included | the format recorded that value. A recorded zero is a fact about the session, and it is shown | +| `-` | **this record** did not record a field its format has. The column stays, because other records in the same file fill it | +| no column at all | **the format** has no such field. ATIF-v1.7 records no cache-*creation* counter of any kind, so a `usage-atif` row has three fields and never a fourth | + +A format-level absence removes the column for the whole file, because alignment +is what a number column is for and alignment is per file; a record-level absence +inside a format that has the field prints `-`. A `0` in the third case would say +"this agent wrote nothing to cache" when the truth is "this format does not +record cache writes", which is a different and false claim. + +**A record with no usage shows its kind and nothing more** — +`file-history-snapshot`, `queue-operation`, `user` — with no number fields at +all, so the numeric column is simply absent where nothing was spent rather than +a row of zeroes. A `type` the dialect has never seen prints its own name rather +than being swallowed. + +**Numbers are floored, never rounded up.** `lens::tokens` is the one spelling +there is: `380`, `1.2k`, `18k`, `1.8M`, never wider than four columns. `999` is +`999` and not `1.0k`; `1999` is `1.9k` and not `2.0k`. A row that says `18k` is +therefore a promise of *at least* 18,000, which is the reading a person makes of +a truncated number and the only one that never overstates what a session spent. + +The cost is real and is stated here rather than discovered: a bucket hides its +magnitude, so `18k` is anything up to 18,999 and **the floored numbers on the +rows will not add up by eye to the total on the group row**. Both totals are the +exact sum, spelled once at the end; and the exact integers of any one record are +one `Enter` away. + +**Where a total lives, and why neither place is in a dialect.** A lens may not +decide a row, so both are in the record seam: + +* the **group row** over a folded run — `lensrow::group_counts` sums the + members' exact counts and `group_text` spells the third clause, + `⟨15 steps · 3 tool calls · 128k tokens⟩`, omitted entirely at zero; +* the **status bar** over the document — `record::view::position_text` appends + ` · ≥1.2M tokens` to its `record 812/≥1204 (indexing 44%)` head. The `≥` is + not cosmetic: classification runs only as far as the reader has scrolled, so an + unqualified total on an 8.8 MB log would be wrong by most of the file. It is a + running sum kept by `Plan::classify`, one add per record and nothing per frame, + and classification runs once per record — so opening a folded run cannot make + the total jump. + +Both are dialect-agnostic: any lens that fills `Summary::tokens` gets them, and a +lens that fills none (`agent`, `atif`) changes not one byte of either. + +**Where the numbers are.** For `usage`, exactly one path — `message.usage` — +mapping `input_tokens` → `in`, `output_tokens` → `out`, +`cache_read_input_tokens` → `read`, `cache_creation_input_tokens` → `new`. For +`usage-atif`, `steps[].metrics`, mapping `prompt_tokens` → `in`, +`completion_tokens` → `out`, `cached_tokens` → `read`. + +Two things are deliberately **not** added to the total, and both would be a +double count: + +* `output_tokens_details.thinking_tokens` and ATIF's + `metrics.extra.reasoning_tokens` are *subsets* of the output count; +* `usage.iterations[]` is a list whose elements repeat the outer counter names + once per attempt. Its **length** is the fact — the request was retried — and + its contents are never summed. This is the mistake the next contributor will + make, so it is in a comment on `usage::Tokens::total` and in a test as well as + here. + +Neither is lost. Both are on the open level, along with the exact integers, the +model the numbers were spent on, the cache-creation breakdown, and — for +`usage` — a `service_tier` or `speed` that is **not** `standard`, which is +exactly the anomaly a reader opened the row to find, and which would be 99.96% +noise as a column. + +**The one decision that shapes the document.** For `usage`, a **human** `user` +turn is a `Class::Message` and everything else is a `Class::Step`, so the run +between two human turns is exactly one turn's mechanics and the group row over +it totals what that turn cost. A `user` record whose content blocks are +`tool_result` is mechanics — the same line `agent` draws — and drawing it +anywhere else would shred every run into pairs and leave no group row totalling +anything. For `usage-atif`, `source` is the discriminator: `user` is the turn +boundary, `agent` is a step. + +**A subagent is `↳assistant`, with no space** — ten columns exactly, which is +what the actor field is wide. `agent`'s `↳ assistant` is eleven and pushes every +column on that row one to the right; more than half the records of a real session +carry `isSidechain`, and on a lens whose whole product is a column of numbers +that is fatal. `Who` still carries the colour, so the row still paints as an +assistant. ATIF has no such flag and no row is marked: inventing one would be a +guess. + +**Neither dialect shows any message text**: `Summary::body` is `None` on every +row, because "what was said" is what `--lens agent` and `--lens atif` are for. +That also means these dialects allocate nothing per record but their own one-line +`what`, so a log whose longest line is most of a megabyte costs them a parse and +no more. + ## Adding a dialect opencode's own logs and OpenAI Codex logs are wanted and **not implemented**. @@ -379,7 +509,9 @@ Adding one is a module and a line, and nothing else: or `RecordsAt::Member("steps")` (a document whose records are one of its keys, every *other* key becoming record 0). This is the only thing a dialect says about files, and the routing in `src/open/lens.rs` turns it - into a format and a refusal. + into a format and a refusal. **It is one answer**: a dialect that wants to + read two file shapes is two entries over one shared module, which is what + `usage` and `usage-atif` are. * `read(&mut self, &Value) -> Option` — called **once per record, in file order**, so a dialect may carry state (the agent lens keeps a bounded ring of `tool_use` ids so a result can name its call). Return `None` for @@ -398,7 +530,7 @@ Adding one is a module and a line, and nothing else: fixtures**. Real session logs are private; read one to learn the shape, never copy it into the repository. -A `Summary` is seven fields and no styling: +A `Summary` is eight fields and no styling: | Field | Meaning | | --- | --- | @@ -408,6 +540,7 @@ A `Summary` is seven fields and no styling: | `time` | `HH:MM`, or `None`; `lens::clock` does ISO-8601 | | `what` | one line: `lens::excerpt` collapses whitespace and cuts to width. What `--toc` prints, and what the row paints for a record with no `body` | | `calls` | tool calls in this record, for a run's `· 4 tool calls` | +| `tokens` | every token unit this record recorded, **exact**; `0` when it records none. The two places a total appears add these up and spell the sum once with `lens::tokens`, because a sum of rounded numbers is not the rounding of a sum | | `body` | the record's own text, for the rows under the summary: what was said on a message, what it was thinking on a step. `None` when it has none. `class` decides how it is painted — a message's body is one wrap split between its row and the rows under it; a step's row keeps saying what it *did* and its text goes wholly underneath | A `Part` is two variants and no styling either: From 26f507e3372e9572df739d7de05f94f7571eacdb Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 15:59:08 +0000 Subject: [PATCH 07/12] lens: measure the usage dialects against a real corpus Fixtures only prove the shapes someone thought of. Two opt-in harnesses in the shape of the existing ones sweep a real corpus and hold the lens to its own contract there: every record with counters totals their exact sum, the numeric block is the same width on every row of every file, a subagent mark never outgrows the actor column, and a line that does not parse reaches the generic row rather than aborting the sweep. They read private files, so they print counts only and nothing from either corpus is copied anywhere. Both are no-ops without their environment variable, so a clean checkout is unaffected. --- CLAUDE.md | 5 ++ src/lens/usage_agent_tests.rs | 97 +++++++++++++++++++++++++++++++++++ src/lens/usage_atif_tests.rs | 48 +++++++++++++++++ 3 files changed, 150 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index ead2434..5146aa2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -159,8 +159,13 @@ TREAD_JAVA_CORPUS=~/some/java_project cargo test --bin tread a_real_java_corpus TREAD_TS_PROJECT=~/some/ts_project cargo test --bin tread a_real_project_resolves -- --nocapture TREAD_ATIF_TRAJECTORY=~/some/trajectory.json \ cargo test --bin tread a_real_atif_trajectory -- --nocapture +TREAD_AGENT_CORPUS=~/.claude/projects cargo test --bin tread a_real_agent_corpus -- --nocapture ``` +The last two sweep **private** files. They read and measure; they print counts +only, and nothing from either is ever copied into a fixture, a doc or a commit +message. + The last reports what fraction of a project's own imports resolve; run it after touching resolution. It caught an absolute path where a relative one was needed, a corpus rooted at the wrong directory, and an extension substituted rather than diff --git a/src/lens/usage_agent_tests.rs b/src/lens/usage_agent_tests.rs index ea4ed45..67282e8 100644 --- a/src/lens/usage_agent_tests.rs +++ b/src/lens/usage_agent_tests.rs @@ -300,3 +300,100 @@ fn the_envelope_carries_what_wrote_the_record() { fn a_record_with_nothing_to_open_has_no_parts() { assert!(detail_text(r#"{"type":"file-history-snapshot"}"#).is_empty()); } + +// -- against a real corpus -------------------------------------------------------- + +/// Columns the numeric block occupies on a `usage` row: four fields of eight, +/// joined by two. The contract the whole lens rests on. +const BLOCK: usize = 4 * 8 + 3 * 2; + +/// Against a real corpus, when one is pointed at: set `TREAD_AGENT_CORPUS` to a +/// directory of session logs (`~/.claude/projects`). +/// +/// Fixtures only prove the shapes someone thought of. Session logs are +/// **private**: this reads them, asserts *structure*, and prints **counts +/// only** — never a byte of their content, and nothing from them is ever copied +/// into this repository. Skipped when the variable is unset, so a clean checkout +/// and CI are unaffected. +#[test] +fn a_real_agent_corpus_is_read_as_what_it_spent() { + let Ok(root) = std::env::var("TREAD_AGENT_CORPUS") else { + return; + }; + let mut files = Vec::new(); + collect(std::path::Path::new(&root), &mut files); + assert!(!files.is_empty(), "no .jsonl under {root}"); + let (mut records, mut unparsed, mut with_usage, mut side, mut zero) = (0, 0, 0, 0, 0); + let mut total = 0u64; + for path in &files { + let Ok(bytes) = std::fs::read(path) else { continue }; + let mut lens = Usage; + for line in bytes.split(|&b| b == b'\n') { + if line.iter().all(u8::is_ascii_whitespace) { + continue; + } + records += 1; + // A line that does not parse must reach the generic row, not abort: + // real corpora carry truncated lines. + let Ok(v) = crate::json::parse(line) else { + unparsed += 1; + continue; + }; + let Some(s) = lens.read(&v) else { continue }; + if s.actor.starts_with('\u{21b3}') { + side += 1; + assert!(str_width(&s.actor) <= ACTOR, "a mark that costs a column"); + } + total = total.saturating_add(s.tokens); + check_row(&v, &s, &mut with_usage, &mut zero); + } + } + println!( + "{} files, {records} records ({unparsed} unparsed), {with_usage} with usage \ + ({zero} recording only zeroes), {side} sidechain, {} tokens", + files.len(), + crate::lens::tokens(total) + ); + assert!(with_usage > 0, "no record in the corpus carried usage"); +} + +/// One record's row, against what the file actually says about it. +fn check_row(v: &crate::json::Value, s: &Summary, with_usage: &mut usize, zero: &mut usize) { + let usage = v.get("message").and_then(|m| m.get("usage")); + let counted: Vec = match usage { + Some(u) => ["input_tokens", "output_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"] + .into_iter() + .filter_map(|k| super::count(u, k)) + .collect(), + None => Vec::new(), + }; + if counted.is_empty() { + // No counters: the kind and nothing more, so no number column anywhere. + assert!(!s.what.starts_with("in "), "a row of numbers with no numbers"); + assert_eq!(s.tokens, 0); + return; + } + *with_usage += 1; + let want: u64 = counted.iter().fold(0u64, |a, b| a.saturating_add(*b)); + assert_eq!(s.tokens, want, "the total is not the sum of the counters"); + if want == 0 { + *zero += 1; + } + // The block is the same width on every row, which is the whole product. + let block = s.what.split(" \u{b7} ").next().unwrap_or(""); + assert_eq!(str_width(block), BLOCK, "the column bends on a real record"); +} + +/// Every `.jsonl` under `dir`, depth-first. No symlink following beyond what +/// `read_dir` gives, and errors are skipped rather than failing the sweep. +fn collect(dir: &std::path::Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect(&path, out); + } else if path.extension().is_some_and(|e| e == "jsonl") { + out.push(path); + } + } +} diff --git a/src/lens/usage_atif_tests.rs b/src/lens/usage_atif_tests.rs index d04e09e..7bcb1ff 100644 --- a/src/lens/usage_atif_tests.rs +++ b/src/lens/usage_atif_tests.rs @@ -183,3 +183,51 @@ fn no_step_ever_opens_into_a_cache_creation_number() { fn the_session_record_has_no_parts() { assert!(detail_text(r#"{"schema_version":"ATIF-v1.7","session_id":"s-1"}"#).is_empty()); } + +// -- against a real trajectory ---------------------------------------------------- + +/// Columns the numeric block occupies on a `usage-atif` row: three fields of +/// eight, joined by two. There is no fourth field in this format. +const BLOCK: usize = 3 * 8 + 2 * 2; + +/// Against a real trajectory, when one is pointed at: set +/// `TREAD_ATIF_TRAJECTORY` — the same variable the `atif` harness uses. +/// +/// A real trajectory is **private**: this reads it, asserts *structure*, and +/// prints **counts only**. Nothing from it is copied into this repository. +/// Skipped when the variable is unset. +#[test] +fn a_real_atif_trajectory_is_read_as_what_it_spent() { + let Ok(path) = std::env::var("TREAD_ATIF_TRAJECTORY") else { + return; + }; + let bytes = std::fs::read(&path).expect("the trajectory reads"); + let doc = crate::json::parse(&bytes).expect("the trajectory parses"); + let steps = doc + .get(super::STEPS) + .and_then(|s| s.as_array()) + .expect("a trajectory with steps"); + let mut lens = UsageAtif; + let (mut read, mut with_metrics) = (0usize, 0usize); + let mut total = 0u64; + for step in steps { + let Some(s) = lens.read(step) else { continue }; + read += 1; + total = total.saturating_add(s.tokens); + // No row of this format ever names a counter it does not have. + assert!(!s.what.contains("new"), "a cache-creation column appeared"); + if s.what.starts_with("in ") { + with_metrics += 1; + let block = s.what.split(" \u{b7} ").next().unwrap_or(""); + assert_eq!(str_width(block), BLOCK, "the column bends on a real step"); + } + assert!(s.body.is_none(), "this lens keeps no message text"); + } + println!( + "{} steps, {read} read by the lens, {with_metrics} with metrics, {} tokens", + steps.len(), + crate::lens::tokens(total) + ); + assert_eq!(read, steps.len(), "the lens left steps unread"); + assert!(with_metrics > 0, "no step carried metrics"); +} From 431748a8ae841d379b67c918fa0c10b953c3efa3 Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 17:44:03 +0000 Subject: [PATCH 08/12] record: keep folding in its own module under the source view The Source impl for a record document had grown past the size limit with every fold question in it. Those are one subject - what folds, what a fold hides, and the fold state a session saves - and they now have their own file, which leaves the view with the layout, the positions and the reads. --- CLAUDE.md | 2 +- src/source/record/folds.rs | 192 ++++++++++++++++++++++++++++++++++++ src/source/record/source.rs | 5 + src/source/record/view.rs | 162 ++++-------------------------- 4 files changed, 215 insertions(+), 146 deletions(-) create mode 100644 src/source/record/folds.rs diff --git a/CLAUDE.md b/CLAUDE.md index 5146aa2..19f41ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,7 +75,7 @@ corpus of linked documents. Crate `tread`, binary `tread`. | `src/source/` | the format seam: `Source`, `markdown.rs`, `csv/`, `json/`, `jsonl/`, `jsonarray/`, `jsonrow.rs`, `record/`, `detect.rs`. Formats are compiled in, never loaded | | `src/source/jsonrow.rs` | the **one** JSON tree-row grammar: indent, fold marker, key spelling, collapsed summary, scalar colours, path steps. Both JSON sources build every row through it, and `tests/json_differential.rs` reads the same content both ways and compares — two renderers would drift and the same object would look different depending on which file it came from | | `src/source/json/` | JSON behind the seam: the lazily indexed `tree.rs` with `ident.rs` (a member's bytes, key and path — **derived** from the parent chain, never stored, or the paths alone cost O(depth²)), the fold state and flatten in `flat.rs`, `render.rs`, the `Source` impl in `view.rs`, and `export.rs` (`--to-jsonl`) | -| `src/source/record/` | records in general, above any one record *format* (SPEC.md §Lenses). There is **one** record source — `source.rs` (`RecordSource`: state, cache, search sweep), `view.rs` (the `Source` impl), `rows.rs` (summary rows, expansion, outline, `--toc`), `ladder.rs` (`Enter`/`za`: the three levels a record has and what descending one costs), `lensstate.rs` (the lens state and its forwarders), `tree.rs` (one record laid out as tree rows, over `jsonrow`'s grammar) — and what a record *format* supplies is `store.rs`'s `Store`: how many records the index has found, how to push it along inside a byte budget, and record `i` as bytes or as a value. Nothing here opens a file, names a format or mentions a line. The lens machinery under it is `plan.rs` (items: which records share a row, and the row arithmetic — a record's own rows depend on the **width**, because its text is wrapped, so `set_width` re-lays every one without reclassifying anything) with `plan_rows.rs` beside it (the level a record is at, its measured heights, and which of its calls is open — all keyed by *record*, because a step inside an open run is as much a thing a reader opens as a message is) and `plan_block.rs` (where a block starts — a member of an open run is a block, and *everything under that member's row* is in it, never in the run's), `rowmap.rs` (which rows an open record owns; `Plan::extra` is a second one of these, holding the rows a member of an open run shows under itself, so `own`, `inside` and `blocks_of_item` keep the shape they had), `lensrow.rs` (the rows a lens paints, the row-to-record translation under them, and the measurement), `body.rs` (one wrap and one clip for every stretch of text this seam shows — a message, a step's reasoning, a call's argument, its output — with the `⋯ +N lines` row that says what the clip left out; a lens never hides anything), `parts.rs` (the **open level**: a record's tool calls as one row apiece, each opening into its arguments and its output under a row that names it. It decides rows — the columns, the glyph, the inset a member of an open run carries — and nothing about what a record contains, which is the dialect's `detail()`), `ops.rs` (what `zR`, `zM`, `Tab`, `Y` and a fold id off the outline do to a plan), and the shared gutter and fold vocabulary: `marker`/`leaf`, `/4` for a record beside `plan::group_id`'s `g4` for a group, the two spellings that must not collide. `Records` is the trait `lensrow`/`ops` reach records through — `with_value` hands the record to a closure, so it is generic rather than `dyn`: a record can be tens of megabytes and lives behind a `RefCell`, and handing out a clone to read one field would undo the cache. A change all lenses need belongs here, not in a dialect | +| `src/source/record/` | records in general, above any one record *format* (SPEC.md §Lenses). There is **one** record source — `source.rs` (`RecordSource`: state, cache, search sweep), `view.rs` (the `Source` impl), `folds.rs` (the fold half of it: what folds, what a fold hides, and the fold state a session saves — `view.rs` forwards to it), `rows.rs` (summary rows, expansion, outline, `--toc`), `ladder.rs` (`Enter`/`za`: the three levels a record has and what descending one costs), `lensstate.rs` (the lens state and its forwarders), `tree.rs` (one record laid out as tree rows, over `jsonrow`'s grammar) — and what a record *format* supplies is `store.rs`'s `Store`: how many records the index has found, how to push it along inside a byte budget, and record `i` as bytes or as a value. Nothing here opens a file, names a format or mentions a line. The lens machinery under it is `plan.rs` (items: which records share a row, and the row arithmetic — a record's own rows depend on the **width**, because its text is wrapped, so `set_width` re-lays every one without reclassifying anything) with `plan_rows.rs` beside it (the level a record is at, its measured heights, and which of its calls is open — all keyed by *record*, because a step inside an open run is as much a thing a reader opens as a message is) and `plan_block.rs` (where a block starts — a member of an open run is a block, and *everything under that member's row* is in it, never in the run's), `rowmap.rs` (which rows an open record owns; `Plan::extra` is a second one of these, holding the rows a member of an open run shows under itself, so `own`, `inside` and `blocks_of_item` keep the shape they had), `lensrow.rs` (the rows a lens paints, the row-to-record translation under them, and the measurement), `body.rs` (one wrap and one clip for every stretch of text this seam shows — a message, a step's reasoning, a call's argument, its output — with the `⋯ +N lines` row that says what the clip left out; a lens never hides anything), `parts.rs` (the **open level**: a record's tool calls as one row apiece, each opening into its arguments and its output under a row that names it. It decides rows — the columns, the glyph, the inset a member of an open run carries — and nothing about what a record contains, which is the dialect's `detail()`), `ops.rs` (what `zR`, `zM`, `Tab`, `Y` and a fold id off the outline do to a plan), and the shared gutter and fold vocabulary: `marker`/`leaf`, `/4` for a record beside `plan::group_id`'s `g4` for a group, the two spellings that must not collide. `Records` is the trait `lensrow`/`ops` reach records through — `with_value` hands the record to a closure, so it is generic rather than `dyn`: a record can be tens of megabytes and lives behind a `RefCell`, and handing out a clone to read one field would undo the cache. A change all lenses need belongs here, not in a dialect | | `src/source/jsonl/` | `.jsonl` / `.ndjson`: the `Store` that says a record is a *line*, over the **CSV** lazy line index (`RowStore::lines` — the one line indexer in the crate, called and not copied), plus the parse of one line and the two constructors. Everything else it used to hold is `record/`, shared | | `src/source/jsonarray/` | records that are the elements of an **array inside a JSON document** (SPEC.md §Lenses, ATIF trajectories): the `Store` over the *existing* lazy structural index (`json::tree::Doc` — a member's byte range, no value built), plus the synthesised record 0 that keeps the document's other top-level keys so a lens hides nothing. Finding the array is a structural scan of the top level — the index knows a member only once it has walked past it — so the first row waits on a byte walk of the file and says so; no record is parsed until it is painted. `At::Root` / `At::Key` is the source's half of `lens::RecordsAt`, and nothing here names a dialect | | `src/source/text/` | plain text behind the seam (SPEC.md §Plain text): the file's lines, verbatim, over the *CSV* lazy line index — `mod.rs` (state, file access, the tab/control-character painting rule) and `view.rs` (the `Source` impl). No outline, no folds, no links: the honest empty answers. Anything whose extension names no parser lands here, and so does a corpus link to a `.sh` | diff --git a/src/source/record/folds.rs b/src/source/record/folds.rs new file mode 100644 index 0000000..d51523e --- /dev/null +++ b/src/source/record/folds.rs @@ -0,0 +1,192 @@ +//! What folds in a record document, and what a fold hides. +//! +//! The fold half of what `view.rs` answers: the outline entry a row belongs to, +//! the three keys that change a fold (`Enter`/`za`, `zR`/`zM`, and an entry +//! toggled off the outline), the fold *state* a session saves and restores, and +//! the two questions a painted row asks about hiding — how much is under this +//! row, and where the next block starts. Split out of `view.rs` so both stay +//! under the size limit: `view.rs` keeps the layout, the positions and the +//! reads, and every method here is the body of one of its trait methods, named +//! apart from it so the forwarder is never the thing it forwards to. +//! +//! Nothing here decides what a *lens* folds — that is `ops.rs` over `plan.rs`, +//! reached through the group forwarders in `lensstate.rs`. This file only knows +//! that a record can be open or shut and that a group id is not a record id. +#![deny(unsafe_code)] + +use super::*; + +impl RecordSource { + /// The outline entry the row under the cursor belongs to. + pub(crate) fn section_of(&self, row: usize) -> Option { + let id = self.fold_id_at(self.spot(row)); + self.outline.iter().position(|e| e.id == id) + } + + /// An outline entry opened or shut by name. + pub(crate) fn fold_entry(&mut self, entry: usize, closed: bool) -> bool { + let id = match self.outline.get(entry) { + Some(e) => e.id.clone(), + None => return false, + }; + // A group id (`g12`) cannot collide with a record id (`/12`); the seam + // tells them apart, and `None` is "not a group" — this file only has to + // answer for its own records, which is the half that costs a parse. + if let Some(changed) = self.set_group_by_id(&id, !closed) { + if changed { + self.mark_entry(entry, closed); + } + return changed; + } + let record: usize = match jsonrow::top_index(&id) { + Some(r) => r, + None => return false, + }; + let changed = match closed { + true => self.map.close(record), + false => self.open_record(record), + }; + if changed { + self.mark_entry(entry, closed); + } + changed + } + + /// The outline is what the last frame painted, so an entry that has since + /// scrolled away simply is not there to update. + fn mark_entry(&mut self, entry: usize, closed: bool) { + if let Some(e) = self.outline.get_mut(entry) { + e.folded = closed; + } + } + + /// `zM` shuts every record, which is free. `zR` opens them as the viewport + /// reaches them, up to [`EXPAND_CAP`]: opening a million records means + /// parsing a million records, and a reader that froze on one keystroke + /// would be worse than one that opens what you can see. + pub(crate) fn fold_every(&mut self, closed: bool) { + self.expand_all = !closed; + self.filled = 0; + self.map.clear(); + // Messages go with it: `zR` is "show me everything here", and a dump + // (`--plain`, which is `fold_all(false)` and nothing else) must print + // the whole of what was said rather than a viewport's clip of it. + self.set_all_bodies(!closed); + if closed { + self.close_groups(); + } + if !closed { + let upto = self.window.end.max(1).saturating_add(LOOKAHEAD); + self.fill_expansion(upto); + } + } + + /// `Enter` / `za`: the record's two levels — clipped and open, and back + /// again (SPEC.md §Lenses). The row's own fold, which is not an outline + /// entry and could not be reached through one. The raw tree is `r`, not a + /// rung, so this never opens or shuts one. A group row falls through to the + /// outline, which is what opens a run. + /// + /// **A record's row claims the key even when it has no rung.** Falling + /// through would hand `Enter` to the outline, and a record's outline entry + /// *is* its raw tree — so on exactly the records with nothing between the + /// headline and the JSON (a message the clip already showed whole, which + /// made no calls) the key would open and shut what `r` owns, and the tree + /// `r` had opened would vanish under a press of `Enter`. Under a lens the + /// answer there is "nothing to descend into", and it is answered here. + /// With **no lens** there is no ladder and no lens rows: a record row is a + /// collapsed tree and the outline is what opens it, so the key falls + /// through exactly as it always did. + pub(crate) fn fold_at_row(&mut self, row: usize) -> Option { + if let Some(was) = self.descend(row) { + return Some(was); + } + // No lens, no ladder: the key belongs to the outline, as it always did. + self.plan.as_ref()?; + match self.spot(row) { + // A run is opened *through* the outline, which is the one thing + // here that is an outline entry in its own right. + Spot::Group { .. } => None, + // Claimed, and nothing moved: no level changed, so "was open" is + // false and the pager repaints the rows it already had. + _ => Some(false), + } + } + + /// The shared fold-id vocabulary ([`jsonrow::ALL_OPEN`]): a default plus the + /// ids that disagree with it, each id a member-index path from the root. + /// A record file's root is the implicit list of records, so record 4 is + /// `/4` — the same id the document reader would give member 4 of its root. + /// + /// The default here is *shut*, so the exceptions are the open records: a + /// million-record file has a million closed ones and listing those is the + /// one thing this source must never do. `zR` is [`jsonrow::ALL_OPEN`] and + /// nothing else, exactly as it is on the document side. + pub(crate) fn fold_ids(&self) -> FoldState { + let mut out: Vec = Vec::new(); + if self.expand_all { + out.push(jsonrow::ALL_OPEN.to_string()); + } + out.extend(self.group_folds()); + out.extend(self.map.records().map(fold_id)); + out + } + + /// The same vocabulary read back: everything shut, then the ids that said + /// otherwise. + pub(crate) fn restore_folds(&mut self, folds: FoldState) { + self.map.clear(); + self.expand_all = folds.iter().any(|s| s == jsonrow::ALL_OPEN); + self.filled = 0; + self.restore_groups(&folds); + for id in &folds { + if let Some(record) = jsonrow::top_index(id) { + self.open_record(record); + } + } + if self.expand_all { + self.fill_expansion(self.window.end.max(1).saturating_add(LOOKAHEAD)); + } + } + + /// How many rows this row is hiding, for the gutter. + pub(crate) fn hidden_under(&self, row: usize) -> Option { + let (record, sub) = match self.spot(row) { + // A body row hides nothing: the clip says what it is not showing. + // Nor does a part row — a shut call carries its own glyph, and the + // gutter marker this answer drives belongs to the record. + Spot::Body { .. } | Spot::Part { .. } => return None, + // A closed group hides one row per record it holds; their trees + // are closed with it, so there is nothing else under it. + Spot::Group { item } => { + let hidden = self.plan.as_ref().map(|p| p.hidden(item)).unwrap_or(0); + return (hidden > 0).then_some(hidden); + } + Spot::Record { record, sub } => (record, sub), + }; + if sub != 0 || self.map.is_open(record) || record >= self.known() { + return None; + } + match self.tree_len(record) { + 0 => None, + n => Some(n), + } + } + + /// Block boundaries — what `Tab` / `S-Tab` jump between; see + /// `plan_block.rs`. With no lens a record is its own block, and the tree + /// rows under an open one are inside it. + pub(crate) fn landmark(&self, row: usize, forward: bool) -> Option { + if self.plan.is_some() { + return self.next_block_row(row, forward); + } + let (record, sub) = self.map.at(row); + match forward { + true if record + 1 < self.known() => Some(self.map.row_of(record + 1)), + true => None, + false if sub > 0 => Some(self.map.row_of(record)), + false if record > 0 => Some(self.map.row_of(record - 1)), + false => None, + } + } +} diff --git a/src/source/record/source.rs b/src/source/record/source.rs index 7d5109f..3bbd769 100644 --- a/src/source/record/source.rs +++ b/src/source/record/source.rs @@ -44,6 +44,11 @@ mod view; #[path = "rows.rs"] mod rows; +/// What folds, what a fold hides, and the fold state a session saves — the half +/// of the `Source` impl `view.rs` forwards to. +#[path = "folds.rs"] +mod folds; + /// `Enter` / `za`, and the levels it walks. #[path = "ladder.rs"] mod ladder; diff --git a/src/source/record/view.rs b/src/source/record/view.rs index 5438a20..abd698b 100644 --- a/src/source/record/view.rs +++ b/src/source/record/view.rs @@ -1,11 +1,13 @@ //! The [`Source`] implementation itself: what the pager sees. //! -//! Split from `source.rs`, which holds the state and the reads, so both stay -//! under the size limit. Written once for every record format: what a `.jsonl` -//! and an array inside a document disagree about is [`Store`] and nothing here. -//! The order of the sections below is the order of the trait's own +//! Split from `source.rs`, which holds the state and the reads, and from +//! `folds.rs`, which holds everything about folding — so all three stay under +//! the size limit. Written once for every record format: what a `.jsonl` and an +//! array inside a document disagree about is [`Store`] and nothing here. The +//! order of the sections below is the order of the trait's own //! (`src/source/mod.rs`), and every method a record file has no honest answer -//! for says so rather than inventing one. +//! for says so rather than inventing one. A method whose body is one call into +//! `folds.rs` is a fold question and is documented there. #![deny(unsafe_code)] use std::ops::Range; @@ -98,7 +100,8 @@ impl Source for RecordSource { Some(lens) => format!( "{lens} \u{b7} record {}/{total}{}{}", record.saturating_add(1), - self.block_text(row), self.tokens_text() + self.block_text(row), + self.tokens_text() ), None => format!("record {}/{total}", record.saturating_add(1)), }; @@ -218,93 +221,19 @@ impl Source for RecordSource { } fn section_at(&self, row: usize) -> Option { - let id = self.fold_id_at(self.spot(row)); - self.outline.iter().position(|e| e.id == id) + self.section_of(row) } fn set_fold(&mut self, entry: usize, closed: bool) -> bool { - let id = match self.outline.get(entry) { - Some(e) => e.id.clone(), - None => return false, - }; - // A group id (`g12`) cannot collide with a record id (`/12`); the seam - // tells them apart, and `None` is "not a group" — this file only has to - // answer for its own records, which is the half that costs a parse. - if let Some(changed) = self.set_group_by_id(&id, !closed) { - if changed { - if let Some(e) = self.outline.get_mut(entry) { - e.folded = closed; - } - } - return changed; - } - let record: usize = match jsonrow::top_index(&id) { - Some(r) => r, - None => return false, - }; - let changed = match closed { - true => self.map.close(record), - false => self.open_record(record), - }; - if changed { - if let Some(e) = self.outline.get_mut(entry) { - e.folded = closed; - } - } - changed + self.fold_entry(entry, closed) } - /// `zM` shuts every record, which is free. `zR` opens them as the viewport - /// reaches them, up to [`EXPAND_CAP`]: opening a million records means - /// parsing a million records, and a reader that froze on one keystroke - /// would be worse than one that opens what you can see. fn fold_all(&mut self, closed: bool) { - self.expand_all = !closed; - self.filled = 0; - self.map.clear(); - // Messages go with it: `zR` is "show me everything here", and a dump - // (`--plain`, which is `fold_all(false)` and nothing else) must print - // the whole of what was said rather than a viewport's clip of it. - self.set_all_bodies(!closed); - if closed { - self.close_groups(); - } - if !closed { - let upto = self.window.end.max(1).saturating_add(LOOKAHEAD); - self.fill_expansion(upto); - } + self.fold_every(closed) } - /// `Enter` / `za`: the record's two levels — clipped and open, and back - /// again (SPEC.md §Lenses). The row's own fold, which is not an outline - /// entry and could not be reached through one. The raw tree is `r`, not a - /// rung, so this never opens or shuts one. A group row falls through to the - /// outline, which is what opens a run. - /// - /// **A record's row claims the key even when it has no rung.** Falling - /// through would hand `Enter` to the outline, and a record's outline entry - /// *is* its raw tree — so on exactly the records with nothing between the - /// headline and the JSON (a message the clip already showed whole, which - /// made no calls) the key would open and shut what `r` owns, and the tree - /// `r` had opened would vanish under a press of `Enter`. Under a lens the - /// answer there is "nothing to descend into", and it is answered here. - /// With **no lens** there is no ladder and no lens rows: a record row is a - /// collapsed tree and the outline is what opens it, so the key falls - /// through exactly as it always did. fn fold_here(&mut self, row: usize) -> Option { - if let Some(was) = self.descend(row) { - return Some(was); - } - // No lens, no ladder: the key belongs to the outline, as it always did. - self.plan.as_ref()?; - match self.spot(row) { - // A run is opened *through* the outline, which is the one thing - // here that is an outline entry in its own right. - Spot::Group { .. } => None, - // Claimed, and nothing moved: no level changed, so "was open" is - // false and the pager repaints the rows it already had. - _ => Some(false), - } + self.fold_at_row(row) } /// `r`: the raw record under the cursor, whatever its body is doing. @@ -312,77 +241,20 @@ impl Source for RecordSource { self.open_tree_at(row) } - /// The shared fold-id vocabulary ([`jsonrow::ALL_OPEN`]): a default plus the - /// ids that disagree with it, each id a member-index path from the root. - /// A record file's root is the implicit list of records, so record 4 is - /// `/4` — the same id the document reader would give member 4 of its root. - /// - /// The default here is *shut*, so the exceptions are the open records: a - /// million-record file has a million closed ones and listing those is the - /// one thing this source must never do. `zR` is [`jsonrow::ALL_OPEN`] and - /// nothing else, exactly as it is on the document side. fn folds(&self) -> FoldState { - let mut out: Vec = Vec::new(); - if self.expand_all { - out.push(jsonrow::ALL_OPEN.to_string()); - } - out.extend(self.group_folds()); - out.extend(self.map.records().map(fold_id)); - out + self.fold_ids() } fn set_folds(&mut self, folds: FoldState) { - self.map.clear(); - self.expand_all = folds.iter().any(|s| s == jsonrow::ALL_OPEN); - self.filled = 0; - self.restore_groups(&folds); - for id in &folds { - if let Some(record) = jsonrow::top_index(id) { - self.open_record(record); - } - } - if self.expand_all { - self.fill_expansion(self.window.end.max(1).saturating_add(LOOKAHEAD)); - } + self.restore_folds(folds) } fn hidden_at(&self, row: usize) -> Option { - let (record, sub) = match self.spot(row) { - // A body row hides nothing: the clip says what it is not showing. - // Nor does a part row — a shut call carries its own glyph, and the - // gutter marker this answer drives belongs to the record. - Spot::Body { .. } | Spot::Part { .. } => return None, - // A closed group hides one row per record it holds; their trees - // are closed with it, so there is nothing else under it. - Spot::Group { item } => { - let hidden = self.plan.as_ref().map(|p| p.hidden(item)).unwrap_or(0); - return (hidden > 0).then_some(hidden); - } - Spot::Record { record, sub } => (record, sub), - }; - if sub != 0 || self.map.is_open(record) || record >= self.known() { - return None; - } - match self.tree_len(record) { - 0 => None, - n => Some(n), - } + self.hidden_under(row) } - /// Block boundaries — what `Tab` / `S-Tab` jump between; see - /// `plan_block.rs`. fn next_landmark(&self, row: usize, forward: bool) -> Option { - if self.plan.is_some() { - return self.next_block_row(row, forward); - } - let (record, sub) = self.map.at(row); - match forward { - true if record + 1 < self.known() => Some(self.map.row_of(record + 1)), - true => None, - false if sub > 0 => Some(self.map.row_of(record)), - false if record > 0 => Some(self.map.row_of(record - 1)), - false => None, - } + self.landmark(row, forward) } /// A record number as an id, so `#41` jumps to line 42's record. From 79e07a7274276e2b848e955bf3a2466862a2be1b Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 17:46:37 +0000 Subject: [PATCH 09/12] lens: say where the usage numbers are, and where they are not read A session log has a second usage object: a subagent result carries one at toolUseResult.usage. The lens still reads message.usage alone - a whole run's total does not belong in a column of per-request numbers - but the module said no other record had numbers at all, which was wrong. The refusal of a counter that is not a non-negative integer also claimed more than it does: it spells such a value the same way it spells a field the record never wrote. Say so rather than imply the two are told apart. --- src/lens/usage_agent.rs | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/lens/usage_agent.rs b/src/lens/usage_agent.rs index f312c4c..7830e59 100644 --- a/src/lens/usage_agent.rs +++ b/src/lens/usage_agent.rs @@ -13,11 +13,21 @@ //! //! # Where the numbers are //! -//! Exactly one path, `message.usage`, and nothing else is read for a number. In -//! the sessions this was written against it is present on every `assistant` -//! record and on no record of any other type, so a record without it is not a -//! record whose numbers are hidden somewhere else — it is a record that spent -//! nothing this file recorded, and it shows its `type` and stops. +//! Exactly one path, `message.usage`, and nothing else is read for a number. +//! Measured over whole session logs it is on every `assistant` record and on no +//! record of any other type — so a row that shows no numbers is never an +//! `assistant` whose counters this lens failed to find. +//! +//! It is **not** the only usage object a session log contains. A `user` record +//! that carries the result of a subagent call has one at +//! `toolUseResult.usage`, with the same key names. This lens does not read it, +//! and such a record shows `user` and no number columns at all. That is the +//! right answer here: every number in this column is one request's spend, and +//! the counters on a subagent result are the *total* of a whole run of them — +//! putting a sum in a column of per-request numbers would make the column mean +//! two things and make it un-addable. Nothing is lost: those records are a +//! handful per corpus, the object is in the record's own tree one `r` away, and +//! the subagent's own session log has the same spend a request at a time. //! //! | key | column | //! | --- | --- | @@ -232,8 +242,16 @@ fn read_usage(v: &Value) -> Option { /// Exact, through the number's own literal text: these are added up into a /// session total, and a count that went through an `f64` would stop being the /// number the file wrote somewhere past 2⁵³. A negative or fractional value is -/// not a token count and is refused rather than clamped — the record then shows -/// `-` in that cell, which is true. +/// not a token count and is refused rather than clamped. +/// +/// **Refusing spells it `-`, the same cell an unwritten field gets**, so a +/// malformed counter is not told apart from one the record never wrote. That is +/// deliberate and it is the one place the three-way distinction in `usage.rs` +/// is blurred: no record of any session measured has ever carried a counter +/// that is not a non-negative integer, and a fourth spelling in a four-column +/// block would be vocabulary a reader has to learn for something they will not +/// see. The value is not hidden either way — `r` shows the record's own tree, +/// with whatever it really wrote in it. pub(super) fn count(usage: &Value, key: &str) -> Option { u64::try_from(usage.get(key)?.as_number()?.as_i64()?).ok() } From 4bf7285ccf28034fdc8f99308a18909f5a8c31a5 Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 17:47:48 +0000 Subject: [PATCH 10/12] lens: say what a retried request costs, not what it repeats usage.iterations[] is one element per attempt, and where a record carries more than one the outer counters are the last element's rather than the sum. Skipping the list stays right; the reason for it was wrong, and the consequence went unsaid: on a retried request every number this lens shows is the surviving attempt's spend, and the file states no total across attempts for it to show instead. --- docs/lenses.md | 16 +++++++++++----- src/lens/usage.rs | 24 +++++++++++++++++------- src/lens/usage_agent.rs | 13 +++++++------ src/lens/usage_agent_tests.rs | 11 ++++++----- 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/lenses.md b/docs/lenses.md index 26e82e5..457f8e0 100644 --- a/docs/lenses.md +++ b/docs/lenses.md @@ -461,11 +461,17 @@ double count: * `output_tokens_details.thinking_tokens` and ATIF's `metrics.extra.reasoning_tokens` are *subsets* of the output count; -* `usage.iterations[]` is a list whose elements repeat the outer counter names - once per attempt. Its **length** is the fact — the request was retried — and - its contents are never summed. This is the mistake the next contributor will - make, so it is in a comment on `usage::Tokens::total` and in a test as well as - here. +* `usage.iterations[]` is one element per attempt at the request, and on a + record that carries more than one the outer counters are the **last** + element's, never the sum of them. Its **length** is the fact — the request was + retried — and its contents are never summed. This is the mistake the next + contributor will make, so it is in a comment on `usage::Tokens::total` and in + a test as well as here. + +What that means for a retried request is that the row, the group row and the +status bar all say what the **last attempt** spent. No field of the file states +what the attempts cost between them, and the lens invents nothing: the row says +the request was retried, and the list itself is one `Enter` (or `r`) away. Neither is lost. Both are on the open level, along with the exact integers, the model the numbers were spent on, the cache-creation breakdown, and — for diff --git a/src/lens/usage.rs b/src/lens/usage.rs index 4309c05..6da43d6 100644 --- a/src/lens/usage.rs +++ b/src/lens/usage.rs @@ -79,13 +79,23 @@ pub struct Tokens { impl Tokens { /// Every unit this record recorded, added once. /// - /// **The four counters and nothing else.** A record's reasoning count is a - /// *subset* of its output and a Claude Code `usage.iterations[]` list - /// repeats the outer counters per attempt — adding either would count the - /// same token twice, and a total that double-counts is worse than no total. - /// That mistake is the one a later reader of this file will be tempted into, - /// which is why it is written here and pinned by a test rather than left in - /// the documentation. + /// **The four counters and nothing else.** That is the one thing a later + /// reader of this file will be tempted to change, so it is written here and + /// pinned by a test rather than left in the documentation. + /// + /// * A record's **reasoning** count is a *subset* of its output, so adding + /// it would count the same token twice. + /// * A Claude Code `usage.iterations[]` list is one element per attempt at + /// the request, and on the rare record with more than one the outer + /// counters are the **last** element's — never the sum of them. Adding + /// the list would therefore count the surviving attempt twice *and* bill + /// the abandoned ones, which is not a number the file states anywhere. + /// + /// What that costs is real and is the point: on a retried request this + /// total is what the **last attempt** spent, not what the attempts spent + /// between them. The file records no total across attempts, so a reader who + /// needs one adds the elements themselves — the row says the request was + /// retried (`iterations` on the open level), and `r` has the list. pub fn total(&self) -> u64 { [self.input, self.output, self.cache_read, self.cache_new] .into_iter() diff --git a/src/lens/usage_agent.rs b/src/lens/usage_agent.rs index 7830e59..509aa17 100644 --- a/src/lens/usage_agent.rs +++ b/src/lens/usage_agent.rs @@ -37,9 +37,10 @@ //! | `cache_creation_input_tokens` | `new` | //! //! `output_tokens_details.thinking_tokens` is a *subset* of `output_tokens` and -//! gets no column. **`usage.iterations[]` is never summed**: it is a list whose -//! elements repeat the outer counters per attempt, so adding both counts every -//! token twice ([`usage::Tokens::total`] says so where the adding happens). +//! gets no column. **`usage.iterations[]` is never summed**: it is one element +//! per attempt at the request, and the outer counters are the last attempt's, +//! so a row here is what the attempt that survived spent +//! ([`usage::Tokens::total`] says what that costs, where the adding happens). //! //! # The one decision that shapes the document //! @@ -187,9 +188,9 @@ fn request_lines(u: &Value) -> Vec { } } } - // A list whose elements repeat the outer counters per attempt. Its *length* - // is the fact — the request was retried — and its contents are never summed - // (`usage::Tokens::total` says why). + // One element per attempt at the request. Its *length* is the fact — the + // request was retried, and the row above is the last attempt alone — and + // its contents are never summed (`usage::Tokens::total` says why). if let Some(n) = u.get("iterations").and_then(|i| i.as_array()).map(|a| a.len()) { if n > 1 { lines.push(usage::line("iterations", n)); diff --git a/src/lens/usage_agent_tests.rs b/src/lens/usage_agent_tests.rs index 67282e8..65df111 100644 --- a/src/lens/usage_agent_tests.rs +++ b/src/lens/usage_agent_tests.rs @@ -146,18 +146,19 @@ fn a_subagent_is_marked_without_costing_a_column() { assert!(str_width(&sum(r#"{"type":"assistant"}"#).actor) <= ACTOR); } -/// `iterations` is a list whose elements repeat the outer counter names. -/// Summing both double-counts every token on the records that carry it, which is -/// the mistake this pins against. +/// `iterations` is one element per attempt at the request, and the outer +/// counters are the **last** attempt's. Adding the list would count that attempt +/// twice and bill the abandoned ones, which is the mistake this pins against — +/// so the total here is the last attempt's numbers and nothing else. #[test] fn iterations_are_never_added_to_the_total() { let s = sum( r#"{"type":"assistant","message":{"role":"assistant","content":[], "usage":{"input_tokens":100,"output_tokens":50, "iterations":[{"input_tokens":60,"output_tokens":30}, - {"input_tokens":40,"output_tokens":20}]}}}"#, + {"input_tokens":100,"output_tokens":50}]}}}"#, ); - assert_eq!(s.tokens, 150, "the outer four once, the iterations not at all"); + assert_eq!(s.tokens, 150, "the last attempt once, the list not at all"); assert!(s.what.starts_with("in 100 out 50"), "{}", s.what); } From 5000e4c1a1ddc4d376558b780e1d080dd7544736 Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 17:48:18 +0000 Subject: [PATCH 11/12] docs: mark the half-recorded counter as a path nothing has taken The three things a usage cell can say are not three things that happen. No record measured has ever carried some of its format's counters and not the others, so the dash is a defensive path held to by hand-written tests. Say that where both documents present the distinction, so a later reader does not go looking for the file that produces it. --- SPEC.md | 5 ++++- docs/lenses.md | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/SPEC.md b/SPEC.md index 53c676e..87d7840 100644 --- a/SPEC.md +++ b/SPEC.md @@ -410,7 +410,10 @@ record **spent**: a fixed-width block of token counters, then what the record did, with a record that spent nothing showing its kind and no number columns at all. A printed number means the format recorded that value, `-` means this record did not record a field its format has, and an absent column means the -format has no such field. Counts are floored to four columns, so a row that says +format has no such field — the middle of those three is a defensive path rather +than observed behaviour: no measured record has yet carried some of its format's +counters and not the others, and it is held to by tests alone. Counts are +floored to four columns, so a row that says `18k` promises at least 18,000; the exact integers are on the level under it, and the exact sums are on the run's row and in the status bar. diff --git a/docs/lenses.md b/docs/lenses.md index 457f8e0..f13a26c 100644 --- a/docs/lenses.md +++ b/docs/lenses.md @@ -415,6 +415,14 @@ inside a format that has the field prints `-`. A `0` in the third case would say "this agent wrote nothing to cache" when the truth is "this format does not record cache writes", which is a different and false claim. +The middle row is a **defensive** path, and a later reader should not take it +for observed behaviour: in every session log and trajectory this has been +measured against, a record that carries a usage object carries either all of its +format's counters or none of them, so no real row has yet printed a `-`. It is +exercised by hand-written tests only. It stays because a counter set is the +schema of another program and the day one goes missing the column must still +line up and must not read as a zero. + **A record with no usage shows its kind and nothing more** — `file-history-snapshot`, `queue-operation`, `user` — with no number fields at all, so the numeric column is simply absent where nothing was spent rather than From 996e04690e9941873f25a8d3f97b767e6ea6a29d Mon Sep 17 00:00:00 2001 From: Victor Simonetti Date: Thu, 13 Aug 2026 17:49:48 +0000 Subject: [PATCH 12/12] lens: ask the numeric block about columns, not the whole row The corpus sweep proved no cache-creation column by searching the row for "new", but the row is the numbers *and* what the step did - a command or a path with "new" in it reddened the sweep with a message about a column that was never there. It now reads the block alone. The plan fixture's spend went through an unchecked i64-to-u64 cast in the one place the running total's arithmetic is exercised, where a negative would have pinned the saturating total at its ceiling. --- src/lens/usage_atif_tests.rs | 26 +++++++++++++++++++++----- src/source/record/plan_fixture.rs | 11 +++++++++-- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/lens/usage_atif_tests.rs b/src/lens/usage_atif_tests.rs index 7bcb1ff..9759038 100644 --- a/src/lens/usage_atif_tests.rs +++ b/src/lens/usage_atif_tests.rs @@ -16,6 +16,20 @@ fn sum(json: &str) -> Summary { read(json).expect("the dialect reads this record") } +/// The numeric block of a row, or `""` when the row has none. +/// +/// The rest of the row is what the record *did* — tool names, a command, a path +/// — so asking about the *columns* by searching the whole row answers about the +/// arguments instead: a step that ran `git checkout -b new` is not a step with a +/// cache-creation column. Every row with counters opens with the first field's +/// label, and the turn separator ends the block. +fn block_of(what: &str) -> &str { + match what.starts_with("in ") { + true => what.split(" \u{b7} ").next().unwrap_or(""), + false => "", + } +} + /// Three fields, twenty-eight columns, and no fourth. #[test] fn a_step_shows_the_three_counters_this_format_has() { @@ -47,7 +61,7 @@ fn no_step_ever_shows_a_cache_creation_column() { r#"{"step_id":3,"source":"agent","metrics":{"cached_tokens":1823744}}"#, ] { let s = sum(fixture); - assert!(!s.what.contains("new"), "{}", s.what); + assert!(!block_of(&s.what).contains("new"), "{}", s.what); } } @@ -214,11 +228,13 @@ fn a_real_atif_trajectory_is_read_as_what_it_spent() { let Some(s) = lens.read(step) else { continue }; read += 1; total = total.saturating_add(s.tokens); - // No row of this format ever names a counter it does not have. - assert!(!s.what.contains("new"), "a cache-creation column appeared"); - if s.what.starts_with("in ") { + // No row of this format ever names a counter it does not have — asked + // of the numeric block alone, because a real step's command or path is + // on the same row and a `new` in one of those is not a column. + let block = block_of(&s.what); + assert!(!block.contains("new"), "a cache-creation column appeared"); + if !block.is_empty() { with_metrics += 1; - let block = s.what.split(" \u{b7} ").next().unwrap_or(""); assert_eq!(str_width(block), BLOCK, "the column bends on a real step"); } assert!(s.body.is_none(), "this lens keeps no message text"); diff --git a/src/source/record/plan_fixture.rs b/src/source/record/plan_fixture.rs index 045f1aa..773f418 100644 --- a/src/source/record/plan_fixture.rs +++ b/src/source/record/plan_fixture.rs @@ -37,8 +37,15 @@ impl crate::lens::Lens for Fake { what: String::new(), calls: 1, // `"t"` is what this record claims to have spent, so the seam's - // running total can be exercised without a dialect. - tokens: v.get("t").and_then(|t| t.as_number()).and_then(|n| n.as_i64()).unwrap_or(0) as u64, + // running total can be exercised without a dialect. A count that is + // not a `u64` is zero here, not a cast: a negative would wrap to + // almost 2⁶⁴ and pin the saturating total at its ceiling, which is + // the one number these tests must be able to trust. + tokens: v + .get("t") + .and_then(|t| t.as_number()) + .and_then(|n| n.as_i64()) + .map_or(0, |n| u64::try_from(n).unwrap_or(0)), // A body is the source's to measure; these tests give one to an // item directly (`with_bodies`), which is the same number the // measurement would have produced and keeps this file file-free.