diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index 90662799..3d11582a 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -68,7 +68,8 @@ methodology: - "Reference on BNB Chain and Robinhood Chain: archive nodes per chain, validated against block hashes. These carry no preconfirmation layer a provider consumes, and their chain-supplied timestamps sit within roughly 60 ms of the moment a trade is observable, so they keep the on-chain timestamp as the zero point." - "Solana (since 2026-09-16): the headline is the lag behind the first feed to report the trade, our own node subscription included in the race. Solana has no on-chain timestamp with sub-second precision, so the timestamps providers send compare conventions: Mobula's `date` (its ingestion time) read as a constant 0.10 s, Serialized's `at` (blockTime, whole seconds) as 0.75 s, while against a common clock the two feeds are 10 to 30 ms apart." - "Solana zero point: no RPC WebSocket we hold, public or keyed (mainnet-beta, Helius, Alchemy), precedes the providers' geyser feeds, so an absolute zero point is not available and the first observation is the ruler. Our node stays in the race: it validates the hash on chain and bounds the field whenever a feed is slower than a plain RPC subscription." - - "Solana race: every feed's arrival for a transaction is compared to the earliest one (ties within 5 ms count as first for each feed involved). Published as `head_lag_first_share_pct` (share of the last 24 h of trades a feed reported first, the First to report view) and as `head_lag_seconds` (lag behind the first, 0 for the leader). GeckoTerminal polls and lands 15 to 100 s later; it is measured against the same first arrival." + - "Race: every feed's arrival for a transaction is compared to the earliest feed's (ties within 5 ms count as first for each feed involved; trades only one feed reported are not scored). Published on every chain as `head_lag_first_share_pct`, the First to report view: the share of the last 24 h of trades a feed reported before every other feed." + - "Solana headline: `head_lag_seconds` is the lag behind the first observation of the trade, our node included, floored at 1 ms. GeckoTerminal polls and lands 15 to 100 s later; it is measured against the same first arrival." - "Regions: us-east, eu-west, sgp. Cross-region median reported in the headline." - "Reference on Base: the sequencer's flashblock preconfirmation stream, held by us in every region. A Base block becomes queryable about 0.36 s after the timestamp it carries, while flashblocks publish every 200 ms inside the 2 s interval, so that timestamp is not the moment a trade becomes knowable." - "Base emissions we cannot match to a flashblock are dropped rather than measured against a different ruler. Measuring against the block timestamp charged a provider reading preconfirmations with a negative lag, and described the chain's stamping convention rather than the provider's pipeline." @@ -106,7 +107,7 @@ metric_panels: metric: avg by (aggregator) (head_lag_first_share_pct{}) unit: pct higher_is_better: true - description: "Share of the last 24 hours of trades this feed delivered before every other feed (ties within 5 ms count for both). Our own node subscription runs in the same race; on Solana it wins a few percent, which is the gap between a plain RPC subscription and a geyser feed." + description: "Share of the last 24 hours of trades this feed delivered before every other feed (ties within 5 ms count for both, so the shares can add up to a little more than 100). Trades only one feed reported are not scored. Our own node is not a competitor here; it sets the zero point of the lag views." prometheus: window: 24h diff --git a/harnesses/aggregator-head-lag/cmd/script/race.go b/harnesses/aggregator-head-lag/cmd/script/race.go index f0e8d8f9..43db8b3e 100644 --- a/harnesses/aggregator-head-lag/cmd/script/race.go +++ b/harnesses/aggregator-head-lag/cmd/script/race.go @@ -17,11 +17,12 @@ import ( // the first arrival, and publishes // // - head_lag_first_total{aggregator,chain,region}: how often each feed -// (the node reference included, as aggregator="reference") reported -// the trade first. Arrivals within raceTie of the earliest count as -// first for every feed involved: below that, the order is network -// jitter between our probe and two servers, not a property of the -// feeds. +// reported the trade before every other feed. Arrivals within raceTie +// of the earliest feed count as first for every feed involved: below +// that, the order is network jitter between our probe and two +// servers, not a property of the feeds. The node reference is scored +// apart (aggregator="reference": trades where it beat every feed) and +// is not a competitor in the share. // - head_lag_races_total{chain,region}: races closed. // - head_lag_first_share_pct{aggregator,chain,region}: the share over a // rolling 24 h, computed in-process so the bench can read it as a @@ -179,9 +180,7 @@ func (b *raceBook) closeLocked(e *raceEntry, k string) { // Our own node takes part when it saw the trade. It also validates // that the hash is real; a race nobody but a single provider saw is // still closed, but carries no reference lag. - refSeen := false if refAt, ok := reference.lookup(e.chain, hash); ok { - refSeen = true if refAt.Before(e.t0) { e.t0 = refAt } @@ -197,24 +196,55 @@ func (b *raceBook) closeLocked(e *raceEntry, k string) { e.closedAt = time.Now() return } + providers := 0 + for _, o := range e.obs { + if o.aggregator != "reference" { + providers++ + } + } + // The share is a race between feeds: the earliest PROVIDER arrival is + // the line, and every provider within raceTie of it is first. Our + // node still sets t0 for the lag figures and is counted on its own + // (head_lag_first_total{aggregator="reference"}) when it beat every + // feed, but it is not a competitor in the share: on Base and BNB the + // flashblock / node reference precedes every feed on nearly every + // trade, and a share where the reference took 95 % told readers + // nothing about the feeds they choose between. + providerT0 := time.Time{} + for _, o := range e.obs { + if o.aggregator == "reference" { + continue + } + if providerT0.IsZero() || o.at.Before(providerT0) { + providerT0 = o.at + } + } winners := []string{} for _, o := range e.obs { - b.note(e.chain, e.region, o.aggregator) delta := o.at.Sub(e.t0) - if delta <= raceTie { + if o.aggregator == "reference" { + if !providerT0.IsZero() && providerT0.Sub(o.at) > raceTie { + headLagFirst.WithLabelValues("reference", e.chain, e.region).Inc() + } + continue + } + b.note(e.chain, e.region, o.aggregator) + if o.at.Sub(providerT0) <= raceTie { winners = append(winners, o.aggregator) headLagFirst.WithLabelValues(o.aggregator, e.chain, e.region).Inc() } - if o.aggregator != "reference" && raceChains[e.chain] { + if raceChains[e.chain] { RecordHeadLag(o.aggregator, e.chain, o.lagBlocks, raceLagSeconds(delta), e.region, hash) } } - if !refSeen { - b.note(e.chain, e.region, "reference") - } - headLagRaces.WithLabelValues(e.chain, e.region).Inc() e.closed = true e.closedAt = time.Now() + if providers < 2 { + // One feed against our node only: lags are recorded above, but + // there was no race between feeds to score. + return + } + headLagRaces.WithLabelValues(e.chain, e.region).Inc() // Rolling 24 h share. pk := e.chain + "|" + e.region diff --git a/src/components/benchmark-body.tsx b/src/components/benchmark-body.tsx index beea871e..96db79a7 100644 --- a/src/components/benchmark-body.tsx +++ b/src/components/benchmark-body.tsx @@ -680,6 +680,10 @@ export function BenchmarkBody({ ...viewBenchmark, metric: activePanel.label, unit: activePanel.unit ?? viewBenchmark.unit, + // A panel carries its own direction (First to report: higher is + // better on a lower-is-better latency bench); without this the + // ranked chart put 0 % rows first. + higherIsBetter: activePanel.higherIsBetter ?? viewBenchmark.higherIsBetter, results: viewBenchmark.results .filter((r) => vals[r.slug] != null && Number.isFinite(vals[r.slug])) .map((r) => ({ ...r, ms: { ...r.ms, p50: vals[r.slug] } })),