diff --git a/Cargo.lock b/Cargo.lock index b45d7b694f..b5a4b0e44d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2911,6 +2911,12 @@ dependencies = [ "yasna", ] +[[package]] +name = "readme-rustdocifier" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ad765b21a08b1a8e5cdce052719188a23772bcbefb3c439f0baaf62c56ceac" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2920,6 +2926,16 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "reed-solomon-simd" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6badd4f4b9c93832eb3707431e8e7bea282fae96801312f0990d48b030f8c5" +dependencies = [ + "fixedbitset", + "readme-rustdocifier", +] + [[package]] name = "regex" version = "1.9.1" @@ -4263,6 +4279,7 @@ dependencies = [ "libp2p-stream", "lru", "rand 0.8.5", + "reed-solomon-simd", "serde", "serde_json", "sha2", diff --git a/README.md b/README.md index 433bc284c9..03d672007d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# stellar-core: leader schedule with terms, leader tx-set push, apply-time tx-set validation, transactions sent to the leaders only (experimental branch) +# stellar-core: leader schedule with terms, leader tx-set push, apply-time tx-set validation, transactions sent to the leaders only, next tx set selected during the previous ballot, the leader's tx set delivered as erasure-coded pieces, level 0 of the bucket list written as shards under a commutative hash (experimental branch) **This branch is an experiment. It is not production code and must not run on a real network.** The leader schedule and the tx-set push have no feature flag and no protocol gate, and the branch @@ -22,13 +22,34 @@ sleeps instead), the same 90 validators at 800 ms hold the target at 4,000 TPS ( 8,000 TPS. One run per point. Results and the ranked bottlenecks: [Real apply at 90 validators and 800 ms](#real-apply-at-90-validators-and-800-ms). +**With the transaction meta off, a bucket list account cache and 256-byte bucket index pages** +(2026-11-13, real execution, one run per point), the 90 validators hold 800 ms at 6,000 TPS +(block time p75 805 to 808 ms, p99 872 to 880 ms) and take 0.87 s per ledger at 8,000 TPS. The +sixth experiment (the leader selects its next tx set while the current ledger is in its ballot, +and pulls less for a build) shortens the leader's tx set build at 8,000 TPS from 176 to 121 ms. +The time per ledger goes from 873 to 858 ms, which one run on each side does not establish. No +tx set was invalid or short. Results, and why the shorter build does not show in the block time: +[Real apply with the meta off, and the tx set build off the critical path](#real-apply-with-the-meta-off-and-the-tx-set-build-off-the-critical-path). + +**With the leader's tx set delivered as erasure-coded pieces** (the seventh experiment, +2026-11-14, real execution, meta off, one run per point, one build for every mode): the 90 +validators **hold the 800 ms target at 8,000 TPS** (a ledger every 804 ms, block time p75 +837 ms, verdict PASS), where the push of the whole set to every peer takes 869 ms and a spanning +tree 832 ms. The leader writes 9.6 MB per set instead of 95.7 MB. With 29 of the 90 validators +withholding every piece they should relay, a ledger takes 837 ms and 1.7% of the deliveries +fall back to a request. **At 10,000 TPS** every ledger is full (10,000 payments) and takes +1.0 s; all 3,600,000 payments are included. What limits it there is no longer the set's way to +the followers but the serial apply path: validation, execution and commit of a ledger take +0.86 s. Results, the comparison tables and the bottlenecks: +[The leader's tx set as erasure-coded pieces](#the-leaders-tx-set-as-erasure-coded-pieces). + The earlier benchmark (48 validators, 5,000 TPS, 1,000 ms, 4-core instances) is kept below. ## What this branch is It is the overlay-v2 stack up to [#5459](https://github.com/stellar/stellar-core/pull/5459) (Rust overlay over QUIC, millisecond close times, compressed tx-set responses, leader-only tx-set -construction) plus five experiments: +construction) plus seven experiments: 1. **No nomination** ([#5495](https://github.com/stellar/stellar-core/pull/5495)). The SCP nomination phase is deleted and replaced by a leader schedule that every node computes locally. @@ -86,8 +107,8 @@ construction) plus five experiments: - **One build.** The first-round leader builds once, a lead time before its trigger, pushes the set as soon as it exists and proposes it unchanged at the trigger. 5. **Transactions go to the upcoming leaders only** - ([#5501](https://github.com/stellar/stellar-core/pull/5501), the latest experiment). No - protocol gate and no version negotiation. + ([#5501](https://github.com/stellar/stellar-core/pull/5501)). No protocol gate and no version + negotiation. - **Targets.** A node that accepts a transaction no longer advertises it to every peer. Its overlay sends the body straight to the leaders of the node's current round and of the next one for the next ledger and, one ledger before a new leader term starts, to that term's @@ -103,11 +124,101 @@ construction) plus five experiments: connection key, signed by the node key. - **Cost.** Followers no longer hold the transactions before the leader's set arrives, so they verify signatures when they check the set at apply instead of in the background. - -Unchanged by all five: the ballot protocol's accept, confirm and commit rules. For a valid tx +6. **The leader's tx set build off the critical path** + ([#5505](https://github.com/stellar/stellar-core/pull/5505)). No protocol change: the sets are ordinary + tx sets, and every node validates them at apply as before. + - **A smaller build.** A build pulls the set's capacity plus 10% from the mempool + (`EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT`); it pulled twice the capacity. A phase that lost + candidates (invalid ones, or a second transaction of an account) and is short pulls again, + twice as deep, as long as a pull brings it something new, up to three pulls. The structural + check of the selection runs beside the XDR write and ends before the set goes to the + overlay. A leader that executes its own vote no longer reads the written set back: that + execution validates it. If the execution does not start, the read-back runs in the + background. + - **Early selection.** The node first in line to lead ledger N+1 selects N+1's set on + background threads while ledger N is in its ballot. It starts from N's tx set, which it + holds at its first vote: it assumes N's transactions applied (sequence numbers and fees), + validates its mempool's top candidates on that view and selects the phases. + - **Completion at the close.** When N has closed with the value the selection was made for, + the main thread validates every selected transaction on the closed ledger, for the set's + real close time, and removes what does not hold. It then tops the set up with what the + mempool received since, validated the same way (the Soroban phase is extended in place, the + classic phase is selected again), computes the base fees, runs the structural check and + writes the set. A selection for another value or other limits, after an upgrade, or one that + is not ready is dropped, and the set is built from scratch. + - `EXPERIMENTAL_EARLY_TX_SET_SELECTION` (default true) turns the early selection off. + - It replaces the selection of [#5503](https://github.com/stellar/stellar-core/pull/5503) + (closed), which waited for the leader's own execution of ledger N, and keeps that attempt's + early execution: a leader starts executing the set it prepared as soon as it is built. + - Two supporting changes came with it: a hit in the bucket list's account cache is read under + the shared lock, and the load generator has the two test-only modes of the real-apply runs + (`sac_setup`, `sac_payment`). + +7. **The leader's tx set travels as erasure-coded pieces, or down a spanning tree** + ([#5506](https://github.com/stellar/stellar-core/pull/5506), the latest experiment). Config-gated, default off: `EXPERIMENTAL_TX_SET_TREE` = + `off`, `star`, `hash`, `rtt` or `coded`. Every node of a network needs the same value. No + protocol change: only who sends the set's bytes to whom. + - **Why.** The leader of experiment 2 writes its whole set to every peer: 89 copies at 90 + validators, 96 MB per ledger at 8,000 TPS. A follower cannot vote before it holds the set. + - **A layout every node derives.** As the leader schedule: from the nodes of the transitive + quorum and the slot's seed. For a slot and a leader, the other nodes are ordered by + `SHA256(seed || leader || node)`. Nothing is negotiated. + - **`hash`: a spanning tree.** The order is laid out as a complete tree of the smallest fanout + that reaches every node in two hops (9 at 90 validators). The leader writes the set to its 9 + children and every node passes it on to its own. `rtt` is the same tree laid out by the + leader from round-trip times it measures; the order travels with the set. + - **`coded`: pieces.** The leader cuts the compressed set into 16 pieces of which any 8 + rebuild it (Reed-Solomon; fewer pieces in a small network), and signs the root of the + pieces' Merkle tree. Every piece has 5 relays, to which the leader writes it; a relay + passes its piece on to a fifth of the other nodes. So a node gets its 16 pieces from 16 + different senders, no piece makes more than two hops, and a node has the set when the 8 + fastest of its 16 pieces have arrived. + - **What a node accepts.** A piece only from the node that owes it that piece in the layout, + only for a leader its Core expects for the slot, and only with the leader's signature and a + Merkle path to the signed root. A relay passes a piece on after those checks. A rebuilt set + then goes the way of every received set: decompressed, parsed strictly and filed under the + hash of its content. + - **What the code protects.** The delivery: up to 8 of a node's 16 senders can be down, slow + or lying without the node having to wait and ask. Not consensus, which does not depend on + it: a set is used only when an SCP statement names the hash of its content, as before. + - **Fallback.** A node that expects the set through relays waits 300 ms before it asks a + peer for it, as it does without the experiment at once. + - `star` is the push of experiment 2 with a header that carries the send time, the baseline + of the measurements. `EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING=N` makes N validators + pass nothing on, for fault runs. + - Wire formats, checks and limits: + [docs/rust-overlay/txset-fetching.md](docs/rust-overlay/txset-fetching.md). + +8. **Level 0 of the live bucket list is written as shards, under a commutative hash** + (`EXPERIMENTAL_SHARDED_TOP_BUCKET = N`, 0 = off; `src/bucket/ShardedTopBucket.h`). + - Today a ledger's entries become one level-0 bucket: sorted, serialized, written and hashed + with SHA-256 by one thread in the ledger's commit (97 ms of a 203 ms commit at 10,000 + payments per ledger). + - With the setting the hash of a level-0 bucket is the **LtHash16 of its records** + (`src/crypto/LtHash.h`), which does not depend on their order or on how they are split. + A ledger's entries are cut into shards by key (N for contract data and code, N for TTL + entries, N for everything else), and every shard is sorted, written and synced, merged + with the shard of the ledger before, hashed and indexed on its own thread; the hashes are + added up. + - What a level-0 bucket **is** does not change, so its single file (written afterwards by a + thread of its own, the shards interleaved by key) is byte for byte the file of today: it + is what spills into level 1 and what is published to history archives. **Levels 1 and + deeper keep SHA-256 and their buckets.** + - The number of shards is each node's choice and changes no hash. The setting itself + changes the bucket list hash of the ledger header: every node of a network has it on, or + none (a real version needs a protocol version; the setting is refused on the public + network). + - A node that stops between a ledger's commit and the end of the bucket's file rebuilds the + file at startup from the ledger's shard files. + +Unchanged by all eight: the ballot protocol's accept, confirm and commit rules. For a valid tx set, ledger state and meta are also unchanged; experiment 3 changes when a set is validated and executed, and what an invalid set does to the ledger; experiment 4 changes who leads; experiment -5 changes which transactions reach a leader, and which submissions a node refuses. +5 changes which transactions reach a leader, and which submissions a node refuses; experiment 6 +changes which candidates a leader looks at and when it selects them, not the checks a set passes; +experiment 7 changes which peers a set's bytes come from, not what a node does with a set; +experiment 8 changes how level 0 of the bucket list is hashed (and so the bucket list hash in +the header) and how it gets to disk, not what it holds. ## Performance results @@ -302,7 +413,7 @@ Ranked by their share of the leader's cycle (90 validators, 16 clusters, local N | # | Step | At 8,000 TPS (1.16 s) | At 6,000 TPS (0.86 s) | Grows with the load | Most plausible fix | |---|---|---:|---:|---|---| | 1 | Ballot, first ballot to externalize | 310 ms, 27% | 307 ms, 36% | no | None found here. It is the floor of the cycle, and the window that speculative execution hides in | -| 2 | Tx set build on the leader's main thread. It needs the previous ledger closed, and once the cycle has no slack the trigger is already due when it ends | 261 ms, 23% | 137 ms, 16% | yes; between 6,000 and 8,000 TPS faster than the load, because of the backlog | Build the next set while the current ledger is in its ballot. Tried once as [#5503](https://github.com/stellar/stellar-core/pull/5503) with simulated apply (slower than the base in its one run, closed); not measured with real apply | +| 2 | Tx set build on the leader's main thread. It needs the previous ledger closed, and once the cycle has no slack the trigger is already due when it ends | 261 ms, 23% | 137 ms, 16% | yes; between 6,000 and 8,000 TPS faster than the load, because of the backlog | Build the next set while the current ledger is in its ballot. Tried once as [#5503](https://github.com/stellar/stellar-core/pull/5503) with simulated apply (slower than the base in its one run, closed). Experiment 6 does it from the current ledger's tx set: measured with real apply in the next section (build 31% shorter, block time 2%) | | 3 | Commit, always after externalize | 244 ms, 21% | 177 ms, 21% | linear, 31 to 35 us per payment | No debug meta on a validator: **measured, 43% less commit per payment** (below) | | 4 | Between the proposal and the followers' vote, beyond the ballot: the set's compression, storage, push, fetch, decoding and hashing, and the main threads' queues | 201 ms, 17% | 142 ms, 17% | yes | Not known. Only the fetch has a timer (88 and 58 ms of it); the rest needs timers first | | 5 | Execution beyond the ballot (from about 4,700 payments per ledger on) | 81 ms, 7%; on a follower 167 ms | 46 ms, 5% | linear, 65 us per payment | Meta off and the account cache: measured (below). Then the serial account prefetch and the tx set validation | @@ -363,14 +474,335 @@ type at 8,000 TPS; apply cost per node does not depend on the network size): prefetch falls by 70%, but an account load on a worker takes twice as long, because a cache hit takes an exclusive lock, and the tx set validation gives back what it gained. Execution is 9% shorter and the block time does not move. -- **Not measured: either setting at 90 validators.** Scaled per payment, meta off would take - about 105 ms off the commit at 8,000 TPS and 70 ms at 6,000 TPS (about 30 and 20 ms of that is - the test build's copy), which puts 6,000 TPS near 800 ms and leaves 8,000 TPS above 1 s. +- **Either setting at 90 validators** was measured after this was written: see the next + section. The estimate made before, scaled per payment: meta off would take about 105 ms off + the commit at 8,000 TPS and 70 ms at 6,000 TPS (about 30 and 20 ms of that is the test build's + copy), which puts 6,000 TPS near 800 ms and leaves 8,000 TPS above 1 s. Measured: 801 ms at + 6,000 TPS and 873 ms at 8,000 TPS. - **At 90 validators one low-priority core thread is busy 100% of a core** in every run, independent of the load (3% at 24 validators). Most likely the quorum intersection checker, which is on by default; inferred from the code, not profiled. It does not sit in the block time. +### Real apply with the meta off, and the tx set build off the critical path + +Runs of 2026-11-13. Every run of this section executes the load, with the transaction meta off +(`METADATA_DEBUG_LEDGERS=0`; in these test builds also `DISABLE_TX_META_FOR_TESTING=true`), +16 clusters, an 800 ms target, m6id.8xlarge with core's data on the local NVMe disk, and the +load and limits of the real-apply runs above. One run per point. + +**Bucket list settings**, on 6 validators at 8,000 TPS offered (apply cost per node does not +depend on the network size). Means over the whole load, medians over the 6 nodes. The table of +the two settings above averages over the load's last five minutes, so compare inside this table +only; its first two rows are the same two runs. + +| Settings beyond meta off | Execution / commit / whole close, per payment | Tx set validation at apply | Time per ledger: mean / p99 / longest | Outcome | +|---|---:|---:|---:|---| +| none | 44.3 / 15.8 / 60.1 us | 77 ms | 802 / 938 / 1,042 ms | all 2,880,000 payments included | +| 4 GB account cache, a hit takes an exclusive lock | 40.2 / 16.4 / 56.5 us | 101 ms | 814 / 1,061 / 1,231 ms | all included (two generators timed out on their last payments) | +| Every live bucket in memory (`BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT=0`) | **33.7** / 17.6 / 51.3 us | 63 ms | 829 / **1,637** / 2,195 ms | all included; **the close stalls** | +| 256-byte index pages (`BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT=8`; the default is 14, 16 KiB) | 38.0 / 16.0 / 54.1 us | 66 ms | 816 / 1,047 / 2,792 ms | all included; one ledger took 2.0 s on every node | +| 4 GB account cache, a hit is read under the shared lock (this branch), and 256-byte index pages | 37.3 / 17.0 / 54.3 us | 73 ms | **802 / 918 / 1,194 ms** | all included, **verdict PASS** | + +- **Every live bucket in memory is the fastest apply and not usable as it is.** Execution per + payment is 24% lower than with the meta off alone, but the time from consensus to the close is + 0.4 to 0.5 s at every 8th ledger and 0.7 to 1.7 s at every 32nd, and grows during the run + (median 186 ms). Inferred from the code, not profiled: with this setting a merged bucket is + loaded into memory entry by entry when it is indexed, and the deeper buckets grow with the run. +- **256-byte index pages: execution 14% lower, the whole close 10% lower, no periodic stalls.** +- **The account cache no longer slows the tx set validation** once a hit is read under the + shared lock (101 ms with the exclusive lock, 73 ms with the shared one, 66 to 77 ms without a + cache). On top of the small index pages it adds nothing these runs can show (37.3 against + 38.0 us of execution per payment). +- The 90-validator runs below use the last row's settings. + +**90 validators**, 30 organizations, as in the real-apply runs above. "Base" is this branch +before experiment 6, with the load generator modes and the cache's shared lock (`54260eafb`). +The meta-on columns are the 16-cluster runs of the table above: another build, and core's +default bucket settings. + +| | Meta on, 6,000 TPS | **Base, 6,000 TPS** | Meta on, 8,000 TPS | **Base, 8,000 TPS** | **Experiment 6, 8,000 TPS** | +|---|---:|---:|---:|---:|---:| +| Payments per ledger | 5,150 | 4,800 | 8,000 (the cap) | 7,085 | 7,076 | +| **Time per ledger: mean / p75 / p99** | 0.86 s | **801 / 806 / 868 ms** | 1.16 s | **873 / 928 / 1,100 ms** | **858 / 908 / 1,107 ms** | +| Block time p75 / p99 of the median node (limits 960 / 1,600 ms), first and second window | 927 / 1,164 ms | 805 / 872 and 808 / 880 ms | 1,236 / 1,482 ms | 946 / 1,143 and 888 / 1,125 ms | 937 / 1,151 and 939 / 1,151 ms | +| **Leader: tx set build** | 137 ms | 122 ms | 261 ms | **176 ms** | **121 ms** | +| Leader: set built to consensus | 449 ms | 392 ms | 511 ms | **468 ms** | **514 ms** | +| Follower: tx set fetch | 58 ms | 27 ms | 88 ms | **98 ms** | **155 ms** | +| Consensus to ledger closed: leader / follower | 223 / 242 ms | 140 / 136 ms | 325 / 453 ms | 196 / 220 ms | 198 / 210 ms | +| Execution / commit, per payment | 65 / 34 us | 40.0 / 16.4 us | 65 / 31 us | 40.0 / 16.1 us | 39.2 / 16.3 us | +| Of the ballot, left after execution | 14 ms | 78 ms | 0 | 9 ms | 13 ms | +| Payments included, of those offered | (inclusion check not run) | 2,159,941 of 2,160,000 | about 6,860 TPS | 2,879,853 of 2,880,000 | **2,880,000 of 2,880,000** | +| Outcome | inside the limits, 16% over the target and rising | **on target on all 90 nodes**; verdict FAIL for the 59 missing payments | window missed on all 90 nodes | inside the limits, **9% over the target**; verdict FAIL for the 147 missing payments | inside the limits, 7% over the target; **verdict PASS** | + +- **With the meta off and these settings the base holds 800 ms at 6,000 TPS with room** (a + quarter of the ballot is left after execution) **and takes 0.87 s per ledger at 8,000 TPS**, + where it took 1.16 s. Per payment, execution is 40 us instead of 65 and the commit 16 us + instead of 31 to 35. +- **At 8,000 TPS the base has no slack left**: 9 ms of the ballot remain after execution, and + the build starts when the previous ledger has closed (trigger to first ballot 0.30 s; 0.09 s at + 6,000 TPS). The leader's cycle is build 176 ms, set built to consensus 468 ms, consensus to + close 196 ms. +- **Both base runs lost payments**: 59 and 147 were never included, and the 3 and 4 generators + that had sent them report a failure, which is the whole reason for the verdict. No node had a + pending transaction of its own left. Not looked into; the base sends a transaction to the + upcoming leaders only (experiment 5). The generators' timeouts of the earlier series are gone: + a generator now waits 150 ledgers for its last payments. + +**Experiment 6: the next tx set selected before the close.** Base and experiment at the same +load and settings; on 6 validators both networks are on target, so only the build can differ. + +| | 6 validators, base | 6 validators, experiment 6 | 90 validators, base | 90 validators, experiment 6 | +|---|---:|---:|---:|---:| +| Leader's tx set build (main thread) | 144 ms | **94 ms** | 176 ms | **121 ms (-31%)** | +| Sets completed from an early selection | | 88% | | 93% | +| Completion: validity pass on the closed ledger | | 14 ms | | 16 ms | +| Completion: top-up from the mempool (share of the final set it adds) | | 39 ms (27%) | | 60 ms (34%) | +| Completion: XDR write, with the structural check beside it | | 23 to 24 ms | | 25 to 27 ms | +| The selection: ready before it is needed / time it takes in the background | | 189 / 108 ms | | 313 / 142 ms | +| Base build: after the set is pushed to the peers (read-back and final check) | 40 ms | | 51 ms | | +| Signature checks inside builds | 411,060 | 48,436 | 452,437 | 32,107 | +| Selected transactions removed at the close; sets short of the cap with more in the mempool; refill pulls | | 0; 0; 0 | | 0; 0; 0 | +| Sets found invalid at apply; executions dropped; payments failed | 0; 0; 0 | 0; 0; 0 | 0; 0; 0 | 0; 0; 0 | +| Time per ledger | 802 ms | 801 ms | 873 ms | 858 ms | + +- **No set of the experiment was invalid, and none was short.** That holds on 6 and on 90 + validators, with every offered payment included. This load has no overlap between consecutive + sets (no selected transaction had to be removed), so the removals and the refill were not + exercised on a network. +- **The leader's build is 55 ms shorter at 90 validators and the time per ledger 15 ms.** Two + runs of one point differed by 4 and by 14 ms in the earlier series, so the block time gain is + not established; it is small at most. +- **Why the build's 55 ms do not arrive in the block time.** The followers' fetch of the set is + 57 ms longer and the leader's way to consensus 46 ms. The base pushes its set to the peers as + soon as the XDR is written and only then reads it back and runs the final check: those 51 ms + pass while the set is already travelling. The experiment has nothing left to do after the + push. What counts is the time from the close of ledger N until the XDR of the next set exists, + and that is about the same: 125 ms in the base (176 less 51), about 110 ms for a completion. + After it the set needs about 0.15 s to reach the followers in both builds. +- **The selection is made too early.** At 90 validators it is ready 0.31 s before it is needed + and takes 0.14 s, so its candidates are pulled about 0.45 s before the close. A third of the + final set arrives after that and is pulled, decoded, validated and added on the main thread at + the close: 60 ms, close to what the base spends on its pull, decoding and trimming together + (68 ms). The + start is timed by the time between a node's vote and its own build, which a node measures only + when it builds: 5 times in 450 ledgers at 90 validators. +- **Next, by what it should buy** (not measured): time the selection's start on every node at + every ledger, so that it pulls about 0.2 s before the close; let the selection take a second, + small pull just before it selects; and the largest piece, push the selected transactions to + the peers while ledger N is in its ballot and send only the difference at the close. The + 0.15 s of delivery are now the largest part of the cycle that is neither ballot nor apply. + +### The leader's tx set as erasure-coded pieces + +Runs of 2026-11-14 (experiment 7). Every run executes the load (real apply), with the settings +of the last row of the bucket list table above: transaction meta off, 4 GB account cache, +256-byte index pages, 16 clusters, an 800 ms target, 90 validators (30 organizations, 4 +regions) on m6id.8xlarge. **One build for every run** (`8015877d9`), so the only difference +between the modes is `EXPERIMENTAL_TX_SET_TREE`. One run per point. + +**The best results: coded pieces at 8,000 and at 10,000 TPS.** + +| Coded pieces (16 pieces, 8 rebuild the set, 5 relays per piece) | 8,000 TPS | 10,000 TPS | +|---|---:|---:| +| Verdict | **PASS** | FAIL (block time p75 over its limit of 960 ms) | +| **Time per ledger: mean / p50 / p75 / p99** | **804 / 800 / 817 / 953 ms** | **978 / 964 / 1,043 / 1,359 ms** | +| Block time p75 / p99 of the median node, first and second window | 837 / 994 and 836 / 994 ms | 1,065 / 1,367 and 1,068 / 1,368 ms | +| Payments per ledger (the cap is 125% of a ledger's offered load) | 6,400 (cap 8,000) | 9,674; **10,000, the cap, in every ledger** once the load is steady | +| Payments included, of those offered; failed | **2,880,000 of 2,880,000**; 0 | **3,600,000 of 3,600,000**; 0 | +| Leader's upload per set | 9.6 MB (80 pieces) | 15.0 MB (80 pieces) | +| Leader's send to the set being in a follower's overlay | 104 ms | 107 ms | +| Sets in hand within 100 / 150 / 200 ms of the send | 47% / 86% / 100% | 46% / 88% / 100% | +| Sets a node had to ask for after the wait | 0 | 0 | +| Pieces invalid, refused or not written; rebuilds failed | 0; 0 | 0; 0 | +| Rebuilding a set from its pieces | 1.3 ms | 2.0 ms | +| Tx set validation at apply / execution / commit, per ledger | 103 / 323 / 133 ms | 164 / 495 / 205 ms | +| The same per payment | 16.0 / 50.4 / 20.7 us | 16.4 / 49.5 / 20.5 us | + +- **At 8,000 TPS the network is on its target**, for the first time in this fixture: the three + runs without the experiment took 873, 858 and 869 ms per ledger. +- **At 10,000 TPS it carries the load at one ledger per second**: the ledgers are full, the + queue does not grow (10,000 payments per 1.007 s in the steady part of the load), nothing is + lost. It does not hold 800 ms. See "Where a ledger's time goes at 10,000 TPS" below. + +**Coded pieces against the spanning tree and the push to every peer**, at 8,000 TPS. "Star" is +the branch as it was: the leader writes the whole set to all 89 peers. + +| 8,000 TPS, same build | Star (push to every peer) | Hash tree (fanout 9) | **Coded pieces** | +|---|---:|---:|---:| +| Verdict | FAIL (p75 over 960 ms on 42 and 83 of 90 nodes) | PASS | **PASS** | +| **Time per ledger: mean / p50 / p75 / p99** | 869 / 853 / 935 / 1,104 ms | 832 / 817 / 868 / 1,055 ms | **804 / 800 / 817 / 953 ms** | +| Block time p75 / p99 of the median node, first and second window | 960 / 1,139 and 965 / 1,140 ms | 901 / 1,136 and 900 / 1,136 ms | **837 / 994 and 836 / 994 ms** | +| Payments per ledger | 6,926 | 6,627 | 6,400 | +| **Leader's upload per set** | **95.7 MB** (89 copies) | 9.1 MB (9 copies) | **9.6 MB** (80 pieces) | +| Written by all relays per set | 0 | 81 MB (80 copies; a relay 9 MB) | 163 MB (1,356 pieces; a relay 2.0 MB) | +| **Leader's send to the set being in a follower's overlay** | 173 ms | 134 ms (72 ms from the leader, 141 ms through a relay) | **104 ms** | +| Sets in hand within 100 / 150 / 200 / 300 ms of the send | 4% / 36% / 65% / 100% | 31% / 61% / 81% / 100% | 47% / 86% / 100% / 100% | +| Core's wait for a set it was told of (`scp.fetch.txset`) | 157 ms | 128 ms | **66 ms** | +| Leader's trigger to the first ballot | 289 ms | 263 ms | **137 ms** | +| Leader: set built to consensus | 516 ms | 493 ms | **431 ms** | +| Sets a node asked for after its 300 ms wait | no wait in star | 149 of 17,176 | **0 of 12,394** | +| All overlay bytes written, per node | 15.7 MB/s | 16.5 MB/s | 18.7 MB/s | +| Payments included, of 2,880,000; failed | all; 0 | all; 0 | all; 0 | + +- **Coded pieces: 65 ms less per ledger than the star (-7.5%), the tree 37 ms.** Since the + coded run sits on the target, 65 ms is the least the coding gains at this load. With shorter + ledgers a ledger also holds fewer payments, so apply is shorter too. +- **Both take 90% of the leader's upload away. The tree gets a third of the way, because a node + still has one path to the set**: its parent. In the coded layout a node needs the 8 fastest + of 16 two-hop paths, and there is no slow tail: every set was in hand within 200 ms of the + leader's send, where the tree had 19% later than that and 149 sets that had to be asked for. + That tail is the tree's p99 (1,136 ms, as the star's), which the coding lowers to 994 ms. +- **The price is bytes**: the network writes 1.8 times the tx-set bytes of the star (172 against + 96 MB per set). Of everything the overlay writes that is 6% more per ledger. +- **Under simulated apply the same change did not move the block time** (10,080 TPS, 90 + validators, three runs of the tree modes on 2026-11-14: the delivery went from 168 to 131 ms + with the hash tree and to 84 ms with the tree by round-trip times, the block time stayed at + 800 ms). That fixture was on its target with slack. With real apply there is none, and + every millisecond of delivery is on the path of the vote. The tree by round-trip times and + a coded layout that uses round-trip times were not run with real apply. + +**Coded pieces when validators withhold.** The same run with +`EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING=29`: the 29 validators with the smallest +node IDs, just under a third of the 90, pass no piece on. They still lead, receive and vote, +so this is the withholding part of a Byzantine fault, not all of it. The prediction was +written down before the run, from a model of the layout fitted to the star run. + +| Coded pieces, 8,000 TPS | Nobody withholding | **29 of 90 validators withholding** | Predicted | Star, nobody withholding | +|---|---:|---:|---:|---:| +| Verdict | PASS | **PASS** | PASS | FAIL | +| Time per ledger: mean / p75 / p99 | 804 / 817 / 953 ms | **837 / 879 / 1,042 ms** | 815 to 850 ms | 869 / 935 / 1,104 ms | +| Block time p75 / p99 of the median node | 837 / 994 ms | 900 / 1,131 ms | | 960 / 1,139 ms | +| Leader's send to the set being in hand | 104 ms | **141 ms** | 130 to 150 ms | 173 ms | +| Sets in hand within 150 / 200 / 300 ms | 86% / 100% / 100% | 54% / 92% / 100% | | 36% / 65% / 100% | +| **Deliveries that ended with a request after the 300 ms wait** | 0 | **543 of 31,506 (1.7%)** | 1 to 3% | no wait in star | +| Core's one-second retries | 0 | **0** | | 1 | +| Pieces a relay passed on, per set | 1,356 | 901 | | | +| Pieces invalid or refused; rebuilds failed | 0; 0 | 0; 0 | 0 | | +| Payments included, of 2,880,000; failed | all; 0 | all; 0 | | all; 0 | + +- **With a third of the relays silent the coded network still closes every ledger without + Core's retry, faster than the healthy star and about as fast as the healthy tree** (832 ms). + 98.3% of the deliveries needed no request. +- The same fault in the hash tree was not run. The model that predicted this run gives it 28% + of deliveries waiting 300 ms and asking; on 7 validators under consensus, with two of them + withholding, the tree asked 39 times in 30 ledgers and the coded network 3 times (one run of + a test). +- Not run on a network: relays that send wrong pieces, a leader that signs pieces that do not + fit together, more than a third withholding. Those are covered by tests with 9 overlay nodes + (a changed piece, a piece from a node that does not owe it, a root signed by another + validator or for another history of the network, a validator that may not lead, a sender + that floods: none is kept or passed on, and the leader's real set still arrives). + +**Where a ledger's time goes at 10,000 TPS** (coded pieces; the steady part of the load, +between the readings at 150 s and at 315 s; means over all nodes). + +| Step, in the order of the leader's cycle | Per 10,000-payment ledger | Share of the 1,007 ms cycle | At 8,000 TPS (6,400 payments) | +|---|---:|---:|---:| +| The leader completes its next tx set once its ledger has closed (main thread: validity pass 33 ms, top-up from the mempool 48 ms, XDR and checks) | 146 ms | 14% | 103 ms | +| Tx set validation at apply (apply thread and workers) | 164 ms | 16% | 103 ms | +| Execution (apply thread; the host calls on 16 workers) | 495 ms | 49% | 323 ms | +| Commit (always after consensus) | 205 ms | 20% | 133 ms | +| **Sum** | **1,010 ms** (measured cycle: 1,007 ms) | | 661 ms (the cycle is the 800 ms target) | +| *Inside the execution, not limiting any more:* | | | | +| The set's way to the followers | 107 ms | | 104 ms | +| Followers: set in hand to consensus | 313 ms | | 307 ms | +| Leader: set built to consensus | 511 ms | | 431 ms | +| Consensus to ledger closed: leader / follower | 285 / 412 ms | | 193 / 197 ms | + +- **The bottleneck at 10,000 TPS is the serial apply path, not consensus and not the network.** + The four steps of the first block follow one another and add up to the cycle within 3 ms. + The execution is still running when consensus arrives: of the ballot, 0.8 ms are left after + it (11 ms at 8,000 TPS), and a ledger closes 0.29 s (leader) to 0.41 s (follower) after + consensus. The ballot and the set's delivery now hide inside the execution, which is what + experiment 3 set out to do. The machines are not busy (6.5 of 32 CPUs). +- **Per payment the apply path costs the same at both loads**: 16 us of validation, 50 us of + execution, 21 us of commit, 86 to 87 us together. So a ledger of n payments needs about + 26 ms + 98 us x n from one close to the next (the completion of the next set included). +- **10,000 TPS at 800 ms is missed by about 2%.** At that pace a ledger holds 8,000 payments, + which by the line above need 813 ms. Once a ledger takes longer than the target the next one + holds more payments and takes longer still, up to the cap: that is why the run sits at full + ledgers and 1.0 s, far from the 813 ms. The same line puts the limit of this build at about + 9,800 TPS (two runs, one line through them: an estimate). +- **Where the 0.86 s of apply go** was profiled at 8,000 TPS before the meta was turned off + (above, "What limits real apply"): the apply thread works serially, 39% of its samples copy, + serialize or free XDR values and 18% are `malloc` and `free`, while each of the 16 workers is + busy a tenth of the time. That profile was not repeated for these runs. + +**Next steps**, by what they should buy at 10,000 TPS (estimates from the table above, not +measured): + +1. **Take the tx set validation off the apply path** (164 ms per full ledger, 16 us per + payment): validate a set while the previous ledger commits, when a node already holds it. + Even a part of it puts 10,000 TPS under 800 ms by the line above. +2. **Commit ledger N while ledger N+1 executes** (205 ms, 21 us per payment): the bucket list + add and the history checkpoint do not need the apply thread's next step to wait. +3. **The serial part of the execution** (495 ms, 50 us per payment, of which the parallel host + calls are a small part): the per-payment copies and allocations of the earlier profile. +4. **Complete the next tx set earlier** (146 ms): time the early selection of experiment 6 so + that less is topped up at the close (its own next step), or complete it while the previous + ledger commits. +5. **Measure the spread and the limit**: a second run of the star and of the coded mode, and + runs at 9,000 and 9,500 TPS. Every point here is one run. +6. **The coded mode itself**: a layout that uses round-trip times (the tree by round-trip + times delivered in 84 ms under simulated apply, with one path per node); a bound from Core + on how large a set a leader may cut into pieces; whom a node asks after its wait. +7. **More of the Byzantine picture on a network**: the tree under the same fault for a measured + contrast, relays that send wrong pieces, a leader that withholds from a part of the network. + +### Level 0 of the bucket list as shards + +Runs of 2026-11-15 (experiment 8), with the fixture of the 10,000 TPS run of experiment 7: 90 +validators (m6id.8xlarge) in four regions, 800 ms target, 10,000 native-asset contract payments +per second executed for real, meta off, 16 clusters, the tx set as coded pieces, no faulty +nodes. One package on both sides (commit `be560d831`); the only difference is +`EXPERIMENTAL_SHARDED_TOP_BUCKET`. One run each. Timers are means per ledger pooled over the 90 +nodes between 150 s and 315 s of the load, where ledgers are full. + +| | Setting off | 8 shards per group | +|---|---|---| +| Verdict (p75 in [640, 960) ms) | FAIL | FAIL | +| Window p75 / p99, mean over nodes | 1,059 / 1,304 ms | 1,035 / 1,356 ms | +| Payments included and applied, of 3,600,000 | 3,600,000, 0 failed | 3,600,000, 0 failed | +| **Level-0 add** (`bucket.batch.addtime`) | 97.6 ms | **21.1 ms (-78%)** | +| its parts: cut / shards / sum; the file (background) | | 0.9 / 19.6 / 0.2 ms; 14.4 ms | +| **Commit** (`ledger.apply.commit`) | 203.2 ms | **161.5 ms (-21%)** | +| Execution (`ledger.apply.execute`) | 484.8 ms | 490.5 ms | +| Apply in all (`ledger.ledger.close`) | 688.0 ms | 652.0 ms (-5%) | +| Followers: consensus to close / wait for the next set | 437 / 258 ms | 404 / 286 ms | +| Leader: build of the next set (its top-up) | 151.9 (44.4) ms | 178.0 (75.4) ms | +| **Time between full ledgers** | 1,009 ms | **1,003.5 ms (-0.5%)** | +| Time between ledgers, whole load to 315 s | 984 ms | 940 ms | +| The first 125 ledgers of the load (one node): interval, payments per ledger | 939 ms, 8,800 to 9,800 | 843 ms, 8,200 to 8,700 | + +Every check other than the windows passed in both runs (inclusion on every node, in sync, no +fork, error counters, no restarts). + +**The add does what it was meant to; the network stays saturated.** +- The add is a fifth of what it was, on every node (20.3 to 24.0 ms per node against 92.8 to + 114.4 ms). +- The commit gains 42 ms, not 76. It waits for two things that run side by side: the add, and + the update of the in-memory Soroban state with the same entries. That update has no timer; + on a development machine it takes 31 ms per ledger for this load, on one thread. It is what + the step waits for now. +- The time between full ledgers gains 5 ms. The 33 ms a follower gains at the close come back + as a longer wait for the next set: the leader's build is 26 ms longer, because without a + backlog the selection made during the previous ballot finds fewer transactions and the + serial completion adds the rest. And a full ledger here (10,000 payments, 125% of the load + per 800 ms) is exactly one second of the offered load: a network that closes full ledgers + cannot close them faster than the load fills them. +- Before it saturates, the network with the setting keeps up with the load for about 105 s at + 843 ms a ledger; without it, it is at 939 ms from the start. Both drift up because the close + after consensus grows during the load (216 to 459 ms and 267 to 436 ms over 300 ledgers). + +**What limits 10,000 TPS after this** (per full ledger, with the setting): execution 490 ms; +the tx set's validation at apply 163 ms; the commit 161 ms, of which the level-0 add is now +21 ms; the leader's build 178 ms; and whatever makes the close grow during a run. + +Local benchmark (a development machine, 30,000 entries per ledger, files synced; hidden tests +tagged `[bucketbench]`): the add 71.7 ms without and 14.4 ms with 8 shards per group; about +130 ms of thread time on 15 threads, a third of it the hash. + ### Earlier benchmark: 1,000 ms target, 5,000 TPS, 48 validators, 4-core instances The measurement of experiment 3 (vote without validating the tx set, validate and execute at @@ -721,10 +1153,11 @@ block_time_check.py \ To reproduce: -- **The load is not in this branch.** The two test-only load generator modes, `sac_setup` (creates - the contract instance) and `sac_payment` (the payments, each to a fresh contract address), are - on the branch `perf/loadgen-real-sac` of `SirTyson/stellar-core` (`e168408bb`, three commits on - `5d24d5494`, in `src/simulation/LoadGenerator.cpp`). +- **The load** is two test-only load generator modes in `src/simulation/LoadGenerator.cpp`: + `sac_setup` (creates the contract instance) and `sac_payment` (the payments, each to a fresh + contract address). They came into this branch with experiment 6; the runs of 2026-11-12 used + the same code as the branch `perf/loadgen-real-sac` of `SirTyson/stellar-core` (`e168408bb`, + three commits on `5d24d5494`). - **The harness flags** (`--real-apply`, `--soroban-clusters`, a list of instance types, the data directory on the local NVMe disk, the cost parameter setting) are in pull request #5 of perfnet's repository (`stellar-experimental/performance-testing`), not merged yet. @@ -735,8 +1168,47 @@ To reproduce: and call graphs at 199 Hz with DWARF unwinding and the package's debug symbols (the packaged binary has no frame pointers). +**The runs with the meta off** (2026-11-13) differ from the real-apply runs above in these +points: + +| | Meta off: 6 and 90 validators | +|---|---| +| Node settings | `--cfg METADATA_DEBUG_LEDGERS=0 --cfg DISABLE_TX_META_FOR_TESTING=true` in every run; from the third settings run on also `--cfg BUCKETLIST_DB_MEMORY_FOR_CACHING=4096 --cfg BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT=8` | +| Builds | The measured commit of experiment 6 is `3973b9321`; the branch has one source change after it (`d33af26e3`: when a short phase pulls deeper; and a formatting commit), whose rules did not come into play in these runs. Base: `54260eafb`, package `30.0.1-3692.54260eafba.noble~perf.txset.build.settings.base~perftests` (this branch at `5d24d5494`, the load generator modes, and the account cache's hits under the shared lock). Experiment 6: `3973b9321`, package `30.0.1-3693.3973b9321a.noble~perf.txset.build.off.critical.path~perftests`. The first two settings runs: `e168408bb` | +| Load generator | A `sac_payment` run waits 150 ledgers for its last payments (20 before). The harness's deadline for the load allows for that wait; that harness change is not published | +| 6 validators | 2 organizations, 8,000 TPS offered, tx set cap 8,000 | +| Timers | Read from every node's metrics while the network was up, 315 s into the load: means since the load's start, medians over the nodes. The leader's build and the timers of experiment 6: the difference between the readings at 150 s and at 315 s, over all builds of all nodes | +| Log times, CPU | From all nodes' logs and per-thread CPU counters, as above | + +**The runs of experiment 7** (2026-11-14) use the fixture of the meta-off runs at 90 +validators with these differences: + +| | Experiment 7: 90 validators | +|---|---| +| Build | `8015877d9` for every run, package `30.0.1-3697.8015877d95.noble~perf.txset.spanning.tree~perftests`. That commit is this branch before its last rebase (kept as branch `perf/txset-spanning-tree-measured` of `SirTyson/stellar-core`; here it is `330d185e5`, with the same content). The branch has one source commit after it (`ea966452d`: which peer a rebuilt set is counted against, a log line, tests), which does not touch what these runs measure | +| Node settings | Those of the meta-off runs, and `--cfg 'EXPERIMENTAL_TX_SET_TREE="star"'`, `"hash"` or `"coded"`; the fault run also `--cfg EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING=29` | +| Offered load | 8,000 TPS (89 per validator; tx set cap 8,000) or 10,000 TPS (111; cap 10,000), for 360 s | +| Order of the runs | Star, coded, hash tree and the fault run within 90 minutes of one morning; the 10,000 TPS run ten hours later | +| Delivery times | The leader's send time travels with the set (in the header of a pushed frame or of a piece); a node records the difference to its own clock when the frame, or the 8th piece, has arrived. The hosts' clock offsets were under 1 ms | +| Timers | Read from every node's metrics while the network was up: the difference between the readings at 150 s and at 315 s into the load, over all nodes. In the coded run at 8,000 TPS the first reading was taken at 205 to 244 s (an operator slip); the time per ledger and the block times do not depend on it | +| Simulated-apply runs of the tree | 2026-11-14, 10,080 TPS, 150 ms sleep per ledger, builds `90c1ee1dc` (star), `b6f32b52b` (hash), `d2dfbe4bb` (rtt) of this experiment before it was rebased; kept as branch `perf/txset-spanning-tree-round1` of `SirTyson/stellar-core` | + +**The runs of experiment 8** (2026-11-15) use the fixture of experiment 7's run at 10,000 TPS: + +| | Experiment 8: 90 validators | +|---|---| +| Build | `be560d831` for both runs, package `30.0.1-3698.be560d8314.noble~perf.sharded.top.bucket.writes~perftests`. The commits after it on this branch were not on a network: fixes from a second review off the measured path (how catchup reads a file it checks, what the bucket manager waits for when it is destroyed), tests, a hidden benchmark and this text | +| Node settings | Those of experiment 7's coded run, and `--cfg EXPERIMENTAL_SHARDED_TOP_BUCKET=8` or `=0` | +| Offered load | 10,000 TPS (111 or 112 per validator; tx set cap 10,000), for 360 s | +| Order of the runs | The setting on, then off, 22 minutes apart | +| Timers | The difference between the readings at 150 s and at 315 to 320 s into the load, over all nodes | +| A ledger's cycle | From every node's log: the time between two consensus lines, the close after consensus of the ledger before, and the time from having a ledger's set to its consensus; ledgers of at least 9,990 payments | + ### Limits of these numbers +- Experiment 8: one run on each side. The size of the gain in the first 105 s is one + observation; the level-0 add is measured on 90 nodes over 165 ledgers each. The window p99 is + higher with the setting (1,356 against 1,304 ms), unexplained. - One run on each side, plus the first run of the change without the IPC fix. The "before" numbers come from a single 5-minute window. - The run with the change is a FAIL on the inclusion check, and the 60 extra transactions are not @@ -770,8 +1242,80 @@ Real apply: - After the one dropped speculative execution, `ledger.transaction.count` and the `ledger.apply` success counters had counted that ledger twice. +Meta off, the bucket list settings and experiment 6: +- One run per point. The block time difference between the base and experiment 6 (15 ms) is + inside the spread of two runs of one point in the earlier series (4 and 14 ms). +- The meta-on runs they are set beside used another build and core's default bucket settings, + and meta, cache and index pages were not separated at 90 validators. +- The load has no overlap between consecutive tx sets: no selected transaction was ever removed + at the close and no refill was needed. Those paths, and every case of requirement "no invalid + set", are covered by unit tests only. +- Both base runs lost payments (59 and 147) and have the verdict FAIL for that. Not explained. + The run of experiment 6 lost none; one run, and nothing in the change is meant to affect it. +- The two-step change was measured as one build. What each step contributes is read from the + timers, not from separate runs. +- The apply numbers of the settings runs are from 6 validators. +- The 6-validator run of experiment 6 and the base's 8,000 TPS run were launched in the same + minute and carry the same network passphrase. They were separate networks, and everything + reported here was read per instance. + +The tx set as coded pieces and the spanning tree: +- One run per point. The 65 ms between the star and the coded mode are four times the spread + of the three runs of this fixture without the experiment (858 to 873 ms, three builds), but + no mode was run twice. The zones the instances landed in differ between the runs. +- The fault run tests relays that stay silent, by a setting on 29 nodes that otherwise behave. + Nothing else of a Byzantine fault was run on a network. +- The delivery time of the coded mode is taken when the 8th piece arrives, before the 1 to 2 ms + of rebuilding; a set that is never completed gives no sample (there were none without + withholders, and 1.7% with them, which were asked for instead). +- The bottleneck table reads timers of one run; the steps were not traced, and that they follow + one another is inferred from their sum and from the execution ending after consensus. The apply + profile it points to is from an earlier build with the meta on. +- The limit of about 9,800 TPS is a line through two runs. +- The hash tree asked for 213 sets after its wait on a healthy network (0.7%); why was not + looked into. + ## Known limitations of the prototype +Level 0 of the bucket list as shards (experiment 8): +- A setting, not a protocol version: a node with it and a node without it compute different + bucket list hashes from the first ledger that changes an entry. +- The hash: LtHash16 with the standard size (1,024 elements of 16 bits) but ChaCha20 in place + of the paper's extendable-output function, and no test vectors from outside. A real version + needs a reviewed construction; it is also the largest cost left in the add. +- Catchup takes a live bucket under either hash when the setting is on (it does not know a + bucket's level), after checking that the file is the canonical file of its records. History + archives hold level-0 files whose names are not their SHA-256. +- Shards are by key, and written at the commit: not one shard per cluster written by the + cluster while it executes. A shard must hold the same keys in every ledger to be merged on + its own with the ledger before, and entries are final only after the steps that follow the + parallel phase. +- The hot archive bucket list is not sharded. POSIX only. State written with the setting by one + commit of this branch is not promised to load with another (the digest's tag changed once). +- Not tested: an upgrade ledger, a failing disk under the file's writer, the bucket garbage + collection racing a missing file. No sanitizer run. + +The tx set as coded pieces and the spanning tree (experiment 7): +- No version negotiation: a node without the change reads a tagged frame, a probe or a piece as + a malformed set. Every node of a network needs the same mode. +- The layout is by hash: the coded mode does not use round-trip times. +- The network writes twice the bytes of a set per node in coded mode. +- A hostile leader that its Core names for the slot can sign frames of up to the general limit + (16 MiB): it writes 10 frame sizes and the network about 178, two roots per slot. Core does + not yet tell the overlay how large a set can be. +- A node connected to fewer than 8 of its 16 senders (a sparse mesh), or with another view of + the nodes, waits 300 ms for every set and then asks. After the wait the request goes to the + peer the SCP statement came from, which may not hold the set and then does not answer; the + next step is Core's retry after one second. The same holds below a failed relay of a tree. +- In a tree a failed relay costs the nodes below it that wait: there is no second path. +- A piece a relay could not write is not sent again; the code absorbs the gap. The stream + library drops inbound streams that are opened in a burst, so pieces can be lost in the first + ledgers after a start; in the runs no piece went unwritten during the load. +- The send time and the hop count in a frame are not signed: a relay can skew the delivery + metric. +- Sanitizer builds were not run for this experiment. + + Transactions to the leaders only: - No protocol gate and no version negotiation: a node without the change ignores the identity stream, and nodes with the change flood toward it. @@ -791,6 +1335,30 @@ Transactions to the leaders only: - Copies of a transaction that reach a node after its ledger closed are counted (`overlay.flood.tx-late-copy`), not dropped. +The next tx set selected before the close, and the smaller build: +- The mempool has one fee order for both phases. A phase with no candidate in the mempool's top + (capacity plus the margin) gets none, where a build looked twice as deep before. The fix is a + pull per phase. +- The start of the selection is timed by an estimate that a node only updates when it builds a + set itself. At 90 validators the selection is ready about 0.3 s before it is needed, and a + third of the set is added on the main thread at the close. +- A Soroban phase that was completed is extended transaction by transaction: clusters are never + merged and no stage is added, so it can be packed less tightly than a built one. With the + conflict-free load of these runs it used 16 clusters where a build used 13 or 14. +- If ledger N's set is not the one that closes, the selection made from it is dropped and the + set is built from scratch. +- The read-back that is left to the leader's execution is matched by tx set hash only. +- A phase that lost a candidate and is short pulls once more even where the mempool holds + nothing else of its kind (one stale classic transaction under a Soroban backlog costs a pull + of twice the size). +- A completed phase that is at its capacity and lost nothing keeps the surge pricing verdict of + the selection, which was made on the predicted state. If everything it left out is invalid on + the closed ledger, the set charges the lowest included fee where a build would charge the + ledger's base fee. The set is valid either way. +- No thread sanitizer run exists for this change (the build box cannot build one); its threads + were reviewed and run under the address sanitizer and the unit tests. +- Not tried with classic transactions or mixed loads on a network. + Leader terms and pre-validation: see [#5502](https://github.com/stellar/stellar-core/pull/5502). The schedule change has no protocol gate, and a term gives one node five ledgers in a row. @@ -841,6 +1409,11 @@ Leader schedule: Unchanged from stellar-core: see [INSTALL.md](./INSTALL.md), [CONTRIBUTING.md](./CONTRIBUTING.md) and [`./docs`](./docs). +Test status with experiment 6 (`f19140f89`, the branch's last source commit): the full test suite +passes, also with `--all-versions` (669 test cases). The address sanitizer ran on the three test +tags of the experiment (at `3973b9321`); there is no thread sanitizer run. A `--disable-tests` +build still fails in `herder/HerderImpl.cpp`, in the code of the leader schedule. + Test status with experiments 4 and 5 (`6ab837c27`, #5501 on top of #5502): the full test suite passes, also with `--all-versions`, including the Rust overlay's tests. diff --git a/docs/rust-overlay/txset-fetching.md b/docs/rust-overlay/txset-fetching.md index a9f3fd306f..0a6e509f21 100644 --- a/docs/rust-overlay/txset-fetching.md +++ b/docs/rust-overlay/txset-fetching.md @@ -139,6 +139,180 @@ Limits on what a peer can make a node decode and keep: fill it and cycle out sets the node built or already uses (on the base one peer could, through prefetches). +## Spanning tree for the push (experimental) + +`EXPERIMENTAL_TX_SET_TREE` (`off`, `star`, `hash`, `rtt`, `coded`; default +`off`) changes who the leader's set travels through. With `off` nothing below +applies. Every node of a network should use the same value. Code: +`overlay/src/flood/txset_tree.rs`, and for `coded` +`overlay/src/flood/txset_coded.rs` (see "Coded pieces" below). + +The tree. Its nodes are the nodes of the leader schedule. For a slot and a +leader the leader is the root and the other nodes follow in the order of +`SHA256(seed || leader || node)`, where `seed` is the seed of the slot's leader +schedule. Position `i` has the children `F*i+1 .. F*i+F`: the complete F-ary +tree, the least depth a fanout of F allows. F is +`EXPERIMENTAL_TX_SET_TREE_FANOUT`; 0 (the default) is the smallest F that +reaches every node in two hops (9 for 90 nodes). Every node derives the same +tree from what it already agrees on; nothing about it is negotiated. + +What Core tells the overlay. At every ledger close Core sends +`SetTxSetTree [mode:1][fanout:1][slots:1]([slot:u32 LE][seed:32][leaders:1][nodeID:32]...)...[count:u32 LE][nodeID:32]...` +for the next two slots: the seed of each, the nodes that may lead it (the +leaders of the next slot's first three rounds; for the slot after it those and +the first three of a new order) and the nodes of the trees. Two slots, because +a leader may push the set of slot N+1 the moment it closes N, when a follower +has only closed N-1. The top bit of the mode byte tells an overlay to pass +nothing on (`EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING = N`: the N +validators with the smallest node IDs; for tests and fault runs). + +On the wire. A tree push is the ordinary tx-set frame with a header in front, +on the same stream: `[tag:u32 = 0xFFFFFFF1][kind:1][fanout:1][hop:1][0][slot:u32]` +`[leader:32][sent:u64 Unix ms][n:u16][n x u16 order][8-byte digest if n > 0]` +(big-endian). The tag stands where an ordinary frame has the length of its +XDR, which is at most 16 MiB, so no ordinary frame starts with it. Kinds: 1 +star (to every peer, never relayed), 2 hash (the tree above), 3 explicit (the +order is in the header, see `rtt`). + +Pushing. `PushTxSet` makes the leader's overlay write the set to its +children in the slot's tree. If it does not know the slot's tree, is not one +of the leaders its Core named for the slot, or a child has no identified +connection, it writes a star to every peer instead. If a +child is left out because an earlier push to it is still in flight, the nodes +below that child get a star from the leader directly. + +Relaying. A node that reads a tagged frame passes the bytes on to its children +before it decompresses or checks them, if all of these hold: its mode is +`hash` or `rtt`; its Core described the tree of the frame's slot; the frame's +leader is one of the leaders Core named for that slot; the sender is its +parent in that tree, by the validator identity of the connection; the body is +of a size `decode` would accept; and it has relayed fewer than two frames of +that slot and leader and not this one. One relay is in flight per child. The +node then verifies and files the set as it does any other, by the hash of its +content. + +Asking. A node of the tree that does not hold a set a proposal names waits +300 ms from the first mention of the hash before it asks a peer (Core's own +one-second retry is not delayed). Without that wait every node the leader did +not write to would ask it when the proposal arrives. + +`star` is the push as it was, with the header (so that the delay from the +leader's send is recorded like a tree's). `rtt`: every node measures the round +trip to each identified peer every two seconds with a 13-byte probe on the +tx-set stream (`[tag:u32 = 0xFFFFFFF2][1 ping | 2 pong][u64]`; the answer must +match the probe sent last); the leader groups its peers by the gaps in the +sorted times, gives every group relays in proportion to its size and fills a +relay's children from its own group, and sends that order in the header. A +receiver follows such an order only in `rtt` mode, only if it has two levels, +and only if it refers to the same list of nodes. + +### Coded pieces (`coded`) + +In a tree a node has one path to the set: its parent. In `coded` mode the +leader cuts the frame it would push into k pieces, adds m-k Reed-Solomon +parity pieces (any k of the m rebuild the frame), and every node gets its m +pieces from m different senders. Up to m-k of a node's senders can be down, +slow or hostile without the node having to wait and ask. + +The layout is derived like the tree. `others` is the hash order of the slot and +leader without the leader. From the number of other nodes alone: m = min(16, +others), k = ceil(m/2), r = others / m relays per piece (90 validators: 16, 8 +and 5; fewer than 4 others: no coding, the set is pushed to everyone). Piece +j's relays are `others[j*r .. j*r+r-1]`; the leader writes them piece j. The +node at position p gets piece j from relay `(p + j) mod r` of that piece, +unless it is one of the piece's relays. No piece makes more than two hops. The +leader writes m*r pieces of a k-th of the frame each (10 frame sizes at 90 +validators), a relay about (others - r) / r of them (2.1 frame sizes); the +network as a whole writes m/k times the frame per node. + +A piece frame, on the tx-set stream (big-endian): +`[tag:u32 = 0xFFFFFFF3][version:1 = 1][hop:1][k:1][m:1][index:1][path length:1][0][0]` +`[slot:u32][leader:32][sent:u64 Unix ms][frame length:u32][root:32]` +`[overlay key:32][binding signature:64][root signature:64][path: 32 each][piece]`. +The leaves of the Merkle tree are `SHA256(0x00 || index || piece)`, inner nodes +`SHA256(0x01 || left || right)`, padded with zero leaves to a power of two. The +root signature is the leader's overlay (libp2p Ed25519) key over +`SHA256("stellar-overlay tx set pieces v1" || network || seed || slot || leader || k || m || frame length || root)` +(`seed` is the seed of the slot's layout, the hash of the ledger two before the +slot: a signature from another history of a network with the same name and +keys is of no use once the two histories differ at that ledger); +the binding signature is the validator's over the peer id of that overlay key, +as nodes exchange it as their identity. So a piece can be checked against the +leader's node ID alone, without a connection to the leader. + +A node looks at a piece only if its mode is `coded`, its Core described the +slot and named the leader as one that may lead it, the sender (by the +validator identity of the connection) has sent fewer than 6 piece frames for +that slot and leader, whatever they were, k and m are what the node derives +itself, and the sender is the node that owes it that index in the layout, +with the hop count that goes with it. The layout is worked out once per slot +and leader. It keeps the piece only if both signatures and the Merkle path check +(the signatures once per root); a leader has room for two roots per slot. A +relay passes its piece on after those checks, once per root, and only if the +leader itself sent it. Pieces do not draw on the per-peer allowance for +unrequested frames: what they can make a node keep and rebuild is bounded by +the above, and nothing is decompressed before k of the node's senders agree on +a root the leader signed. + +A node that its Core does not name as a leader of the slot (the leader of a +fourth or later round, a node with another view) does not code: every node +would refuse its pieces. It pushes its set to everyone, as in `star`, and logs +`TXSET_CODED_STAR` (in `hash` and `rtt`: `TXSET_TREE_STAR`). + +With k pieces under one root the frame is rebuilt (no decoding if they are +the k data pieces) and goes the way of every received frame: decompress, +strict parse, content hash, cache. It counts as received from the overlay that +signed its root (the leader's, connected or not), not from the sender of the +piece that completed it. Afterwards the node encodes the frame again +and compares the root: a leader whose pieces do not fit together is counted +(`sets-inconsistent`); the set is still what its content hash says. A node +waits for pieces as it does for a tree frame (300 ms) before it asks. + +What the code protects is the delivery. A set is identified by the hash of +its content, as before; a wrong piece, root or rebuild gives bytes that fail +the decompression or the parse, or a set with another hash that no SCP +statement names. + +Metrics: `overlay.txset-coded.*` (sets cut into pieces; pieces sent, relayed, +received, late, refused, invalid, duplicate, skipped; sets rebuilt, rebuilt +with parity, failed, inconsistent; microseconds of rebuilding) and +`overlay.txset-tree.withheld`. The leader's piece bytes count in +`overlay.txset-push.sent-bytes`, a relay's in `overlay.txset-tree.relayed-bytes`, +and a rebuilt set in `overlay.txset-tree.received` and the delay metrics (at +the arrival of the k-th piece). + +Metrics: `overlay.txset-tree.*` (copies written as root and as relay, frames +received, refused, duplicates, relays over the limit, busy and unreachable +children, requests put off and those sent after the wait, the delay since the +leader's send in total, per hop count and in ranges) and +`overlay.txset-push.sent-bytes`. + +Limits of the prototype: + +- No version negotiation: a node without this change reads a tagged frame or a + probe as a malformed set. +- In `hash` and `rtt` a relay that is down, slow, two ledgers behind or has + another view of the nodes costs the nodes below it the 300 ms wait: there is + no second path. `coded` is the mode with several paths. +- `coded` writes twice the bytes of a set per node (m/k), and its layout is by + hash only: it does not use round-trip times. +- A piece a relay could not write (the receiver has four in flight, or no + identified connection) is not sent later; the code absorbs the gap. +- `coded`: the only bound on a frame's size is the general one (16 MiB). For + a frame a leader signs, it writes 10 frame sizes itself and the network about + 178 in all (18 times the leader's own), per root, two roots per slot; Core + does not yet tell the overlay how large a set can be. +- `coded`: a node connected to fewer than k of its senders (a sparse mesh), or + with another list of nodes, waits 300 ms for every set and then asks. The + request goes to the peer the SCP statement came from, which may not hold + the set and then does not answer; the next step is Core's one-second retry + (the same holds for the nodes below a failed relay of a tree). +- The delay metric compares two hosts' clocks and only counts tagged frames + that arrived. +- A leader of a fourth or later round is not one of the named leaders: its set + goes to every peer as a star, in every mode. +- Relays have no write timeout (neither has the leader's push). + ## Core requests and network prefetches `RequestTxSet` IPC carries `[hash:32][slot:u32 LE]`. App records the newest @@ -203,8 +377,9 @@ See [mempool](mempool.md). ## Remaining limitations -- The push is not relayed and carries the full compressed set; nodes that are - not connected to the leader, and a compact (hash-list) form, are future work. +- Without the tree the push is not relayed; with or without it, it carries the + full compressed set. Nodes that are not connected to the leader (or, in a + tree, to their parent), and a compact (hash-list) form, are future work. - A peer whose first copy of a proposal came through a relay requests the set from that relay, which answers from its cache while the leader's push may still be in flight: one duplicate transfer. diff --git a/docs/software/commands.md b/docs/software/commands.md index 052437d90d..a554112799 100644 --- a/docs/software/commands.md +++ b/docs/software/commands.md @@ -537,7 +537,7 @@ this survey mechanism, just set `SURVEYOR_KEYS` to `$self` or a bogus key ### The following HTTP commands are exposed on test instances * **generateload** `generateload[?mode= - (pay|soroban_upload|soroban_invoke_setup|soroban_invoke|upgrade_setup|create_upgrade|mixed_classic_soroban|pay_pregenerated|stop)&accounts=N&offset=K&txs=M&txrate=R&spikesize=S&spikeinterval=I&maxfeerate=F&skiplowfeetxs=(0|1)&minpercentsuccess=S&instances=Y&wasms=Z&payweight=P&sorobanuploadweight=Q&sorobaninvokeweight=R&file=F]` + (pay|soroban_upload|soroban_invoke_setup|soroban_invoke|upgrade_setup|create_upgrade|mixed_classic_soroban|pay_pregenerated|stop)&accounts=N&offset=K&txs=M&txrate=R&spikesize=S&spikeinterval=I&maxfeerate=F&skiplowfeetxs=(0|1)&completiontimeout=L&minpercentsuccess=S&instances=Y&wasms=Z&payweight=P&sorobanuploadweight=Q&sorobaninvokeweight=R&file=F]` Artificially generate load for testing; must be used with `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING` set to true. @@ -608,6 +608,9 @@ this survey mechanism, just set `SURVEYOR_KEYS` to `$self` or a bogus key * when `skiplowfeetxs` is set to `true` the transactions that are not accepted by the node due to having too low fee to pass the rate limiting are silently skipped. Otherwise (by default), such transactions would cause load generation to fail. + * `completiontimeout` is the number of ledgers a run waits, after it has + submitted its last transaction, for its transactions to be applied before + it reports failure. The default is 20 ledgers, and 150 for `sac_payment`. Soroban load generation also makes use of the `minpercentsuccess` parameter, which determines the minimum percentage of Soroban transactions that must diff --git a/overlay/Cargo.toml b/overlay/Cargo.toml index 1ad9ab16af..9b4a7eff2f 100644 --- a/overlay/Cargo.toml +++ b/overlay/Cargo.toml @@ -32,6 +32,10 @@ futures = "0.3" # TX set response compression zstd = { version = "0.13.3", default-features = false } +# Erasure code for the pieces of a pushed TX set (EXPERIMENTAL_TX_SET_TREE = "coded") +# (3.0: its dependencies are in the lock file already, apart from one build-time crate.) +reed-solomon-simd = "~3.0.1" + # Logging tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/overlay/src/flood/mod.rs b/overlay/src/flood/mod.rs index 6bce126a8f..db8ad8f307 100644 --- a/overlay/src/flood/mod.rs +++ b/overlay/src/flood/mod.rs @@ -13,6 +13,8 @@ mod txset; pub(crate) mod txset_encoding; pub(crate) mod txset_push; mod txset_store; +pub mod txset_coded; +pub mod txset_tree; pub use inv_batcher::InvBatcher; pub use inv_messages::{GetData, InvBatch, InvEntry, TxStreamMessage, INV_BATCH_MAX_SIZE}; diff --git a/overlay/src/flood/tx_router.rs b/overlay/src/flood/tx_router.rs index 494b4f21f3..1ebd3c3319 100644 --- a/overlay/src/flood/tx_router.rs +++ b/overlay/src/flood/tx_router.rs @@ -226,6 +226,15 @@ impl TxRouter

{ self.node_of_peer.get(peer) } + /// The connected peer of every validator that proved who it is. (If a + /// validator has two such peers, one of them.) + pub fn identified_peers(&self) -> HashMap { + self.node_of_peer + .iter() + .map(|(peer, node)| (*node, *peer)) + .collect() + } + /// The peer is gone. Whatever was sent to it may not have arrived, and a /// restarted node has an empty mempool: forget what it was sent. pub fn peer_disconnected(&mut self, peer: &P) { diff --git a/overlay/src/flood/txset_coded.rs b/overlay/src/flood/txset_coded.rs new file mode 100644 index 0000000000..6c564bcc1c --- /dev/null +++ b/overlay/src/flood/txset_coded.rs @@ -0,0 +1,1216 @@ +//! Erasure-coded pieces of the leader's tx set (mode `coded`). +//! +//! In a tree a node has one path to the leader's set: its parent. Here the +//! leader cuts the frame it would push into `k` pieces, adds `m - k` +//! Reed-Solomon parity pieces, and every node gets its `m` pieces from `m` +//! different senders. Any `k` of them rebuild the frame, so up to `m - k` of a +//! node's senders can be down, slow or hostile without the node having to +//! wait and ask. +//! +//! * [`coded_params`]: `k`, `m` and the relays per piece, from the node count +//! alone (every node derives the same). +//! * [`CodedLayout`]: who sends which piece to whom, from the hash order of +//! the slot's tree (`txset_tree::tree_order`). +//! * [`encode_pieces`], [`rebuild`]: the code. +//! * [`merkle_tree`], [`merkle_root_from`], [`root_message`]: how a piece +//! proves that it is the leader's. The leader's overlay key signs the root; +//! a piece carries its path to that root. +//! * [`PieceHeader`]: in front of a piece on the tx-set stream. +//! * [`CodedState`]: the pieces a node holds, and its limits. +//! +//! As with the tree, nothing here is trusted by the user of a set: a rebuilt +//! frame goes down the path every received frame takes (decompress, strict +//! parse, content hash), and Core gets a set only when an SCP statement names +//! that hash. The code protects the delivery, not the set. + +use super::txset_tree::NodeId; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io; +use std::sync::Arc; + +/// Stands where an ordinary tx-set frame has the big-endian length of its +/// XDR, as `TREE_TAG` does. No length can take this value. +pub const PIECE_TAG: u32 = 0xFFFF_FFF3; + +/// Version of the piece frame. +pub const PIECE_VERSION: u8 = 1; + +/// Pieces a set is cut into, at most. +pub const MAX_PIECES: usize = 16; + +/// With fewer other nodes than this a set is not coded (the leader pushes +/// it to everyone). +pub const MIN_CODED_OTHERS: usize = 4; + +/// Roots a node keeps pieces for, per slot and leader. A leader pushes one +/// set per slot it leads, two if it built one early and another at the +/// trigger (as `MAX_RELAYS_PER_SLOT_AND_LEADER` for the tree). +pub const MAX_ROOTS_PER_SLOT_AND_LEADER: usize = 2; + +/// Piece frames a node looks at from one sender for one slot and leader, +/// whatever they turn out to be (counted before the layout is consulted). A +/// sender owes it one piece per root; the rest is room for a repeat. +pub const MAX_FRAMES_PER_SENDER: u8 = 6; + +/// Pieces queued or in flight to one peer at a time. A peer that stops +/// reading its tx-set stream pins at most this many pieces in the sender's +/// memory; further pieces are not sent to it (the code absorbs the gap). +pub const MAX_PIECES_IN_FLIGHT_PER_PEER: u32 = 4; + +/// Roots, senders and relayed pieces a node remembers. +const CODED_MEMORY: usize = 256; + +/// The shape of the code for a network. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CodedParams { + /// Pieces that rebuild the frame (`k`). + pub data: usize, + /// Pieces in all (`m`). + pub pieces: usize, + /// Relays per piece (`r`). + pub relays: usize, +} + +/// The code for a network with `others` nodes besides the leader: up to +/// [`MAX_PIECES`] pieces, half of them (rounded up) enough to rebuild, and as +/// many relays per piece as the nodes allow with every node relaying one +/// piece at most. `None`: too few nodes to code for. +pub fn coded_params(others: usize) -> Option { + if others < MIN_CODED_OTHERS { + return None; + } + let pieces = others.min(MAX_PIECES); + Some(CodedParams { + data: pieces.div_ceil(2), + pieces, + relays: others / pieces, + }) +} + +/// Who sends which piece to whom, for one slot and leader. +/// +/// `others` is the hash order of the slot's tree without the leader. Piece +/// `j`'s relays are `others[j*r .. j*r + r]`: the leader sends them piece `j`. +/// The node at position `p` gets piece `j` from relay `(p + j) mod r` of that +/// piece, unless it is one of the piece's relays itself. So the `m` pieces of +/// a node come from `m` different senders, and no piece makes more than two +/// hops. +#[derive(Clone, Debug)] +pub struct CodedLayout { + pub leader: NodeId, + pub params: CodedParams, + others: Vec, + position: HashMap, +} + +impl CodedLayout { + /// From the order of the slot's tree (`tree_order`: the leader first). + pub fn new(order: &[NodeId]) -> Option { + let (leader, others) = order.split_first()?; + let params = coded_params(others.len())?; + Some(Self { + leader: *leader, + params, + others: others.to_vec(), + position: others.iter().enumerate().map(|(i, n)| (*n, i)).collect(), + }) + } + + /// The relays of `piece`. + pub fn relays_of(&self, piece: usize) -> &[NodeId] { + let r = self.params.relays; + if piece >= self.params.pieces { + return &[]; + } + &self.others[piece * r..piece * r + r] + } + + /// The piece `node` relays, if any. + pub fn relayed_piece(&self, node: &NodeId) -> Option { + let p = *self.position.get(node)?; + let piece = p / self.params.relays; + (piece < self.params.pieces).then_some(piece) + } + + /// The node that owes `node` the piece `piece`: the leader if `node` is + /// one of the piece's relays, one of those relays otherwise. + pub fn sender_of(&self, node: &NodeId, piece: usize) -> Option { + let p = *self.position.get(node)?; + if piece >= self.params.pieces { + return None; + } + if self.relayed_piece(node) == Some(piece) { + return Some(self.leader); + } + let relays = self.relays_of(piece); + Some(relays[(p + piece) % relays.len()]) + } + + /// The nodes `relay` passes its piece on to. + pub fn receivers_of(&self, relay: &NodeId) -> Vec { + let Some(piece) = self.relayed_piece(relay) else { + return Vec::new(); + }; + self.others + .iter() + .filter(|node| *node != relay && self.sender_of(node, piece) == Some(*relay)) + .copied() + .collect() + } + + /// Every piece the leader writes, with the relay it goes to. + pub fn leader_targets(&self) -> Vec<(usize, NodeId)> { + (0..self.params.pieces) + .flat_map(|piece| self.relays_of(piece).iter().map(move |n| (piece, *n))) + .collect() + } + + pub fn contains(&self, node: &NodeId) -> bool { + self.position.contains_key(node) + } +} + +/// Bytes of one piece of a frame of `total_len` bytes cut into `k`: the +/// frame is padded with zeros to `k` pieces of equal, even length (the code +/// works on 16-bit symbols). +pub fn piece_len(total_len: usize, k: usize) -> usize { + let len = total_len.div_ceil(k.max(1)).max(2); + len + len % 2 +} + +fn invalid(what: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, what) +} + +/// Cut `frame` into `params.data` pieces and add the parity pieces: all +/// `params.pieces` of them, the data pieces first. +pub fn encode_pieces(frame: &[u8], params: CodedParams) -> io::Result>> { + let (k, m) = (params.data, params.pieces); + if k == 0 || m <= k || frame.is_empty() { + return Err(invalid("no code for these parameters")); + } + let len = piece_len(frame.len(), k); + let mut pieces: Vec> = Vec::with_capacity(m); + for i in 0..k { + let start = (i * len).min(frame.len()); + let end = ((i + 1) * len).min(frame.len()); + let mut piece = Vec::with_capacity(len); + piece.extend_from_slice(&frame[start..end]); + piece.resize(len, 0); + pieces.push(piece); + } + let parity = reed_solomon_simd::encode(k, m - k, &pieces) + .map_err(|_| invalid("Reed-Solomon encoding failed"))?; + pieces.extend(parity); + Ok(pieces) +} + +/// Rebuild a frame of `total_len` bytes from at least `params.data` of its +/// pieces (`pieces[i]` is piece `i`, if held). Returns the frame and whether +/// a parity piece was needed. +pub fn rebuild( + pieces: &[Option<&[u8]>], + params: CodedParams, + total_len: usize, +) -> io::Result<(Vec, bool)> { + let (k, m) = (params.data, params.pieces); + if k == 0 || m <= k || pieces.len() != m { + return Err(invalid("no code for these parameters")); + } + let len = piece_len(total_len, k); + if total_len == 0 || pieces.iter().flatten().any(|p| p.len() != len) { + return Err(invalid("piece of the wrong length")); + } + let missing = pieces[..k].iter().filter(|p| p.is_none()).count(); + let restored = if missing == 0 { + None + } else { + let originals = pieces[..k] + .iter() + .enumerate() + .filter_map(|(i, p)| p.map(|p| (i, p))); + let recovery = pieces[k..] + .iter() + .enumerate() + .filter_map(|(i, p)| p.map(|p| (i, p))); + Some( + reed_solomon_simd::decode(k, m - k, originals, recovery) + .map_err(|_| invalid("too few pieces to rebuild"))?, + ) + }; + let mut frame = Vec::with_capacity(k * len); + for (i, piece) in pieces[..k].iter().enumerate() { + match piece { + Some(piece) => frame.extend_from_slice(piece), + None => frame.extend_from_slice( + restored + .as_ref() + .and_then(|restored| restored.get(&i)) + .ok_or_else(|| invalid("a piece was not rebuilt"))?, + ), + } + } + // What was cut off is the padding, and padding is zeros. + if frame[total_len..].iter().any(|b| *b != 0) { + return Err(invalid("padding is not zero")); + } + frame.truncate(total_len); + Ok((frame, missing > 0)) +} + +/// The leaf of piece `index` in the Merkle tree of a set's pieces. +pub fn leaf_hash(index: u8, piece: &[u8]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update([0u8, index]); + hasher.update(piece); + hasher.finalize().into() +} + +fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update([1u8]); + hasher.update(left); + hasher.update(right); + hasher.finalize().into() +} + +/// Levels above the leaves of a tree over `pieces` leaves. +pub fn merkle_depth(pieces: usize) -> usize { + pieces.max(1).next_power_of_two().trailing_zeros() as usize +} + +/// The root of the tree over `leaves` (padded with zero leaves to a power of +/// two) and, for every leaf, its path to the root (the sibling at each +/// level, bottom up). +pub fn merkle_tree(leaves: &[[u8; 32]]) -> ([u8; 32], Vec>) { + let width = leaves.len().max(1).next_power_of_two(); + let mut level: Vec<[u8; 32]> = leaves.to_vec(); + level.resize(width, [0u8; 32]); + let mut paths: Vec> = vec![Vec::new(); leaves.len()]; + let mut span = 1; + while level.len() > 1 { + for (leaf, path) in paths.iter_mut().enumerate() { + path.push(level[(leaf / span) ^ 1]); + } + level = level + .chunks_exact(2) + .map(|pair| node_hash(&pair[0], &pair[1])) + .collect(); + span *= 2; + } + (level[0], paths) +} + +/// The root a leaf at `index` reaches along `path`. +pub fn merkle_root_from(leaf: [u8; 32], index: usize, path: &[[u8; 32]]) -> [u8; 32] { + let mut at = index; + let mut hash = leaf; + for sibling in path { + hash = if at % 2 == 0 { + node_hash(&hash, sibling) + } else { + node_hash(sibling, &hash) + }; + at /= 2; + } + hash +} + +/// What the leader's overlay key signs for a set it cut into pieces: the +/// network, the slot and the seed of its layout (the hash of the ledger two +/// before the slot, so that a signature from another history of a network +/// with the same name and keys is of no use once the histories differ +/// there), the leader, the shape of the code, the frame's length and the +/// root of its pieces. +pub fn root_message( + network_id: &[u8; 32], + seed: &[u8; 32], + slot: u32, + leader: &NodeId, + params: CodedParams, + total_len: u32, + root: &[u8; 32], +) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"stellar-overlay tx set pieces v1"); + hasher.update(network_id); + hasher.update(seed); + hasher.update(slot.to_be_bytes()); + hasher.update(leader); + hasher.update([params.data as u8, params.pieces as u8]); + hasher.update(total_len.to_be_bytes()); + hasher.update(root); + hasher.finalize().into() +} + +/// Bytes of a piece header without its Merkle path. +pub const PIECE_HEADER_FIXED: usize = 4 + 8 + 4 + 32 + 8 + 4 + 32 + 32 + 64 + 64; + +/// The largest piece header. +pub const MAX_PIECE_HEADER: usize = PIECE_HEADER_FIXED + 32 * 8; + +/// In front of a piece. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PieceHeader { + /// 1 = straight from the leader, 2 = through a relay. + pub hop: u8, + /// `k` and `m` of the code. + pub data: u8, + pub pieces: u8, + /// Which piece this is. + pub index: u8, + pub slot: u32, + pub leader: NodeId, + /// When the leader sent the set, in Unix milliseconds by its clock. + pub sent_unix_ms: u64, + /// Bytes of the frame the pieces rebuild. + pub total_len: u32, + /// Root of the Merkle tree over the set's pieces. + pub root: [u8; 32], + /// The Ed25519 key of the leader's overlay. + pub overlay_key: [u8; 32], + /// The leader's (validator's) signature that binds that overlay to it: + /// what nodes exchange as their identity (`node_id_binding`). + pub binding_sig: [u8; 64], + /// The overlay key's signature over [`root_message`]. + pub root_sig: [u8; 64], + /// The piece's path to the root. + pub path: Vec<[u8; 32]>, +} + +impl PieceHeader { + pub fn params(&self) -> (usize, usize) { + (self.data as usize, self.pieces as usize) + } + + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(PIECE_HEADER_FIXED + 32 * self.path.len()); + out.extend_from_slice(&PIECE_TAG.to_be_bytes()); + out.push(PIECE_VERSION); + out.push(self.hop); + out.push(self.data); + out.push(self.pieces); + out.push(self.index); + out.push(self.path.len() as u8); + out.extend_from_slice(&[0, 0]); + out.extend_from_slice(&self.slot.to_be_bytes()); + out.extend_from_slice(&self.leader); + out.extend_from_slice(&self.sent_unix_ms.to_be_bytes()); + out.extend_from_slice(&self.total_len.to_be_bytes()); + out.extend_from_slice(&self.root); + out.extend_from_slice(&self.overlay_key); + out.extend_from_slice(&self.binding_sig); + out.extend_from_slice(&self.root_sig); + for hash in &self.path { + out.extend_from_slice(hash); + } + out + } + + /// Whether a frame starts with the piece tag. + pub fn is_tagged(frame: &[u8]) -> bool { + frame.len() >= 4 && frame[..4] == PIECE_TAG.to_be_bytes() + } + + /// Parse the header of a piece frame. Returns it with the offset at + /// which the piece begins, or `None` if the frame is malformed: a field + /// out of range, a path of the wrong length for `m`, or a piece that + /// does not have the length the frame's length and `k` give. + pub fn parse(frame: &[u8]) -> Option<(Self, usize)> { + if !Self::is_tagged(frame) || frame.len() < PIECE_HEADER_FIXED { + return None; + } + if frame[4] != PIECE_VERSION || frame[10] != 0 || frame[11] != 0 { + return None; + } + let (hop, data, pieces, index, path_len) = + (frame[5], frame[6], frame[7], frame[8], frame[9] as usize); + if !(1..=2).contains(&hop) + || data == 0 + || pieces <= data + || pieces as usize > MAX_PIECES + || index >= pieces + || path_len != merkle_depth(pieces as usize) + { + return None; + } + let take32 = |at: usize| -> [u8; 32] { frame[at..at + 32].try_into().unwrap() }; + let take64 = |at: usize| -> [u8; 64] { frame[at..at + 64].try_into().unwrap() }; + let slot = u32::from_be_bytes(frame[12..16].try_into().unwrap()); + let leader = take32(16); + let sent_unix_ms = u64::from_be_bytes(frame[48..56].try_into().unwrap()); + let total_len = u32::from_be_bytes(frame[56..60].try_into().unwrap()); + let root = take32(60); + let overlay_key = take32(92); + let binding_sig = take64(124); + let root_sig = take64(188); + let body = PIECE_HEADER_FIXED + 32 * path_len; + if frame.len() < body + || (total_len as usize) < 4 + || total_len as usize > super::txset_encoding::max_encoded_size() + || frame.len() - body != piece_len(total_len as usize, data as usize) + { + return None; + } + let path = (0..path_len) + .map(|i| take32(PIECE_HEADER_FIXED + 32 * i)) + .collect(); + Some(( + Self { + hop, + data, + pieces, + index, + slot, + leader, + sent_unix_ms, + total_len, + root, + overlay_key, + binding_sig, + root_sig, + path, + }, + body, + )) + } +} + +/// A piece as it arrived: the frame, and where the piece begins in it. +#[derive(Clone, Debug)] +pub struct PieceBytes { + pub frame: Arc>, + pub at: usize, +} + +impl PieceBytes { + pub fn as_slice(&self) -> &[u8] { + &self.frame[self.at..] + } +} + +/// A set's pieces are told apart by slot, leader and root. +pub type RootKey = (u32, NodeId, [u8; 32]); + +struct Assembly { + data: usize, + total_len: u32, + pieces: Vec>, + /// Indices seen, also after the pieces were handed out. + seen: Vec, + done: bool, + /// The overlay key that signed the root (the leader's, as checked when + /// the root was added). + origin: [u8; 32], +} + +/// Enough pieces of a set to rebuild it. +#[derive(Debug)] +pub struct ReadySet { + pub pieces: Vec>, + pub params: CodedParams, + pub total_len: u32, + /// The overlay key of the leader that signed the set's root. + pub origin: [u8; 32], +} + +/// What became of a valid piece. +#[derive(Debug)] +pub enum PieceOutcome { + /// Kept; more are needed. + Stored, + /// With this one the set can be rebuilt. + Ready(ReadySet), + /// This index is held already. + Duplicate, + /// The set was rebuilt before this piece came. + Late, + /// The root is not one this node keeps pieces for (see + /// [`CodedState::add_root`]), or the piece does not fit it. + Unknown, +} + +/// The pieces a node holds and the limits on what peers can make it hold. +pub struct CodedState { + assemblies: lru::LruCache, + /// Roots kept per slot and leader. + roots: lru::LruCache<(u32, NodeId), Vec<[u8; 32]>>, + /// Piece frames looked at per slot, leader and sender. + frames: lru::LruCache<(u32, NodeId, NodeId), u8>, + /// Roots whose piece this node passed on as a relay. + relayed: lru::LruCache, + /// Sets this node cut into pieces as their leader, by content hash. + pushed: lru::LruCache<[u8; 32], ()>, + /// Layouts worked out already, by slot, seed and leader (see + /// `TreeState::coded_layout`). + layouts: lru::LruCache<(u32, [u8; 32], NodeId), Arc>, + /// Pieces not yet handed to the transport, per receiver. + in_flight: HashMap, +} + +impl Default for CodedState { + fn default() -> Self { + Self::new() + } +} + +impl CodedState { + pub fn new() -> Self { + let memory = std::num::NonZeroUsize::new(CODED_MEMORY).unwrap(); + Self { + assemblies: lru::LruCache::new(memory), + roots: lru::LruCache::new(memory), + frames: lru::LruCache::new(std::num::NonZeroUsize::new(16 * CODED_MEMORY).unwrap()), + relayed: lru::LruCache::new(memory), + pushed: lru::LruCache::new(memory), + layouts: lru::LruCache::new(std::num::NonZeroUsize::new(32).unwrap()), + in_flight: HashMap::new(), + } + } + + /// The layout of `slot` (with `seed`) and `leader`, if it was worked out + /// before. + pub fn cached_layout( + &mut self, + slot: u32, + seed: &[u8; 32], + leader: &NodeId, + ) -> Option> { + self.layouts.get(&(slot, *seed, *leader)).cloned() + } + + pub fn cache_layout(&mut self, slot: u32, seed: [u8; 32], layout: Arc) { + self.layouts.put((slot, seed, layout.leader), layout); + } + + /// The nodes changed: the layouts are others now. + pub fn forget_layouts(&mut self) { + self.layouts.clear(); + } + + /// A piece frame for `slot` and `leader` came from `sender`. False: that + /// sender used up what this node looks at for that slot and leader. + pub fn admit_frame(&mut self, slot: u32, leader: NodeId, sender: NodeId) -> bool { + let count = self.frames.get_or_insert_mut((slot, leader, sender), || 0); + if *count >= MAX_FRAMES_PER_SENDER { + return false; + } + *count += 1; + true + } + + /// Whether pieces are kept for this root with this shape: `Some(true)`. + /// `Some(false)`: the root is known with another shape or length (a + /// frame that says so is wrong). `None`: not known; its signature has to + /// be checked before [`Self::add_root`]. + pub fn knows_root(&self, key: &RootKey, params: CodedParams, total_len: u32) -> Option { + self.assemblies.peek(key).map(|a| { + // (`seen`, not `pieces`: the pieces leave with the rebuild.) + a.data == params.data && a.seen.len() == params.pieces && a.total_len == total_len + }) + } + + /// Start keeping pieces for a root whose signature by the slot's leader + /// was checked; `origin` is the overlay key that signed it. False: the + /// leader has [`MAX_ROOTS_PER_SLOT_AND_LEADER`] other roots for that slot + /// already. + pub fn add_root( + &mut self, + key: RootKey, + params: CodedParams, + total_len: u32, + origin: [u8; 32], + ) -> bool { + if self.assemblies.contains(&key) { + return true; + } + let roots = self.roots.get_or_insert_mut((key.0, key.1), Vec::new); + if roots.len() >= MAX_ROOTS_PER_SLOT_AND_LEADER { + return false; + } + roots.push(key.2); + self.assemblies.put( + key, + Assembly { + data: params.data, + total_len, + pieces: vec![None; params.pieces], + seen: vec![false; params.pieces], + done: false, + origin, + }, + ); + true + } + + /// Keep a piece whose path to the root was checked. + pub fn add_piece(&mut self, key: &RootKey, index: usize, piece: PieceBytes) -> PieceOutcome { + let Some(assembly) = self.assemblies.get_mut(key) else { + return PieceOutcome::Unknown; + }; + if index >= assembly.seen.len() { + return PieceOutcome::Unknown; + } + if std::mem::replace(&mut assembly.seen[index], true) { + return PieceOutcome::Duplicate; + } + if assembly.done { + return PieceOutcome::Late; + } + assembly.pieces[index] = Some(piece); + let have = assembly.pieces.iter().flatten().count(); + if have < assembly.data { + return PieceOutcome::Stored; + } + assembly.done = true; + let pieces = std::mem::take(&mut assembly.pieces); + PieceOutcome::Ready(ReadySet { + params: CodedParams { + data: assembly.data, + pieces: assembly.seen.len(), + relays: 0, + }, + total_len: assembly.total_len, + origin: assembly.origin, + pieces, + }) + } + + /// This node is about to pass on its piece of this root. False: it did + /// so already. + pub fn mark_relayed(&mut self, key: RootKey) -> bool { + self.relayed.put(key, ()).is_none() + } + + /// This node is about to cut the set `hash` into pieces as its leader. + /// False: it did so already. + pub fn mark_pushed(&mut self, hash: [u8; 32]) -> bool { + self.pushed.put(hash, ()).is_none() + } + + /// A piece is about to be written to `node`. False: too many are in + /// flight to it, and this one is not sent. + pub fn start_send(&mut self, node: NodeId) -> bool { + let in_flight = self.in_flight.entry(node).or_insert(0); + if *in_flight >= MAX_PIECES_IN_FLIGHT_PER_PEER { + return false; + } + *in_flight += 1; + true + } + + /// The piece to `node` was handed to the transport, or failed. + pub fn finish_send(&mut self, node: &NodeId) { + if let Some(count) = self.in_flight.get_mut(node) { + *count = count.saturating_sub(1); + if *count == 0 { + self.in_flight.remove(node); + } + } + } + + /// Drop the pieces of slots below `min_slot` (their sets are decided). + pub fn prune(&mut self, min_slot: u32) { + let old: Vec = self + .assemblies + .iter() + .filter(|(key, _)| key.0 < min_slot) + .map(|(key, _)| *key) + .collect(); + for key in old { + self.assemblies.pop(&key); + } + let old: Vec<(u32, NodeId)> = self + .roots + .iter() + .filter(|(key, _)| key.0 < min_slot) + .map(|(key, _)| *key) + .collect(); + for key in old { + self.roots.pop(&key); + } + } + + /// Bytes of pieces held (for tests). + #[cfg(test)] + pub fn held_bytes(&self) -> usize { + self.assemblies + .iter() + .flat_map(|(_, a)| a.pieces.iter().flatten()) + .map(|p| p.as_slice().len()) + .sum() + } +} + +#[cfg(test)] +mod tests { + use super::super::txset_tree::tree_order; + use super::*; + use rand::{RngCore, SeedableRng}; + + fn node(n: u16) -> NodeId { + let mut id = [0u8; 32]; + id[..2].copy_from_slice(&n.to_be_bytes()); + id[31] = 0xAA; + id + } + + fn nodes(count: u16) -> Vec { + (0..count).map(node).collect() + } + + fn params(data: usize, pieces: usize) -> CodedParams { + CodedParams { + data, + pieces, + relays: 1, + } + } + + #[test] + fn the_code_is_derived_from_the_node_count() { + for (others, expected) in [ + (0, None), + (3, None), + (4, Some((2, 4, 1))), + (6, Some((3, 6, 1))), + (7, Some((4, 7, 1))), + (16, Some((8, 16, 1))), + (17, Some((8, 16, 1))), + (31, Some((8, 16, 1))), + (32, Some((8, 16, 2))), + (89, Some((8, 16, 5))), + (1023, Some((8, 16, 63))), + ] { + let got = coded_params(others).map(|p| (p.data, p.pieces, p.relays)); + assert_eq!(got, expected, "{others} others"); + if let Some((k, m, r)) = got { + assert!(k < m && m * r <= others); + } + } + } + + #[test] + fn every_node_gets_every_piece_from_another_sender() { + for count in [5u16, 7, 17, 33, 90, 200] { + let all = nodes(count); + let order = tree_order(&[3u8; 32], &all[2], &all); + let layout = CodedLayout::new(&order).unwrap(); + let p = layout.params; + // The relay sets of the pieces are disjoint, and the leader + // writes each piece to its relays. + let targets = layout.leader_targets(); + assert_eq!(targets.len(), p.pieces * p.relays); + let mut relays: Vec = targets.iter().map(|(_, n)| *n).collect(); + relays.sort(); + relays.dedup(); + assert_eq!(relays.len(), p.pieces * p.relays); + assert!(!relays.contains(&all[2])); + for me in order.iter().skip(1) { + let senders: Vec = (0..p.pieces) + .map(|j| layout.sender_of(me, j).unwrap()) + .collect(); + // m different senders, never the node itself, the leader + // for the one piece the node relays (if it relays one). + let mut distinct = senders.clone(); + distinct.sort(); + distinct.dedup(); + assert_eq!(distinct.len(), p.pieces, "{count} nodes"); + assert!(!senders.contains(me)); + let from_leader = senders.iter().filter(|s| **s == all[2]).count(); + assert_eq!(from_leader, usize::from(layout.relayed_piece(me).is_some())); + // A relay sends exactly to the nodes that expect it to. + for (j, sender) in senders.iter().enumerate() { + if *sender != all[2] { + assert_eq!(layout.relayed_piece(sender), Some(j)); + assert!(layout.receivers_of(sender).contains(me)); + } + } + } + // The receivers of a piece's relays are everyone but its relays, + // each once; the load of a relay is even within one. + for j in 0..p.pieces { + let mut got: Vec = Vec::new(); + let mut loads = Vec::new(); + for relay in layout.relays_of(j) { + let receivers = layout.receivers_of(relay); + loads.push(receivers.len()); + got.extend(receivers); + } + got.sort(); + let before = got.len(); + got.dedup(); + assert_eq!(got.len(), before); + assert_eq!(got.len(), count as usize - 1 - p.relays); + assert!(loads.iter().max().unwrap() - loads.iter().min().unwrap() <= 2); + } + // A node outside the layout is owed nothing. + assert_eq!(layout.sender_of(&node(9999), 0), None); + assert!(layout.receivers_of(&node(9999)).is_empty()); + assert!(layout.receivers_of(&all[2]).is_empty()); + } + // Too few nodes: no layout. + let few = nodes(4); + assert!(CodedLayout::new(&tree_order(&[3u8; 32], &few[0], &few)).is_none()); + } + + #[test] + fn any_k_pieces_rebuild_the_frame() { + let mut rng = rand::rngs::StdRng::seed_from_u64(11); + for (len, k, m) in [ + (5usize, 2usize, 4usize), + (4, 3, 6), + (1, 4, 7), + (1000, 3, 6), + (1001, 8, 16), + (123_457, 8, 16), + (1_000_003, 8, 16), + ] { + let mut frame = vec![0u8; len]; + rng.fill_bytes(&mut frame); + let p = params(k, m); + let pieces = encode_pieces(&frame, p).unwrap(); + assert_eq!(pieces.len(), m); + let piece = piece_len(len, k); + assert!(piece % 2 == 0 && piece * k >= len && piece * k < len + 2 * k + 2); + assert!(pieces.iter().all(|p| p.len() == piece)); + // The data pieces alone: no parity needed. + let held: Vec> = (0..m) + .map(|i| (i < k).then(|| pieces[i].as_slice())) + .collect(); + assert_eq!(rebuild(&held, p, len).unwrap(), (frame.clone(), false)); + // The parity pieces and as few data pieces as that leaves. + let held: Vec> = (0..m) + .map(|i| (i >= m - k).then(|| pieces[i].as_slice())) + .collect(); + assert_eq!(rebuild(&held, p, len).unwrap(), (frame.clone(), true)); + // Random subsets of exactly k pieces. + for _ in 0..8 { + let mut idx: Vec = (0..m).collect(); + for i in (1..m).rev() { + idx.swap(i, (rng.next_u32() as usize) % (i + 1)); + } + let chosen = &idx[..k]; + let held: Vec> = (0..m) + .map(|i| chosen.contains(&i).then(|| pieces[i].as_slice())) + .collect(); + assert_eq!(rebuild(&held, p, len).unwrap().0, frame); + } + // One piece too few. + let held: Vec> = (0..m) + .map(|i| (i >= m - k + 1).then(|| pieces[i].as_slice())) + .collect(); + assert!(rebuild(&held, p, len).is_err()); + // A piece of another length. + let short = &pieces[0][..piece - 2]; + let mut held: Vec> = (0..m) + .map(|i| (i < k).then(|| pieces[i].as_slice())) + .collect(); + held[0] = Some(short); + assert!(rebuild(&held, p, len).is_err()); + } + assert!(encode_pieces(&[], params(2, 4)).is_err()); + assert!(encode_pieces(&[1, 2, 3], params(4, 4)).is_err()); + } + + #[test] + fn a_wrong_piece_gives_a_wrong_frame_and_the_path_tells() { + let mut rng = rand::rngs::StdRng::seed_from_u64(5); + let mut frame = vec![0u8; 40_000]; + rng.fill_bytes(&mut frame); + let p = params(8, 16); + let pieces = encode_pieces(&frame, p).unwrap(); + let leaves: Vec<[u8; 32]> = pieces + .iter() + .enumerate() + .map(|(i, piece)| leaf_hash(i as u8, piece)) + .collect(); + let (root, paths) = merkle_tree(&leaves); + assert_eq!(paths.len(), 16); + assert!(paths.iter().all(|path| path.len() == merkle_depth(16))); + for (i, piece) in pieces.iter().enumerate() { + assert_eq!( + merkle_root_from(leaf_hash(i as u8, piece), i, &paths[i]), + root + ); + // The same bytes at another index, or another path, do not + // reach the root. + let other = (i + 1) % 16; + assert_ne!( + merkle_root_from(leaf_hash(other as u8, piece), other, &paths[other]), + root + ); + assert_ne!( + merkle_root_from(leaf_hash(i as u8, piece), i, &paths[other]), + root + ); + } + // A changed parity piece: the rebuild that uses it gives another + // frame (or none), and its path no longer reaches the root. + let mut bad = pieces[9].clone(); + bad[100] ^= 1; + assert_ne!(merkle_root_from(leaf_hash(9, &bad), 9, &paths[9]), root); + let held: Vec> = (0..16) + .map(|i| match i { + 9 => Some(bad.as_slice()), + 1..=7 => Some(pieces[i].as_slice()), + _ => None, + }) + .collect(); + match rebuild(&held, p, frame.len()) { + Ok((rebuilt, _)) => assert_ne!(rebuilt, frame), + Err(_) => {} + } + } + + #[test] + fn merkle_trees_of_other_sizes() { + for count in [2usize, 3, 4, 5, 6, 7, 9, 16] { + let leaves: Vec<[u8; 32]> = (0..count) + .map(|i| leaf_hash(i as u8, &[i as u8; 10])) + .collect(); + let (root, paths) = merkle_tree(&leaves); + for i in 0..count { + assert_eq!(paths[i].len(), merkle_depth(count)); + assert_eq!(merkle_root_from(leaves[i], i, &paths[i]), root); + } + // A padding leaf is not a piece: no bytes hash to it. + if count < count.next_power_of_two() { + assert_ne!(leaf_hash(count as u8, &[]), [0u8; 32]); + } + } + assert_eq!(merkle_depth(2), 1); + assert_eq!(merkle_depth(6), 3); + assert_eq!(merkle_depth(16), 4); + } + + fn header(index: u8, total_len: u32) -> PieceHeader { + PieceHeader { + hop: 1, + data: 8, + pieces: 16, + index, + slot: 77, + leader: node(3), + sent_unix_ms: 1_700_000_000_123, + total_len, + root: [9u8; 32], + overlay_key: [8u8; 32], + binding_sig: [7u8; 64], + root_sig: [6u8; 64], + path: vec![[1u8; 32], [2u8; 32], [3u8; 32], [4u8; 32]], + } + } + + #[test] + fn a_piece_header_round_trips_and_malformed_ones_are_refused() { + let h = header(5, 1000); + let piece = vec![0x55u8; piece_len(1000, 8)]; + let mut frame = h.encode(); + assert_eq!(frame.len(), PIECE_HEADER_FIXED + 4 * 32); + frame.extend_from_slice(&piece); + assert!(PieceHeader::is_tagged(&frame)); + let (parsed, at) = PieceHeader::parse(&frame).unwrap(); + assert_eq!(parsed, h); + assert_eq!(&frame[at..], piece.as_slice()); + // Not a tree frame and not a length. + assert!(!crate::flood::txset_tree::TreeHeader::is_tagged(&frame)); + assert!(PIECE_TAG as usize > super::super::txset_encoding::MAX_TXSET_XDR_SIZE); + + let broken = |change: &dyn Fn(&mut Vec)| { + let mut f = frame.clone(); + change(&mut f); + PieceHeader::parse(&f).is_none() + }; + assert!(broken(&|f| f[4] = 2)); // version + assert!(broken(&|f| f[5] = 0)); // hop + assert!(broken(&|f| f[5] = 3)); + assert!(broken(&|f| f[6] = 0)); // k + assert!(broken(&|f| f[6] = 16)); // k = m + assert!(broken(&|f| f[7] = 17)); // m over the limit + assert!(broken(&|f| f[8] = 16)); // index = m + assert!(broken(&|f| f[9] = 3)); // path length + assert!(broken(&|f| f[10] = 1)); // reserved + assert!(broken(&|f| f.push(0))); // piece too long + assert!(broken(&|f| { + f.pop(); + })); + assert!(broken(&|f| f[56..60].copy_from_slice(&0u32.to_be_bytes()))); + assert!(broken( + &|f| f[56..60].copy_from_slice(&u32::MAX.to_be_bytes()) + )); + assert!(broken(&|f| f.truncate(100))); + assert!(PieceHeader::parse(&frame[..PIECE_HEADER_FIXED]).is_none()); + } + + fn piece_bytes(len: usize, fill: u8) -> PieceBytes { + PieceBytes { + frame: Arc::new(vec![fill; len + 3]), + at: 3, + } + } + + #[test] + fn a_node_keeps_pieces_until_it_can_rebuild() { + let mut state = CodedState::new(); + let p = params(3, 6); + let key: RootKey = (10, node(1), [1u8; 32]); + // Nothing is kept for a root nobody vouched for. + assert!(matches!( + state.add_piece(&key, 0, piece_bytes(8, 0)), + PieceOutcome::Unknown + )); + assert_eq!(state.knows_root(&key, p, 20), None); + assert!(state.add_root(key, p, 20, [5u8; 32])); + assert_eq!(state.knows_root(&key, p, 20), Some(true)); + assert_eq!(state.knows_root(&key, p, 21), Some(false)); + assert_eq!(state.knows_root(&key, params(2, 6), 20), Some(false)); + assert!(matches!( + state.add_piece(&key, 4, piece_bytes(8, 4)), + PieceOutcome::Stored + )); + assert!(matches!( + state.add_piece(&key, 4, piece_bytes(8, 4)), + PieceOutcome::Duplicate + )); + assert!(matches!( + state.add_piece(&key, 6, piece_bytes(8, 6)), + PieceOutcome::Unknown + )); + assert!(matches!( + state.add_piece(&key, 0, piece_bytes(8, 0)), + PieceOutcome::Stored + )); + assert_eq!(state.held_bytes(), 16); + match state.add_piece(&key, 5, piece_bytes(8, 5)) { + PieceOutcome::Ready(ready) => { + assert_eq!( + (ready.params.data, ready.params.pieces, ready.total_len), + (3, 6, 20) + ); + let held: Vec = ready + .pieces + .iter() + .enumerate() + .filter_map(|(i, p)| p.as_ref().map(|_| i)) + .collect(); + assert_eq!(held, vec![0, 4, 5]); + assert_eq!(ready.pieces[4].as_ref().unwrap().as_slice(), &[4u8; 8]); + } + other => panic!("{other:?}"), + } + // The pieces left with the rebuild; later ones are not kept. + assert_eq!(state.held_bytes(), 0); + assert_eq!(state.knows_root(&key, p, 20), Some(true)); + assert!(matches!( + state.add_piece(&key, 1, piece_bytes(8, 1)), + PieceOutcome::Late + )); + assert!(matches!( + state.add_piece(&key, 1, piece_bytes(8, 1)), + PieceOutcome::Duplicate + )); + assert!(matches!( + state.add_piece(&key, 0, piece_bytes(8, 0)), + PieceOutcome::Duplicate + )); + assert_eq!(state.held_bytes(), 0); + } + + #[test] + fn the_limits_of_what_peers_can_make_a_node_hold() { + let mut state = CodedState::new(); + let p = params(3, 6); + // Two roots per slot and leader. + assert!(state.add_root((10, node(1), [1u8; 32]), p, 20, [5u8; 32])); + assert!(state.add_root((10, node(1), [2u8; 32]), p, 20, [5u8; 32])); + assert!(state.add_root((10, node(1), [1u8; 32]), p, 20, [5u8; 32])); + assert!(!state.add_root((10, node(1), [3u8; 32]), p, 20, [5u8; 32])); + assert_eq!(state.knows_root(&(10, node(1), [3u8; 32]), p, 20), None); + // Another leader or slot has its own. + assert!(state.add_root((10, node(2), [3u8; 32]), p, 20, [5u8; 32])); + assert!(state.add_root((11, node(1), [3u8; 32]), p, 20, [5u8; 32])); + // A sender is listened to a few times per slot and leader. + for _ in 0..MAX_FRAMES_PER_SENDER { + assert!(state.admit_frame(10, node(1), node(7))); + } + assert!(!state.admit_frame(10, node(1), node(7))); + assert!(state.admit_frame(10, node(1), node(8))); + assert!(state.admit_frame(11, node(1), node(7))); + // Pieces in flight per receiver. + for _ in 0..MAX_PIECES_IN_FLIGHT_PER_PEER { + assert!(state.start_send(node(5))); + } + assert!(!state.start_send(node(5))); + assert!(state.start_send(node(6))); + state.finish_send(&node(5)); + assert!(state.start_send(node(5))); + // A relay passes a root's piece on once; a leader cuts a set once. + assert!(state.mark_relayed((10, node(1), [1u8; 32]))); + assert!(!state.mark_relayed((10, node(1), [1u8; 32]))); + assert!(state.mark_pushed([4u8; 32])); + assert!(!state.mark_pushed([4u8; 32])); + // Old slots are dropped with their pieces, and their roots' places. + assert!(matches!( + state.add_piece(&(10, node(1), [1u8; 32]), 0, piece_bytes(8, 0)), + PieceOutcome::Stored + )); + assert!(matches!( + state.add_piece(&(11, node(1), [3u8; 32]), 0, piece_bytes(8, 0)), + PieceOutcome::Stored + )); + assert_eq!(state.held_bytes(), 16); + state.prune(11); + assert_eq!(state.held_bytes(), 8); + assert_eq!(state.knows_root(&(10, node(1), [1u8; 32]), p, 20), None); + assert_eq!( + state.knows_root(&(11, node(1), [3u8; 32]), p, 20), + Some(true) + ); + } + + #[test] + fn the_signed_message_binds_every_field() { + let p = params(8, 16); + let message = + |network: u8, seed: u8, slot: u32, leader: u16, p: CodedParams, len: u32, root: u8| { + root_message( + &[network; 32], + &[seed; 32], + slot, + &node(leader), + p, + len, + &[root; 32], + ) + }; + let base = message(1, 4, 5, 1, p, 1000, 2); + assert_ne!(base, message(9, 4, 5, 1, p, 1000, 2)); + assert_ne!(base, message(1, 5, 5, 1, p, 1000, 2)); + assert_ne!(base, message(1, 4, 6, 1, p, 1000, 2)); + assert_ne!(base, message(1, 4, 5, 2, p, 1000, 2)); + assert_ne!(base, message(1, 4, 5, 1, params(7, 16), 1000, 2)); + assert_ne!(base, message(1, 4, 5, 1, params(8, 15), 1000, 2)); + assert_ne!(base, message(1, 4, 5, 1, p, 1001, 2)); + assert_ne!(base, message(1, 4, 5, 1, p, 1000, 3)); + // The relays per piece are not part of the code. + let other = CodedParams { relays: 5, ..p }; + assert_eq!(base, message(1, 4, 5, 1, other, 1000, 2)); + } + + #[test] + fn layouts_are_kept_by_slot_seed_and_leader() { + let all = nodes(9); + let layout = Arc::new(CodedLayout::new(&tree_order(&[3u8; 32], &all[2], &all)).unwrap()); + let mut state = CodedState::new(); + assert!(state.cached_layout(10, &[3u8; 32], &all[2]).is_none()); + state.cache_layout(10, [3u8; 32], Arc::clone(&layout)); + assert!(state.cached_layout(10, &[3u8; 32], &all[2]).is_some()); + assert!(state.cached_layout(11, &[3u8; 32], &all[2]).is_none()); + assert!(state.cached_layout(10, &[4u8; 32], &all[2]).is_none()); + assert!(state.cached_layout(10, &[3u8; 32], &all[3]).is_none()); + state.forget_layouts(); + assert!(state.cached_layout(10, &[3u8; 32], &all[2]).is_none()); + } +} diff --git a/overlay/src/flood/txset_push.rs b/overlay/src/flood/txset_push.rs index e09567e637..8aba21517f 100644 --- a/overlay/src/flood/txset_push.rs +++ b/overlay/src/flood/txset_push.rs @@ -95,16 +95,27 @@ impl PushRecord { hash: Hash256, connected: impl IntoIterator, ) -> (Vec, usize) { + let (targets, busy) = self.start_push_listing_busy(hash, connected); + (targets, busy.len()) + } + + /// As [`Self::start_push`]; also returns the peers that were left out + /// because an earlier push to them is still in flight. + pub fn start_push_listing_busy( + &mut self, + hash: Hash256, + connected: impl IntoIterator, + ) -> (Vec, Vec) { let peers = self.by_hash.get_or_insert_mut(hash, HashMap::new); let mut targets = Vec::new(); - let mut busy = 0; + let mut busy = Vec::new(); for peer in connected { if peers.contains_key(&peer) { continue; } let in_flight = self.in_flight.entry(peer).or_insert(0); if *in_flight >= MAX_IN_FLIGHT_PUSHES_PER_PEER { - busy += 1; + busy.push(peer); continue; } *in_flight += 1; diff --git a/overlay/src/flood/txset_tree.rs b/overlay/src/flood/txset_tree.rs new file mode 100644 index 0000000000..65c9edbbf3 --- /dev/null +++ b/overlay/src/flood/txset_tree.rs @@ -0,0 +1,1720 @@ +//! Spanning tree for the leader's tx-set push. +//! +//! Without a tree the round's leader writes its set to every peer. With one, +//! it writes it to a few (its children in the tree) and every node that +//! receives it passes the bytes on to its own children. The tree of a slot +//! and leader is a function of things every node already agrees on, as the +//! leader schedule is: the nodes of the transitive quorum and a hash. +//! +//! * [`tree_order`]: the leader first, the other nodes by +//! `SHA256(seed || leader || node)`. +//! * Position `i` of the order has the children `F*i+1 ..= F*i+F`: the +//! complete F-ary tree, the least depth a fanout of F allows. +//! * [`TreeHeader`]: what a pushed frame says about itself, in front of the +//! ordinary tx-set frame. +//! * [`relay_plan`]: whether this node passes a frame on, and to whom. +//! * [`rtt_order`]: an order chosen by the leader from its round-trip times +//! (it travels in the header, because no other node can recompute it). +//! +//! Nothing here is trusted by the receiver of a set: a set is still verified +//! and filed by the hash of its content. The tree only decides who sends the +//! bytes to whom. + +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::time::Duration; + +pub type NodeId = [u8; 32]; + +/// Stands where an ordinary tx-set frame has the big-endian length of its +/// XDR. No length can take this value (the limit is 16 MiB). +pub const TREE_TAG: u32 = 0xFFFF_FFF1; + +/// Stands in the same place in a ping or a pong: a frame of 13 bytes on the +/// tx-set stream by which a node measures the round trip to a peer. +pub const TREE_PING_TAG: u32 = 0xFFFF_FFF2; + +/// A round-trip probe: the peer sends back what it was sent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Probe { + /// Answer with a pong that carries the same number. + Ping(u64), + /// The answer to a ping this node sent with that number. + Pong(u64), +} + +impl Probe { + pub fn encode(self) -> [u8; 13] { + let (kind, value) = match self { + Probe::Ping(value) => (1u8, value), + Probe::Pong(value) => (2u8, value), + }; + let mut out = [0u8; 13]; + out[..4].copy_from_slice(&TREE_PING_TAG.to_be_bytes()); + out[4] = kind; + out[5..].copy_from_slice(&value.to_be_bytes()); + out + } + + /// `Some(None)`: tagged as a probe but malformed. + pub fn parse(frame: &[u8]) -> Option> { + if frame.len() < 4 || frame[..4] != TREE_PING_TAG.to_be_bytes() { + return None; + } + if frame.len() != 13 { + return Some(None); + } + let value = u64::from_be_bytes(frame[5..].try_into().unwrap()); + Some(match frame[4] { + 1 => Some(Probe::Ping(value)), + 2 => Some(Probe::Pong(value)), + _ => None, + }) + } +} + +/// Bytes of a header without an explicit order. +pub const TREE_HEADER_FIXED: usize = 4 + 1 + 1 + 1 + 1 + 4 + 32 + 8 + 2; + +/// Nodes a tree may have (an explicit order is two bytes per node). +pub const MAX_TREE_NODES: usize = 1024; + +/// Bytes of the node-list digest behind an explicit order. +pub const TREE_NODES_DIGEST: usize = 8; + +/// The largest header a frame can carry. +pub const MAX_TREE_HEADER: usize = TREE_HEADER_FIXED + 2 * MAX_TREE_NODES + TREE_NODES_DIGEST; + +/// Frames a node relays for one slot and leader. A leader pushes one set per +/// slot it leads, two if it built one early and another at the trigger. +pub const MAX_RELAYS_PER_SLOT_AND_LEADER: u8 = 2; + +/// Relays that may be queued or in flight to one child at a time. A child +/// that stops reading its tx-set stream then pins at most this many frames +/// in the relay's memory; further frames are not relayed to it, and it asks +/// for them instead. +pub const MAX_IN_FLIGHT_RELAYS_PER_CHILD: u32 = 1; + +/// How a node pushes the set it leads with, as its Core configures it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TreeMode { + /// As before the tree: an untagged frame to every peer. + Off, + /// To every peer, tagged (the frame says when it was sent), never + /// relayed. The baseline for measurements. + Star, + /// Down the tree derived from the slot's seed. + Hash, + /// Down a tree the leader lays out by its round-trip times. + Rtt, + /// As erasure-coded pieces, each through its own relays (see + /// `txset_coded`). + Coded, +} + +impl TreeMode { + pub fn from_u8(value: u8) -> Option { + match value { + 0 => Some(Self::Off), + 1 => Some(Self::Star), + 2 => Some(Self::Hash), + 3 => Some(Self::Rtt), + 4 => Some(Self::Coded), + _ => None, + } + } + + /// Whether nodes in this mode relay tagged frames down a tree. + pub fn relays(self) -> bool { + matches!(self, Self::Hash | Self::Rtt) + } + + /// Whether nodes in this mode expect a set to reach them through other + /// nodes, and so wait before they ask for it. + pub fn waits(self) -> bool { + matches!(self, Self::Hash | Self::Rtt | Self::Coded) + } +} + +/// What a tagged frame is, from its header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameKind { + /// Sent to every peer by the leader; not to be relayed. + Star = 1, + /// Travels down the tree every node derives from the slot's seed. + Hash = 2, + /// Travels down the tree whose order the header carries. + Explicit = 3, +} + +/// The smallest fanout that reaches `nodes` nodes (the leader included) in +/// two hops: the least F with `F + F*F >= nodes - 1`. +pub fn default_fanout(nodes: usize) -> usize { + let others = nodes.saturating_sub(1); + let mut fanout = 1; + while fanout + fanout * fanout < others { + fanout += 1; + } + fanout +} + +/// The order of the tree of `leader`: the leader, then the other nodes by +/// `SHA256(seed || leader || node)` (ties, impossible in practice, by node +/// ID). Duplicates in `nodes` count once; the leader need not be in `nodes`. +pub fn tree_order(seed: &[u8; 32], leader: &NodeId, nodes: &[NodeId]) -> Vec { + let mut keyed: Vec<([u8; 32], NodeId)> = nodes + .iter() + .filter(|node| *node != leader) + .map(|node| { + let mut hasher = Sha256::new(); + hasher.update(seed); + hasher.update(leader); + hasher.update(node); + (hasher.finalize().into(), *node) + }) + .collect(); + keyed.sort(); + keyed.dedup_by(|a, b| a.1 == b.1); + let mut order = Vec::with_capacity(keyed.len() + 1); + order.push(*leader); + order.extend(keyed.into_iter().map(|(_, node)| node)); + order +} + +/// Position of `node` in `order`. +fn position(order: &[NodeId], node: &NodeId) -> Option { + order.iter().position(|n| n == node) +} + +/// The children of `node` in the complete `fanout`-ary tree over `order` +/// (empty for a leaf, and for a node that is not in the order). +pub fn children_of(order: &[NodeId], node: &NodeId, fanout: usize) -> Vec { + let Some(i) = position(order, node) else { + return Vec::new(); + }; + children_at(order, i, fanout) +} + +fn children_at(order: &[NodeId], i: usize, fanout: usize) -> Vec { + if fanout == 0 { + return Vec::new(); + } + let first = match i.checked_mul(fanout).and_then(|v| v.checked_add(1)) { + Some(first) if first < order.len() => first, + _ => return Vec::new(), + }; + let end = first.saturating_add(fanout).min(order.len()); + order[first..end].to_vec() +} + +/// The parent of `node` (none for the root and for a node not in the order). +pub fn parent_of(order: &[NodeId], node: &NodeId, fanout: usize) -> Option { + let i = position(order, node)?; + if i == 0 || fanout == 0 { + return None; + } + Some(order[(i - 1) / fanout]) +} + +/// Hops from the root to `node` (0 for the root). +pub fn depth_of(order: &[NodeId], node: &NodeId, fanout: usize) -> Option { + let mut i = position(order, node)?; + if fanout == 0 { + return (i == 0).then_some(0); + } + let mut depth = 0; + while i > 0 { + i = (i - 1) / fanout; + depth += 1; + } + Some(depth) +} + +/// In front of a pushed tx-set frame. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TreeHeader { + pub kind: FrameKind, + /// Fanout of the tree (explicit orders: the leader's choice). + pub fanout: u8, + /// Hops this copy has made (1 = straight from the leader). + pub hop: u8, + pub slot: u32, + pub leader: NodeId, + /// When the leader sent it, in Unix milliseconds by the leader's clock. + pub sent_unix_ms: u64, + /// Explicit orders only: positions in the node list sorted by node ID, + /// the leader's first. + pub order: Vec, + /// Explicit orders only: [`TreeConfig::nodes_digest`] of the node list + /// the positions refer to (zero otherwise). A receiver whose list is + /// another does not follow the order. + pub nodes_digest: [u8; TREE_NODES_DIGEST], +} + +impl TreeHeader { + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(TREE_HEADER_FIXED + 2 * self.order.len()); + out.extend_from_slice(&TREE_TAG.to_be_bytes()); + out.push(self.kind as u8); + out.push(self.fanout); + out.push(self.hop); + out.push(0); + out.extend_from_slice(&self.slot.to_be_bytes()); + out.extend_from_slice(&self.leader); + out.extend_from_slice(&self.sent_unix_ms.to_be_bytes()); + out.extend_from_slice(&(self.order.len() as u16).to_be_bytes()); + for index in &self.order { + out.extend_from_slice(&index.to_be_bytes()); + } + if self.kind == FrameKind::Explicit { + out.extend_from_slice(&self.nodes_digest); + } + out + } + + /// Whether a frame starts with the tag (and so is not an ordinary one). + pub fn is_tagged(frame: &[u8]) -> bool { + frame.len() >= 4 && frame[..4] == TREE_TAG.to_be_bytes() + } + + /// Parse the header of a tagged frame. Returns it with the offset at + /// which the ordinary tx-set frame begins, or `None` if it is malformed. + pub fn parse(frame: &[u8]) -> Option<(Self, usize)> { + if !Self::is_tagged(frame) || frame.len() < TREE_HEADER_FIXED { + return None; + } + let kind = match frame[4] { + 1 => FrameKind::Star, + 2 => FrameKind::Hash, + 3 => FrameKind::Explicit, + _ => return None, + }; + let fanout = frame[5]; + let hop = frame[6]; + if frame[7] != 0 { + return None; + } + let slot = u32::from_be_bytes(frame[8..12].try_into().unwrap()); + let mut leader = [0u8; 32]; + leader.copy_from_slice(&frame[12..44]); + let sent_unix_ms = u64::from_be_bytes(frame[44..52].try_into().unwrap()); + let count = u16::from_be_bytes(frame[52..54].try_into().unwrap()) as usize; + if count > MAX_TREE_NODES { + return None; + } + // Only an explicit order carries one, with the digest of the node + // list it refers to behind it. + if (kind == FrameKind::Explicit) != (count != 0) { + return None; + } + let digest_at = TREE_HEADER_FIXED + 2 * count; + let body = digest_at + + if kind == FrameKind::Explicit { + TREE_NODES_DIGEST + } else { + 0 + }; + if frame.len() < body { + return None; + } + let mut nodes_digest = [0u8; TREE_NODES_DIGEST]; + if kind == FrameKind::Explicit { + nodes_digest.copy_from_slice(&frame[digest_at..body]); + } + let order = (0..count) + .map(|i| { + let at = TREE_HEADER_FIXED + 2 * i; + u16::from_be_bytes(frame[at..at + 2].try_into().unwrap()) + }) + .collect(); + Some(( + Self { + kind, + fanout, + hop, + slot, + leader, + sent_unix_ms, + order, + nodes_digest, + }, + body, + )) + } +} + +/// What this node's Core told it about the trees of the next slots. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TreeConfig { + pub mode: TreeMode, + /// 0 = [`default_fanout`] of the node count. + pub fanout: u8, + /// The slots a tree is known for, with the seed of each. + pub slots: Vec<(u32, [u8; 32])>, + /// For each of `slots`: the nodes that may lead it, and so root a tree + /// (the leaders of its first rounds, as Core expects them). `None`: any + /// node of the tree. + pub leaders: Vec>>, + /// The nodes of the tree, sorted by node ID, without duplicates. + pub nodes: Vec, + /// For tests and fault runs: this node passes nothing on (tree frames + /// or pieces). It still leads, receives and asks. + pub withhold: bool, +} + +impl TreeConfig { + pub fn new( + mode: TreeMode, + fanout: u8, + slots: Vec<(u32, [u8; 32])>, + mut nodes: Vec, + ) -> Self { + nodes.sort(); + nodes.dedup(); + let leaders = vec![None; slots.len()]; + Self { + mode, + fanout, + slots, + leaders, + nodes, + withhold: false, + } + } + + /// Make this node withhold what it should relay (tests, fault runs). + pub fn with_withhold(mut self, withhold: bool) -> Self { + self.withhold = withhold; + self + } + + /// Restrict who may lead each slot (one entry per slot, in order). + pub fn with_leaders(mut self, leaders: Vec>>) -> Self { + assert_eq!(leaders.len(), self.slots.len()); + self.leaders = leaders; + self + } + + /// Whether a frame may name `leader` as the root of the tree of `slot`. + pub fn may_lead(&self, slot: u32, leader: &NodeId) -> bool { + match self.slots.iter().position(|(s, _)| *s == slot) { + Some(i) => match &self.leaders[i] { + Some(leaders) => leaders.contains(leader), + None => true, + }, + None => false, + } + } + + /// Stands for this node's list of the tree's nodes in a header with an + /// explicit order: the first bytes of the SHA-256 of the sorted node IDs. + pub fn nodes_digest(&self) -> [u8; TREE_NODES_DIGEST] { + let mut hasher = Sha256::new(); + for node in &self.nodes { + hasher.update(node); + } + let digest: [u8; 32] = hasher.finalize().into(); + digest[..TREE_NODES_DIGEST].try_into().unwrap() + } + + /// Whether a tree of `fanout` over this node's nodes has two levels at + /// most (a leader that lays out its own tree may not make it deeper). + pub fn two_levels(&self, fanout: usize) -> bool { + fanout + fanout.saturating_mul(fanout) >= self.nodes.len().saturating_sub(1) + } + + pub fn seed_of(&self, slot: u32) -> Option<&[u8; 32]> { + self.slots + .iter() + .find(|(s, _)| *s == slot) + .map(|(_, seed)| seed) + } + + /// The fanout of this node's own trees. + pub fn effective_fanout(&self) -> usize { + if self.fanout == 0 { + default_fanout(self.nodes.len()) + } else { + self.fanout as usize + } + } + + pub fn contains(&self, node: &NodeId) -> bool { + self.nodes.binary_search(node).is_ok() + } + + /// The order of a hash tree, if the slot is known and the leader is one + /// of the nodes. + pub fn hash_order(&self, slot: u32, leader: &NodeId) -> Option> { + let seed = self.seed_of(slot)?; + self.contains(leader) + .then(|| tree_order(seed, leader, &self.nodes)) + } + + /// The order a header's indices stand for, if they are a permutation of + /// this node's node list that starts with the leader. + pub fn explicit_order(&self, header: &TreeHeader) -> Option> { + if header.order.len() != self.nodes.len() || header.nodes_digest != self.nodes_digest() { + return None; + } + let mut seen = vec![false; self.nodes.len()]; + let mut order = Vec::with_capacity(self.nodes.len()); + for index in &header.order { + let index = *index as usize; + if index >= self.nodes.len() || std::mem::replace(&mut seen[index], true) { + return None; + } + order.push(self.nodes[index]); + } + (order.first() == Some(&header.leader)).then_some(order) + } + + /// The indices that stand for `order` in a header. + pub fn indices_of(&self, order: &[NodeId]) -> Option> { + order + .iter() + .map(|node| self.nodes.binary_search(node).ok().map(|i| i as u16)) + .collect() + } +} + +/// What a node does with a tagged frame beyond filing its set. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RelayPlan { + /// Pass the bytes on to these nodes. + Relay(Vec), + /// Nothing to pass on: a leaf of the tree. + Leaf, + /// Not relayed, and why. + Refused(RelayRefusal), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RelayRefusal { + /// The frame is a star push, or this node does not relay (its mode). + NotATree, + /// This node's Core has not described the tree of that slot. + UnknownSlot, + /// The leader, this node or the header's order does not fit the nodes + /// this node knows. + UnknownNodes, + /// The sender is not this node's parent in that tree (or did not prove + /// which validator it is). + NotParent, + /// The frame's leader is not one this node's Core expects to lead the + /// slot. + NotALeader, + /// An explicit order reached a node that is not in rtt mode, or makes a + /// tree of more than two levels. + NotAllowed, +} + +/// Decide whether `me` relays a frame with `header` that came from the +/// validator `sender`. +pub fn relay_plan( + config: &TreeConfig, + header: &TreeHeader, + me: &NodeId, + sender: Option<&NodeId>, +) -> RelayPlan { + if !config.mode.relays() || header.kind == FrameKind::Star { + return RelayPlan::Refused(RelayRefusal::NotATree); + } + if config.seed_of(header.slot).is_none() { + return RelayPlan::Refused(RelayRefusal::UnknownSlot); + } + if !config.may_lead(header.slot, &header.leader) { + return RelayPlan::Refused(RelayRefusal::NotALeader); + } + // An order chosen by the sender is followed only where every node is + // configured for it, and only for the shape a leader is meant to choose: + // otherwise a validator could send any node's traffic down a chain of + // its choosing. + if header.kind == FrameKind::Explicit + && (config.mode != TreeMode::Rtt || !config.two_levels(header.fanout as usize)) + { + return RelayPlan::Refused(RelayRefusal::NotAllowed); + } + let (order, fanout) = match header.kind { + FrameKind::Hash => match config.hash_order(header.slot, &header.leader) { + // The fanout is this node's own: every node of the network is + // configured alike, and a node that is not refuses below. + Some(order) => (order, config.effective_fanout()), + None => return RelayPlan::Refused(RelayRefusal::UnknownNodes), + }, + FrameKind::Explicit => match config.explicit_order(header) { + Some(order) if header.fanout != 0 => (order, header.fanout as usize), + _ => return RelayPlan::Refused(RelayRefusal::UnknownNodes), + }, + FrameKind::Star => unreachable!(), + }; + if !config.contains(me) { + return RelayPlan::Refused(RelayRefusal::UnknownNodes); + } + match (parent_of(&order, me, fanout), sender) { + (Some(parent), Some(sender)) if parent == *sender => {} + _ => return RelayPlan::Refused(RelayRefusal::NotParent), + } + let children = children_of(&order, me, fanout); + if children.is_empty() { + RelayPlan::Leaf + } else { + RelayPlan::Relay(children) + } +} + +/// Nodes whose round-trip times follow each other within this are taken to be +/// in one place. (Inside a region the times differ by a few milliseconds; +/// between regions by tens.) Two regions at the same distance from the leader +/// end up in one group; see `rtt_order` for what is done about that. +const RTT_GROUP_GAP: Duration = Duration::from_millis(8); + +/// Split nodes sorted by round-trip time into groups at the gaps. +fn rtt_groups(mut measured: Vec<(Duration, NodeId)>) -> Vec> { + measured.sort(); + let mut groups: Vec> = Vec::new(); + let mut last: Option = None; + for (rtt, node) in measured { + let same = last.is_some_and(|prev| rtt.saturating_sub(prev) <= RTT_GROUP_GAP); + if same { + groups.last_mut().unwrap().push(node); + } else { + groups.push(vec![node]); + } + last = Some(rtt); + } + groups +} + +/// An order for the leader's own tree from its round-trip times, so that a +/// set makes one long hop to a relay near its destination and a short one +/// from there, instead of two hops of any length. +/// +/// * Nodes at about the same distance are a group (`rtt_groups`). Every +/// group gets relays in proportion to its size. +/// * A group's nodes, sorted by round-trip time, are cut into consecutive +/// runs, one per relay. The relay of a run is its middle node, its children +/// the rest of the run. Two regions at the same distance from the leader +/// are one group, but their times still differ by a little, so the runs +/// mostly fall apart along the regions, and a run that straddles two is +/// relayed by a node of the part it holds more of. +/// * What does not fit under its own run's relay (a relay has `fanout` +/// children) goes where there is room. +/// +/// `nodes` are the other nodes of the tree with the round-trip time to each +/// (`None`: not measured; those come last, as one group). The result starts +/// with `leader` and holds every node once, laid out as the complete tree +/// `children_of` reads: only the last relays may have fewer than `fanout` +/// children. +pub fn rtt_order( + leader: &NodeId, + nodes: &[(NodeId, Option)], + fanout: usize, +) -> Vec { + let mut unique: HashMap> = HashMap::new(); + for (node, rtt) in nodes { + if node != leader { + unique.entry(*node).or_insert(*rtt); + } + } + let mut measured = Vec::new(); + let mut unmeasured = Vec::new(); + for (node, rtt) in unique { + match rtt { + Some(rtt) => measured.push((rtt, node)), + None => unmeasured.push(node), + } + } + unmeasured.sort(); + let mut groups = rtt_groups(measured); + if !unmeasured.is_empty() { + groups.push(unmeasured); + } + let total: usize = groups.iter().map(Vec::len).sum(); + let mut order = vec![*leader]; + if total == 0 { + return order; + } + let fanout = fanout.max(1); + let relays = fanout.min(total); + // A tree of two levels holds `relays` relays and `relays * fanout` + // leaves. If the nodes do not fit, the order is the nodes from the + // nearest to the farthest (a deeper tree; `relay_plan` only follows two + // levels). With more groups than relays the groups cannot each have a + // relay: all nodes are one group then. + if total > relays + relays * fanout { + order.extend(groups.into_iter().flatten()); + return order; + } + if groups.len() > relays { + groups = vec![groups.into_iter().flatten().collect()]; + } + // Relays per group: one each, the rest to the group with the most nodes + // per relay, never more than the group's size. + let mut seats: Vec = vec![1; groups.len()]; + let mut left = relays - groups.len(); + while left > 0 { + let Some(next) = (0..groups.len()) + .filter(|g| seats[*g] < groups[*g].len()) + .max_by(|a, b| { + let load = |g: usize| groups[g].len() as f64 / seats[g] as f64; + load(*a) + .partial_cmp(&load(*b)) + .unwrap() + // Equal load: the nearer group (the lower index). + .then(b.cmp(a)) + }) + else { + break; + }; + seats[next] += 1; + left -= 1; + } + // Cut every group into consecutive runs, one per relay. + let mut relay_nodes: Vec = Vec::new(); + let mut children: Vec> = Vec::new(); + let mut leftovers: std::collections::VecDeque = std::collections::VecDeque::new(); + for (g, group) in groups.iter().enumerate() { + let mut start = 0; + for j in 0..seats[g] { + // Runs of nearly equal size (the earlier ones get the remainder). + let rest = group.len() - start; + let size = rest.div_ceil(seats[g] - j); + let run = &group[start..start + size]; + start += size; + let middle = run.len() / 2; + relay_nodes.push(run[middle]); + let mut kids: Vec = run + .iter() + .enumerate() + .filter(|(i, _)| *i != middle) + .map(|(_, node)| *node) + .collect(); + // A relay has `fanout` children; the farthest of a longer run + // go where there is room. + if kids.len() > fanout { + leftovers.extend(kids.drain(fanout..)); + } + children.push(kids); + } + } + let relay_count = relay_nodes.len(); + for slot in children.iter_mut() { + while slot.len() < fanout { + match leftovers.pop_front() { + Some(node) => slot.push(node), + None => break, + } + } + } + debug_assert!(leftovers.is_empty()); + // In the complete tree, position i's children are the positions + // F*i+1 ..= F*i+F, so only the last relays may have fewer than F + // children. Put the relays with a full set of children first; if more + // than one is short, the later ones hand children to the earlier. + let mut by_fullness: Vec = (0..relay_count).collect(); + by_fullness.sort_by_key(|j| children[*j].len() < fanout); + for at in 0..relay_count { + let j = by_fullness[at]; + while children[j].len() < fanout { + let donor = by_fullness[at + 1..] + .iter() + .rev() + .copied() + .find(|k| !children[*k].is_empty()); + match donor { + Some(k) => { + let node = children[k].pop().unwrap(); + children[j].push(node); + } + None => break, + } + } + } + order.extend(by_fullness.iter().map(|j| relay_nodes[*j])); + for j in &by_fullness { + order.extend(children[*j].iter().copied()); + } + order +} + +/// How long a node that expects a set through the tree waits, from the first +/// mention of the set's hash, before it asks a peer for it. Long enough for +/// two hops of a large set between the farthest regions; short against +/// Core's own retry (one second). +pub const TREE_FETCH_GRACE: Duration = Duration::from_millis(300); + +/// Frames a node remembers having relayed, hashes it waits for, and sets it +/// saw arrive. +const TREE_MEMORY: usize = 256; + +/// Round-trip samples kept per node; the smallest counts (a sample taken +/// behind a large transfer on the same stream is too long, never too short). +const RTT_SAMPLES: usize = 8; + +/// What to do about a set this node does not hold. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FetchDecision { + /// Ask a peer now. + Proceed, + /// The tree should bring it: ask only if it is still missing after this + /// long (the caller sets the timer). + Wait(Duration), + /// A timer for this set is already running. + AlreadyWaiting, +} + +type RelayKey = (u32, NodeId, usize, [u8; 32]); + +/// What a node does with a frame it could relay. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RelayAdmission { + Relay, + /// A copy of a frame this node relayed already. + Duplicate, + /// The node relayed [`MAX_RELAYS_PER_SLOT_AND_LEADER`] other frames of + /// this slot and leader already. + OverLimit, +} + +/// A node's view of the trees: what its Core told it, what it relayed, what +/// it waits for, and how far away its peers are. +pub struct TreeState { + config: Option, + relayed: lru::LruCache, + waiting: lru::LruCache<[u8; 32], std::time::Instant>, + delivered: lru::LruCache<[u8; 32], ()>, + rtts: HashMap>, + /// This node's own validator key, once Core has sent it. + local: Option, + /// Frames relayed per slot and leader. + relays: lru::LruCache<(u32, NodeId), u8>, + /// Relays not yet handed to the transport, per child. + relays_in_flight: HashMap, + /// The probe this node last sent each peer and has no answer to. + probes: HashMap, + /// Pieces of coded sets (mode `coded`). + pub coded: super::txset_coded::CodedState, + #[cfg(test)] + probing_paused: bool, +} + +impl Default for TreeState { + fn default() -> Self { + Self::new() + } +} + +impl TreeState { + pub fn new() -> Self { + let memory = std::num::NonZeroUsize::new(TREE_MEMORY).unwrap(); + Self { + config: None, + relayed: lru::LruCache::new(memory), + waiting: lru::LruCache::new(memory), + delivered: lru::LruCache::new(memory), + rtts: HashMap::new(), + local: None, + relays: lru::LruCache::new(memory), + relays_in_flight: HashMap::new(), + probes: HashMap::new(), + coded: super::txset_coded::CodedState::new(), + #[cfg(test)] + probing_paused: false, + } + } + + pub fn set_local_node(&mut self, node: NodeId) { + self.local = Some(node); + } + + pub fn set_config(&mut self, config: TreeConfig) { + // Pieces of slots Core no longer describes are of no use. + if let Some(oldest) = config.slots.iter().map(|(slot, _)| *slot).min() { + self.coded.prune(oldest); + } + if self.config.as_ref().map(|old| &old.nodes) != Some(&config.nodes) { + self.coded.forget_layouts(); + } + self.config = Some(config); + } + + /// The coded layout of `slot` and `leader` (see `txset_coded`), if this + /// node's Core described the slot, the leader is one of the nodes and + /// there are enough nodes to code for. Worked out once per slot and + /// leader. + pub fn coded_layout( + &mut self, + slot: u32, + leader: &NodeId, + ) -> Option> { + let config = self.config.as_ref()?; + let seed = *config.seed_of(slot)?; + if let Some(layout) = self.coded.cached_layout(slot, &seed, leader) { + return Some(layout); + } + let order = config.hash_order(slot, leader)?; + let layout = std::sync::Arc::new(super::txset_coded::CodedLayout::new(&order)?); + self.coded + .cache_layout(slot, seed, std::sync::Arc::clone(&layout)); + Some(layout) + } + + /// This node's own validator key, once Core has sent it. + pub fn local_node(&self) -> Option { + self.local + } + + pub fn config(&self) -> Option<&TreeConfig> { + self.config.as_ref() + } + + pub fn mode(&self) -> TreeMode { + self.config.as_ref().map_or(TreeMode::Off, |c| c.mode) + } + + /// Whether this node relays this copy of a frame: only the first copy + /// it sees, and only [`MAX_RELAYS_PER_SLOT_AND_LEADER`] different frames + /// of a slot and leader. Copies are told apart by slot, leader, length + /// and the first and last bytes of the body, without hashing megabytes + /// before the bytes go on; a sender that varies those bytes runs into + /// the limit. + pub fn admit_relay(&mut self, header: &TreeHeader, body: &[u8]) -> RelayAdmission { + let mut ends = [0u8; 32]; + let head = body.len().min(16); + ends[..head].copy_from_slice(&body[..head]); + let tail = body.len().min(16); + ends[32 - tail..].copy_from_slice(&body[body.len() - tail..]); + let key = (header.slot, header.leader, body.len(), ends); + if self.relayed.contains(&key) { + return RelayAdmission::Duplicate; + } + let count = self + .relays + .get_or_insert_mut((header.slot, header.leader), || 0); + if *count >= MAX_RELAYS_PER_SLOT_AND_LEADER { + return RelayAdmission::OverLimit; + } + *count += 1; + self.relayed.put(key, ()); + RelayAdmission::Relay + } + + /// A relay to `child` is about to be written. False: an earlier one is + /// still in flight, and this one is not sent (the child asks instead). + pub fn start_relay(&mut self, child: NodeId) -> bool { + let in_flight = self.relays_in_flight.entry(child).or_insert(0); + if *in_flight >= MAX_IN_FLIGHT_RELAYS_PER_CHILD { + return false; + } + *in_flight += 1; + true + } + + /// The relay to `child` was handed to the transport, or failed. + pub fn finish_relay(&mut self, child: &NodeId) { + if let Some(count) = self.relays_in_flight.get_mut(child) { + *count = count.saturating_sub(1); + if *count == 0 { + self.relays_in_flight.remove(child); + } + } + } + + /// This node sends `node` a probe with `value`. + pub fn probe_sent(&mut self, node: NodeId, value: u64) { + self.probes.insert(node, value); + } + + /// `node` answered with `value`. True if that is the probe this node + /// last sent it and has no answer to yet: only then is the answer a + /// measurement (a peer cannot make itself look near with an answer + /// nobody asked for). + pub fn probe_answered(&mut self, node: &NodeId, value: u64) -> bool { + if self.probes.get(node) == Some(&value) { + self.probes.remove(node); + true + } else { + false + } + } + + /// A proposal (or Core) named the set `hash` of `slot`, and this node + /// does not hold it. + pub fn fetch_decision( + &mut self, + hash: &[u8; 32], + slot: u32, + now: std::time::Instant, + ) -> FetchDecision { + // Only a node of the tree is sent the set through it. (A node that + // is not, a watcher for one, asks at once as before.) + let local = self.local; + let expects_tree = self.config.as_ref().is_some_and(|c| { + c.mode.waits() && c.seed_of(slot).is_some() && local.is_some_and(|me| c.contains(&me)) + }); + if !expects_tree || self.delivered.contains(hash) { + return FetchDecision::Proceed; + } + match self.waiting.get(hash) { + Some(deadline) if *deadline > now => FetchDecision::AlreadyWaiting, + Some(_) => FetchDecision::Proceed, + None => { + self.waiting.put(*hash, now + TREE_FETCH_GRACE); + FetchDecision::Wait(TREE_FETCH_GRACE) + } + } + } + + /// The set `hash` arrived from a peer. + pub fn set_delivered(&mut self, hash: [u8; 32]) { + self.delivered.put(hash, ()); + } + + pub fn is_delivered(&self, hash: &[u8; 32]) -> bool { + self.delivered.contains(hash) + } + + /// Tests set round-trip times by hand; the probe task then leaves them + /// alone. + #[cfg(test)] + pub fn pause_probing(&mut self) { + self.probing_paused = true; + } + + /// Whether the probe task measures (always, outside tests). + pub fn probing(&self) -> bool { + #[cfg(test)] + if self.probing_paused { + return false; + } + true + } + + /// A round trip to the validator `node` took `rtt`. + pub fn note_rtt(&mut self, node: NodeId, rtt: Duration) { + let samples = self.rtts.entry(node).or_default(); + if samples.len() == RTT_SAMPLES { + samples.pop_front(); + } + samples.push_back(rtt); + } + + /// The peer is gone; a new connection is measured afresh. + pub fn forget_rtt(&mut self, node: &NodeId) { + self.rtts.remove(node); + self.probes.remove(node); + } + + /// The round-trip time to `node`: the smallest of the last samples. + pub fn rtt_of(&self, node: &NodeId) -> Option { + self.rtts.get(node).and_then(|s| s.iter().min().copied()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(n: u16) -> NodeId { + let mut id = [0u8; 32]; + id[..2].copy_from_slice(&n.to_be_bytes()); + id[31] = 0xAA; + id + } + + fn nodes(count: u16) -> Vec { + (0..count).map(node).collect() + } + + #[test] + fn default_fanout_reaches_everyone_in_two_hops() { + for (count, fanout) in [ + (1, 1), + (2, 1), + (3, 1), + (4, 2), + (7, 2), + (8, 3), + (13, 3), + (48, 7), + (90, 9), + (91, 9), + (92, 10), + ] { + assert_eq!(default_fanout(count), fanout, "{count} nodes"); + let f = default_fanout(count); + assert!(f + f * f >= count - 1); + if f > 1 { + assert!((f - 1) + (f - 1) * (f - 1) < count - 1); + } + } + } + + #[test] + fn the_order_starts_with_the_leader_and_depends_on_seed_and_leader() { + let all = nodes(90); + let seed = [7u8; 32]; + let order = tree_order(&seed, &all[5], &all); + assert_eq!(order.len(), 90); + assert_eq!(order[0], all[5]); + let mut sorted = order.clone(); + sorted.sort(); + assert_eq!(sorted, all); + // The same for every node that agrees on the inputs, whatever order + // (and duplicates) the nodes come in. + let mut shuffled = all.clone(); + shuffled.reverse(); + shuffled.push(all[3]); + assert_eq!(tree_order(&seed, &all[5], &shuffled), order); + // Another seed or another leader gives another order. + assert_ne!(tree_order(&[8u8; 32], &all[5], &all), order); + let other = tree_order(&seed, &all[6], &all); + assert_eq!(other[0], all[6]); + assert_ne!(other[1..], order[1..]); + // A leader that is not among the nodes still roots its tree. + let outsider = node(999); + let with_outsider = tree_order(&seed, &outsider, &all); + assert_eq!(with_outsider.len(), 91); + assert_eq!(with_outsider[0], outsider); + } + + #[test] + fn parents_and_children_form_one_tree_of_least_depth() { + for (count, fanout) in [(90u16, 9usize), (90, 4), (90, 1), (48, 7), (2, 3), (1, 3)] { + let all = nodes(count); + let order = tree_order(&[1u8; 32], &all[0], &all); + let mut reached = vec![order[0]]; + let mut frontier = vec![order[0]]; + let mut depth = 0; + while !frontier.is_empty() { + let mut next = Vec::new(); + for parent in &frontier { + let children = children_of(&order, parent, fanout); + assert!(children.len() <= fanout); + for child in &children { + assert_eq!(parent_of(&order, child, fanout), Some(*parent)); + assert_eq!(depth_of(&order, child, fanout), Some(depth + 1)); + assert!(!reached.contains(child)); + reached.push(*child); + } + next.extend(children); + } + if !next.is_empty() { + depth += 1; + } + frontier = next; + } + // Everyone is reached exactly once. + assert_eq!(reached.len(), count as usize); + // No tree of that fanout is shallower: the levels above the last + // are full. + let mut capacity = 1usize; + let mut level = 1usize; + let mut least = 0; + while capacity < count as usize { + level *= fanout; + capacity += level; + least += 1; + } + assert_eq!(depth, least, "{count} nodes, fanout {fanout}"); + assert_eq!(parent_of(&order, &order[0], fanout), None); + assert_eq!(depth_of(&order, &order[0], fanout), Some(0)); + } + let all = nodes(10); + let order = tree_order(&[1u8; 32], &all[0], &all); + assert!(children_of(&order, &node(77), 3).is_empty()); + assert_eq!(parent_of(&order, &node(77), 3), None); + assert!(children_of(&order, &order[0], 0).is_empty()); + } + + fn header(kind: FrameKind, slot: u32, leader: NodeId) -> TreeHeader { + TreeHeader { + kind, + fanout: 3, + hop: 1, + slot, + leader, + sent_unix_ms: 1_800_000_000_123, + order: Vec::new(), + nodes_digest: [0; TREE_NODES_DIGEST], + } + } + + #[test] + fn a_header_round_trips_and_malformed_ones_are_refused() { + let mut h = header(FrameKind::Hash, 77, node(3)); + let mut frame = h.encode(); + assert_eq!(frame.len(), TREE_HEADER_FIXED); + frame.extend_from_slice(b"body"); + assert!(TreeHeader::is_tagged(&frame)); + let (parsed, body) = TreeHeader::parse(&frame).unwrap(); + assert_eq!(parsed, h); + assert_eq!(&frame[body..], b"body"); + + h.kind = FrameKind::Explicit; + h.order = vec![3, 0, 1, 2]; + h.nodes_digest = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut frame = h.encode(); + frame.extend_from_slice(b"x"); + let (parsed, body) = TreeHeader::parse(&frame).unwrap(); + assert_eq!(parsed, h); + assert_eq!(body, TREE_HEADER_FIXED + 8 + TREE_NODES_DIGEST); + // The largest header fits the limit the reader allows for it. + let mut largest = h.clone(); + largest.order = (0..MAX_TREE_NODES as u16).collect(); + assert_eq!(largest.encode().len(), MAX_TREE_HEADER); + + // An ordinary frame starts with a length and is not tagged. + assert!(!TreeHeader::is_tagged(&[0, 0x10, 0, 0, 1, 2, 3])); + assert!(TreeHeader::parse(&[0, 0x10, 0, 0, 1, 2, 3]).is_none()); + // Truncated, unknown kind, an order where none belongs, none where + // one does, an order longer than the frame. + let good = header(FrameKind::Hash, 1, node(1)).encode(); + assert!(TreeHeader::parse(&good[..good.len() - 1]).is_none()); + let mut bad = good.clone(); + bad[4] = 9; + assert!(TreeHeader::parse(&bad).is_none()); + // The reserved byte must be zero. + let mut bad = good.clone(); + bad[7] = 1; + assert!(TreeHeader::parse(&bad).is_none()); + let mut bad = good.clone(); + bad[53] = 2; + bad.extend_from_slice(&[0, 0, 0, 1]); + assert!(TreeHeader::parse(&bad).is_none()); + let mut explicit = header(FrameKind::Explicit, 1, node(1)); + explicit.order = Vec::new(); + assert!(TreeHeader::parse(&explicit.encode()).is_none()); + explicit.order = vec![0, 1]; + let encoded = explicit.encode(); + assert!(TreeHeader::parse(&encoded[..encoded.len() - 1]).is_none()); + } + + fn config(mode: TreeMode, fanout: u8, count: u16) -> TreeConfig { + TreeConfig::new( + mode, + fanout, + vec![(10, [1u8; 32]), (11, [2u8; 32])], + nodes(count), + ) + } + + #[test] + fn a_node_relays_only_what_its_parent_sent_for_a_tree_it_knows() { + let cfg = config(TreeMode::Hash, 3, 20); + let leader = node(4); + let order = cfg.hash_order(10, &leader).unwrap(); + let h = header(FrameKind::Hash, 10, leader); + // A child of the root relays to its own children. + let child = order[1]; + let expected = children_of(&order, &child, 3); + assert_eq!(expected.len(), 3); + assert_eq!( + relay_plan(&cfg, &h, &child, Some(&leader)), + RelayPlan::Relay(expected) + ); + // A leaf has nothing to pass on. + let leaf = order[19]; + let parent = parent_of(&order, &leaf, 3).unwrap(); + assert_eq!(relay_plan(&cfg, &h, &leaf, Some(&parent)), RelayPlan::Leaf); + // From anyone but the parent: refused. Also from a peer that did not + // prove which validator it is. + assert_eq!( + relay_plan(&cfg, &h, &child, Some(&order[2])), + RelayPlan::Refused(RelayRefusal::NotParent) + ); + assert_eq!( + relay_plan(&cfg, &h, &child, None), + RelayPlan::Refused(RelayRefusal::NotParent) + ); + // The root itself never relays what comes back to it. + assert_eq!( + relay_plan(&cfg, &h, &leader, Some(&child)), + RelayPlan::Refused(RelayRefusal::NotParent) + ); + // A slot this node's Core has not described. + assert_eq!( + relay_plan( + &cfg, + &header(FrameKind::Hash, 12, leader), + &child, + Some(&leader) + ), + RelayPlan::Refused(RelayRefusal::UnknownSlot) + ); + // A leader, or a relay, that is not one of the nodes. + assert_eq!( + relay_plan( + &cfg, + &header(FrameKind::Hash, 10, node(500)), + &child, + Some(&node(500)) + ), + RelayPlan::Refused(RelayRefusal::UnknownNodes) + ); + assert_eq!( + relay_plan(&cfg, &h, &node(500), Some(&leader)), + RelayPlan::Refused(RelayRefusal::UnknownNodes) + ); + // Star frames, and nodes whose mode does not relay. + assert_eq!( + relay_plan( + &cfg, + &header(FrameKind::Star, 10, leader), + &child, + Some(&leader) + ), + RelayPlan::Refused(RelayRefusal::NotATree) + ); + for mode in [TreeMode::Off, TreeMode::Star] { + assert_eq!( + relay_plan(&config(mode, 3, 20), &h, &child, Some(&leader)), + RelayPlan::Refused(RelayRefusal::NotATree) + ); + } + // A frame whose leader this node's Core does not expect to lead the + // slot, also from the node that would be the parent in that tree. + let restricted = + config(TreeMode::Hash, 3, 20).with_leaders(vec![Some(vec![node(7), node(8)]), None]); + assert!(!restricted.may_lead(10, &leader)); + assert!(restricted.may_lead(10, &node(8))); + assert!(restricted.may_lead(11, &leader)); + assert!(!restricted.may_lead(12, &leader)); + assert_eq!( + relay_plan(&restricted, &h, &child, Some(&leader)), + RelayPlan::Refused(RelayRefusal::NotALeader) + ); + let h8 = header(FrameKind::Hash, 10, node(8)); + let order8 = restricted.hash_order(10, &node(8)).unwrap(); + assert_eq!( + relay_plan(&restricted, &h8, &order8[1], Some(&node(8))), + RelayPlan::Relay(children_of(&order8, &order8[1], 3)) + ); + // The second slot has another seed, so another tree. + assert_ne!(cfg.hash_order(11, &leader).unwrap(), order); + // A fanout of 0 is the default for the node count. + assert_eq!(config(TreeMode::Hash, 0, 90).effective_fanout(), 9); + } + + #[test] + fn an_explicit_order_is_followed_if_it_is_a_permutation_led_by_the_leader() { + let cfg = config(TreeMode::Rtt, 0, 8); + let leader = node(2); + // leader, then 7 6 5 | 4 3 1 | 0 + let order: Vec = [2u16, 7, 6, 5, 4, 3, 1, 0] + .iter() + .map(|n| node(*n)) + .collect(); + let mut h = header(FrameKind::Explicit, 10, leader); + h.fanout = 3; + h.order = cfg.indices_of(&order).unwrap(); + h.nodes_digest = cfg.nodes_digest(); + assert_eq!(cfg.explicit_order(&h).unwrap(), order); + // node 7 is the first child of the root; its children are 4, 3, 1. + assert_eq!( + relay_plan(&cfg, &h, &node(7), Some(&leader)), + RelayPlan::Relay(vec![node(4), node(3), node(1)]) + ); + assert_eq!( + relay_plan(&cfg, &h, &node(6), Some(&leader)), + RelayPlan::Relay(vec![node(0)]) + ); + assert_eq!( + relay_plan(&cfg, &h, &node(4), Some(&node(7))), + RelayPlan::Leaf + ); + assert_eq!( + relay_plan(&cfg, &h, &node(4), Some(&node(6))), + RelayPlan::Refused(RelayRefusal::NotParent) + ); + // Not a permutation, too short, not led by the leader, an index out + // of range. + let mut bad = h.clone(); + bad.order[3] = bad.order[4]; + assert_eq!( + relay_plan(&cfg, &bad, &node(7), Some(&leader)), + RelayPlan::Refused(RelayRefusal::UnknownNodes) + ); + let mut bad = h.clone(); + bad.order.pop(); + assert!(cfg.explicit_order(&bad).is_none()); + let mut bad = h.clone(); + bad.order.swap(0, 1); + assert!(cfg.explicit_order(&bad).is_none()); + let mut bad = h.clone(); + bad.order[1] = 99; + assert!(cfg.explicit_order(&bad).is_none()); + // An order for another list of nodes (the same number of them). + let mut bad = h.clone(); + bad.nodes_digest[0] ^= 1; + assert_eq!( + relay_plan(&cfg, &bad, &node(7), Some(&leader)), + RelayPlan::Refused(RelayRefusal::UnknownNodes) + ); + let mut other_nodes = nodes(7); + other_nodes.push(node(100)); + assert_ne!( + TreeConfig::new(TreeMode::Rtt, 0, vec![(10, [1u8; 32])], other_nodes).nodes_digest(), + cfg.nodes_digest() + ); + // A leader lays out two levels, not a chain: a fanout that needs + // more is refused (8 nodes need a fanout of 3). + assert!(cfg.two_levels(3)); + assert!(!cfg.two_levels(2)); + for fanout in [0u8, 1, 2] { + let mut deep = h.clone(); + deep.fanout = fanout; + assert_eq!( + relay_plan(&cfg, &deep, &node(7), Some(&leader)), + RelayPlan::Refused(RelayRefusal::NotAllowed) + ); + } + // A node that is not in rtt mode does not follow a sender's order. + let hash_cfg = config(TreeMode::Hash, 0, 8); + assert_eq!( + relay_plan(&hash_cfg, &h, &node(7), Some(&leader)), + RelayPlan::Refused(RelayRefusal::NotAllowed) + ); + // A node in rtt mode follows the common tree as well. + let hash_frame = header(FrameKind::Hash, 10, leader); + let common = cfg.hash_order(10, &leader).unwrap(); + assert_eq!( + relay_plan(&cfg, &hash_frame, &common[1], Some(&leader)), + RelayPlan::Relay(children_of(&common, &common[1], cfg.effective_fanout())) + ); + } + + /// Four places, as on perfnet: 22 nodes with the leader (21 others), and + /// 23, 22, 23 at three distances. + fn four_regions() -> (NodeId, Vec<(NodeId, Option)>, Vec>) { + let leader = node(0); + let mut measured = Vec::new(); + let mut regions = Vec::new(); + let mut next = 1u16; + for (size, base_ms) in [(21u16, 1u64), (23, 62), (22, 91), (23, 215)] { + let mut region = Vec::new(); + for i in 0..size { + let n = node(next); + next += 1; + // A few milliseconds of spread inside a region. + let rtt = Duration::from_micros(base_ms * 1000 + (i as u64 * 137) % 2500); + measured.push((n, Some(rtt))); + region.push(n); + } + regions.push(region); + } + (leader, measured, regions) + } + + #[test] + fn an_rtt_order_keeps_a_relays_children_in_its_own_group() { + let (leader, mut measured, regions) = four_regions(); + // The input order must not matter. + measured.reverse(); + let fanout = default_fanout(90); + assert_eq!(fanout, 9); + let order = rtt_order(&leader, &measured, fanout); + assert_eq!(order.len(), 90); + assert_eq!(order[0], leader); + let mut sorted = order.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), 90); + + let region_of = |n: &NodeId| regions.iter().position(|r| r.contains(n)).unwrap(); + let relays = children_of(&order, &leader, fanout); + assert_eq!(relays.len(), 9); + // Every region has at least two relays (9 relays for 4 regions of + // about equal size). + for r in 0..4 { + let count = relays.iter().filter(|n| region_of(n) == r).count(); + assert!(count >= 2, "region {r} has {count} relays"); + } + // Nearly every node is a relay or sits under a relay of its region. + let mut foreign = 0; + for relay in &relays { + for child in children_of(&order, relay, fanout) { + assert_eq!(depth_of(&order, &child, fanout), Some(2)); + if region_of(&child) != region_of(relay) { + foreign += 1; + } + } + } + // A relay and its nine children are ten nodes; regions of 21 to 23 + // overflow by one to three, and the overflow sits under the relays + // of the region that has a third relay. + assert!( + foreign <= 7, + "{foreign} nodes under a relay of another region" + ); + // And the tree is still complete: two hops for everyone. + for n in &order[1..] { + assert!(depth_of(&order, n, fanout).unwrap() <= 2); + } + } + + /// Two regions at the same distance from the leader (as eu-central-1 + /// and us-west-2 are from ap-southeast-1): they are one group, and most + /// of their nodes still sit under a relay of their own region. + #[test] + fn an_rtt_order_keeps_regions_at_the_same_distance_apart() { + let leader = node(0); + let mut measured = Vec::new(); + let mut regions: Vec> = Vec::new(); + let mut next = 1u16; + // 22 with the leader, 22 at 160-162 ms, 23 at 165-167 ms, 22 at 215. + for (size, base_us) in [(22u16, 900u64), (22, 160_000), (23, 165_200), (22, 215_000)] { + let mut region = Vec::new(); + for i in 0..size { + let n = node(next); + next += 1; + let rtt = Duration::from_micros(base_us + (i as u64 * 173) % 2000); + measured.push((n, Some(rtt))); + region.push(n); + } + regions.push(region); + } + let fanout = 9; + let order = rtt_order(&leader, &measured, fanout); + assert_eq!(order.len(), 90); + let mut sorted = order.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), 90); + let region_of = |n: &NodeId| regions.iter().position(|r| r.contains(n)).unwrap(); + let relays = children_of(&order, &leader, fanout); + assert_eq!(relays.len(), 9); + // Both regions of the merged group have relays of their own. + assert!(relays.iter().filter(|n| region_of(n) == 1).count() >= 2); + assert!(relays.iter().filter(|n| region_of(n) == 2).count() >= 2); + let mut foreign = 0; + for relay in &relays { + for child in children_of(&order, relay, fanout) { + assert_eq!(depth_of(&order, &child, fanout), Some(2)); + if region_of(&child) != region_of(relay) { + foreign += 1; + } + } + } + // Of 80 leaves; the runs that straddle two regions and the overflow + // of the larger groups. + assert!( + foreign <= 12, + "{foreign} nodes under a relay of another region" + ); + } + + #[test] + fn an_rtt_order_copes_with_unmeasured_nodes_small_networks_and_many_groups() { + // Nothing measured: one group, still every node once. + let leader = node(0); + let unmeasured: Vec<_> = (1..20u16).map(|n| (node(n), None)).collect(); + let order = rtt_order(&leader, &unmeasured, default_fanout(20)); + assert_eq!(order.len(), 20); + assert_eq!(order[0], leader); + // Alone. + assert_eq!(rtt_order(&leader, &[], 3), vec![leader]); + // The leader in its own list, and duplicates, count once. + let dup = vec![ + (leader, Some(Duration::from_millis(1))), + (node(1), Some(Duration::from_millis(5))), + (node(1), Some(Duration::from_millis(50))), + ]; + assert_eq!(rtt_order(&leader, &dup, 2), vec![leader, node(1)]); + // More groups than relays: one group of all, every node once, and + // still two levels. + let spread: Vec<_> = (1..=12u16) + .map(|n| (node(n), Some(Duration::from_millis(n as u64 * 40)))) + .collect(); + let order = rtt_order(&leader, &spread, 3); + assert_eq!(order.len(), 13); + for n in &order[1..] { + assert!(depth_of(&order, n, 3).unwrap() <= 2); + } + // Each relay's children are its neighbours in distance: the three + // runs are nodes 1-4, 5-8 and 9-12. + for relay in children_of(&order, &leader, 3) { + let run = |n: &NodeId| (u16::from_be_bytes([n[0], n[1]]) - 1) / 4; + for child in children_of(&order, &relay, 3) { + assert_eq!(run(&child), run(&relay)); + } + } + let mut sorted = order.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), 13); + // A fanout too small for two levels: nearest first as well. + let (leader, measured, _) = four_regions(); + let order = rtt_order(&leader, &measured, 4); + assert_eq!(order.len(), 90); + let mut sorted = order.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), 90); + // Mixed: measured groups first, the unmeasured ones are still placed. + let mut mixed = four_regions().1; + mixed[5].1 = None; + mixed[40].1 = None; + let order = rtt_order(&leader, &mixed, 9); + assert_eq!(order.len(), 90); + let mut sorted = order.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), 90); + } + + #[test] + fn a_probe_round_trips() { + for probe in [ + Probe::Ping(0), + Probe::Ping(u64::MAX), + Probe::Pong(123_456_789), + ] { + assert_eq!(Probe::parse(&probe.encode()), Some(Some(probe))); + } + // Not a probe at all; a probe of the wrong length or kind. + assert_eq!(Probe::parse(&[0, 0, 1, 0, 9, 9]), None); + assert_eq!(Probe::parse(&TREE_TAG.to_be_bytes()), None); + let mut bad = Probe::Ping(5).encode().to_vec(); + bad.push(0); + assert_eq!(Probe::parse(&bad), Some(None)); + let mut bad = Probe::Ping(5).encode(); + bad[4] = 3; + assert_eq!(Probe::parse(&bad), Some(None)); + // A probe is not mistaken for a tree frame. + assert!(!TreeHeader::is_tagged(&Probe::Ping(1).encode())); + } + + #[test] + fn a_node_that_expects_the_tree_waits_once_before_it_asks() { + let mut state = TreeState::new(); + let now = std::time::Instant::now(); + let hash = [9u8; 32]; + // No tree: ask at once. + assert_eq!(state.fetch_decision(&hash, 10, now), FetchDecision::Proceed); + for mode in [TreeMode::Off, TreeMode::Star] { + state.set_config(config(mode, 0, 20)); + assert_eq!(state.fetch_decision(&hash, 10, now), FetchDecision::Proceed); + } + state.set_config(config(TreeMode::Hash, 0, 20)); + assert_eq!(state.mode(), TreeMode::Hash); + // A node that does not know which validator it is, or is not one of + // the tree's nodes, is not sent the set through the tree: no wait. + assert_eq!(state.fetch_decision(&hash, 10, now), FetchDecision::Proceed); + state.set_local_node(node(500)); + assert_eq!(state.fetch_decision(&hash, 10, now), FetchDecision::Proceed); + state.set_local_node(node(3)); + // A slot the tree is not known for (catching up): ask at once. + assert_eq!(state.fetch_decision(&hash, 9, now), FetchDecision::Proceed); + // Otherwise wait, once. + assert_eq!( + state.fetch_decision(&hash, 10, now), + FetchDecision::Wait(TREE_FETCH_GRACE) + ); + assert_eq!( + state.fetch_decision(&hash, 10, now + Duration::from_millis(100)), + FetchDecision::AlreadyWaiting + ); + assert_eq!( + state.fetch_decision(&hash, 10, now + TREE_FETCH_GRACE), + FetchDecision::Proceed + ); + // Another set has its own wait. + assert_eq!( + state.fetch_decision(&[8u8; 32], 11, now), + FetchDecision::Wait(TREE_FETCH_GRACE) + ); + // A set that arrived is not waited for. + assert!(!state.is_delivered(&[7u8; 32])); + state.set_delivered([7u8; 32]); + assert!(state.is_delivered(&[7u8; 32])); + assert_eq!( + state.fetch_decision(&[7u8; 32], 10, now), + FetchDecision::Proceed + ); + } + + #[test] + fn a_frame_is_relayed_once_and_two_frames_per_slot_and_leader() { + let mut state = TreeState::new(); + let h = header(FrameKind::Hash, 10, node(1)); + let body = vec![5u8; 100]; + assert_eq!(state.admit_relay(&h, &body), RelayAdmission::Relay); + assert_eq!(state.admit_relay(&h, &body), RelayAdmission::Duplicate); + // Another body is another frame: the second one of this slot and + // leader, and the last. + let mut other = body.clone(); + other[99] = 6; + assert_eq!(state.admit_relay(&h, &other), RelayAdmission::Relay); + assert_eq!(state.admit_relay(&h, &other), RelayAdmission::Duplicate); + assert_eq!( + state.admit_relay(&h, &body[..50]), + RelayAdmission::OverLimit + ); + other[0] = 6; + assert_eq!(state.admit_relay(&h, &other), RelayAdmission::OverLimit); + // The first copies are still known as copies. + assert_eq!(state.admit_relay(&h, &body), RelayAdmission::Duplicate); + // Another slot or leader has its own count. + let h11 = header(FrameKind::Hash, 11, node(1)); + let h2 = header(FrameKind::Hash, 10, node(2)); + assert_eq!(state.admit_relay(&h11, &body), RelayAdmission::Relay); + assert_eq!(state.admit_relay(&h2, &body), RelayAdmission::Relay); + // Short bodies do not panic. + assert_eq!(state.admit_relay(&h11, &[1, 2, 3]), RelayAdmission::Relay); + assert_eq!(state.admit_relay(&h2, &[]), RelayAdmission::Relay); + assert_eq!(state.admit_relay(&h2, &[9]), RelayAdmission::OverLimit); + } + + #[test] + fn one_relay_at_a_time_per_child() { + let mut state = TreeState::new(); + assert!(state.start_relay(node(1))); + assert!(!state.start_relay(node(1))); + assert!(state.start_relay(node(2))); + state.finish_relay(&node(1)); + assert!(state.start_relay(node(1))); + // Finishing what was never started changes nothing. + state.finish_relay(&node(9)); + state.finish_relay(&node(1)); + state.finish_relay(&node(1)); + assert!(state.start_relay(node(1))); + } + + #[test] + fn round_trip_times_keep_their_minimum_and_need_a_matching_probe() { + let mut state = TreeState::new(); + assert_eq!(state.rtt_of(&node(1)), None); + for ms in [40u64, 35, 90, 37] { + state.note_rtt(node(1), Duration::from_millis(ms)); + } + assert_eq!(state.rtt_of(&node(1)), Some(Duration::from_millis(35))); + // Old samples leave: after eight longer ones the minimum moves. + for _ in 0..RTT_SAMPLES { + state.note_rtt(node(1), Duration::from_millis(60)); + } + assert_eq!(state.rtt_of(&node(1)), Some(Duration::from_millis(60))); + state.forget_rtt(&node(1)); + assert_eq!(state.rtt_of(&node(1)), None); + + // An answer counts once, and only to the probe that was sent last. + assert!(!state.probe_answered(&node(1), 7)); + state.probe_sent(node(1), 7); + assert!(!state.probe_answered(&node(1), 8)); + assert!(!state.probe_answered(&node(2), 7)); + assert!(state.probe_answered(&node(1), 7)); + assert!(!state.probe_answered(&node(1), 7)); + state.probe_sent(node(1), 9); + state.probe_sent(node(1), 10); + assert!(!state.probe_answered(&node(1), 9)); + assert!(state.probe_answered(&node(1), 10)); + // A peer that left has no probe outstanding. + state.probe_sent(node(1), 11); + state.forget_rtt(&node(1)); + assert!(!state.probe_answered(&node(1), 11)); + assert!(state.probing()); + state.pause_probing(); + assert!(!state.probing()); + } +} diff --git a/overlay/src/ipc/messages.rs b/overlay/src/ipc/messages.rs index 7ffb64682a..f7cda428ed 100644 --- a/overlay/src/ipc/messages.rs +++ b/overlay/src/ipc/messages.rs @@ -73,6 +73,16 @@ pub enum MessageType { /// `everyone` = 1: flood to all peers (the list is ignored). SetTxTargets = 16, + /// The spanning trees the leader's tx set travels down in the next slots + /// (see `flood::txset_tree`). + /// Payload: [mode:1][fanout:1][slots:1] + /// ([slot:4 LE][seed:32][leaders:1][nodeID:32]...)... + /// [count:4 LE][nodeID:32]... + /// mode: 0 off, 1 star, 2 hash, 3 rtt. fanout 0 = the smallest that + /// reaches every node in two hops. `leaders` of a slot: the nodes that + /// may lead it (255 = any node of the tree, no IDs follow). + SetTxSetTree = 17, + // ═══ Overlay → Core (Critical Path) ═══ /// Received SCP envelope from network ScpReceived = 100, @@ -118,6 +128,7 @@ impl TryFrom for MessageType { 14 => Ok(MessageType::PushTxSet), 15 => Ok(MessageType::SetNodeIdentity), 16 => Ok(MessageType::SetTxTargets), + 17 => Ok(MessageType::SetTxSetTree), 100 => Ok(MessageType::ScpReceived), 101 => Ok(MessageType::TopTxsResponse), 102 => Ok(MessageType::PeerRequestsScpState), @@ -323,6 +334,7 @@ mod tests { MessageType::PushTxSet, MessageType::SetNodeIdentity, MessageType::SetTxTargets, + MessageType::SetTxSetTree, MessageType::OverlayIdentity, MessageType::ScpReceived, MessageType::TopTxsResponse, @@ -400,6 +412,7 @@ mod tests { MessageType::SetNodeIdentity ); assert_eq!(MessageType::try_from(16).unwrap(), MessageType::SetTxTargets); + assert_eq!(MessageType::try_from(17).unwrap(), MessageType::SetTxSetTree); assert_eq!( MessageType::try_from(106).unwrap(), MessageType::OverlayIdentity @@ -432,7 +445,7 @@ mod tests { assert!(MessageType::try_from(9).is_err()); // gap between 8 and 10 assert!(MessageType::try_from(99).is_err()); assert!(MessageType::try_from(104).is_err()); - assert!(MessageType::try_from(17).is_err()); + assert!(MessageType::try_from(18).is_err()); assert!(MessageType::try_from(107).is_err()); assert!(MessageType::try_from(u32::MAX).is_err()); } diff --git a/overlay/src/libp2p_overlay.rs b/overlay/src/libp2p_overlay.rs index effb1017ea..a07638845d 100644 --- a/overlay/src/libp2p_overlay.rs +++ b/overlay/src/libp2p_overlay.rs @@ -11,6 +11,13 @@ //! Each stream is opened once per peer and kept alive. //! QUIC provides independent loss recovery per stream. +use crate::flood::txset_coded::{ + self, CodedLayout, CodedParams, PieceBytes, PieceHeader, PieceOutcome, ReadySet, RootKey, +}; +use crate::flood::txset_tree::{ + self, FetchDecision, FrameKind, Probe, RelayAdmission, RelayPlan, TreeConfig, TreeHeader, + TreeMode, TreeState, +}; use crate::flood::{ GetData, InvBatch, InvBatcher, InvEntry, InvTracker, NodeId, PendingRequests, PushRateLimiter, PushRecord, SendPlan, TxBuffer, TxRouter, TxSetData, TxStreamMessage, @@ -132,6 +139,38 @@ fn verify_node_id_binding( key.verify(&node_id_binding(network_id, peer), signature) } +/// Whether a piece is of a set its leader cut up: the header's overlay key +/// belongs to the leader (the validator's own signature over +/// [`node_id_binding`] of that overlay's peer id, as peers exchange it), and +/// that key signed the root of the set's pieces with the slot, the shape of +/// the code and the frame's length (`txset_coded::root_message`). Needs no +/// connection to the leader. `seed` is this node's seed of the header's slot. +fn verify_piece_signatures(network_id: &[u8; 32], seed: &[u8; 32], header: &PieceHeader) -> bool { + let Ok(overlay_key) = libp2p::identity::ed25519::PublicKey::try_from_bytes(&header.overlay_key) + else { + return false; + }; + let peer = libp2p::identity::PublicKey::from(overlay_key.clone()).to_peer_id(); + if !verify_node_id_binding(network_id, &peer, &header.leader, &header.binding_sig) { + return false; + } + let (data, pieces) = header.params(); + let message = txset_coded::root_message( + network_id, + seed, + header.slot, + &header.leader, + CodedParams { + data, + pieces, + relays: 0, + }, + header.total_len, + &header.root, + ); + overlay_key.verify(&message, &header.root_sig) +} + /// Identity state: ours, and claims from peers that arrived before Core told /// us the network id needed to check them. #[derive(Default)] @@ -213,6 +252,9 @@ pub enum OverlayCommand { PushTxSet { hash: [u8; 32], data: Arc, + /// The slot the set is for, if the caller knows it: with a tree + /// configured for that slot the set goes down the tree. + slot: Option, }, /// Core accepted this transaction. The router already holds it; this /// carries out what it decided at that moment. @@ -302,6 +344,10 @@ pub struct OverlayHandle { /// Core's messages (a transaction accepted, transactions externalized, /// new targets), then sends a command to act on it. tx_router: Arc>>, + /// Shared with the overlay task: the trees the leader's tx set travels + /// down (see `flood::txset_tree`). + txset_tree: Arc>, + metrics: Arc, local_peer_id: PeerId, } @@ -406,8 +452,47 @@ impl OverlayHandle { self.tx_router.lock().unwrap().len() } + /// Ask a peer for a set this node does not hold. If the set is expected + /// through the tree, the request waits (`TREE_FETCH_GRACE` from the first + /// mention of the hash) and is only sent if the set has not arrived by + /// then: otherwise every node the leader did not push to would ask it the + /// moment the proposal arrives. pub async fn fetch_txset(&self, hash: [u8; 32], slot: u32) { - self.send_fetch_txset(hash, slot, false).await; + let decision = self + .txset_tree + .lock() + .unwrap() + .fetch_decision(&hash, slot, Instant::now()); + match decision { + FetchDecision::Proceed => self.send_fetch_txset(hash, slot, false).await, + FetchDecision::AlreadyWaiting => {} + FetchDecision::Wait(wait) => { + self.metrics + .txset_tree_fetch_deferred + .fetch_add(1, Ordering::Relaxed); + let this = self.clone(); + tokio::spawn(async move { + tokio::time::sleep(wait).await; + if this.txset_tree.lock().unwrap().is_delivered(&hash) { + return; + } + this.metrics + .txset_tree_fetch_fallback + .fetch_add(1, Ordering::Relaxed); + info!( + "TXSET_TREE_FALLBACK: TX set {:02x?}... for slot {} did not arrive through the tree, asking for it", + &hash[..4], + slot + ); + this.send_fetch_txset(hash, slot, false).await; + }); + } + } + } + + /// Core described the trees of the next slots. + pub fn set_txset_tree(&self, config: TreeConfig) { + self.txset_tree.lock().unwrap().set_config(config); } /// Re-request a set an earlier request has not delivered. Unlike @@ -485,9 +570,20 @@ impl OverlayHandle { /// Push a TX set to every connected peer without waiting for requests. /// Each peer is sent a given set once; see `PushRecord`. pub async fn push_txset(&self, hash: [u8; 32], data: Arc) { + self.send_push_txset(hash, data, None).await; + } + + /// As [`Self::push_txset`] for the set of `slot`: if Core configured a + /// tree for that slot, the set goes to this node's children in it (or, + /// in star mode, to every peer with a header that says when). + pub async fn push_txset_for_slot(&self, hash: [u8; 32], data: Arc, slot: u32) { + self.send_push_txset(hash, data, Some(slot)).await; + } + + async fn send_push_txset(&self, hash: [u8; 32], data: Arc, slot: Option) { if let Err(e) = self .cmd_tx - .send(OverlayCommand::PushTxSet { hash, data }) + .send(OverlayCommand::PushTxSet { hash, data, slot }) .await { warn!( @@ -635,6 +731,10 @@ struct SharedState { /// Where this node's own transactions go (shared with the App through /// `OverlayHandle`). Never held across an await. tx_router: Arc>>, + /// The trees the leader's tx set travels down (shared with the App + /// through `OverlayHandle`). Never held across an await, and never + /// together with another of these locks. + txset_tree: Arc>, /// Validator identities: ours and unchecked claims. Never held across /// an await. identity: std::sync::Mutex, @@ -643,6 +743,9 @@ struct SharedState { direct_out: std::sync::Mutex, /// Overlay metrics (shared with App for IPC reporting) metrics: Arc, + /// This overlay's own key: it signs the root of the pieces of a set + /// this node leads with (see `txset_coded`). + keypair: Keypair, } impl SharedState { @@ -652,6 +755,8 @@ impl SharedState { control: Control, metrics: Arc, tx_router: Arc>>, + txset_tree: Arc>, + keypair: Keypair, ) -> Self { Self { peer_streams: RwLock::new(HashMap::new()), @@ -680,9 +785,11 @@ impl SharedState { pending_getdata: RwLock::new(PendingRequests::new()), tx_buffer: RwLock::new(TxBuffer::new()), tx_router, + txset_tree, identity: std::sync::Mutex::new(IdentityState::default()), direct_out: std::sync::Mutex::new(DirectOut::default()), metrics, + keypair, } } } @@ -786,12 +893,15 @@ pub fn create_overlay( let (event_tx, event_rx) = mpsc::unbounded_channel(); let tx_router = Arc::new(std::sync::Mutex::new(TxRouter::new())); + let txset_tree = Arc::new(std::sync::Mutex::new(TreeState::new())); let state = Arc::new(SharedState::new( event_tx, mempool, control.clone(), - metrics, + Arc::clone(&metrics), Arc::clone(&tx_router), + Arc::clone(&txset_tree), + keypair.clone(), )); let overlay = StellarOverlay { @@ -807,6 +917,8 @@ pub fn create_overlay( txset_send_slots: Arc::new(Semaphore::new(MAX_OUTSTANDING_TXSET_SENDS)), txset_send_bytes: Arc::new(Semaphore::new(MAX_OUTSTANDING_TXSET_BYTES)), tx_router, + txset_tree, + metrics, local_peer_id: peer_id, }; @@ -864,6 +976,7 @@ impl StellarOverlay { // Spawn INV/GETDATA housekeeping task tokio::spawn(inv_getdata_housekeeping_task(state.clone())); tokio::spawn(direct_send_flush_task(state.clone())); + tokio::spawn(txset_tree_probe_task(state.clone())); loop { tokio::select! { @@ -902,8 +1015,8 @@ impl StellarOverlay { sources.put(hash, peer); debug!("Recorded peer {} as source for TX set {:02x?}...", peer, &hash[..4]); } - OverlayCommand::PushTxSet { hash, data } => { - self.push_txset(hash, data).await; + OverlayCommand::PushTxSet { hash, data, slot } => { + self.push_txset(hash, data, slot).await; } OverlayCommand::SendLocalTx(plan) => { self.send_local_tx_plan(plan).await; @@ -1117,11 +1230,16 @@ impl StellarOverlay { // Whatever was queued for the peer is not sent; the // router forgets its sends to the peer just below. self.state.direct_out.lock().unwrap().queued.remove(&peer_id); - let identified = { + let (identified, gone) = { let mut router = self.state.tx_router.lock().unwrap(); + let gone = router.node_of(&peer_id).copied(); router.peer_disconnected(&peer_id); - router.identified_peer_count() + (router.identified_peer_count(), gone) }; + if let Some(node) = gone { + // A new connection to it is measured afresh. + self.state.txset_tree.lock().unwrap().forget_rtt(&node); + } self.state .metrics .node_id_identified @@ -1324,6 +1442,11 @@ impl StellarOverlay { .lock() .unwrap() .set_local_node(identity.node_id); + self.state + .txset_tree + .lock() + .unwrap() + .set_local_node(identity.node_id); for (peer, (node_id, signature)) in waiting { accept_node_id_claim(&self.state, peer, node_id, signature).await; } @@ -1451,10 +1574,279 @@ impl StellarOverlay { }); } - /// Push a TX set to every connected peer that has not been sent it. The - /// frame is the one a request is answered with, sent unsolicited on the - /// TX-set stream; receivers verify and cache it by its content hash. - async fn push_txset(&mut self, hash: [u8; 32], data: Arc) { + /// Whom the set of `slot` goes to when this node leads, and the header it + /// carries (none: an ordinary frame, as before the tree). + /// + /// * No tree configured, or mode off: every connected peer, no header. + /// * Star: every connected peer, with a header. + /// * Hash, rtt: this node's children in the tree. If the tree of the slot + /// is not known here, this node is not one of the leaders its Core + /// named for the slot (every relay would refuse its frame), or a child + /// has no identified connection (right after a start, before the peers + /// proved who they are), the set goes to every peer as a star instead: + /// a subtree must not depend on a child the root cannot reach. + fn tree_push_plan(&self, slot: Option, connected: &[PeerId]) -> TreePushPlan { + let everyone = |header: Option| TreePushPlan { + header, + targets: connected.to_vec(), + subtrees: HashMap::new(), + star: None, + }; + let Some(slot) = slot else { + return everyone(None); + }; + let Some(me) = self + .state + .identity + .lock() + .unwrap() + .ours + .as_ref() + .map(|i| i.node_id) + else { + return everyone(None); + }; + // One lock at a time: the identified peers first, then the tree. + let peer_of: HashMap = { + let router = self.state.tx_router.lock().unwrap(); + connected + .iter() + .filter_map(|peer| router.node_of(peer).map(|node| (*node, *peer))) + .collect() + }; + let sent_unix_ms = unix_ms_now(); + let star = TreeHeader { + kind: FrameKind::Star, + fanout: 0, + hop: 1, + slot, + leader: me, + sent_unix_ms, + order: Vec::new(), + nodes_digest: [0; txset_tree::TREE_NODES_DIGEST], + }; + // Copy what the layout needs and let go of the lock before computing. + let (config, measured) = { + let tree = self.state.txset_tree.lock().unwrap(); + let Some(config) = tree.config().cloned() else { + return everyone(None); + }; + let measured: Vec<(NodeId, Option)> = if config.mode == TreeMode::Rtt { + config + .nodes + .iter() + .filter(|node| **node != me) + .map(|node| (*node, tree.rtt_of(node))) + .collect() + } else { + Vec::new() + }; + (config, measured) + }; + let fanout = config.effective_fanout(); + // A node the others do not expect to lead the slot (a later round, + // another view) would have its frames refused by every relay: it + // pushes to everyone. + let known = + config.seed_of(slot).is_some() && config.contains(&me) && config.may_lead(slot, &me); + if config.mode.relays() && config.seed_of(slot).is_some() && !config.may_lead(slot, &me) { + info!( + "TXSET_TREE_STAR: This node is not one of the leaders its Core named for slot {}, pushing to every peer", + slot + ); + } + let (kind, order, indices) = match config.mode { + TreeMode::Off => return everyone(None), + // Coded: `coded_push` did not take the set (see there). + TreeMode::Star | TreeMode::Coded => return everyone(Some(star)), + _ if !known => return everyone(Some(star)), + TreeMode::Hash => match config.hash_order(slot, &me) { + Some(order) => (FrameKind::Hash, order, Vec::new()), + None => return everyone(Some(star)), + }, + TreeMode::Rtt => { + let order = txset_tree::rtt_order(&me, &measured, fanout); + match config.indices_of(&order) { + Some(indices) => (FrameKind::Explicit, order, indices), + None => return everyone(Some(star)), + } + } + }; + let children = txset_tree::children_of(&order, &me, fanout); + let targets: Vec = children + .iter() + .filter_map(|node| peer_of.get(node).copied()) + .collect(); + if targets.len() < children.len() { + self.state + .metrics + .txset_tree_unreachable + .fetch_add((children.len() - targets.len()) as u64, Ordering::Relaxed); + warn!( + "TXSET_TREE_STAR: {} of {} children for slot {} have no identified connection, pushing to every peer", + children.len() - targets.len(), + children.len(), + slot + ); + return everyone(Some(star)); + } + // Who sits below each child: if a child cannot be sent the set now + // (an earlier push to it is still in flight), they get it from this + // node directly. + let mut subtrees: HashMap> = HashMap::new(); + for child in &children { + let mut below = Vec::new(); + let mut frontier = vec![*child]; + while let Some(node) = frontier.pop() { + for descendant in txset_tree::children_of(&order, &node, fanout) { + if let Some(peer) = peer_of.get(&descendant) { + below.push(*peer); + } + frontier.push(descendant); + } + } + subtrees.insert(peer_of[child], below); + } + TreePushPlan { + header: Some(TreeHeader { + kind, + fanout: fanout.min(u8::MAX as usize) as u8, + hop: 1, + slot, + leader: me, + sent_unix_ms, + order: indices, + nodes_digest: if kind == FrameKind::Explicit { + config.nodes_digest() + } else { + [0; txset_tree::TREE_NODES_DIGEST] + }, + }), + targets, + subtrees, + star: Some(star), + } + } + + /// In coded mode, cut the set of `slot` into pieces and write every piece + /// to its relays (see `txset_coded`). False: the set is not coded (another + /// mode, the slot's layout is not known here, this node is not one of the + /// leaders its Core named for the slot, too few nodes, or more than a + /// quarter of the relays have no identified connection, as right after a + /// start), and the caller pushes it as before, to every peer. + /// + /// The pieces are not registered as a push of the set to their receivers + /// (`PushRecord`): no peer gets the whole set from this node, so a + /// request for it is answered. + fn coded_push( + &mut self, + hash: [u8; 32], + data: &Arc, + slot: u32, + connected: &[PeerId], + ) -> bool { + let Some(identity) = self.state.identity.lock().unwrap().ours.clone() else { + return false; + }; + let me = identity.node_id; + // One lock at a time. A node the others do not expect to lead the + // slot would have its pieces refused by every node: it does not code. + let (planned, unexpected) = { + let mut tree = self.state.txset_tree.lock().unwrap(); + let (seed, unexpected) = match tree.config() { + Some(config) if config.mode == TreeMode::Coded && config.contains(&me) => { + let named = config.may_lead(slot, &me); + ( + config.seed_of(slot).copied().filter(|_| named), + config.seed_of(slot).is_some() && !named, + ) + } + _ => (None, false), + }; + ( + seed.and_then(|seed| tree.coded_layout(slot, &me).map(|layout| (seed, layout))), + unexpected, + ) + }; + if unexpected { + info!( + "TXSET_CODED_STAR: This node is not one of the leaders its Core named for slot {}, pushing to every peer", + slot + ); + } + let Some((seed, layout)) = planned else { + return false; + }; + // The root of the pieces is signed with an Ed25519 key. + if self.state.keypair.public().try_into_ed25519().is_err() { + return false; + } + let peer_of: HashMap = { + let router = self.state.tx_router.lock().unwrap(); + connected + .iter() + .filter_map(|peer| router.node_of(peer).map(|node| (*node, *peer))) + .collect() + }; + let targets = layout.leader_targets(); + let unreachable = targets + .iter() + .filter(|(_, node)| !peer_of.contains_key(node)) + .count(); + if unreachable * 4 > targets.len() { + self.state + .metrics + .txset_tree_unreachable + .fetch_add(unreachable as u64, Ordering::Relaxed); + warn!( + "TXSET_CODED_STAR: {} of {} relays for slot {} have no identified connection, pushing to every peer", + unreachable, + targets.len(), + slot + ); + return false; + } + if !self + .state + .txset_tree + .lock() + .unwrap() + .coded + .mark_pushed(hash) + { + // This set went out as pieces already. + return true; + } + info!( + "TXSET_CODED_PUSH: Pushing TX set {:02x?}... ({} bytes, {} on the wire) for slot {} as {} pieces ({} rebuild it) to {} relays", + &hash[..4], + data.len(), + data.encoded().len(), + slot, + layout.params.pieces, + layout.params.data, + targets.len() - unreachable + ); + tokio::spawn(send_coded_set( + Arc::clone(&self.state), + hash, + Arc::clone(data), + slot, + seed, + layout, + identity, + peer_of, + unix_ms_now(), + )); + true + } + + /// Push a TX set this node leads with. Without a tree it goes to every + /// connected peer that has not been sent it, as the frame a request is + /// answered with, unsolicited on the TX-set stream; with one, to this + /// node's children in the tree, with a header in front (see + /// `tree_push_plan`). Receivers verify and cache it by its content hash. + async fn push_txset(&mut self, hash: [u8; 32], data: Arc, slot: Option) { // Include the StellarMessage discriminant in the existing wire limit. if data.len() > MAX_MESSAGE_SIZE - 4 { warn!( @@ -1472,21 +1864,53 @@ impl StellarOverlay { .keys() .cloned() .collect(); + if let Some(slot) = slot { + if self.coded_push(hash, &data, slot, &connected) { + return; + } + } + let plan = self.tree_push_plan(slot, &connected); + let tagged = plan.header.is_some(); + let header: Arc> = Arc::new(plan.header.map(|h| h.encode()).unwrap_or_default()); let (targets, busy) = self .state .txset_pushes .lock() .unwrap() - .start_push_counting_busy(hash, connected); - if busy > 0 { + .start_push_listing_busy(hash, plan.targets); + if !busy.is_empty() { // Those peers request the set when they see the proposal. warn!( "TXSET_PUSH_BUSY: Not pushing TX set {:02x?}... to {} peer(s): an earlier push to them is still in flight", &hash[..4], - busy + busy.len() ); } - if targets.is_empty() { + // In a tree a child that is left out would leave everyone below it + // without the set: they get it from this node directly, as a star + // (which nobody relays). + let mut direct: Vec = Vec::new(); + for child in &busy { + if let Some(below) = plan.subtrees.get(child) { + direct.extend(below.iter().copied()); + } + } + if !plan.subtrees.is_empty() && !busy.is_empty() { + self.state + .metrics + .txset_tree_child_busy + .fetch_add(busy.len() as u64, Ordering::Relaxed); + } + let direct = if direct.is_empty() { + direct + } else { + self.state + .txset_pushes + .lock() + .unwrap() + .start_push(hash, direct) + }; + if targets.is_empty() && direct.is_empty() { debug!( "TXSET_PUSH_SKIP: TX set {:02x?}... has no peer left to push to", &hash[..4] @@ -1494,30 +1918,70 @@ impl StellarOverlay { return; } info!( - "TXSET_PUSH: Pushing TX set {:02x?}... ({} bytes, {} on the wire) to {} peers", + "TXSET_PUSH: Pushing TX set {:02x?}... ({} bytes, {} on the wire) to {} of {} peers{}{}", &hash[..4], data.len(), data.encoded().len(), - targets.len() + targets.len() + direct.len(), + connected.len(), + if tagged { " (tagged)" } else { "" }, + if direct.is_empty() { + String::new() + } else { + format!(", {} of them below a busy child", direct.len()) + } ); - for peer in targets { + self.spawn_pushes(hash, &data, header, targets); + if let (false, Some(star)) = (direct.is_empty(), plan.star) { + self.spawn_pushes(hash, &data, Arc::new(star.encode()), direct); + } + } + + /// Write the set to `peers`, each on its TX-set stream, with `header` in + /// front if it is not empty. The peers were registered with + /// `PushRecord::start_push*`. + fn spawn_pushes( + &mut self, + hash: [u8; 32], + data: &Arc, + header: Arc>, + peers: Vec, + ) { + for peer in peers { let state = Arc::clone(&self.state); - let data = Arc::clone(&data); + let data = Arc::clone(data); + let header = Arc::clone(&header); self.sends.spawn(async move { - match send_to_peer_stream(&state, peer, StreamType::TxSet, data.encoded()).await { + let sent = if header.is_empty() { + send_to_peer_stream(&state, peer, StreamType::TxSet, data.encoded()).await + } else { + send_to_peer_stream_parts( + &state, + peer, + StreamType::TxSet, + &[header.as_slice(), data.encoded()], + ) + .await + }; + match sent { Ok(_) => { state.txset_pushes.lock().unwrap().push_written( &hash, &peer, Instant::now(), ); + let bytes = (header.len() + data.encoded().len()) as u64; let metrics = &state.metrics; metrics.txset_push_sent.fetch_add(1, Ordering::Relaxed); + metrics + .txset_push_sent_bytes + .fetch_add(bytes, Ordering::Relaxed); + if !header.is_empty() { + metrics.txset_tree_root_sent.fetch_add(1, Ordering::Relaxed); + } metrics.send_txset.fetch_add(1, Ordering::Relaxed); metrics.message_write.fetch_add(1, Ordering::Relaxed); - metrics - .byte_write - .fetch_add(data.encoded().len() as u64, Ordering::Relaxed); + metrics.byte_write.fetch_add(bytes, Ordering::Relaxed); debug!( "TXSET_PUSH_OK: Queued TX set {:02x?}... to {}", &hash[..4], @@ -3124,6 +3588,8 @@ fn handle_txset_request(state: &SharedState, peer_id: PeerId, hash: [u8; 32]) { /// Deliver verified XDR together with its original compressed encoding. async fn deliver_txset(state: &SharedState, peer_id: PeerId, txset_data: Arc) { let hash = crate::xdr::sha256_hash(txset_data.as_slice()); + // A node that waits for this set through the tree need not ask for it. + state.txset_tree.lock().unwrap().set_delivered(hash); // Clear pending request flag and measure fetch latency let slot = { @@ -3171,6 +3637,808 @@ async fn deliver_txset(state: &SharedState, peer_id: PeerId, txset_data: Arc u64 { + static START: std::sync::OnceLock = std::sync::OnceLock::new(); + START.get_or_init(Instant::now).elapsed().as_micros() as u64 +} + +/// In rtt mode, measure the round trip to every identified peer: a ping on +/// the tx-set stream, which the peer sends back (`Probe`). The leader lays +/// its tree out by these times (`txset_tree::rtt_order`). A probe that waits +/// behind a set on the same stream comes back late; the smallest of the last +/// few samples is used. +async fn txset_tree_probe_task(state: Arc) { + let mut interval = tokio::time::interval(TREE_PROBE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + interval.tick().await; + { + let tree = state.txset_tree.lock().unwrap(); + if tree.mode() != TreeMode::Rtt || !tree.probing() { + continue; + } + } + let peers = state.tx_router.lock().unwrap().identified_peers(); + for (node, peer) in peers { + let state = state.clone(); + tokio::spawn(async move { + // The clock starts when the stream is this probe's to write. + let value = monotonic_micros(); + state.txset_tree.lock().unwrap().probe_sent(node, value); + let ping = Probe::Ping(value).encode(); + let _ = send_to_peer_stream(&state, peer, StreamType::TxSet, &ping).await; + }); + } + } +} + +/// Where a set this node leads goes (see `StellarOverlay::tree_push_plan`). +struct TreePushPlan { + /// In front of the frame; none for an ordinary frame. + header: Option, + /// The peers to write it to. + targets: Vec, + /// In a tree: for each target, the peers below it. + subtrees: HashMap>, + /// In a tree: the header of a copy that goes to a node directly (not to + /// be relayed). + star: Option, +} + +/// Milliseconds since the Unix epoch by this node's clock. +fn unix_ms_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis() as u64) +} + +/// A received tx-set frame with the offset at which the ordinary frame +/// begins (0 if it had no tree header). +struct TaggedBody { + frame: Arc>, + body: usize, +} + +impl TaggedBody { + fn plain(frame: Vec) -> Self { + Self { + frame: Arc::new(frame), + body: 0, + } + } + + /// The ordinary frame as an allocation of its own (what the cache keeps + /// to answer requests with). Free when there was no header and nobody + /// else holds the bytes; one copy otherwise. + fn into_frame(self) -> Vec { + if self.body == 0 { + Arc::try_unwrap(self.frame).unwrap_or_else(|shared| shared.as_ref().clone()) + } else { + self.frame[self.body..].to_vec() + } + } +} + +/// Record how long a set took from its leader's send to this node: now less +/// `sent_unix_ms`, by the leader's clock and ours (the nodes of a network +/// keep theirs within a few milliseconds). A nonsensical difference is not +/// recorded. `hop`: how the frame came (1 = straight from the leader), for +/// the per-hop sums; none for a set rebuilt from pieces. +fn record_tree_delay(metrics: &OverlayMetrics, sent_unix_ms: u64, hop: Option) { + let delay_ms = unix_ms_now().saturating_sub(sent_unix_ms); + if sent_unix_ms == 0 || delay_ms >= 60_000 { + return; + } + metrics + .txset_tree_delay_ms_sum + .fetch_add(delay_ms, Ordering::Relaxed); + metrics + .txset_tree_delay_count + .fetch_add(1, Ordering::Relaxed); + if let Some(hop) = hop { + let (sum, count) = if hop <= 1 { + ( + &metrics.txset_tree_delay_hop1_ms_sum, + &metrics.txset_tree_delay_hop1_count, + ) + } else { + ( + &metrics.txset_tree_delay_hop2_ms_sum, + &metrics.txset_tree_delay_hop2_count, + ) + }; + sum.fetch_add(delay_ms, Ordering::Relaxed); + count.fetch_add(1, Ordering::Relaxed); + } + let bucket = match delay_ms { + 0..=50 => &metrics.txset_tree_delay_le_50, + 51..=100 => &metrics.txset_tree_delay_le_100, + 101..=150 => &metrics.txset_tree_delay_le_150, + 151..=200 => &metrics.txset_tree_delay_le_200, + 201..=300 => &metrics.txset_tree_delay_le_300, + _ => &metrics.txset_tree_delay_gt_300, + }; + bucket.fetch_add(1, Ordering::Relaxed); +} + +/// Handle the tree header of a frame from `peer_id`: record how long the +/// frame took, and pass it on to this node's children in the tree if it has +/// any (see `txset_tree::relay_plan`). The bytes go on as they came, before +/// they are decompressed or checked: every receiver checks them itself, and +/// a node only relays what its parent in the tree sent it, once. +/// +/// Returns the frame with the offset of the ordinary frame in it, or `None` +/// if the header is malformed. +fn relay_tagged_frame( + state: &Arc, + peer_id: PeerId, + frame: Vec, +) -> Option { + let (header, body) = TreeHeader::parse(&frame)?; + let metrics = &state.metrics; + metrics.txset_tree_received.fetch_add(1, Ordering::Relaxed); + record_tree_delay(metrics, header.sent_unix_ms, Some(header.hop)); + let frame = Arc::new(frame); + if header.kind == FrameKind::Star { + return Some(TaggedBody { frame, body }); + } + // What `decode` will refuse anyway is not passed on: a body that is too + // large, or that announces more XDR than a set may have. + let body_len = frame.len() - body; + let plausible = body_len >= 4 + && body_len <= crate::flood::txset_encoding::max_encoded_size() + && u32::from_be_bytes(frame[body..body + 4].try_into().unwrap()) as usize + <= crate::flood::txset_encoding::MAX_TXSET_XDR_SIZE; + if !plausible { + metrics.txset_tree_refused.fetch_add(1, Ordering::Relaxed); + return Some(TaggedBody { frame, body }); + } + // One lock at a time. + let me = state + .identity + .lock() + .unwrap() + .ours + .as_ref() + .map(|i| i.node_id); + let sender = state.tx_router.lock().unwrap().node_of(&peer_id).copied(); + let plan = { + let mut tree = state.txset_tree.lock().unwrap(); + let withhold = tree.config().is_some_and(|config| config.withhold); + let plan = match (tree.config(), me) { + (Some(config), Some(me)) => { + txset_tree::relay_plan(config, &header, &me, sender.as_ref()) + } + _ => RelayPlan::Refused(txset_tree::RelayRefusal::NotATree), + }; + match plan { + // A node set up to withhold (tests, fault runs) passes nothing on. + RelayPlan::Relay(_) if withhold => { + metrics.txset_tree_withheld.fetch_add(1, Ordering::Relaxed); + RelayPlan::Leaf + } + RelayPlan::Relay(children) => match tree.admit_relay(&header, &frame[body..]) { + RelayAdmission::Relay => RelayPlan::Relay(children), + RelayAdmission::Duplicate => { + metrics.txset_tree_duplicate.fetch_add(1, Ordering::Relaxed); + RelayPlan::Leaf + } + RelayAdmission::OverLimit => { + metrics + .txset_tree_relay_limited + .fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_TREE_LIMIT: Not relaying another frame of slot {} from {}: this node relayed {} for that slot and leader already", + header.slot, + peer_id, + txset_tree::MAX_RELAYS_PER_SLOT_AND_LEADER + ); + RelayPlan::Leaf + } + }, + other => other, + } + }; + match plan { + RelayPlan::Leaf => {} + RelayPlan::Refused(why) => { + if why != txset_tree::RelayRefusal::NotATree { + metrics.txset_tree_refused.fetch_add(1, Ordering::Relaxed); + debug!( + "TXSET_TREE_REFUSED: Not relaying the tree frame for slot {} from {}: {:?}", + header.slot, peer_id, why + ); + } + } + RelayPlan::Relay(children) => { + let peer_of: HashMap = { + let router = state.tx_router.lock().unwrap(); + router.identified_peers() + }; + let relay_header = Arc::new( + TreeHeader { + hop: header.hop.saturating_add(1), + ..header.clone() + } + .encode(), + ); + let mut unreachable = 0u64; + let mut busy = 0u64; + for child in &children { + let Some(peer) = peer_of.get(child).copied() else { + unreachable += 1; + continue; + }; + // A child that does not read its stream holds one frame in + // this node's memory, not one per slot: while a relay to it + // is in flight it is sent no other (it asks for the set). + if !state.txset_tree.lock().unwrap().start_relay(*child) { + busy += 1; + continue; + } + let child = *child; + let state = Arc::clone(state); + let frame = Arc::clone(&frame); + let relay_header = Arc::clone(&relay_header); + tokio::spawn(async move { + let parts = [relay_header.as_slice(), &frame[body..]]; + let sent = + send_to_peer_stream_parts(&state, peer, StreamType::TxSet, &parts).await; + state.txset_tree.lock().unwrap().finish_relay(&child); + match sent { + Ok(_) => { + let bytes = (relay_header.len() + frame.len() - body) as u64; + let metrics = &state.metrics; + metrics.txset_tree_relayed.fetch_add(1, Ordering::Relaxed); + metrics + .txset_tree_relayed_bytes + .fetch_add(bytes, Ordering::Relaxed); + metrics.send_txset.fetch_add(1, Ordering::Relaxed); + metrics.message_write.fetch_add(1, Ordering::Relaxed); + metrics.byte_write.fetch_add(bytes, Ordering::Relaxed); + } + Err(e) => { + state.metrics.error_write.fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_TREE_RELAY_FAIL: Failed to relay a tree frame to {}: {}", + peer, e + ); + } + } + }); + } + if unreachable > 0 { + metrics + .txset_tree_unreachable + .fetch_add(unreachable, Ordering::Relaxed); + } + if busy > 0 { + metrics + .txset_tree_relay_busy + .fetch_add(busy, Ordering::Relaxed); + warn!( + "TXSET_TREE_RELAY_BUSY: Not relaying the tree frame for slot {} to {} child(ren): an earlier relay to them is still in flight", + header.slot, busy + ); + } + debug!( + "TXSET_TREE_RELAY: Relaying the tree frame for slot {} (hop {}) to {} children", + header.slot, + header.hop, + children.len() as u64 - unreachable - busy + ); + } + } + Some(TaggedBody { frame, body }) +} + +/// The pieces of a set with their headers, as its leader sends them. +struct SignedPieces { + headers: Vec, + pieces: Vec>>, +} + +/// Sign the root of the Merkle tree over `pieces` with this overlay's key, +/// and make every piece its header (hop 1). `None`: the overlay's key is not +/// an Ed25519 key, or the signing failed. +#[allow(clippy::too_many_arguments)] +fn sign_pieces( + keypair: &Keypair, + identity: &NodeIdentity, + slot: u32, + seed: &[u8; 32], + params: CodedParams, + total_len: u32, + pieces: Vec>, + sent_unix_ms: u64, +) -> Option { + let overlay_key = keypair.public().try_into_ed25519().ok()?.to_bytes(); + let leaves: Vec<[u8; 32]> = pieces + .iter() + .enumerate() + .map(|(i, piece)| txset_coded::leaf_hash(i as u8, piece)) + .collect(); + let (root, paths) = txset_coded::merkle_tree(&leaves); + let message = txset_coded::root_message( + &identity.network_id, + seed, + slot, + &identity.node_id, + params, + total_len, + &root, + ); + let root_sig: [u8; 64] = keypair.sign(&message).ok()?.try_into().ok()?; + let headers = paths + .into_iter() + .enumerate() + .map(|(index, path)| PieceHeader { + hop: 1, + data: params.data as u8, + pieces: params.pieces as u8, + index: index as u8, + slot, + leader: identity.node_id, + sent_unix_ms, + total_len, + root, + overlay_key, + binding_sig: identity.signature, + root_sig, + path, + }) + .collect(); + Some(SignedPieces { + headers, + pieces: pieces.into_iter().map(Arc::new).collect(), + }) +} + +/// As the leader of `slot`: cut the set's frame into pieces, sign the root of +/// their Merkle tree, and write every piece to its relays. Runs as a task of +/// its own, so that the coding does not hold up the overlay's command loop. +#[allow(clippy::too_many_arguments)] +async fn send_coded_set( + state: Arc, + hash: [u8; 32], + data: Arc, + slot: u32, + seed: [u8; 32], + layout: Arc, + identity: NodeIdentity, + peer_of: HashMap, + sent_unix_ms: u64, +) { + let params = layout.params; + let signed = { + let keypair = state.keypair.clone(); + let identity = identity.clone(); + tokio::task::spawn_blocking(move || { + let frame = data.encoded(); + let pieces = txset_coded::encode_pieces(frame, params).ok()?; + sign_pieces( + &keypair, + &identity, + slot, + &seed, + params, + frame.len() as u32, + pieces, + sent_unix_ms, + ) + }) + .await + }; + let Ok(Some(SignedPieces { headers, pieces })) = signed else { + // Nothing was sent: the other nodes ask for the set after their wait. + warn!( + "TXSET_CODED_ERR: Could not cut TX set {:02x?}... for slot {} into pieces", + &hash[..4], + slot + ); + return; + }; + let metrics = &state.metrics; + metrics + .txset_coded_sets_encoded + .fetch_add(1, Ordering::Relaxed); + let mut skipped = 0u64; + for (index, node) in layout.leader_targets() { + let Some(peer) = peer_of.get(&node).copied() else { + skipped += 1; + continue; + }; + if !state.txset_tree.lock().unwrap().coded.start_send(node) { + skipped += 1; + continue; + } + let header = headers[index].encode(); + let piece = Arc::clone(&pieces[index]); + let state = Arc::clone(&state); + tokio::spawn(async move { + let parts = [header.as_slice(), piece.as_slice()]; + let sent = send_to_peer_stream_parts(&state, peer, StreamType::TxSet, &parts).await; + state.txset_tree.lock().unwrap().coded.finish_send(&node); + let metrics = &state.metrics; + match sent { + Ok(_) => { + let bytes = (header.len() + piece.len()) as u64; + metrics + .txset_coded_pieces_sent + .fetch_add(1, Ordering::Relaxed); + metrics + .txset_push_sent_bytes + .fetch_add(bytes, Ordering::Relaxed); + metrics.message_write.fetch_add(1, Ordering::Relaxed); + metrics.byte_write.fetch_add(bytes, Ordering::Relaxed); + } + Err(e) => { + metrics.error_write.fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_CODED_SEND_FAIL: Failed to send a piece to {}: {}", + peer, e + ); + } + } + }); + } + if skipped > 0 { + metrics + .txset_coded_pieces_skipped + .fetch_add(skipped, Ordering::Relaxed); + } +} + +/// A piece of a coded set arrived from `peer_id` (see `txset_coded`). +/// +/// It is looked at only if this node's Core described the slot and names the +/// leader as one that may lead it, the code is the one this node derives, and +/// the sender is the node that owes this node that piece in the layout (by +/// the validator identity of the connection). It is kept only if the leader's +/// signatures and the piece's Merkle path check. A relay passes on what the +/// leader itself sent it, once, and only after those checks. With enough +/// pieces under one root the frame is rebuilt and goes the way of every +/// received frame. +fn handle_piece_frame(state: &Arc, peer_id: PeerId, frame: Vec) { + let metrics = &state.metrics; + let Some((header, at)) = PieceHeader::parse(&frame) else { + metrics + .txset_coded_pieces_invalid + .fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_CODED_ERR: Dropping a malformed piece frame ({} bytes) from {}", + frame.len(), + peer_id + ); + return; + }; + let refuse = |why: &str| { + metrics + .txset_coded_pieces_refused + .fetch_add(1, Ordering::Relaxed); + debug!( + "TXSET_CODED_REFUSED: Piece {} of slot {} from {}: {}", + header.index, header.slot, peer_id, why + ); + }; + let invalid = |why: &str| { + metrics + .txset_coded_pieces_invalid + .fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_CODED_INVALID: Dropping piece {} of slot {} from {}: {}", + header.index, header.slot, peer_id, why + ); + }; + // One lock at a time. + let Some(identity) = state.identity.lock().unwrap().ours.clone() else { + return refuse("this node's identity is not known yet"); + }; + let me = identity.node_id; + let Some(sender) = state.tx_router.lock().unwrap().node_of(&peer_id).copied() else { + return refuse("the sender did not prove which validator it is"); + }; + let key: RootKey = (header.slot, header.leader, header.root); + let index = header.index as usize; + // Cheap checks first; then the frame is counted against its sender, + // whatever it turns out to be; only then is the layout consulted (worked + // out once per slot and leader). + type Checked = (Arc, [u8; 32], bool, Option); + let checked: Result = { + let mut tree = state.txset_tree.lock().unwrap(); + let config = match tree.config() { + None => Err("no layout from Core"), + Some(config) if config.mode != TreeMode::Coded => Err("not in coded mode"), + Some(config) if !config.may_lead(header.slot, &header.leader) => { + Err("unknown slot, or not a leader of it") + } + Some(config) if !config.contains(&me) || !config.contains(&header.leader) => { + Err("this node or the leader is not in the layout") + } + Some(config) => config + .seed_of(header.slot) + .map(|seed| (*seed, config.withhold)) + .ok_or("unknown slot"), + }; + match config { + Err(why) => Err(why), + Ok(_) if !tree.coded.admit_frame(header.slot, header.leader, sender) => { + Err("too many frames from this sender for the slot and leader") + } + Ok((seed, withhold)) => match tree.coded_layout(header.slot, &header.leader) { + None => Err("no layout for these nodes"), + Some(layout) if (layout.params.data, layout.params.pieces) != header.params() => { + Err("another code than this node derives") + } + Some(layout) if layout.sender_of(&me, index) != Some(sender) => { + Err("not from the node that owes this piece") + } + Some(_) if (sender == header.leader) != (header.hop == 1) => Err("wrong hop count"), + Some(layout) => { + let known = tree.coded.knows_root(&key, layout.params, header.total_len); + Ok((layout, seed, withhold, known)) + } + }, + } + }; + let (layout, seed, withhold, known) = match checked { + Ok(checked) => checked, + Err(why) => return refuse(why), + }; + match known { + Some(true) => {} + Some(false) => return invalid("its root is known with another shape or length"), + None if !verify_piece_signatures(&identity.network_id, &seed, &header) => { + return invalid("the root is not signed by the leader"); + } + None => {} + } + let leaf = txset_coded::leaf_hash(header.index, &frame[at..]); + if txset_coded::merkle_root_from(leaf, index, &header.path) != header.root { + return invalid("its Merkle path does not reach the root"); + } + let frame = Arc::new(frame); + let (outcome, relay) = { + let mut tree = state.txset_tree.lock().unwrap(); + // A root gets its place only from a frame whose signatures were + // checked just now (`known` was none): the key that signed it is kept + // with it. + if known.is_none() + && !tree + .coded + .add_root(key, layout.params, header.total_len, header.overlay_key) + { + return refuse("the leader has two other roots for the slot already"); + } + match tree.coded.knows_root(&key, layout.params, header.total_len) { + Some(true) => {} + Some(false) => return invalid("its root is known with another shape or length"), + None => return refuse("its root is no longer kept"), + } + let outcome = tree.coded.add_piece( + &key, + index, + PieceBytes { + frame: Arc::clone(&frame), + at, + }, + ); + // Passed on: what the leader itself sent this node as the relay of + // this piece, once per root. + let relay = header.hop == 1 + && layout.relayed_piece(&me) == Some(index) + && !matches!(outcome, PieceOutcome::Duplicate | PieceOutcome::Unknown) + && tree.coded.mark_relayed(key); + (outcome, relay) + }; + match &outcome { + PieceOutcome::Stored | PieceOutcome::Ready(_) => { + metrics + .txset_coded_pieces_received + .fetch_add(1, Ordering::Relaxed); + } + PieceOutcome::Late => { + metrics + .txset_coded_pieces_received + .fetch_add(1, Ordering::Relaxed); + metrics + .txset_coded_pieces_late + .fetch_add(1, Ordering::Relaxed); + } + PieceOutcome::Duplicate => { + metrics + .txset_coded_pieces_duplicate + .fetch_add(1, Ordering::Relaxed); + } + PieceOutcome::Unknown => refuse("no place for it"), + } + if relay && withhold { + metrics.txset_tree_withheld.fetch_add(1, Ordering::Relaxed); + } else if relay { + relay_piece(state, &layout, &me, &header, &frame, at); + } + if let PieceOutcome::Ready(ready) = outcome { + // The set is in hand when the last piece needed is: the rebuild that + // follows is this node's own work. + record_tree_delay(metrics, header.sent_unix_ms, None); + metrics.txset_tree_received.fetch_add(1, Ordering::Relaxed); + // The set is its leader's, whoever sent the piece that completed + // it: what a peer may have this node keep unclaimed is counted + // against the overlay that signed the root (the leader's, connected + // to this node or not), not against a relay the leader could pick by + // sending that relay's piece last. + let from = libp2p::identity::ed25519::PublicKey::try_from_bytes(&ready.origin) + .map(|key| libp2p::identity::PublicKey::from(key).to_peer_id()) + .unwrap_or(peer_id); + tokio::spawn(rebuild_coded_set( + Arc::clone(state), + from, + ready, + header.slot, + header.root, + )); + } +} + +/// Pass this node's piece on to the nodes that expect it from this node. +fn relay_piece( + state: &Arc, + layout: &CodedLayout, + me: &NodeId, + header: &PieceHeader, + frame: &Arc>, + at: usize, +) { + let peer_of: HashMap = state.tx_router.lock().unwrap().identified_peers(); + let relay_header = Arc::new( + PieceHeader { + hop: 2, + ..header.clone() + } + .encode(), + ); + let mut skipped = 0u64; + for node in layout.receivers_of(me) { + let Some(peer) = peer_of.get(&node).copied() else { + skipped += 1; + continue; + }; + if !state.txset_tree.lock().unwrap().coded.start_send(node) { + skipped += 1; + continue; + } + let state = Arc::clone(state); + let frame = Arc::clone(frame); + let relay_header = Arc::clone(&relay_header); + tokio::spawn(async move { + let parts = [relay_header.as_slice(), &frame[at..]]; + let sent = send_to_peer_stream_parts(&state, peer, StreamType::TxSet, &parts).await; + state.txset_tree.lock().unwrap().coded.finish_send(&node); + let metrics = &state.metrics; + match sent { + Ok(_) => { + let bytes = (relay_header.len() + frame.len() - at) as u64; + metrics + .txset_coded_pieces_relayed + .fetch_add(1, Ordering::Relaxed); + metrics + .txset_tree_relayed_bytes + .fetch_add(bytes, Ordering::Relaxed); + metrics.message_write.fetch_add(1, Ordering::Relaxed); + metrics.byte_write.fetch_add(bytes, Ordering::Relaxed); + } + Err(e) => { + metrics.error_write.fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_CODED_RELAY_FAIL: Failed to relay a piece to {}: {}", + peer, e + ); + } + } + }); + } + if skipped > 0 { + state + .metrics + .txset_coded_pieces_skipped + .fetch_add(skipped, Ordering::Relaxed); + } +} + +/// Rebuild a set's frame from its pieces and hand it on as a received frame: +/// decompress, strict parse, then `deliver_txset` (which files it by the hash +/// of its content). Afterwards, off the path of the set, encode the frame +/// again and compare the root: a leader that signed pieces which do not fit +/// together is counted (the set is still what its content hash says). +async fn rebuild_coded_set( + state: Arc, + peer_id: PeerId, + ready: ReadySet, + slot: u32, + root: [u8; 32], +) { + let params = ready.params; + let rebuilt = tokio::task::spawn_blocking(move || { + use stellar_xdr::{GeneralizedTransactionSet, Limits, ReadXdr}; + let started = Instant::now(); + let held: Vec> = ready + .pieces + .iter() + .map(|piece| piece.as_ref().map(|piece| piece.as_slice())) + .collect(); + let (frame, with_parity) = + txset_coded::rebuild(&held, ready.params, ready.total_len as usize)?; + let rebuild_us = started.elapsed().as_micros() as u64; + let data = crate::flood::txset_encoding::decode(frame)?; + GeneralizedTransactionSet::from_xdr(data.as_slice(), Limits::none()) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok::<_, io::Error>((Arc::new(data), with_parity, rebuild_us)) + }) + .await; + let metrics = &state.metrics; + let data = match rebuilt { + Ok(Ok((data, with_parity, rebuild_us))) => { + metrics + .txset_coded_sets_rebuilt + .fetch_add(1, Ordering::Relaxed); + if with_parity { + metrics + .txset_coded_sets_rebuilt_with_parity + .fetch_add(1, Ordering::Relaxed); + } + metrics + .txset_coded_rebuild_us_sum + .fetch_add(rebuild_us, Ordering::Relaxed); + data + } + result => { + metrics + .txset_coded_sets_failed + .fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_CODED_ERR: The pieces of a set for slot {} do not rebuild a TX set: {:?}", + slot, + result.map(|inner| inner.map(|_| ())) + ); + return; + } + }; + deliver_txset(&state, peer_id, Arc::clone(&data)).await; + let consistent = tokio::task::spawn_blocking(move || { + txset_coded::encode_pieces(data.encoded(), params).is_ok_and(|pieces| { + let leaves: Vec<[u8; 32]> = pieces + .iter() + .enumerate() + .map(|(i, piece)| txset_coded::leaf_hash(i as u8, piece)) + .collect(); + txset_coded::merkle_tree(&leaves).0 == root + }) + }) + .await; + if !matches!(consistent, Ok(true)) { + state + .metrics + .txset_coded_sets_inconsistent + .fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_CODED_INCONSISTENT: The leader of slot {} signed pieces that do not fit together", + slot + ); + } +} + async fn handle_inbound_txset_streams(mut incoming: IncomingStreams, state: Arc) { while let Some((peer_id, mut stream)) = incoming.next().await { debug!("Accepted inbound TxSet stream from {}", peer_id); @@ -3178,10 +4446,13 @@ async fn handle_inbound_txset_streams(mut incoming: IncomingStreams, state: Arc< let state = state.clone(); tokio::spawn(async move { + let mut probe_window = Instant::now(); + let mut probes_answered = 0u32; + let pong_in_flight = Arc::new(std::sync::atomic::AtomicBool::new(false)); loop { match read_framed_with_limit( &mut stream, - crate::flood::txset_encoding::max_encoded_size(), + crate::flood::txset_encoding::max_encoded_size() + txset_tree::MAX_TREE_HEADER, ) .await { @@ -3191,6 +4462,73 @@ async fn handle_inbound_txset_streams(mut incoming: IncomingStreams, state: Arc< .metrics .byte_read .fetch_add(data.len() as u64, Ordering::Relaxed); + // A round-trip probe (see `txset_tree_probe_task`): + // answer a ping, record a pong. Neither is a set. + if let Some(probe) = Probe::parse(&data) { + match probe { + Some(Probe::Ping(value)) => { + // A peer gets a few answers a second. + let now = Instant::now(); + if now.duration_since(probe_window) >= Duration::from_secs(1) { + probe_window = now; + probes_answered = 0; + } + // And one answer at a time: a peer + // that does not read leaves one small + // write waiting, not one per ping. + if probes_answered < MAX_PROBES_ANSWERED_PER_SEC + && !pong_in_flight.swap(true, Ordering::AcqRel) + { + probes_answered += 1; + let state = state.clone(); + let pong_in_flight = Arc::clone(&pong_in_flight); + tokio::spawn(async move { + let pong = Probe::Pong(value).encode(); + let _ = send_to_peer_stream( + &state, + peer_id, + StreamType::TxSet, + &pong, + ) + .await; + pong_in_flight.store(false, Ordering::Release); + }); + } + } + Some(Probe::Pong(sent_micros)) => { + let now = monotonic_micros(); + let node = + state.tx_router.lock().unwrap().node_of(&peer_id).copied(); + // Only the answer to the probe this node + // sent that validator last counts. + if let Some(node) = node { + let mut tree = state.txset_tree.lock().unwrap(); + if tree.probe_answered(&node, sent_micros) + && sent_micros <= now + && now - sent_micros < 10_000_000 + { + tree.note_rtt( + node, + Duration::from_micros(now - sent_micros), + ); + } + } + } + None => {} + } + continue; + } + // A piece of a coded set (see `txset_coded`). It + // does not draw on the allowance below: what a peer + // can make this node keep and rebuild is bounded by + // the layout (one piece index per sender, slot and + // leader) and by `CodedState`'s limits, and nothing + // is decompressed before pieces from half of this + // node's senders agree on a root its leader signed. + if PieceHeader::is_tagged(&data) { + handle_piece_frame(&state, peer_id, data); + continue; + } // Bound what a peer can make this node decompress // and parse. Answers to this node's requests are // free (the token is returned in `deliver_txset`); @@ -3213,11 +4551,26 @@ async fn handle_inbound_txset_streams(mut incoming: IncomingStreams, state: Arc< ); continue; } + // A frame with a tree header: pass the bytes on to + // this node's children first (if it has any in that + // tree), then go on with the ordinary frame behind + // the header. + let data = if TreeHeader::is_tagged(&data) { + match relay_tagged_frame(&state, peer_id, data) { + Some(body) => body, + None => { + warn!("TXSET_TREE_ERR: Dropping TxSet frame with a malformed tree header from {}", peer_id); + continue; + } + } + } else { + TaggedBody::plain(data) + }; // Preserve the received encoding through verification and // cache insertion. Forwarding never compresses it again. let decoded = tokio::task::spawn_blocking(move || { use stellar_xdr::{GeneralizedTransactionSet, Limits, ReadXdr}; - let data = crate::flood::txset_encoding::decode(data)?; + let data = crate::flood::txset_encoding::decode(data.into_frame())?; GeneralizedTransactionSet::from_xdr(data.as_slice(), Limits::none()) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; Ok::<_, io::Error>(Arc::new(data)) @@ -3424,6 +4777,12 @@ mod txset_push_tests; #[cfg(test)] mod tx_direct_tests; +#[cfg(test)] +mod txset_tree_tests; + +#[cfg(test)] +mod txset_coded_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/overlay/src/libp2p_overlay/txset_coded_tests.rs b/overlay/src/libp2p_overlay/txset_coded_tests.rs new file mode 100644 index 0000000000..a70a9d6d17 --- /dev/null +++ b/overlay/src/libp2p_overlay/txset_coded_tests.rs @@ -0,0 +1,709 @@ +//! Tests for the erasure-coded pieces of the leader's tx set (mode `coded`): +//! nine overlay nodes on loopback, fully connected, each with a validator +//! identity. With eight other nodes a set is cut into eight pieces, four of +//! which rebuild it; every other node relays one piece to the seven others. + +use super::txset_tree_tests::{ + config, expect_set, full_mesh, index_of, serve_request, set_events_within, test_set, Node, + NETWORK_ID, SEED, SLOT, +}; +use super::*; +use crate::flood::txset_tree::TREE_FETCH_GRACE; + +const NODES: usize = 9; +const PIECES: u64 = 8; +const DATA: usize = 4; + +/// A full mesh whose tx-set streams are known to work in both directions. +/// +/// The stream library hands inbound streams to the overlay through a channel +/// without a buffer and drops a stream that arrives while another is being +/// handed over; the sender notices at its first write and opens the stream +/// again (three attempts). When the mesh forms, and when every node writes to +/// every other at once for the first time, as here, three attempts are not +/// always enough. So every pair exchanges a ping first, one after the other. +async fn coded_mesh(first_port: u16) -> Vec { + let nodes = full_mesh(first_port, NODES).await; + for (a, from) in nodes.iter().enumerate() { + for (b, to) in nodes.iter().enumerate() { + if a != b { + let ping = Probe::Ping(1).encode(); + send_to_peer_stream(&from.state, to.peer_id, StreamType::TxSet, &ping) + .await + .unwrap(); + } + } + // The answers come back on streams that may have to be opened too. + tokio::time::sleep(Duration::from_millis(60)).await; + } + tokio::time::sleep(Duration::from_millis(300)).await; + for node in &nodes { + node.metrics.error_write.store(0, Ordering::Relaxed); + } + nodes +} + +fn coded_config(nodes: &[Node]) -> TreeConfig { + config(nodes, TreeMode::Coded, 0) +} + +fn layout_of(nodes: &[Node], leader: usize) -> CodedLayout { + let order = coded_config(nodes) + .hash_order(SLOT, &nodes[leader].node_id()) + .unwrap(); + let layout = CodedLayout::new(&order).unwrap(); + assert_eq!( + ( + layout.params.data, + layout.params.pieces as u64, + layout.params.relays + ), + (DATA, PIECES, 1) + ); + layout +} + +fn identity_of(node: &Node) -> NodeIdentity { + node.state.identity.lock().unwrap().ours.clone().unwrap() +} + +fn sum(nodes: &[Node], pick: impl Fn(&OverlayMetrics) -> &AtomicU64 + Copy) -> u64 { + nodes.iter().map(|n| n.count(pick)).sum() +} + +/// The pieces of `data` as `leader` would send them for `SLOT`, after `spoil` +/// had its way with them (before the root is computed and signed). +fn pieces_of( + leader: &Node, + data: &TxSetData, + params: CodedParams, + spoil: impl FnOnce(&mut Vec>), +) -> SignedPieces { + let mut pieces = txset_coded::encode_pieces(data.encoded(), params).unwrap(); + spoil(&mut pieces); + sign_pieces( + &leader.state.keypair, + &identity_of(leader), + SLOT, + &SEED, + params, + data.encoded().len() as u32, + pieces, + unix_ms_now(), + ) + .unwrap() +} + +/// Write piece `index` of `signed` to `to` from the node the layout says +/// owes it (with the hop count that goes with that sender). +async fn deliver_piece( + nodes: &[Node], + layout: &CodedLayout, + signed: &SignedPieces, + index: usize, + to: usize, +) { + let sender = layout.sender_of(&nodes[to].node_id(), index).unwrap(); + let from = index_of(nodes, &sender); + let header = PieceHeader { + hop: if sender == layout.leader { 1 } else { 2 }, + ..signed.headers[index].clone() + } + .encode(); + send_to_peer_stream_parts( + &nodes[from].state, + nodes[to].peer_id, + StreamType::TxSet, + &[header.as_slice(), signed.pieces[index].as_slice()], + ) + .await + .unwrap(); +} + +/// The leader writes one piece to each of the eight other nodes, each passes +/// its piece on to the seven others, and every node rebuilds the set from the +/// first four pieces it holds, without asking. +#[tokio::test] +async fn a_coded_set_reaches_every_node_as_pieces() { + let mut nodes = coded_mesh(27101).await; + for node in &nodes { + node.handle.set_txset_tree(coded_config(&nodes)); + } + let leader = 2; + let leader_peer = nodes[leader].peer_id; + let (hash, data) = test_set(71); + // The proposal names the hash before the pieces come: nobody asks. + for i in (0..NODES).filter(|i| *i != leader) { + nodes[i].handle.record_txset_source(hash, leader_peer).await; + nodes[i].handle.fetch_txset(hash, SLOT).await; + } + nodes[leader] + .handle + .push_txset_for_slot(hash, Arc::clone(&data), SLOT) + .await; + for i in (0..NODES).filter(|i| *i != leader) { + let (slot, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(slot, None, "node {i} had not asked"); + // The set counts as its leader's, whoever sent the fourth piece. + assert_eq!(from, leader_peer, "node {i}"); + } + // Let the pieces nobody needed any more arrive, and the waits end. + tokio::time::sleep(TREE_FETCH_GRACE + Duration::from_millis(300)).await; + + // The leader: eight pieces, no whole copy, nothing relayed. + assert_eq!(nodes[leader].count(|m| &m.txset_coded_sets_encoded), 1); + assert_eq!(nodes[leader].count(|m| &m.txset_coded_pieces_sent), PIECES); + assert_eq!(nodes[leader].count(|m| &m.txset_push_sent), 0); + assert_eq!(nodes[leader].count(|m| &m.txset_tree_root_sent), 0); + assert_eq!(nodes[leader].count(|m| &m.txset_coded_pieces_relayed), 0); + assert_eq!(nodes[leader].count(|m| &m.txset_coded_pieces_received), 0); + let piece = txset_coded::piece_len(data.encoded().len(), DATA) as u64; + let header = (txset_coded::PIECE_HEADER_FIXED + 32 * txset_coded::merkle_depth(8)) as u64; + assert_eq!( + nodes[leader].count(|m| &m.txset_push_sent_bytes), + PIECES * (piece + header) + ); + for i in (0..NODES).filter(|i| *i != leader) { + let node = &nodes[i]; + // Its own piece to the seven others; all eight pieces received, four + // of them after the set was rebuilt. + assert_eq!( + node.count(|m| &m.txset_coded_pieces_relayed), + PIECES - 1, + "node {i}" + ); + assert_eq!( + node.count(|m| &m.txset_tree_relayed_bytes), + (PIECES - 1) * (piece + header) + ); + assert_eq!( + node.count(|m| &m.txset_coded_pieces_received), + PIECES, + "node {i}" + ); + assert_eq!( + node.count(|m| &m.txset_coded_pieces_late), + PIECES - DATA as u64 + ); + assert_eq!(node.count(|m| &m.txset_coded_sets_rebuilt), 1); + assert_eq!(node.count(|m| &m.txset_tree_received), 1); + assert_eq!(node.count(|m| &m.txset_tree_delay_count), 1); + // Nothing wrong, nothing asked for. + assert_eq!(node.count(|m| &m.txset_coded_pieces_refused), 0, "node {i}"); + assert_eq!(node.count(|m| &m.txset_coded_pieces_invalid), 0); + assert_eq!(node.count(|m| &m.txset_coded_pieces_duplicate), 0); + assert_eq!(node.count(|m| &m.txset_coded_pieces_skipped), 0); + assert_eq!(node.count(|m| &m.error_write), 0, "node {i}"); + assert_eq!(node.count(|m| &m.txset_coded_sets_failed), 0); + assert_eq!(node.count(|m| &m.txset_coded_sets_inconsistent), 0); + assert_eq!(node.count(|m| &m.txset_tree_fetch_deferred), 1); + assert_eq!(node.count(|m| &m.txset_tree_fetch_fallback), 0); + assert_eq!(node.count(|m| &m.fetch_txset_count), 0); + } + // Nobody asked the leader, and no pieces are left in memory. + let (_, requested) = set_events_within(&mut nodes[leader], Duration::from_millis(100)).await; + assert!(requested.is_empty()); + // A second push of the same set sends nothing. + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(nodes[leader].count(|m| &m.txset_coded_pieces_sent), PIECES); + assert_eq!(nodes[leader].count(|m| &m.txset_push_sent), 0); +} + +/// Half of the relays pass nothing on: every node still has four pieces (its +/// own from the leader and those of the relays that work), rebuilds the set +/// with parity where it must, and nobody waits or asks. In a tree every child +/// of such a relay would wait and ask. +#[tokio::test] +async fn relays_that_withhold_cost_no_wait() { + let mut nodes = coded_mesh(27201).await; + let leader = 5; + let layout = layout_of(&nodes, leader); + // The relays of pieces 0, 2, 5 and 7 withhold: data and parity pieces. + let withholders: Vec = [0usize, 2, 5, 7] + .iter() + .map(|piece| index_of(&nodes, &layout.relays_of(*piece)[0])) + .collect(); + for (i, node) in nodes.iter().enumerate() { + node.handle + .set_txset_tree(coded_config(&nodes).with_withhold(withholders.contains(&i))); + } + let leader_peer = nodes[leader].peer_id; + let (hash, data) = test_set(72); + for i in (0..NODES).filter(|i| *i != leader) { + nodes[i].handle.record_txset_source(hash, leader_peer).await; + nodes[i].handle.fetch_txset(hash, SLOT).await; + } + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + for i in (0..NODES).filter(|i| *i != leader) { + let (slot, _) = expect_set(&mut nodes[i], hash).await; + assert_eq!(slot, None, "node {i} had not asked"); + } + tokio::time::sleep(TREE_FETCH_GRACE + Duration::from_millis(300)).await; + for i in (0..NODES).filter(|i| *i != leader) { + let node = &nodes[i]; + let withholds = withholders.contains(&i); + assert_eq!( + node.count(|m| &m.txset_tree_withheld), + u64::from(withholds), + "node {i}" + ); + assert_eq!( + node.count(|m| &m.txset_coded_pieces_relayed), + if withholds { 0 } else { PIECES - 1 } + ); + // Its own piece and those of the four relays that work (three, for + // a node that is one of them). + assert_eq!( + node.count(|m| &m.txset_coded_pieces_received), + if withholds { 5 } else { 4 }, + "node {i}" + ); + assert_eq!(node.count(|m| &m.txset_coded_sets_rebuilt), 1); + assert_eq!(node.count(|m| &m.txset_coded_sets_failed), 0); + assert_eq!(node.count(|m| &m.txset_tree_fetch_deferred), 1); + assert_eq!(node.count(|m| &m.txset_tree_fetch_fallback), 0, "node {i}"); + assert_eq!(node.count(|m| &m.fetch_txset_count), 0); + } + // Two of the four data pieces never left their relays: all but those + // two relays needed parity. + assert!(sum(&nodes, |m| &m.txset_coded_sets_rebuilt_with_parity) >= 6); + let (_, requested) = set_events_within(&mut nodes[leader], Duration::from_millis(100)).await; + assert!(requested.is_empty()); +} + +/// With five of the eight relays withholding, a node that relays honestly +/// holds three pieces, one too few: it asks after the wait, as in the tree, +/// and gets the set as an answer. +#[tokio::test] +async fn with_too_few_pieces_a_node_asks_after_the_wait() { + let mut nodes = coded_mesh(27301).await; + let leader = 0; + let layout = layout_of(&nodes, leader); + let withholders: Vec = [0usize, 1, 3, 4, 6] + .iter() + .map(|piece| index_of(&nodes, &layout.relays_of(*piece)[0])) + .collect(); + let starved: Vec = (0..NODES) + .filter(|i| *i != leader && !withholders.contains(i)) + .collect(); + assert_eq!(starved.len(), 3); + for (i, node) in nodes.iter().enumerate() { + node.handle + .set_txset_tree(coded_config(&nodes).with_withhold(withholders.contains(&i))); + } + let leader_peer = nodes[leader].peer_id; + let (hash, data) = test_set(73); + let asked_at = Instant::now(); + for i in (0..NODES).filter(|i| *i != leader) { + nodes[i].handle.record_txset_source(hash, leader_peer).await; + nodes[i].handle.fetch_txset(hash, SLOT).await; + } + nodes[leader] + .handle + .push_txset_for_slot(hash, Arc::clone(&data), SLOT) + .await; + // The withholders hold their own piece and three more. + for i in &withholders { + let (slot, _) = expect_set(&mut nodes[*i], hash).await; + assert_eq!(slot, None); + } + // The others ask the leader after the wait: pieces are not a push of the + // set to them, so the leader answers. + let mut askers = Vec::new(); + for _ in 0..starved.len() { + askers.push(serve_request(&mut nodes[leader], hash, &data).await); + } + assert!(asked_at.elapsed() >= TREE_FETCH_GRACE); + askers.sort(); + let mut expected: Vec = starved.iter().map(|i| nodes[*i].peer_id).collect(); + expected.sort(); + assert_eq!(askers, expected); + for i in &starved { + let (slot, from) = expect_set(&mut nodes[*i], hash).await; + assert_eq!(slot, Some(SLOT), "node {i} had to ask"); + assert_eq!(from, leader_peer); + assert_eq!(nodes[*i].count(|m| &m.txset_coded_pieces_received), 3); + assert_eq!(nodes[*i].count(|m| &m.txset_coded_sets_rebuilt), 0); + assert_eq!(nodes[*i].count(|m| &m.txset_tree_fetch_fallback), 1); + } + assert_eq!(nodes[leader].count(|m| &m.txset_push_request_skipped), 0); +} + +/// What a node does with pieces that are wrong: a changed piece, a piece +/// from a node that does not owe it, a piece signed by another validator in +/// the leader's name, and the pieces of a validator that may not lead the +/// slot. None is kept or passed on, and the leader's real set then arrives +/// everywhere. +#[tokio::test] +async fn wrong_pieces_are_dropped_and_not_passed_on() { + let mut nodes = coded_mesh(27401).await; + let leader = 4; + let leader_id = nodes[leader].node_id(); + let layout = layout_of(&nodes, leader); + // Only the leader may lead the slot. + for node in &nodes { + node.handle + .set_txset_tree(coded_config(&nodes).with_leaders(vec![Some(vec![leader_id])])); + } + let (hash, data) = test_set(74); + let good = pieces_of(&nodes[leader], &data, layout.params, |_| {}); + // The relay of piece 1, and an attacker: the relay of piece 6. + let target = index_of(&nodes, &layout.relays_of(1)[0]); + let attacker = index_of(&nodes, &layout.relays_of(6)[0]); + let invalid = |nodes: &[Node]| nodes[target].count(|m| &m.txset_coded_pieces_invalid); + let refused = |nodes: &[Node]| nodes[target].count(|m| &m.txset_coded_pieces_refused); + let settle = || tokio::time::sleep(Duration::from_millis(200)); + + // 1. The target's own piece from the leader, one bit changed: its path + // does not reach the signed root. + let mut changed = good.pieces[1].as_ref().clone(); + changed[0] ^= 1; + send_to_peer_stream_parts( + &nodes[leader].state, + nodes[target].peer_id, + StreamType::TxSet, + &[good.headers[1].encode().as_slice(), changed.as_slice()], + ) + .await + .unwrap(); + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (1, 0)); + + // 2. The right piece 1, but from the attacker: the leader owes the + // target that piece, nobody else. + send_to_peer_stream_parts( + &nodes[attacker].state, + nodes[target].peer_id, + StreamType::TxSet, + &[ + good.headers[1].encode().as_slice(), + good.pieces[1].as_slice(), + ], + ) + .await + .unwrap(); + // And piece 6, which the attacker does owe the target, marked as if it + // came straight from the leader. + send_to_peer_stream_parts( + &nodes[attacker].state, + nodes[target].peer_id, + StreamType::TxSet, + &[ + good.headers[6].encode().as_slice(), + good.pieces[6].as_slice(), + ], + ) + .await + .unwrap(); + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (1, 2)); + + // 3. The attacker cuts a set of its own and signs it with its own keys, + // in the leader's name: its overlay is not the leader's. + let (_, other) = test_set(75); + let mut forged = pieces_of(&nodes[attacker], &other, layout.params, |_| {}); + for header in &mut forged.headers { + header.leader = leader_id; + } + deliver_piece(&nodes, &layout, &forged, 6, target).await; + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (2, 2)); + // The leader's signature over another root does not help either. + let mut grafted = pieces_of(&nodes[attacker], &other, layout.params, |_| {}); + for header in &mut grafted.headers { + header.leader = leader_id; + header.overlay_key = good.headers[0].overlay_key; + header.binding_sig = good.headers[0].binding_sig; + header.root_sig = good.headers[0].root_sig; + } + deliver_piece(&nodes, &layout, &grafted, 6, target).await; + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (3, 2)); + + // 4. The attacker cuts a set of its own and signs it honestly, as the + // leader of a slot it may not lead, and sends the target the piece + // the target would relay in the attacker's layout: refused. + let own_layout = layout_of(&nodes, attacker); + let own = pieces_of(&nodes[attacker], &other, own_layout.params, |_| {}); + let owed = own_layout.relayed_piece(&nodes[target].node_id()).unwrap(); + deliver_piece(&nodes, &own_layout, &own, owed, target).await; + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (3, 3)); + + // 5. A piece signed for another history of the same network (another + // seed for the slot): the signature is of no use here. The pieces + // and the root are those of the real set, so only the signature + // differs. + let replayed = txset_coded::encode_pieces(data.encoded(), layout.params) + .ok() + .and_then(|pieces| { + sign_pieces( + &nodes[leader].state.keypair, + &identity_of(&nodes[leader]), + SLOT, + &[0x77; 32], + layout.params, + data.encoded().len() as u32, + pieces, + unix_ms_now(), + ) + }) + .unwrap(); + assert_eq!(replayed.headers[1].root, good.headers[1].root); + deliver_piece(&nodes, &layout, &replayed, 1, target).await; + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (4, 3)); + + // 6. A sender is listened to six times per slot and leader, whatever it + // sends: the attacker has used four; after two more, even the piece + // it owes the target is not looked at. + for _ in 0..3 { + send_to_peer_stream_parts( + &nodes[attacker].state, + nodes[target].peer_id, + StreamType::TxSet, + &[ + good.headers[1].encode().as_slice(), + good.pieces[1].as_slice(), + ], + ) + .await + .unwrap(); + } + deliver_piece(&nodes, &layout, &good, 6, target).await; + settle().await; + assert_eq!((invalid(&nodes), refused(&nodes)), (4, 7)); + assert_eq!(sum(&nodes, |m| &m.txset_coded_pieces_refused), 7); + + // Nothing of all this was kept, passed on or rebuilt. + assert_eq!(sum(&nodes, |m| &m.txset_coded_pieces_received), 0); + assert_eq!(sum(&nodes, |m| &m.txset_coded_pieces_relayed), 0); + assert_eq!(sum(&nodes, |m| &m.txset_coded_sets_rebuilt), 0); + for node in &nodes { + assert_eq!(node.state.txset_tree.lock().unwrap().coded.held_bytes(), 0); + } + + // The leader's set still arrives everywhere, the target relaying its + // piece as if nothing had happened. + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + for i in (0..NODES).filter(|i| *i != leader) { + expect_set(&mut nodes[i], hash).await; + } + settle().await; + assert_eq!( + nodes[target].count(|m| &m.txset_coded_pieces_relayed), + PIECES - 1 + ); + // (The attacker's own piece for the target is one frame too many now: + // the target makes do with the other seven.) + assert_eq!( + nodes[target].count(|m| &m.txset_coded_pieces_received), + PIECES - 1 + ); + assert_eq!( + sum(&nodes, |m| &m.txset_coded_sets_rebuilt), + NODES as u64 - 1 + ); + assert_eq!(sum(&nodes, |m| &m.txset_coded_sets_failed), 0); + assert_eq!(sum(&nodes, |m| &m.txset_coded_sets_inconsistent), 0); +} + +/// A leader that signs pieces which do not fit together (here: one parity +/// piece is not the code's) harms only the delivery of its own set: a node +/// that holds the data pieces files the set and counts the mismatch; a node +/// that needs the bad piece rebuilds bytes that are no tx set, counts that, +/// and files nothing. +#[tokio::test] +async fn pieces_that_do_not_fit_together_are_counted() { + let mut nodes = coded_mesh(27501).await; + for node in &nodes { + node.handle.set_txset_tree(coded_config(&nodes)); + } + let leader = 1; + let layout = layout_of(&nodes, leader); + let (hash, data) = test_set(77); + let bad = pieces_of(&nodes[leader], &data, layout.params, |pieces| { + for byte in pieces[DATA].iter_mut() { + *byte ^= 0x5A; + } + }); + // Two nodes that relay pieces which are not sent here (6 and 7): what + // they get comes through relays, and they pass nothing on. + let with_data = index_of(&nodes, &layout.relays_of(7)[0]); + let with_parity = index_of(&nodes, &layout.relays_of(6)[0]); + + // The four data pieces: the set, and a root that is not the code's. + for index in 0..DATA { + deliver_piece(&nodes, &layout, &bad, index, with_data).await; + } + let (slot, _) = expect_set(&mut nodes[with_data], hash).await; + assert_eq!(slot, None); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(nodes[with_data].count(|m| &m.txset_coded_sets_rebuilt), 1); + assert_eq!( + nodes[with_data].count(|m| &m.txset_coded_sets_rebuilt_with_parity), + 0 + ); + assert_eq!( + nodes[with_data].count(|m| &m.txset_coded_sets_inconsistent), + 1 + ); + + // Three data pieces and the bad parity piece: no tx set. + for index in 1..=DATA { + deliver_piece(&nodes, &layout, &bad, index, with_parity).await; + } + let (received, _) = + set_events_within(&mut nodes[with_parity], Duration::from_millis(400)).await; + assert!(received.is_empty()); + assert_eq!( + nodes[with_parity].count(|m| &m.txset_coded_pieces_received), + DATA as u64 + ); + assert_eq!( + nodes[with_parity].count(|m| &m.txset_coded_pieces_invalid), + 0 + ); + assert_eq!(nodes[with_parity].count(|m| &m.txset_coded_sets_failed), 1); + assert_eq!(nodes[with_parity].count(|m| &m.txset_coded_sets_rebuilt), 0); +} + +/// The signatures of a piece: the validator vouches for its overlay's key, +/// that key for the root, the slot, the code and the length. +#[test] +fn a_piece_is_checked_against_the_leaders_node_id() { + use libp2p::identity::ed25519; + let validator = ed25519::Keypair::generate(); + let overlay = Keypair::generate_ed25519(); + let peer = overlay.public().to_peer_id(); + let identity = NodeIdentity { + network_id: NETWORK_ID, + node_id: validator.public().to_bytes(), + signature: validator + .sign(&node_id_binding(&NETWORK_ID, &peer)) + .try_into() + .unwrap(), + }; + let params = CodedParams { + data: 4, + pieces: 8, + relays: 1, + }; + let frame = vec![7u8; 1001]; + let pieces = txset_coded::encode_pieces(&frame, params).unwrap(); + let signed = sign_pieces(&overlay, &identity, SLOT, &SEED, params, 1001, pieces, 5).unwrap(); + let header = signed.headers[3].clone(); + assert!(verify_piece_signatures(&NETWORK_ID, &SEED, &header)); + // The path of every piece reaches the signed root. + for (i, piece) in signed.pieces.iter().enumerate() { + let h = &signed.headers[i]; + assert_eq!( + txset_coded::merkle_root_from(txset_coded::leaf_hash(i as u8, piece), i, &h.path), + h.root + ); + } + let wrong = |change: &dyn Fn(&mut PieceHeader)| { + let mut h = header.clone(); + change(&mut h); + !verify_piece_signatures(&NETWORK_ID, &SEED, &h) + }; + assert!(wrong(&|h| h.slot += 1)); + assert!(wrong(&|h| h.total_len += 1)); + assert!(wrong(&|h| h.data = 3)); + assert!(wrong(&|h| h.pieces = 9)); + assert!(wrong(&|h| h.root[0] ^= 1)); + assert!(wrong(&|h| h.root_sig[0] ^= 1)); + assert!(wrong(&|h| h.binding_sig[0] ^= 1)); + assert!(wrong( + &|h| h.leader = ed25519::Keypair::generate().public().to_bytes() + )); + assert!(wrong( + &|h| h.overlay_key = ed25519::Keypair::generate().public().to_bytes() + )); + assert!(!verify_piece_signatures(&[1u8; 32], &SEED, &header)); + // Another seed for the slot: another history of the network. + assert!(!verify_piece_signatures(&NETWORK_ID, &[4u8; 32], &header)); + // What is not signed: where the piece is in the set and how it came. + let mut other = header.clone(); + other.index = 5; + other.hop = 2; + other.sent_unix_ms = 99; + assert!(verify_piece_signatures(&NETWORK_ID, &SEED, &other)); +} + +/// A node the others do not expect to lead the slot (a leader of a later +/// round, or one with another view) does not cut its set into pieces, which +/// every node would refuse: it pushes the set to everyone, as a star. +#[tokio::test] +async fn a_leader_nobody_expects_pushes_to_everyone() { + let mut nodes = coded_mesh(27601).await; + let expected = nodes[3].node_id(); + for node in &nodes { + node.handle + .set_txset_tree(coded_config(&nodes).with_leaders(vec![Some(vec![expected])])); + } + let leader = 6; + let (hash, data) = test_set(78); + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + for i in (0..NODES).filter(|i| *i != leader) { + let (slot, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(slot, None); + assert_eq!(from, nodes[leader].peer_id); + } + assert_eq!(nodes[leader].count(|m| &m.txset_coded_sets_encoded), 0); + assert_eq!(nodes[leader].count(|m| &m.txset_coded_pieces_sent), 0); + assert_eq!( + nodes[leader].count(|m| &m.txset_push_sent), + NODES as u64 - 1 + ); + assert_eq!(sum(&nodes, |m| &m.txset_coded_pieces_refused), 0); + // The expected leader does code. + let (hash, data) = test_set(79); + nodes[3].handle.push_txset_for_slot(hash, data, SLOT).await; + for i in (0..NODES).filter(|i| *i != 3) { + expect_set(&mut nodes[i], hash).await; + } + // (A node has the set after four pieces: give the leader's other + // writes time to be counted.) + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(nodes[3].count(|m| &m.txset_coded_pieces_sent), PIECES); + assert_eq!(nodes[3].count(|m| &m.txset_push_sent), 0); +} + +/// A set rebuilt from pieces counts as received from the overlay whose key +/// signed its root, also at a node that got none of its pieces from the +/// leader: the leader cannot have a relay charged for its set by sending that +/// relay's piece last. +#[tokio::test] +async fn a_rebuilt_set_counts_as_its_leaders() { + let mut nodes = coded_mesh(27701).await; + for node in &nodes { + node.handle.set_txset_tree(coded_config(&nodes)); + } + let leader = 0; + let layout = layout_of(&nodes, leader); + let (hash, data) = test_set(80); + let signed = pieces_of(&nodes[leader], &data, layout.params, |_| {}); + // The relay of piece 7 gets four pieces, every one through a relay. + let target = index_of(&nodes, &layout.relays_of(7)[0]); + for index in 0..DATA { + deliver_piece(&nodes, &layout, &signed, index, target).await; + } + let (slot, from) = expect_set(&mut nodes[target], hash).await; + assert_eq!(slot, None); + assert_eq!(from, nodes[leader].peer_id); + assert_eq!(nodes[target].count(|m| &m.txset_coded_sets_rebuilt), 1); +} diff --git a/overlay/src/libp2p_overlay/txset_tree_tests.rs b/overlay/src/libp2p_overlay/txset_tree_tests.rs new file mode 100644 index 0000000000..1040d31ad8 --- /dev/null +++ b/overlay/src/libp2p_overlay/txset_tree_tests.rs @@ -0,0 +1,940 @@ +//! Tests for the spanning tree of the leader's tx-set push: several overlay +//! nodes on loopback, fully connected, each with a validator identity. + +use super::*; +use crate::flood::txset_tree::{children_of, parent_of, TREE_FETCH_GRACE}; +use libp2p::identity::ed25519; + +pub(super) const NETWORK_ID: [u8; 32] = [9; 32]; +pub(super) const SLOT: u32 = 40; +pub(super) const SEED: [u8; 32] = [3; 32]; + +pub(super) struct Node { + pub(super) handle: OverlayHandle, + pub(super) events: mpsc::UnboundedReceiver, + pub(super) state: Arc, + pub(super) metrics: Arc, + pub(super) peer_id: PeerId, + pub(super) port: u16, + pub(super) validator: ed25519::Keypair, + pub(super) _admissions: mpsc::UnboundedReceiver, +} + +impl Node { + pub(super) fn node_id(&self) -> NodeId { + self.validator.public().to_bytes() + } + + pub(super) fn count(&self, pick: impl Fn(&OverlayMetrics) -> &AtomicU64) -> u64 { + pick(&self.metrics).load(Ordering::Relaxed) + } +} + +pub(super) async fn start_node(port: u16) -> Node { + let keypair = Keypair::generate_ed25519(); + let peer_id = keypair.public().to_peer_id(); + let metrics = Arc::new(OverlayMetrics::new()); + let (handle, events, admissions, overlay) = + create_test_overlay(keypair, Arc::clone(&metrics)).unwrap(); + let state = Arc::clone(&overlay.state); + tokio::spawn(async move { overlay.run("127.0.0.1", port).await }); + tokio::time::sleep(Duration::from_millis(100)).await; + Node { + handle, + events, + state, + metrics, + peer_id, + port, + validator: ed25519::Keypair::generate(), + _admissions: admissions, + } +} + +/// `count` nodes, every pair connected, every node knowing which validator +/// each of its peers is. +pub(super) async fn full_mesh(first_port: u16, count: usize) -> Vec { + let mut nodes = Vec::new(); + for i in 0..count { + nodes.push(start_node(first_port + i as u16).await); + } + for a in 0..count { + for b in a + 1..count { + let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{}/quic-v1", nodes[b].port) + .parse() + .unwrap(); + nodes[a].handle.dial(addr).await; + } + } + for node in &nodes { + let signature: [u8; 64] = node + .validator + .sign(&node_id_binding(&NETWORK_ID, &node.peer_id)) + .try_into() + .unwrap(); + node.handle + .set_node_identity(NodeIdentity { + network_id: NETWORK_ID, + node_id: node.node_id(), + signature, + }) + .await; + } + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let mut ready = true; + for node in &nodes { + let streams = node.state.peer_streams.read().await.len(); + let identified = node.state.tx_router.lock().unwrap().identified_peer_count(); + ready &= streams == count - 1 && identified == count - 1; + } + if ready { + break; + } + assert!(Instant::now() < deadline, "the mesh did not form"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + // Let the SCP state exchange of the new connections settle. + tokio::time::sleep(Duration::from_millis(300)).await; + nodes +} + +pub(super) fn node_ids(nodes: &[Node]) -> Vec { + nodes.iter().map(Node::node_id).collect() +} + +pub(super) fn config(nodes: &[Node], mode: TreeMode, fanout: u8) -> TreeConfig { + TreeConfig::new(mode, fanout, vec![(SLOT, SEED)], node_ids(nodes)) +} + +pub(super) fn configure(nodes: &[Node], mode: TreeMode, fanout: u8) { + for node in nodes { + node.handle.set_txset_tree(config(nodes, mode, fanout)); + } +} + +pub(super) fn index_of(nodes: &[Node], id: &NodeId) -> usize { + nodes.iter().position(|n| n.node_id() == *id).unwrap() +} + +pub(super) fn test_set(seed: u8) -> ([u8; 32], Arc) { + let (hash, xdr) = test_txset_xdr(seed); + (hash, Arc::new(TxSetData::from_local(xdr).unwrap())) +} + +/// Wait for a `TxSetReceived` of `hash`; returns the slot it carries (None: +/// nobody on this node had asked) and the sender. +pub(super) async fn expect_set(node: &mut Node, hash: [u8; 32]) -> (Option, PeerId) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + match tokio::time::timeout_at(deadline, node.events.recv()).await { + Ok(Some(OverlayEvent::TxSetReceived { + hash: got, + slot, + from, + data, + })) if got == hash => { + assert_eq!(crate::xdr::sha256_hash(data.as_slice()), hash); + return (slot, from); + } + Ok(Some(_)) => {} + _ => panic!("no TxSetReceived for {:02x?}", &hash[..4]), + } + } +} + +/// Wait until a peer asks `node` for `hash`, and answer as the App does +/// (these tests have no App behind the overlay). Returns the requester. +pub(super) async fn serve_request( + node: &mut Node, + hash: [u8; 32], + data: &Arc, +) -> PeerId { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + match tokio::time::timeout_at(deadline, node.events.recv()).await { + Ok(Some(OverlayEvent::TxSetRequested { hash: got, from })) if got == hash => { + node.handle.send_txset(hash, Arc::clone(data), from).await; + return from; + } + Ok(Some(_)) => {} + _ => panic!("nobody asked for {:02x?}", &hash[..4]), + } + } +} + +/// The hashes of the sets received and of the sets peers asked for within +/// `window`. +pub(super) async fn set_events_within( + node: &mut Node, + window: Duration, +) -> (Vec<[u8; 32]>, Vec<[u8; 32]>) { + let deadline = tokio::time::Instant::now() + window; + let (mut received, mut requested) = (Vec::new(), Vec::new()); + loop { + match tokio::time::timeout_at(deadline, node.events.recv()).await { + Ok(Some(OverlayEvent::TxSetReceived { hash, .. })) => received.push(hash), + Ok(Some(OverlayEvent::TxSetRequested { hash, .. })) => requested.push(hash), + Ok(Some(_)) => {} + _ => return (received, requested), + } + } +} + +/// With a hash tree the leader writes its set to its children only, the +/// children pass it on, and every node receives it exactly once from its +/// parent in the tree, without asking. +#[tokio::test] +async fn a_set_reaches_every_node_down_the_hash_tree() { + let mut nodes = full_mesh(24201, 7).await; + configure(&nodes, TreeMode::Hash, 2); + let leader = 3; + let leader_id = nodes[leader].node_id(); + let order = config(&nodes, TreeMode::Hash, 2) + .hash_order(SLOT, &leader_id) + .unwrap(); + let (hash, data) = test_set(51); + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + + for i in 0..nodes.len() { + if i == leader { + continue; + } + let id = nodes[i].node_id(); + let parent = parent_of(&order, &id, 2).unwrap(); + let parent_peer = nodes[index_of(&nodes, &parent)].peer_id; + let (slot, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(slot, None, "node {i} had not asked"); + assert_eq!(from, parent_peer, "node {i} gets the set from its parent"); + } + // Give stray copies time to show up. + tokio::time::sleep(Duration::from_millis(300)).await; + // The leader wrote two copies; the two relays two each; nobody else any. + assert_eq!(nodes[leader].count(|m| &m.txset_tree_root_sent), 2); + assert_eq!(nodes[leader].count(|m| &m.txset_push_sent), 2); + assert_eq!(nodes[leader].count(|m| &m.txset_tree_relayed), 0); + let mut relayed = 0; + for i in 0..nodes.len() { + if i == leader { + continue; + } + let id = nodes[i].node_id(); + let children = children_of(&order, &id, 2).len() as u64; + assert_eq!( + nodes[i].count(|m| &m.txset_tree_relayed), + children, + "node {i}" + ); + relayed += children; + // One tagged frame each, none refused, none asked for. + assert_eq!(nodes[i].count(|m| &m.txset_tree_received), 1, "node {i}"); + assert_eq!(nodes[i].count(|m| &m.txset_tree_refused), 0); + assert_eq!(nodes[i].count(|m| &m.txset_tree_duplicate), 0); + assert_eq!(nodes[i].count(|m| &m.fetch_txset_count), 0); + assert_eq!(nodes[i].count(|m| &m.txset_tree_delay_count), 1); + assert_eq!(nodes[i].count(|m| &m.txset_tree_root_sent), 0); + } + assert_eq!(relayed, 4); + // Two frames came straight from the leader, four through a relay. + let hop1: u64 = nodes + .iter() + .map(|n| n.count(|m| &m.txset_tree_delay_hop1_count)) + .sum(); + let hop2: u64 = nodes + .iter() + .map(|n| n.count(|m| &m.txset_tree_delay_hop2_count)) + .sum(); + assert_eq!((hop1, hop2), (2, 4)); + // A second push of the same set goes nowhere (each peer gets a set once). + let (_, again) = test_set(51); + nodes[leader] + .handle + .push_txset_for_slot(hash, again, SLOT) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(nodes[leader].count(|m| &m.txset_tree_root_sent), 2); +} + +/// A node that expects the set through the tree does not ask for it when the +/// proposal names it; it asks after the wait if the set has not come (here: +/// its parent does not relay). +#[tokio::test] +async fn a_node_waits_for_the_tree_and_asks_if_it_fails() { + let mut nodes = full_mesh(24301, 7).await; + configure(&nodes, TreeMode::Hash, 2); + let leader = 0; + let leader_id = nodes[leader].node_id(); + let leader_peer = nodes[leader].peer_id; + let order = config(&nodes, TreeMode::Hash, 2) + .hash_order(SLOT, &leader_id) + .unwrap(); + // The first relay is switched off: it files the set and passes nothing on. + let dead = index_of(&nodes, &order[1]); + nodes[dead] + .handle + .set_txset_tree(config(&nodes, TreeMode::Off, 2)); + let starved: Vec = children_of(&order, &order[1], 2) + .iter() + .map(|id| index_of(&nodes, id)) + .collect(); + assert_eq!(starved.len(), 2); + let fed: Vec = children_of(&order, &order[2], 2) + .iter() + .map(|id| index_of(&nodes, id)) + .collect(); + + let (hash, data) = test_set(52); + // The proposal reaches every follower first (it names the hash and comes + // from the leader), then the leader pushes. + let asked_at = Instant::now(); + for i in starved.iter().chain(fed.iter()) { + nodes[*i] + .handle + .record_txset_source(hash, leader_peer) + .await; + nodes[*i].handle.fetch_txset(hash, SLOT).await; + // Core asks too; that changes nothing while the wait runs. + nodes[*i].handle.fetch_txset(hash, SLOT).await; + } + nodes[leader] + .handle + .push_txset_for_slot(hash, Arc::clone(&data), SLOT) + .await; + + // The children of the working relay get the set through the tree and + // never ask. + for i in &fed { + let (slot, _) = expect_set(&mut nodes[*i], hash).await; + assert_eq!(slot, None); + } + // The children of the dead relay ask the leader after the wait, and get + // the set as an answer. + let mut askers = vec![ + serve_request(&mut nodes[leader], hash, &data).await, + serve_request(&mut nodes[leader], hash, &data).await, + ]; + assert!(asked_at.elapsed() >= TREE_FETCH_GRACE); + askers.sort(); + let mut expected: Vec = starved.iter().map(|i| nodes[*i].peer_id).collect(); + expected.sort(); + assert_eq!(askers, expected); + for i in &starved { + let (slot, from) = expect_set(&mut nodes[*i], hash).await; + assert_eq!(slot, Some(SLOT), "node {i} had to ask"); + assert_eq!(from, leader_peer); + } + assert!(asked_at.elapsed() >= TREE_FETCH_GRACE); + tokio::time::sleep(Duration::from_millis(200)).await; + for i in &fed { + assert_eq!(nodes[*i].count(|m| &m.txset_tree_fetch_deferred), 1); + assert_eq!(nodes[*i].count(|m| &m.txset_tree_fetch_fallback), 0); + assert_eq!(nodes[*i].count(|m| &m.fetch_txset_count), 0); + } + for i in &starved { + assert_eq!(nodes[*i].count(|m| &m.txset_tree_fetch_deferred), 1); + assert_eq!(nodes[*i].count(|m| &m.txset_tree_fetch_fallback), 1); + assert_eq!(nodes[*i].count(|m| &m.fetch_txset_count), 1); + } + // Nobody else asked the leader. + let (_, requested) = set_events_within(&mut nodes[leader], Duration::from_millis(200)).await; + assert!(requested.is_empty()); + // The switched-off relay got its copy and relayed nothing. + assert_eq!(nodes[dead].count(|m| &m.txset_tree_received), 1); + assert_eq!(nodes[dead].count(|m| &m.txset_tree_relayed), 0); +} + +/// A node passes on only what its parent in the tree sent it: the same frame +/// from another validator is filed and not relayed. +#[tokio::test] +async fn a_frame_from_a_node_that_is_not_the_parent_is_not_relayed() { + let mut nodes = full_mesh(24401, 7).await; + configure(&nodes, TreeMode::Hash, 2); + let leader = 0; + let leader_id = nodes[leader].node_id(); + let order = config(&nodes, TreeMode::Hash, 2) + .hash_order(SLOT, &leader_id) + .unwrap(); + let relay = index_of(&nodes, &order[1]); + let child = index_of(&nodes, &children_of(&order, &order[1], 2)[0]); + // A leaf of the other relay sends the relay a frame that claims to be + // the leader's tree frame for the slot. + let impostor = index_of(&nodes, &children_of(&order, &order[2], 2)[0]); + let (hash, data) = test_set(53); + let header = TreeHeader { + kind: FrameKind::Hash, + fanout: 2, + hop: 1, + slot: SLOT, + leader: leader_id, + sent_unix_ms: unix_ms_now(), + order: Vec::new(), + nodes_digest: [0; txset_tree::TREE_NODES_DIGEST], + } + .encode(); + send_to_peer_stream_parts( + &nodes[impostor].state, + nodes[relay].peer_id, + StreamType::TxSet, + &[header.as_slice(), data.encoded()], + ) + .await + .unwrap(); + let (slot, from) = expect_set(&mut nodes[relay], hash).await; + assert_eq!((slot, from), (None, nodes[impostor].peer_id)); + let (received, _) = set_events_within(&mut nodes[child], Duration::from_millis(400)).await; + assert!( + received.is_empty(), + "the relay passed the impostor's frame on" + ); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_refused), 1); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relayed), 0); + + // The same holds for a slot the relay's Core has not described, also + // from the true parent. + let (hash2, data2) = test_set(54); + let header = TreeHeader { + kind: FrameKind::Hash, + fanout: 2, + hop: 1, + slot: SLOT + 5, + leader: leader_id, + sent_unix_ms: unix_ms_now(), + order: Vec::new(), + nodes_digest: [0; txset_tree::TREE_NODES_DIGEST], + } + .encode(); + send_to_peer_stream_parts( + &nodes[leader].state, + nodes[relay].peer_id, + StreamType::TxSet, + &[header.as_slice(), data2.encoded()], + ) + .await + .unwrap(); + expect_set(&mut nodes[relay], hash2).await; + let (received, _) = set_events_within(&mut nodes[child], Duration::from_millis(400)).await; + assert!(received.is_empty()); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_refused), 2); + + // A frame with a broken header is dropped whole. + let mut broken = header.clone(); + broken[4] = 77; + send_to_peer_stream_parts( + &nodes[leader].state, + nodes[relay].peer_id, + StreamType::TxSet, + &[broken.as_slice(), data2.encoded()], + ) + .await + .unwrap(); + let (received, _) = set_events_within(&mut nodes[relay], Duration::from_millis(400)).await; + assert!(received.is_empty()); +} + +/// Star mode: the leader writes to every peer, with a header that says when; +/// nobody relays and nobody waits before asking. A slot whose tree the leader +/// does not know is pushed the same way in hash mode. +#[tokio::test] +async fn star_mode_and_an_unknown_slot_push_to_every_peer() { + let mut nodes = full_mesh(24501, 5).await; + configure(&nodes, TreeMode::Star, 0); + let (hash, data) = test_set(55); + nodes[1].handle.push_txset_for_slot(hash, data, SLOT).await; + let leader_peer = nodes[1].peer_id; + for i in [0usize, 2, 3, 4] { + let (slot, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!((slot, from), (None, leader_peer)); + assert_eq!(nodes[i].count(|m| &m.txset_tree_received), 1); + assert_eq!(nodes[i].count(|m| &m.txset_tree_delay_hop1_count), 1); + assert_eq!(nodes[i].count(|m| &m.txset_tree_relayed), 0); + assert_eq!(nodes[i].count(|m| &m.txset_tree_refused), 0); + } + assert_eq!(nodes[1].count(|m| &m.txset_tree_root_sent), 4); + assert!(nodes[1].count(|m| &m.txset_push_sent_bytes) > 0); + // No wait in star mode: a request goes out at once. + let (other, _) = test_set(56); + nodes[0].handle.fetch_txset(other, SLOT).await; + assert_eq!(nodes[0].count(|m| &m.txset_tree_fetch_deferred), 0); + + // Hash mode, a slot without a tree: a star as well. + configure(&nodes, TreeMode::Hash, 2); + let (hash, data) = test_set(57); + nodes[2] + .handle + .push_txset_for_slot(hash, data, SLOT + 9) + .await; + for i in [0usize, 1, 3, 4] { + let (_, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(from, nodes[2].peer_id); + } + assert_eq!(nodes[2].count(|m| &m.txset_tree_root_sent), 4); + // Without a slot (callers that do not know it) the frame is the old one. + let (hash, data) = test_set(58); + nodes[3].handle.push_txset(hash, data).await; + for i in [0usize, 1, 2, 4] { + expect_set(&mut nodes[i], hash).await; + } + assert_eq!(nodes[3].count(|m| &m.txset_tree_root_sent), 0); + assert_eq!(nodes[3].count(|m| &m.txset_push_sent), 4); +} + +/// Rtt mode: the leader lays the tree out by its round-trip times and sends +/// the order along; the other nodes follow it. +#[tokio::test] +async fn an_order_chosen_by_the_leader_is_followed() { + let mut nodes = full_mesh(24601, 7).await; + configure(&nodes, TreeMode::Rtt, 2); + let leader = 6; + // Two places: nodes 0..2 near (1 ms), nodes 3..5 far (80 ms). + { + let mut tree = nodes[leader].state.txset_tree.lock().unwrap(); + // The times are set by hand; real probes on loopback would be + // shorter and take their place. + tree.pause_probing(); + for i in 0..6 { + tree.forget_rtt(&nodes[i].node_id()); + } + for i in 0..6 { + let rtt = if i < 3 { 1 } else { 80 }; + tree.note_rtt( + nodes[i].node_id(), + Duration::from_millis(rtt + i as u64 % 2), + ); + } + } + let (hash, data) = test_set(59); + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + let mut from_leader = Vec::new(); + let mut parent_of_node = HashMap::new(); + for i in 0..6 { + let (slot, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(slot, None); + if from == nodes[leader].peer_id { + from_leader.push(i); + } + parent_of_node.insert(i, from); + } + tokio::time::sleep(Duration::from_millis(300)).await; + // One relay per place, and each relay's children are in its own place. + assert_eq!(from_leader.len(), 2); + assert!(from_leader.iter().any(|i| *i < 3)); + assert!(from_leader.iter().any(|i| *i >= 3)); + for i in 0..6 { + if from_leader.contains(&i) { + continue; + } + let parent = parent_of_node[&i]; + let parent_index = nodes.iter().position(|n| n.peer_id == parent).unwrap(); + assert_eq!( + parent_index < 3, + i < 3, + "node {i} sits under node {parent_index}" + ); + } + assert_eq!(nodes[leader].count(|m| &m.txset_tree_root_sent), 2); + let relayed: u64 = nodes + .iter() + .map(|n| n.count(|m| &m.txset_tree_relayed)) + .sum(); + assert_eq!(relayed, 4); + for node in &nodes { + assert_eq!(node.count(|m| &m.txset_tree_refused), 0); + } +} + +/// In rtt mode a node measures the round trip to its identified peers. +#[tokio::test] +async fn round_trips_are_measured_in_rtt_mode_only() { + let nodes = full_mesh(24801, 3).await; + tokio::time::sleep(Duration::from_millis(700)).await; + for node in &nodes { + let tree = node.state.txset_tree.lock().unwrap(); + for other in &nodes { + assert_eq!(tree.rtt_of(&other.node_id()), None); + } + } + configure(&nodes, TreeMode::Rtt, 0); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let mut measured = 0; + for node in &nodes { + let tree = node.state.txset_tree.lock().unwrap(); + for other in &nodes { + if let Some(rtt) = tree.rtt_of(&other.node_id()) { + assert!(rtt < Duration::from_secs(1)); + measured += 1; + } + } + } + if measured == 6 { + break; + } + assert!( + Instant::now() < deadline, + "only {measured} of 6 round trips measured" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + // Probes are not sets: nothing was received, dropped or refused. + for node in &nodes { + assert_eq!(node.count(|m| &m.txset_tree_received), 0); + assert_eq!(node.count(|m| &m.txset_push_dropped), 0); + assert_eq!(node.count(|m| &m.txset_push_received), 0); + } +} + +/// The cached frame of a set with a tree header in front, sent by `from` to +/// `to` as if `from` had pushed it. +async fn send_tagged(from: &Node, to: &Node, header: &TreeHeader, data: &Arc) { + let header = header.encode(); + send_to_peer_stream_parts( + &from.state, + to.peer_id, + StreamType::TxSet, + &[header.as_slice(), data.encoded()], + ) + .await + .unwrap(); +} + +/// Only a node that this node's Core expects to lead the slot can root a +/// tree: a frame that names another validator as leader is filed and not +/// passed on, also when it comes from that validator and the receiver is +/// its child in the tree it claims. +#[tokio::test] +async fn a_validator_that_does_not_lead_the_slot_cannot_root_a_tree() { + let mut nodes = full_mesh(24901, 7).await; + let true_leader = nodes[0].node_id(); + for node in &nodes { + node.handle.set_txset_tree( + config(&nodes, TreeMode::Hash, 2).with_leaders(vec![Some(vec![true_leader])]), + ); + } + // Node 5 roots a tree at itself and pushes down it. + let pretender = 5; + let pretender_id = nodes[pretender].node_id(); + let order = config(&nodes, TreeMode::Hash, 2) + .hash_order(SLOT, &pretender_id) + .unwrap(); + let relay = index_of(&nodes, &order[1]); + let child = index_of(&nodes, &children_of(&order, &order[1], 2)[0]); + let (hash, data) = test_set(61); + let header = TreeHeader { + kind: FrameKind::Hash, + fanout: 2, + hop: 1, + slot: SLOT, + leader: pretender_id, + sent_unix_ms: unix_ms_now(), + order: Vec::new(), + nodes_digest: [0; txset_tree::TREE_NODES_DIGEST], + }; + send_tagged(&nodes[pretender], &nodes[relay], &header, &data).await; + expect_set(&mut nodes[relay], hash).await; + let (received, _) = set_events_within(&mut nodes[child], Duration::from_millis(400)).await; + assert!( + received.is_empty(), + "a tree rooted at a non-leader was relayed" + ); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_refused), 1); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relayed), 0); + // The real leader's push still travels down its tree. + let (hash, data) = test_set(62); + nodes[0].handle.push_txset_for_slot(hash, data, SLOT).await; + for i in 1..7 { + expect_set(&mut nodes[i], hash).await; + } + let relayed: u64 = nodes + .iter() + .map(|n| n.count(|m| &m.txset_tree_relayed)) + .sum(); + assert_eq!(relayed, 4); +} + +/// A node in hash mode does not follow an order a sender chose, and a node +/// in rtt mode does not follow one that makes a chain. +#[tokio::test] +async fn an_order_from_the_sender_is_followed_in_rtt_mode_and_two_levels_only() { + let mut nodes = full_mesh(25001, 7).await; + configure(&nodes, TreeMode::Hash, 2); + let cfg = config(&nodes, TreeMode::Hash, 2); + let sender = 0; + // The sender first, then the others as they come: node 1 would relay to + // nodes 3 and 4. + let order: Vec = node_ids(&nodes); + let mut header = TreeHeader { + kind: FrameKind::Explicit, + fanout: 2, + hop: 1, + slot: SLOT, + leader: nodes[sender].node_id(), + sent_unix_ms: unix_ms_now(), + order: cfg.indices_of(&order).unwrap(), + nodes_digest: cfg.nodes_digest(), + }; + let (hash, data) = test_set(63); + send_tagged(&nodes[sender], &nodes[1], &header, &data).await; + expect_set(&mut nodes[1], hash).await; + let (received, _) = set_events_within(&mut nodes[3], Duration::from_millis(400)).await; + assert!( + received.is_empty(), + "a node in hash mode followed a sender's order" + ); + assert_eq!(nodes[1].count(|m| &m.txset_tree_refused), 1); + + // In rtt mode the same order is followed ... + configure(&nodes, TreeMode::Rtt, 2); + let (hash, data) = test_set(64); + send_tagged(&nodes[sender], &nodes[1], &header, &data).await; + expect_set(&mut nodes[1], hash).await; + expect_set(&mut nodes[3], hash).await; + expect_set(&mut nodes[4], hash).await; + assert_eq!(nodes[1].count(|m| &m.txset_tree_relayed), 2); + // ... but not as a chain (fanout 1: six levels). + header.fanout = 1; + let (hash, data) = test_set(65); + send_tagged(&nodes[sender], &nodes[1], &header, &data).await; + expect_set(&mut nodes[1], hash).await; + let (received, _) = set_events_within(&mut nodes[2], Duration::from_millis(400)).await; + assert!(received.is_empty(), "a chain was followed"); + assert_eq!(nodes[1].count(|m| &m.txset_tree_refused), 2); + assert_eq!(nodes[1].count(|m| &m.txset_tree_relayed), 2); +} + +/// A relay passes on two different frames of a slot and leader, not a third: +/// a parent cannot use it to multiply traffic by varying the bytes. +#[tokio::test] +async fn a_relay_passes_on_two_frames_per_slot_and_leader() { + let mut nodes = full_mesh(25101, 7).await; + configure(&nodes, TreeMode::Hash, 2); + let leader = 0; + let leader_id = nodes[leader].node_id(); + let order = config(&nodes, TreeMode::Hash, 2) + .hash_order(SLOT, &leader_id) + .unwrap(); + let relay = index_of(&nodes, &order[1]); + let child = index_of(&nodes, &children_of(&order, &order[1], 2)[0]); + let header = TreeHeader { + kind: FrameKind::Hash, + fanout: 2, + hop: 1, + slot: SLOT, + leader: leader_id, + sent_unix_ms: unix_ms_now(), + order: Vec::new(), + nodes_digest: [0; txset_tree::TREE_NODES_DIGEST], + }; + let both: Vec = children_of(&order, &order[1], 2) + .iter() + .map(|id| index_of(&nodes, id)) + .collect(); + // Two different frames, one after the other (a relay has one frame in + // flight per child): both are passed on to both children. + for seed in [66u8, 67] { + let (hash, data) = test_set(seed); + send_tagged(&nodes[leader], &nodes[relay], &header, &data).await; + expect_set(&mut nodes[relay], hash).await; + for i in &both { + let (_, from) = expect_set(&mut nodes[*i], hash).await; + assert_eq!(from, nodes[relay].peer_id); + } + } + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relayed), 4); + // A third one is filed by the relay and goes no further. + let (hash, data) = test_set(68); + send_tagged(&nodes[leader], &nodes[relay], &header, &data).await; + expect_set(&mut nodes[relay], hash).await; + let (received, _) = set_events_within(&mut nodes[child], Duration::from_millis(500)).await; + assert!( + received.is_empty(), + "a third frame of the slot and leader was relayed" + ); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relayed), 4); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relay_limited), 1); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relay_busy), 0); + // A second copy of a frame it relayed is a duplicate, not a third frame. + let (_, again) = test_set(66); + send_tagged(&nodes[leader], &nodes[relay], &header, &again).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(nodes[relay].count(|m| &m.txset_tree_duplicate), 1); + assert_eq!(nodes[relay].count(|m| &m.txset_tree_relayed), 4); +} + +/// If the leader cannot send a child the set now (an earlier push to it is +/// still in flight), the nodes below that child get it from the leader +/// directly. +#[tokio::test] +async fn the_nodes_below_a_busy_child_get_the_set_from_the_leader() { + let mut nodes = full_mesh(25201, 7).await; + configure(&nodes, TreeMode::Hash, 2); + let leader = 2; + let leader_id = nodes[leader].node_id(); + let leader_peer = nodes[leader].peer_id; + let order = config(&nodes, TreeMode::Hash, 2) + .hash_order(SLOT, &leader_id) + .unwrap(); + let busy_child = index_of(&nodes, &order[1]); + let below: Vec = children_of(&order, &order[1], 2) + .iter() + .map(|id| index_of(&nodes, id)) + .collect(); + // An earlier push to that child is still in flight. + let (earlier, _) = test_set(69); + let started = nodes[leader] + .state + .txset_pushes + .lock() + .unwrap() + .start_push(earlier, vec![nodes[busy_child].peer_id]); + assert_eq!(started.len(), 1); + + let (hash, data) = test_set(70); + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + for i in &below { + let (slot, from) = expect_set(&mut nodes[*i], hash).await; + assert_eq!((slot, from), (None, leader_peer), "node {i}"); + } + // The other child and its subtree are served through the tree. + let other = index_of(&nodes, &order[2]); + expect_set(&mut nodes[other], hash).await; + for id in children_of(&order, &order[2], 2) { + let i = index_of(&nodes, &id); + let (_, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(from, nodes[other].peer_id); + } + // The busy child itself got nothing (it asks when it sees the proposal). + let (received, _) = set_events_within(&mut nodes[busy_child], Duration::from_millis(300)).await; + assert!(received.is_empty()); + assert_eq!(nodes[leader].count(|m| &m.txset_tree_child_busy), 1); + // One child and the two nodes below the busy one. + assert_eq!(nodes[leader].count(|m| &m.txset_tree_root_sent), 3); +} + +/// An answer to a probe this node did not send is not a measurement. +#[tokio::test] +async fn an_unsolicited_probe_answer_is_ignored() { + let nodes = full_mesh(25301, 2).await; + configure(&nodes, TreeMode::Rtt, 0); + nodes[0].state.txset_tree.lock().unwrap().pause_probing(); + nodes[1].state.txset_tree.lock().unwrap().pause_probing(); + tokio::time::sleep(Duration::from_millis(300)).await; + { + let mut tree = nodes[0].state.txset_tree.lock().unwrap(); + tree.forget_rtt(&nodes[1].node_id()); + } + // Node 1 claims a round trip of nothing: an answer "sent" in the future. + for value in [u64::MAX, 0, monotonic_micros()] { + let pong = Probe::Pong(value).encode(); + send_to_peer_stream(&nodes[1].state, nodes[0].peer_id, StreamType::TxSet, &pong) + .await + .unwrap(); + } + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + nodes[0] + .state + .txset_tree + .lock() + .unwrap() + .rtt_of(&nodes[1].node_id()), + None + ); +} + +/// A node whose mode is off still reads tagged frames (and files the set); +/// it relays nothing and asks at once. +#[tokio::test] +async fn a_node_without_the_tree_reads_tagged_frames() { + let mut nodes = full_mesh(24701, 3).await; + nodes[0] + .handle + .set_txset_tree(config(&nodes, TreeMode::Hash, 1)); + // Nodes 1 and 2 were never told about trees. + let order = config(&nodes, TreeMode::Hash, 1) + .hash_order(SLOT, &nodes[0].node_id()) + .unwrap(); + let first = index_of(&nodes, &order[1]); + let second = index_of(&nodes, &order[2]); + let (hash, data) = test_set(60); + nodes[0] + .handle + .push_txset_for_slot(hash, Arc::clone(&data), SLOT) + .await; + let (slot, _) = expect_set(&mut nodes[first], hash).await; + assert_eq!(slot, None); + assert_eq!(nodes[first].count(|m| &m.txset_tree_received), 1); + assert_eq!(nodes[first].count(|m| &m.txset_tree_relayed), 0); + // The second node is not reached by the tree and asks without waiting. + let (received, _) = set_events_within(&mut nodes[second], Duration::from_millis(300)).await; + assert!(received.is_empty()); + let leader_peer = nodes[0].peer_id; + nodes[second] + .handle + .record_txset_source(hash, leader_peer) + .await; + let asked_at = Instant::now(); + nodes[second].handle.fetch_txset(hash, SLOT).await; + assert_eq!( + serve_request(&mut nodes[0], hash, &data).await, + nodes[second].peer_id + ); + let (slot, from) = expect_set(&mut nodes[second], hash).await; + assert_eq!((slot, from), (Some(SLOT), leader_peer)); + assert!(asked_at.elapsed() < TREE_FETCH_GRACE); + assert_eq!(nodes[second].count(|m| &m.txset_tree_fetch_deferred), 0); +} + +/// A node that the others do not expect to lead the slot does not root a +/// tree, whose frames every relay would refuse: it pushes its set to every +/// peer as a star. +#[tokio::test] +async fn a_leader_nobody_expects_pushes_a_star() { + let mut nodes = full_mesh(25601, 7).await; + let expected = nodes[2].node_id(); + for node in &nodes { + node.handle.set_txset_tree( + config(&nodes, TreeMode::Hash, 2).with_leaders(vec![Some(vec![expected])]), + ); + } + let leader = 5; + let (hash, data) = test_set(61); + nodes[leader] + .handle + .push_txset_for_slot(hash, data, SLOT) + .await; + let leader_peer = nodes[leader].peer_id; + for i in (0..nodes.len()).filter(|i| *i != leader) { + let (slot, from) = expect_set(&mut nodes[i], hash).await; + assert_eq!(slot, None); + assert_eq!(from, leader_peer, "node {i} gets it from the leader itself"); + } + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(nodes[leader].count(|m| &m.txset_push_sent), 6); + let sum = |pick: fn(&OverlayMetrics) -> &AtomicU64| -> u64 { + nodes.iter().map(|n| n.count(pick)).sum() + }; + assert_eq!(sum(|m| &m.txset_tree_relayed), 0); + assert_eq!(sum(|m| &m.txset_tree_refused), 0); + // The expected leader roots its tree: two copies, four relayed. + let (hash, data) = test_set(62); + nodes[2].handle.push_txset_for_slot(hash, data, SLOT).await; + for i in (0..nodes.len()).filter(|i| *i != 2) { + expect_set(&mut nodes[i], hash).await; + } + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(nodes[2].count(|m| &m.txset_push_sent), 2); + let relayed: u64 = nodes + .iter() + .map(|n| n.count(|m| &m.txset_tree_relayed)) + .sum(); + assert_eq!(relayed, 4); +} diff --git a/overlay/src/main.rs b/overlay/src/main.rs index 7b542a118b..dc04eb140e 100644 --- a/overlay/src/main.rs +++ b/overlay/src/main.rs @@ -23,6 +23,7 @@ use stellar_overlay::config::Config; use stellar_overlay::flood::{ CachedTxSet, Hash256, TxSetCache, TxSetData, TxSetStore, UnclaimedSetBudget, }; +use stellar_overlay::flood::txset_tree::{TreeConfig, TreeMode, MAX_TREE_NODES}; use stellar_overlay::integrated::{Overlay, OverlayHandle}; use stellar_overlay::ipc::{CoreIpc, Message, MessageType}; use stellar_overlay::libp2p_overlay::{ @@ -1405,7 +1406,7 @@ impl App { // Spawn so the main loop never awaits on the bounded libp2p // command channel. tokio::spawn(async move { - handle.push_txset(hash, data).await; + handle.push_txset_for_slot(hash, data, slot).await; }); } @@ -1490,6 +1491,28 @@ impl App { } } + MessageType::SetTxSetTree => { + let Some(config) = parse_txset_tree(&msg.payload) else { + warn!( + "SetTxSetTree payload malformed ({} bytes)", + msg.payload.len() + ); + return true; + }; + debug!( + "TXSET_TREE: mode {:?}, fanout {}, {} nodes, slots {:?}", + config.mode, + config.effective_fanout(), + config.nodes.len(), + config + .slots + .iter() + .map(|(slot, _)| *slot) + .collect::>() + ); + self.libp2p_handle.set_txset_tree(config); + } + MessageType::RequestScpState => { // Core is asking us to request SCP state from peers // Payload is ledger sequence (u32, 4 bytes) @@ -1833,6 +1856,58 @@ fn parse_tx_targets(payload: &[u8]) -> Option>> { Some(Some(nodes)) } +/// Parse a `SetTxSetTree` payload: +/// `[mode:1][fanout:1][slots:1]([slot:4 LE][seed:32][leaders:1][nodeID:32]...)...[count:4 LE][nodeID:32]...` +/// (`leaders` = 255: any node of the tree may lead the slot). +fn parse_txset_tree(payload: &[u8]) -> Option { + if payload.len() < 3 { + return None; + } + // The top bit of the mode byte: this node withholds what it should + // relay (EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING). + let mode = TreeMode::from_u8(payload[0] & 0x7F)?; + let withhold = payload[0] & 0x80 != 0; + let fanout = payload[1]; + let slot_count = payload[2] as usize; + let mut at = 3; + let mut slots = Vec::with_capacity(slot_count); + let mut leaders = Vec::with_capacity(slot_count); + for _ in 0..slot_count { + let entry = payload.get(at..at + 37)?; + let slot = u32::from_le_bytes(entry[..4].try_into().unwrap()); + let mut seed = [0u8; 32]; + seed.copy_from_slice(&entry[4..36]); + let leader_count = entry[36]; + at += 37; + slots.push((slot, seed)); + if leader_count == u8::MAX { + leaders.push(None); + continue; + } + let ids = payload.get(at..at + leader_count as usize * 32)?; + at += ids.len(); + leaders.push(Some( + ids.chunks_exact(32) + .map(|chunk| chunk.try_into().unwrap()) + .collect(), + )); + } + let count = u32::from_le_bytes(payload.get(at..at + 4)?.try_into().unwrap()) as usize; + at += 4; + if count > MAX_TREE_NODES || payload.len() != at + count.checked_mul(32)? { + return None; + } + let nodes = payload[at..] + .chunks_exact(32) + .map(|chunk| chunk.try_into().unwrap()) + .collect(); + Some( + TreeConfig::new(mode, fanout, slots, nodes) + .with_leaders(leaders) + .with_withhold(withhold), + ) +} + fn setup_logging(level: &str) { use tracing_subscriber::{fmt, EnvFilter}; @@ -3287,6 +3362,65 @@ mod tests { assert!(app.pending_core_tx_sets.contains_key(&sets[0].0)); } + /// The layout of `SetTxSetTree`, as Core writes it + /// (`OverlayIPC::setTxSetTree`). + #[test] + fn set_txset_tree_payload_is_parsed() { + let mut payload = vec![2u8, 9, 2]; + // Slot 40: seed 3s, two leaders. + payload.extend_from_slice(&40u32.to_le_bytes()); + payload.extend_from_slice(&[3u8; 32]); + payload.push(2); + payload.extend_from_slice(&[7u8; 32]); + payload.extend_from_slice(&[8u8; 32]); + // Slot 41: seed 4s, any leader. + payload.extend_from_slice(&41u32.to_le_bytes()); + payload.extend_from_slice(&[4u8; 32]); + payload.push(255); + // Three nodes, not sorted, one twice. + payload.extend_from_slice(&4u32.to_le_bytes()); + for id in [9u8, 7, 8, 7] { + payload.extend_from_slice(&[id; 32]); + } + let config = parse_txset_tree(&payload).unwrap(); + assert_eq!(config.mode, TreeMode::Hash); + assert_eq!(config.fanout, 9); + assert_eq!(config.slots, vec![(40, [3u8; 32]), (41, [4u8; 32])]); + assert_eq!(config.leaders, vec![Some(vec![[7u8; 32], [8u8; 32]]), None]); + assert_eq!(config.nodes, vec![[7u8; 32], [8u8; 32], [9u8; 32]]); + assert!(config.may_lead(40, &[8u8; 32])); + assert!(!config.may_lead(40, &[9u8; 32])); + assert!(config.may_lead(41, &[9u8; 32])); + + // Truncated anywhere, an unknown mode, trailing bytes. + for cut in 0..payload.len() { + assert!(parse_txset_tree(&payload[..cut]).is_none(), "cut at {cut}"); + } + let mut bad = payload.clone(); + bad[0] = 5; + assert!(parse_txset_tree(&bad).is_none()); + assert!(!config.withhold); + // Coded mode, and the bit that makes a node withhold its relays. + let mut coded = payload.clone(); + coded[0] = 4; + let coded = parse_txset_tree(&coded).unwrap(); + assert_eq!((coded.mode, coded.withhold), (TreeMode::Coded, false)); + let mut withholding = payload.clone(); + withholding[0] = 0x80 | 2; + let withholding = parse_txset_tree(&withholding).unwrap(); + assert_eq!( + (withholding.mode, withholding.withhold), + (TreeMode::Hash, true) + ); + let mut bad = payload.clone(); + bad.push(0); + assert!(parse_txset_tree(&bad).is_none()); + // No slots and no nodes is a valid (empty) description. + let empty = parse_txset_tree(&[0, 0, 0, 0, 0, 0, 0]).unwrap(); + assert_eq!(empty.mode, TreeMode::Off); + assert!(empty.slots.is_empty() && empty.nodes.is_empty()); + } + /// PushTxSet for a set the overlay does not hold, or with a short /// payload, is ignored: no crash, no shutdown, nothing sent to Core. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/overlay/src/metrics.rs b/overlay/src/metrics.rs index 2b2555baa7..57658cebe6 100644 --- a/overlay/src/metrics.rs +++ b/overlay/src/metrics.rs @@ -111,6 +111,90 @@ pub struct OverlayMetrics { /// overlay.txset-push.evicted — unrequested tx sets evicted by the unclaimed-set limits pub txset_push_evicted: AtomicU64, + // Spanning tree for the leader's tx-set push + /// overlay.txset-push.sent-bytes — bytes of tx sets written to peers as round leader (pushes, tagged or not) + pub txset_push_sent_bytes: AtomicU64, + /// overlay.txset-tree.root-sent — tagged copies of a set written as its leader (star or tree) + pub txset_tree_root_sent: AtomicU64, + /// overlay.txset-tree.relayed — copies of a set written as a relay of its tree + pub txset_tree_relayed: AtomicU64, + /// overlay.txset-tree.relayed-bytes — bytes written as a relay + pub txset_tree_relayed_bytes: AtomicU64, + /// overlay.txset-tree.received — tagged frames received + pub txset_tree_received: AtomicU64, + /// overlay.txset-tree.duplicate — tagged frames that were a second copy (not relayed again) + pub txset_tree_duplicate: AtomicU64, + /// overlay.txset-tree.refused — tree frames not relayed: unknown slot or nodes, or not from the parent + pub txset_tree_refused: AtomicU64, + /// overlay.txset-tree.unreachable — children a set could not be sent to (no identified connection) + pub txset_tree_unreachable: AtomicU64, + /// overlay.txset-tree.child-busy — children the leader could not send a set to because an earlier push to them was in flight (everyone below them gets it from the leader directly) + pub txset_tree_child_busy: AtomicU64, + /// overlay.txset-tree.relay-busy — children a relay did not pass a frame on to because an earlier relay to them was in flight + pub txset_tree_relay_busy: AtomicU64, + /// overlay.txset-tree.relay-limited — frames not relayed because this node relayed two for that slot and leader already + pub txset_tree_relay_limited: AtomicU64, + /// overlay.txset-tree.fetch-deferred — requests for a set put off because the tree should bring it + pub txset_tree_fetch_deferred: AtomicU64, + /// overlay.txset-tree.fetch-fallback — sets asked for after the wait, because the tree did not bring them + pub txset_tree_fetch_fallback: AtomicU64, + /// overlay.txset-tree.delay-ms-sum — sum of the delays from the leader's send to the tagged frame's arrival here, by the two nodes' clocks (a set that came by request, or not at all, is not in it) + pub txset_tree_delay_ms_sum: AtomicU64, + /// overlay.txset-tree.delay-count — frames in delay-ms-sum + pub txset_tree_delay_count: AtomicU64, + /// overlay.txset-tree.delay-hop1-ms-sum — the same for frames straight from the leader + pub txset_tree_delay_hop1_ms_sum: AtomicU64, + /// overlay.txset-tree.delay-hop1-count — frames in delay-hop1-ms-sum + pub txset_tree_delay_hop1_count: AtomicU64, + /// overlay.txset-tree.delay-hop2-ms-sum — the same for frames that came through a relay + pub txset_tree_delay_hop2_ms_sum: AtomicU64, + /// overlay.txset-tree.delay-hop2-count — frames in delay-hop2-ms-sum + pub txset_tree_delay_hop2_count: AtomicU64, + /// overlay.txset-tree.delay-le-50ms — frames that arrived 0 to 50 ms after the leader's send (the delay-* buckets are disjoint ranges, not cumulative) + pub txset_tree_delay_le_50: AtomicU64, + /// overlay.txset-tree.delay-le-100ms — 51 to 100 ms after + pub txset_tree_delay_le_100: AtomicU64, + /// overlay.txset-tree.delay-le-150ms — 101 to 150 ms after + pub txset_tree_delay_le_150: AtomicU64, + /// overlay.txset-tree.delay-le-200ms — 151 to 200 ms after + pub txset_tree_delay_le_200: AtomicU64, + /// overlay.txset-tree.delay-le-300ms — 201 to 300 ms after + pub txset_tree_delay_le_300: AtomicU64, + /// overlay.txset-tree.delay-gt-300ms — later than 300 ms + pub txset_tree_delay_gt_300: AtomicU64, + + // Erasure-coded pieces of the leader's tx set (mode `coded`) + /// overlay.txset-tree.withheld — relays this node did not make because it is configured to withhold them (testing) + pub txset_tree_withheld: AtomicU64, + /// overlay.txset-coded.sets-encoded — sets this node cut into coded pieces as their leader + pub txset_coded_sets_encoded: AtomicU64, + /// overlay.txset-coded.pieces-sent — pieces written as the set's leader + pub txset_coded_pieces_sent: AtomicU64, + /// overlay.txset-coded.pieces-relayed — pieces passed on as a relay + pub txset_coded_pieces_relayed: AtomicU64, + /// overlay.txset-coded.pieces-received — valid pieces received (signed by an allowed leader, from the sender that owes them, with a Merkle path that verifies), late ones included + pub txset_coded_pieces_received: AtomicU64, + /// overlay.txset-coded.pieces-late — valid pieces that arrived after the set was rebuilt + pub txset_coded_pieces_late: AtomicU64, + /// overlay.txset-coded.pieces-refused — pieces not looked at: unknown slot, leader or layout, not from the sender that owes that piece, or over a limit + pub txset_coded_pieces_refused: AtomicU64, + /// overlay.txset-coded.pieces-invalid — pieces dropped because the frame, a signature or the Merkle path is wrong + pub txset_coded_pieces_invalid: AtomicU64, + /// overlay.txset-coded.pieces-duplicate — pieces received a second time + pub txset_coded_pieces_duplicate: AtomicU64, + /// overlay.txset-coded.pieces-skipped — pieces not written: the receiver has no identified connection or too many pieces in flight + pub txset_coded_pieces_skipped: AtomicU64, + /// overlay.txset-coded.sets-rebuilt — sets rebuilt from pieces + pub txset_coded_sets_rebuilt: AtomicU64, + /// overlay.txset-coded.sets-rebuilt-with-parity — of those, sets that needed a parity piece + pub txset_coded_sets_rebuilt_with_parity: AtomicU64, + /// overlay.txset-coded.sets-failed — rebuilds whose result could not be decoded, decompressed or parsed + pub txset_coded_sets_failed: AtomicU64, + /// overlay.txset-coded.sets-inconsistent — rebuilt sets whose pieces, encoded again, do not give the root the leader signed + pub txset_coded_sets_inconsistent: AtomicU64, + /// overlay.txset-coded.rebuild-us-sum — microseconds spent rebuilding sets from pieces (decoding and joining, before decompression) + pub txset_coded_rebuild_us_sum: AtomicU64, + // Transactions sent to the target leaders only /// overlay.flood.tx-direct-sent — transaction bodies sent straight to a /// target (no advert) @@ -194,6 +278,46 @@ impl Default for OverlayMetrics { txset_push_request_skipped: AtomicU64::new(0), txset_push_dropped: AtomicU64::new(0), txset_push_evicted: AtomicU64::new(0), + txset_push_sent_bytes: AtomicU64::new(0), + txset_tree_root_sent: AtomicU64::new(0), + txset_tree_relayed: AtomicU64::new(0), + txset_tree_relayed_bytes: AtomicU64::new(0), + txset_tree_received: AtomicU64::new(0), + txset_tree_duplicate: AtomicU64::new(0), + txset_tree_refused: AtomicU64::new(0), + txset_tree_unreachable: AtomicU64::new(0), + txset_tree_child_busy: AtomicU64::new(0), + txset_tree_relay_busy: AtomicU64::new(0), + txset_tree_relay_limited: AtomicU64::new(0), + txset_tree_fetch_deferred: AtomicU64::new(0), + txset_tree_fetch_fallback: AtomicU64::new(0), + txset_tree_delay_ms_sum: AtomicU64::new(0), + txset_tree_delay_count: AtomicU64::new(0), + txset_tree_delay_hop1_ms_sum: AtomicU64::new(0), + txset_tree_delay_hop1_count: AtomicU64::new(0), + txset_tree_delay_hop2_ms_sum: AtomicU64::new(0), + txset_tree_delay_hop2_count: AtomicU64::new(0), + txset_tree_delay_le_50: AtomicU64::new(0), + txset_tree_delay_le_100: AtomicU64::new(0), + txset_tree_delay_le_150: AtomicU64::new(0), + txset_tree_delay_le_200: AtomicU64::new(0), + txset_tree_delay_le_300: AtomicU64::new(0), + txset_tree_delay_gt_300: AtomicU64::new(0), + txset_tree_withheld: AtomicU64::new(0), + txset_coded_sets_encoded: AtomicU64::new(0), + txset_coded_pieces_sent: AtomicU64::new(0), + txset_coded_pieces_relayed: AtomicU64::new(0), + txset_coded_pieces_received: AtomicU64::new(0), + txset_coded_pieces_late: AtomicU64::new(0), + txset_coded_pieces_refused: AtomicU64::new(0), + txset_coded_pieces_invalid: AtomicU64::new(0), + txset_coded_pieces_duplicate: AtomicU64::new(0), + txset_coded_pieces_skipped: AtomicU64::new(0), + txset_coded_sets_rebuilt: AtomicU64::new(0), + txset_coded_sets_rebuilt_with_parity: AtomicU64::new(0), + txset_coded_sets_failed: AtomicU64::new(0), + txset_coded_sets_inconsistent: AtomicU64::new(0), + txset_coded_rebuild_us_sum: AtomicU64::new(0), flood_tx_direct_sent: AtomicU64::new(0), flood_tx_direct_sent_bytes: AtomicU64::new(0), flood_tx_direct_recv: AtomicU64::new(0), @@ -268,6 +392,48 @@ impl OverlayMetrics { txset_push_request_skipped: self.txset_push_request_skipped.load(ORD), txset_push_dropped: self.txset_push_dropped.load(ORD), txset_push_evicted: self.txset_push_evicted.load(ORD), + txset_push_sent_bytes: self.txset_push_sent_bytes.load(ORD), + txset_tree_root_sent: self.txset_tree_root_sent.load(ORD), + txset_tree_relayed: self.txset_tree_relayed.load(ORD), + txset_tree_relayed_bytes: self.txset_tree_relayed_bytes.load(ORD), + txset_tree_received: self.txset_tree_received.load(ORD), + txset_tree_duplicate: self.txset_tree_duplicate.load(ORD), + txset_tree_refused: self.txset_tree_refused.load(ORD), + txset_tree_unreachable: self.txset_tree_unreachable.load(ORD), + txset_tree_child_busy: self.txset_tree_child_busy.load(ORD), + txset_tree_relay_busy: self.txset_tree_relay_busy.load(ORD), + txset_tree_relay_limited: self.txset_tree_relay_limited.load(ORD), + txset_tree_fetch_deferred: self.txset_tree_fetch_deferred.load(ORD), + txset_tree_fetch_fallback: self.txset_tree_fetch_fallback.load(ORD), + txset_tree_delay_ms_sum: self.txset_tree_delay_ms_sum.load(ORD), + txset_tree_delay_count: self.txset_tree_delay_count.load(ORD), + txset_tree_delay_hop1_ms_sum: self.txset_tree_delay_hop1_ms_sum.load(ORD), + txset_tree_delay_hop1_count: self.txset_tree_delay_hop1_count.load(ORD), + txset_tree_delay_hop2_ms_sum: self.txset_tree_delay_hop2_ms_sum.load(ORD), + txset_tree_delay_hop2_count: self.txset_tree_delay_hop2_count.load(ORD), + txset_tree_delay_le_50: self.txset_tree_delay_le_50.load(ORD), + txset_tree_delay_le_100: self.txset_tree_delay_le_100.load(ORD), + txset_tree_delay_le_150: self.txset_tree_delay_le_150.load(ORD), + txset_tree_delay_le_200: self.txset_tree_delay_le_200.load(ORD), + txset_tree_delay_le_300: self.txset_tree_delay_le_300.load(ORD), + txset_tree_delay_gt_300: self.txset_tree_delay_gt_300.load(ORD), + txset_tree_withheld: self.txset_tree_withheld.load(ORD), + txset_coded_sets_encoded: self.txset_coded_sets_encoded.load(ORD), + txset_coded_pieces_sent: self.txset_coded_pieces_sent.load(ORD), + txset_coded_pieces_relayed: self.txset_coded_pieces_relayed.load(ORD), + txset_coded_pieces_received: self.txset_coded_pieces_received.load(ORD), + txset_coded_pieces_late: self.txset_coded_pieces_late.load(ORD), + txset_coded_pieces_refused: self.txset_coded_pieces_refused.load(ORD), + txset_coded_pieces_invalid: self.txset_coded_pieces_invalid.load(ORD), + txset_coded_pieces_duplicate: self.txset_coded_pieces_duplicate.load(ORD), + txset_coded_pieces_skipped: self.txset_coded_pieces_skipped.load(ORD), + txset_coded_sets_rebuilt: self.txset_coded_sets_rebuilt.load(ORD), + txset_coded_sets_rebuilt_with_parity: self + .txset_coded_sets_rebuilt_with_parity + .load(ORD), + txset_coded_sets_failed: self.txset_coded_sets_failed.load(ORD), + txset_coded_sets_inconsistent: self.txset_coded_sets_inconsistent.load(ORD), + txset_coded_rebuild_us_sum: self.txset_coded_rebuild_us_sum.load(ORD), flood_tx_direct_sent: self.flood_tx_direct_sent.load(ORD), flood_tx_direct_sent_bytes: self.flood_tx_direct_sent_bytes.load(ORD), flood_tx_direct_recv: self.flood_tx_direct_recv.load(ORD), @@ -350,6 +516,46 @@ pub struct MetricsSnapshot { pub txset_push_request_skipped: u64, pub txset_push_dropped: u64, pub txset_push_evicted: u64, + pub txset_push_sent_bytes: u64, + pub txset_tree_root_sent: u64, + pub txset_tree_relayed: u64, + pub txset_tree_relayed_bytes: u64, + pub txset_tree_received: u64, + pub txset_tree_duplicate: u64, + pub txset_tree_refused: u64, + pub txset_tree_unreachable: u64, + pub txset_tree_child_busy: u64, + pub txset_tree_relay_busy: u64, + pub txset_tree_relay_limited: u64, + pub txset_tree_fetch_deferred: u64, + pub txset_tree_fetch_fallback: u64, + pub txset_tree_delay_ms_sum: u64, + pub txset_tree_delay_count: u64, + pub txset_tree_delay_hop1_ms_sum: u64, + pub txset_tree_delay_hop1_count: u64, + pub txset_tree_delay_hop2_ms_sum: u64, + pub txset_tree_delay_hop2_count: u64, + pub txset_tree_delay_le_50: u64, + pub txset_tree_delay_le_100: u64, + pub txset_tree_delay_le_150: u64, + pub txset_tree_delay_le_200: u64, + pub txset_tree_delay_le_300: u64, + pub txset_tree_delay_gt_300: u64, + pub txset_tree_withheld: u64, + pub txset_coded_sets_encoded: u64, + pub txset_coded_pieces_sent: u64, + pub txset_coded_pieces_relayed: u64, + pub txset_coded_pieces_received: u64, + pub txset_coded_pieces_late: u64, + pub txset_coded_pieces_refused: u64, + pub txset_coded_pieces_invalid: u64, + pub txset_coded_pieces_duplicate: u64, + pub txset_coded_pieces_skipped: u64, + pub txset_coded_sets_rebuilt: u64, + pub txset_coded_sets_rebuilt_with_parity: u64, + pub txset_coded_sets_failed: u64, + pub txset_coded_sets_inconsistent: u64, + pub txset_coded_rebuild_us_sum: u64, pub flood_tx_direct_sent: u64, pub flood_tx_direct_sent_bytes: u64, pub flood_tx_direct_recv: u64, diff --git a/src/bucket/BucketBase.cpp b/src/bucket/BucketBase.cpp index 4d80327a72..9c9bd880f9 100644 --- a/src/bucket/BucketBase.cpp +++ b/src/bucket/BucketBase.cpp @@ -71,6 +71,32 @@ BucketBase::BucketBase(std::string const& filename, } } +template +BucketBase::BucketBase(std::string const& filename, + Hash const& hash, + std::shared_ptr&& index, + size_t fileSize, + std::shared_future fileReady) + : mFilename(filename) + , mHash(hash) + , mSize(fileSize) + , mIndex(std::move(index)) + , mFileReady(std::move(fileReady)) +{ + releaseAssert(!filename.empty() && !isZero(hash)); + releaseAssert(mFileReady.valid()); +} + +template +void +BucketBase::waitForFile() const +{ + if (mFileReady.valid()) + { + mFileReady.get(); + } +} + template BucketBase::BucketBase() { } @@ -86,6 +112,9 @@ template std::filesystem::path const& BucketBase::getFilename() const { + // Whoever asks for the file's name is about to open, copy or remove the + // file: let them find it. + waitForFile(); return mFilename; } diff --git a/src/bucket/BucketBase.h b/src/bucket/BucketBase.h index 84f55d0096..54c663bf7c 100644 --- a/src/bucket/BucketBase.h +++ b/src/bucket/BucketBase.h @@ -10,6 +10,7 @@ #include "util/ProtocolVersion.h" #include "xdr/Stellar-types.h" #include +#include #include namespace asio @@ -76,6 +77,12 @@ class BucketBase : public NonMovableOrCopyable Hash const mHash; size_t mSize{0}; + // Set for a bucket that was created before its file was written (level + // 0 of the live bucket list with EXPERIMENTAL_SHARDED_TOP_BUCKET): ready + // when the file is at mFilename. `getFilename` waits for it, so that + // whoever is about to open the file finds it. + std::shared_future mFileReady{}; + std::shared_ptr mIndex{}; // Unconditionally resets mIndex. Must only be called via the static @@ -103,6 +110,16 @@ class BucketBase : public NonMovableOrCopyable BucketBase(std::string const& filename, Hash const& hash, std::shared_ptr&& index); + // A bucket whose file, of `fileSize` bytes, will be at `filename` when + // `fileReady` is. + BucketBase(std::string const& filename, Hash const& hash, + std::shared_ptr&& index, size_t fileSize, + std::shared_future fileReady); + + // Blocks until the bucket's file is on disk (at once, unless the bucket + // was created ahead of its file). Rethrows what the writer threw. + void waitForFile() const; + Hash const& getHash() const; std::filesystem::path const& getFilename() const; size_t getSize() const; diff --git a/src/bucket/BucketListBase.cpp b/src/bucket/BucketListBase.cpp index ee7c04cfd3..386a7844fa 100644 --- a/src/bucket/BucketListBase.cpp +++ b/src/bucket/BucketListBase.cpp @@ -7,6 +7,7 @@ #include "bucket/BucketManager.h" #include "bucket/HotArchiveBucket.h" #include "bucket/LiveBucket.h" +#include "bucket/ShardedTopBucket.h" #include "crypto/SHA.h" #include "ledger/LedgerTxn.h" #include "main/Application.h" @@ -210,6 +211,22 @@ BucketLevel::prepareFirstLevel(Application& app, ? std::make_shared() : mCurr; + // The ledger's entries as shards, written in parallel, under the + // commutative hash. + if (auto shardsPerGroup = app.getConfig().EXPERIMENTAL_SHARDED_TOP_BUCKET; + shardsPerGroup > 0) + { + if (!mShardedTop || mShardedTop->shardsPerGroup() != shardsPerGroup) + { + mShardedTop = + std::make_shared(app, shardsPerGroup); + } + setNextInMemory( + mShardedTop->addBatch(app, currLedger, currLedgerProtocol, curr, + inputVectors..., countMergeEvents, doFsync)); + return; + } + // On startup, the current bucket may not be initialized in memory, so // fallback to normal prepare if (!curr->hasInMemoryEntries()) @@ -764,6 +781,10 @@ BucketListBase::addBatchInternal(Application& app, uint32_t currLedger, * else spilled/added to it. */ + // (With EXPERIMENTAL_SHARDED_TOP_BUCKET level 0's current bucket + // may still be on its way to disk: the merge that starts below + // reads its file.) + mLevels[i - 1].getCurr()->waitForFile(); auto snap = mLevels[i - 1].snap(); mLevels[i].commit(); mLevels[i].prepare(app, currLedger, currLedgerProtocol, snap, diff --git a/src/bucket/BucketListBase.h b/src/bucket/BucketListBase.h index 4f57dcc111..e25a2fc71c 100644 --- a/src/bucket/BucketListBase.h +++ b/src/bucket/BucketListBase.h @@ -356,6 +356,8 @@ namespace testutil template class BucketListDepthModifier; } +class ShardedTopBucket; + template class BucketLevel { BUCKET_TYPE_ASSERT(BucketT); @@ -367,6 +369,10 @@ template class BucketLevel std::shared_ptr mCurr; std::shared_ptr mSnap; + // Level 0 of the live bucket list with EXPERIMENTAL_SHARDED_TOP_BUCKET: + // the shards of mCurr. + std::shared_ptr mShardedTop; + void setNextInMemory(std::shared_ptr&& b); bool hasInMemoryMerge() const; diff --git a/src/bucket/BucketManager.cpp b/src/bucket/BucketManager.cpp index 72e138661d..e85022fc72 100644 --- a/src/bucket/BucketManager.cpp +++ b/src/bucket/BucketManager.cpp @@ -252,7 +252,8 @@ BucketManager::getBucketDir() const BucketManager::~BucketManager() { - + // The writer of level 0's file writes into the directories that go next. + waitForTopBucketFile(); deleteTmpDirAndUnlockBucketDir(); } @@ -457,6 +458,30 @@ BucketManager::adoptFileAsBucket( std::move(inMemoryState)); } +std::shared_ptr +BucketManager::adoptLiveBucketAheadOfItsFile( + uint256 const& hash, std::shared_ptr index, + size_t fileSize, std::shared_future fileReady, uint32_t bucketVersion, + bool& created) +{ + RecursiveMutexLocker lock(mBucketMutex); + std::shared_ptr b = + getBucketByHashInternal(hash, mSharedLiveBuckets); + if (b) + { + maybeSetIndex(b, std::move(index)); + created = false; + return b; + } + created = true; + b = std::make_shared(bucketFilename(hash), hash, + std::move(index), fileSize, + std::move(fileReady), bucketVersion); + mSharedLiveBuckets.emplace(hash, b); + updateSharedBucketSize(); + return b; +} + template <> std::shared_ptr BucketManager::adoptFileAsBucket( @@ -924,6 +949,19 @@ void BucketManager::forgetUnreferencedBuckets(HistoryArchiveState const& has) { ZoneScoped; + // With EXPERIMENTAL_SHARDED_TOP_BUCKET level 0's current bucket may still + // be on its way to disk. Until it is there, a restart rebuilds it from + // the ledger's shard files and the bucket before it, which is no longer + // referenced: it must not be dropped yet. (Before taking the bucket + // mutex. The caller holds the ledger state mutex, on the main thread: a + // slow disk shows in this timer.) + if (mLiveBucketList && mConfig.EXPERIMENTAL_SHARDED_TOP_BUCKET > 0) + { + auto timer = mAppConnector.getMetrics() + .NewTimer({"bucket", "top-shard", "gc-wait"}) + .TimeScope(); + mLiveBucketList->getLevel(0).getCurr()->waitForFile(); + } RecursiveMutexLocker lock(mBucketMutex); auto referenced = getAllReferencedBuckets(has, lock); auto blReferenced = getBucketListReferencedBuckets(); @@ -1324,6 +1362,14 @@ BucketManager::checkForMissingBucketsFiles(HistoryArchiveState const& has) std::vector result; std::copy_if(buckets.begin(), buckets.end(), std::back_inserter(result), [&](std::string b) { + // (A level-0 bucket made with + // EXPERIMENTAL_SHARDED_TOP_BUCKET may still be on its + // way to disk.) + if (auto live = + getBucketIfExists(hexToBin256(b))) + { + live->waitForFile(); + } auto filename = bucketFilename(b); return !isZero(hexToBin256(b)) && !fs::exists(filename); }); @@ -1418,6 +1464,35 @@ BucketManager::shutdown() mIsShutdown = true; } +void +BucketManager::waitForTopBucketFile() +{ + // Every bucket made ahead of its file is in the map, whether or not it + // is still level 0's current bucket (a bucket list may have been assumed + // over it, or it belongs to a list of a test's own). + std::vector> buckets; + { + RecursiveMutexLocker lock(mBucketMutex); + buckets.reserve(mSharedLiveBuckets.size()); + for (auto const& [_, bucket] : mSharedLiveBuckets) + { + buckets.emplace_back(bucket); + } + } + for (auto const& bucket : buckets) + { + try + { + bucket->waitForFile(); + } + catch (std::exception const& e) + { + CLOG_ERROR(Bucket, "The file of bucket {} was not written: {}", + binToHex(bucket->getHash()), e.what()); + } + } +} + bool BucketManager::isShutdown() const { diff --git a/src/bucket/BucketManager.h b/src/bucket/BucketManager.h index ebe02a839f..ac8081fd4e 100644 --- a/src/bucket/BucketManager.h +++ b/src/bucket/BucketManager.h @@ -214,6 +214,9 @@ class BucketManager : NonMovableOrCopyable std::atomic mDelayMergesForTesting{false}; #endif + // Writes the files of level 0's buckets (see ShardedTopBucket). + friend class ShardedTopBucket; + protected: BucketManager(AppConnector& appConnector); void calculateSkipValues(LedgerHeader& currentHeader); @@ -227,6 +230,11 @@ class BucketManager : NonMovableOrCopyable void initialize(); void maybeDropAndCreateNew(); std::string bucketIndexFilename(Hash const& hash) const; + // Blocks until every live bucket that was made ahead of its file has its + // file (see ShardedTopBucket; at once without it). Logs a file that + // could not be written. + void waitForTopBucketFile(); + std::string const& getTmpDir(); TmpDirManager& getTmpDirManager(); std::string const& getBucketDir() const; @@ -264,6 +272,16 @@ class BucketManager : NonMovableOrCopyable std::shared_ptr index, std::unique_ptr> inMemoryState = nullptr); + // Registers a live bucket whose file is still being written (see + // ShardedTopBucket): it will be at `bucketFilename(hash)` when + // `fileReady` is. Returns the bucket that already has this hash, if there + // is one (its file is then there, or on its way), and sets `created` to + // whether the bucket is a new one, whose file the caller has to write. + std::shared_ptr adoptLiveBucketAheadOfItsFile( + uint256 const& hash, std::shared_ptr index, + size_t fileSize, std::shared_future fileReady, + uint32_t bucketVersion, bool& created); + // Companion method to `adoptFileAsLiveBucket` also called from the // `BucketOutputIterator::getBucket` merge-completion path. This method // however should be called when the output bucket is _empty_ and thereby diff --git a/src/bucket/InMemoryIndex.cpp b/src/bucket/InMemoryIndex.cpp index b055c9b341..7f28d30991 100644 --- a/src/bucket/InMemoryIndex.cpp +++ b/src/bucket/InMemoryIndex.cpp @@ -61,10 +61,45 @@ InMemoryBucketState::insert(BucketEntry const& be) } // Perform a binary search using start iter as lower bound for search key. +void +InMemoryBucketState::insertInto(SetT& set, BucketEntry const& be) +{ + auto [_, inserted] = set.insert( + InternalInMemoryBucketEntry(std::make_shared(be))); + releaseAssertOrThrow(inserted); +} + +void +InMemoryBucketState::setShards(std::vector&& shards, + std::function shardOf) +{ + releaseAssertOrThrow(mEntries.empty() && mShards.empty()); + mShards = std::move(shards); + mShardOf = std::move(shardOf); + for (auto const& shard : mShards) + { + if (!shard.empty()) + { + mEntries.insert(InternalInMemoryBucketEntry(shard.begin()->get())); + break; + } + } +} + std::pair InMemoryBucketState::scan(IterT start, LedgerKey const& searchKey) const { ZoneScoped; + if (!mShards.empty()) + { + auto const& shard = mShards.at(mShardOf(searchKey)); + auto shardIt = shard.find(InternalInMemoryBucketEntry(searchKey)); + if (shardIt != shard.end()) + { + return {IndexReturnT(shardIt->get()), mEntries.begin()}; + } + return {IndexReturnT(), mEntries.begin()}; + } auto it = mEntries.find(InternalInMemoryBucketEntry(searchKey)); // If we found the key if (it != mEntries.end()) @@ -116,6 +151,70 @@ InMemoryIndex::InMemoryIndex(BucketManager& bm, mTypeRanges = buildTypeRangesMap(typeStartOffsets, typeEndOffsets); } +void +InMemoryIndexShard::add(BucketEntry const& be, size_t xdrSize) +{ + releaseAssertOrThrow(be.type() != METAENTRY); + counters.template count(be); + LedgerKey lk = getBucketLedgerKey(be); + if (be.type() == INITENTRY && lk.type() == LIQUIDITY_POOL) + { + auto const& poolParams = + be.liveEntry().data.liquidityPool().body.constantProduct().params; + assetPoolIDMap[poolParams.assetA].emplace_back( + lk.liquidityPool().liquidityPoolID); + assetPoolIDMap[poolParams.assetB].emplace_back( + lk.liquidityPool().liquidityPoolID); + } + InMemoryBucketState::insertInto(entries, be); + // 4 bytes of size info in front of every entry on disk + typeBytes[lk.type()] += static_cast(xdrSize + 4); +} + +InMemoryIndex::InMemoryIndex(std::vector&& shards, + std::function shardOf, + BucketMetadata const& metadata, bool hasMetaEntry) +{ + ZoneScoped; + std::map typeBytes; + std::vector sets; + sets.reserve(shards.size()); + for (auto& shard : shards) + { + mCounters += shard.counters; + for (auto& [asset, pools] : shard.assetPoolIDMap) + { + auto& all = mAssetPoolIDMap[asset]; + all.insert(all.end(), pools.begin(), pools.end()); + } + for (auto const& [type, bytes] : shard.typeBytes) + { + typeBytes[type] += bytes; + } + sets.emplace_back(std::move(shard.entries)); + } + mInMemoryState.setShards(std::move(sets), std::move(shardOf)); + + // The file holds the entries sorted by key, and keys sort by type first: + // a type's entries are one run, as long as the sum of their sizes. The + // offsets are those the other constructors arrive at entry by entry. + std::streamoff offset = 0; + if (hasMetaEntry) + { + offset = xdr::xdr_size(metadata) + 4 + 4; + } + std::map typeStartOffsets; + std::map typeEndOffsets; + std::optional lastTypeSeen = std::nullopt; + for (auto const& [type, bytes] : typeBytes) + { + updateTypeBoundaries(type, offset, typeStartOffsets, typeEndOffsets, + lastTypeSeen); + offset += bytes; + } + mTypeRanges = buildTypeRangesMap(typeStartOffsets, typeEndOffsets); +} + InMemoryIndex::InMemoryIndex(BucketManager const& bm, std::filesystem::path const& filename, SHA256* hasher) diff --git a/src/bucket/InMemoryIndex.h b/src/bucket/InMemoryIndex.h index be3c3ea02c..f674150b68 100644 --- a/src/bucket/InMemoryIndex.h +++ b/src/bucket/InMemoryIndex.h @@ -9,6 +9,7 @@ #include "xdr/Stellar-ledger-entries.h" #include "ledger/LedgerHashUtils.h" +#include #include namespace stellar @@ -149,10 +150,27 @@ class InMemoryBucketState : public NonMovableOrCopyable using InMemorySet = std::unordered_set; + // Every entry, or, when the state is sharded (below), one entry of it: + // enough for `begin() != end()` to say whether the state holds anything, + // which is all the iterators are used for. InMemorySet mEntries; + // A sharded state: every entry is in the set of its key's shard, and a + // lookup goes to that set alone. The sets are built by as many threads + // (see ShardedTopBucket) and never change afterwards. + std::vector mShards; + std::function mShardOf; + public: using IterT = InMemorySet::const_iterator; + using SetT = InMemorySet; + + // Adds an entry to a set that is being built for `setShards`. + static void insertInto(SetT& set, BucketEntry const& be); + + // Makes this (empty) state a sharded one. + void setShards(std::vector&& shards, + std::function shardOf); // Insert a LedgerEntry (INIT/LIVE) into the cache. void insert(BucketEntry const& be); @@ -183,6 +201,20 @@ class InMemoryBucketState : public NonMovableOrCopyable #endif }; +// What one shard contributes to the index of a bucket that is built shard by +// shard (see ShardedTopBucket): its entries, their counts, the pools its new +// liquidity pool entries belong to, and how many bytes of the bucket file its +// entries of each type take (their XDR and the 4 bytes in front of each). +struct InMemoryIndexShard +{ + InMemoryBucketState::SetT entries; + BucketEntryCounters counters; + AssetPoolIDMap assetPoolIDMap; + std::map typeBytes; + + void add(BucketEntry const& be, size_t xdrSize); +}; + class InMemoryIndex { private: @@ -195,6 +227,14 @@ class InMemoryIndex public: using IterT = InMemoryBucketState::IterT; + // The index of the bucket whose entries are those of `shards`, which a + // lookup of key k finds in shard `shardOf(k)`. The type ranges are those + // of the bucket's file: the entries of all shards sorted by key behind + // the metadata entry. + InMemoryIndex(std::vector&& shards, + std::function shardOf, + BucketMetadata const& metadata, bool hasMetaEntry); + InMemoryIndex(BucketManager const& bm, std::filesystem::path const& filename, SHA256* hasher); diff --git a/src/bucket/LiveBucket.cpp b/src/bucket/LiveBucket.cpp index e26adfc24a..cd83bc3dbf 100644 --- a/src/bucket/LiveBucket.cpp +++ b/src/bucket/LiveBucket.cpp @@ -561,6 +561,16 @@ LiveBucket::LiveBucket(std::string const& filename, Hash const& hash, { } +LiveBucket::LiveBucket(std::string const& filename, Hash const& hash, + std::shared_ptr&& index, + size_t fileSize, std::shared_future fileReady, + uint32_t bucketVersion) + : BucketBase(filename, hash, std::move(index), fileSize, + std::move(fileReady)) + , mKnownBucketVersion(bucketVersion) +{ +} + LiveBucket::LiveBucket() : BucketBase() { // Empty bucket is trivially stored in memory @@ -575,6 +585,10 @@ LiveBucket::LiveBucket(std::unique_ptr> entries) uint32_t LiveBucket::getBucketVersion() const { + if (mKnownBucketVersion) + { + return *mKnownBucketVersion; + } LiveBucketInputIterator it(shared_from_this()); return it.getMetadata().ledgerVersion; } @@ -587,6 +601,32 @@ LiveBucket::maybeInitializeCache(size_t totalBucketListAccountsSizeBytes, mIndex->maybeInitializeCache(totalBucketListAccountsSizeBytes, cfg); } +std::vector +LiveBucket::mergeEntriesInMemory(BucketManager& bucketManager, + uint32_t maxProtocolVersion, + std::vector const& oldEntries, + std::vector const& newEntries, + MergeCounters& mc) +{ + std::vector mergedEntries; + mergedEntries.reserve(oldEntries.size() + newEntries.size()); + + // First level never has shadows + std::vector shadowIterators{}; + bool keepShadowedLifecycleEntries = updateMergeCountersForProtocolVersion( + mc, maxProtocolVersion, shadowIterators); + + MemoryMergeInput inputSource(oldEntries, newEntries); + std::function putFunc = + [&mergedEntries](BucketEntry const& entry) { + mergedEntries.emplace_back(entry); + }; + + mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, mc, + shadowIterators, keepShadowedLifecycleEntries); + return mergedEntries; +} + std::shared_ptr LiveBucket::mergeInMemory(BucketManager& bucketManager, uint32_t maxProtocolVersion, @@ -602,9 +642,6 @@ LiveBucket::mergeInMemory(BucketManager& bucketManager, auto const& oldEntries = oldBucket->getInMemoryEntries(); auto const& newEntries = newBucket->getInMemoryEntries(); - std::vector mergedEntries; - mergedEntries.reserve(oldEntries.size() + newEntries.size()); - // Prepare metadata for the merged bucket BucketMetadata meta; meta.ledgerVersion = maxProtocolVersion; @@ -616,20 +653,9 @@ LiveBucket::mergeInMemory(BucketManager& bucketManager, meta.ext.bucketListType() = BucketListType::LIVE; } - // First level never has shadows - std::vector shadowIterators{}; MergeCounters mc; - bool keepShadowedLifecycleEntries = updateMergeCountersForProtocolVersion( - mc, maxProtocolVersion, shadowIterators); - - MemoryMergeInput inputSource(oldEntries, newEntries); - std::function putFunc = - [&mergedEntries](BucketEntry const& entry) { - mergedEntries.emplace_back(entry); - }; - - mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, mc, - shadowIterators, keepShadowedLifecycleEntries); + std::vector mergedEntries = mergeEntriesInMemory( + bucketManager, maxProtocolVersion, oldEntries, newEntries, mc); if (countMergeEvents) { diff --git a/src/bucket/LiveBucket.h b/src/bucket/LiveBucket.h index bc0ae0fb70..d62a0b68c9 100644 --- a/src/bucket/LiveBucket.h +++ b/src/bucket/LiveBucket.h @@ -39,6 +39,10 @@ class LiveBucket : public BucketBase, // in-memory entries to produce the new bucket instead of file IO. std::unique_ptr> mEntries{}; + // The bucket's protocol version, where it is known without reading the + // file (a bucket created ahead of its file). + std::optional mKnownBucketVersion{}; + public: // Entry type that this bucket stores using EntryT = BucketEntry; @@ -65,6 +69,13 @@ class LiveBucket : public BucketBase, std::shared_ptr&& index, std::unique_ptr> inMemoryState = nullptr); + // A bucket of protocol version `bucketVersion` whose file, of `fileSize` + // bytes, is still being written and will be at `filename` when + // `fileReady` is (see ShardedTopBucket). Lookups are answered by `index`. + LiveBucket(std::string const& filename, Hash const& hash, + std::shared_ptr&& index, size_t fileSize, + std::shared_future fileReady, uint32_t bucketVersion); + // Returns true if a BucketEntry that is key-wise identical to the given // BucketEntry exists in the bucket. For testing. bool containsBucketIdentity(BucketEntry const& id) const; @@ -161,6 +172,14 @@ class LiveBucket : public BucketBase, std::vector& shadowIterators, bool keepShadowedLifecycleEntries); + // The entries of the merge of two sorted lists of entries (the older + // first), by the rules of every live bucket merge at level 0: no shadows, + // tombstones kept. + static std::vector mergeEntriesInMemory( + BucketManager& bucketManager, uint32_t maxProtocolVersion, + std::vector const& oldEntries, + std::vector const& newEntries, MergeCounters& mc); + // Merge two buckets in memory without using FutureBucket. // This is used only for level 0 merges. Note that the resulting Bucket is // still written to disk. diff --git a/src/bucket/LiveBucketIndex.cpp b/src/bucket/LiveBucketIndex.cpp index e0122b2340..6022907d71 100644 --- a/src/bucket/LiveBucketIndex.cpp +++ b/src/bucket/LiveBucketIndex.cpp @@ -91,6 +91,15 @@ LiveBucketIndex::LiveBucketIndex(BucketManager& bm, { } +LiveBucketIndex::LiveBucketIndex( + BucketManager& bm, std::unique_ptr inMemoryIndex) + : mInMemoryIndex(std::move(inMemoryIndex)) + , mCacheHitMeter(bm.getCacheHitMeter()) + , mCacheMissMeter(bm.getCacheMissMeter()) +{ + releaseAssertOrThrow(mInMemoryIndex); +} + void LiveBucketIndex::maybeInitializeCache(size_t totalBucketListAccountsSizeBytes, Config const& cfg) const @@ -200,13 +209,27 @@ LiveBucketIndex::markBloomMiss() const std::shared_ptr LiveBucketIndex::getCachedEntry(LedgerKey const& k) const { - if (shouldUseCache() && isCachedType(k)) + if (mDiskIndex && isCachedType(k)) { - SharedLockExclusive lock(mCacheMutex); - auto cachePtr = mCache->maybeGet(k); + // Lookups only read the cache (`peek` records nothing), so they run + // under the shared lock, side by side: ledger apply loads accounts + // from many threads at once, and each load asks the cache of every + // bucket on its way down the bucket list. Only inserting an entry + // takes the lock exclusively (maybeAddToCache). What a lookup gives + // up is the access time of the entry it hits: when the cache is + // smaller than the bucket's accounts, eviction goes by when an entry + // was put in, not by when it was last read. + SharedLockShared lock(mCacheMutex); + if (!mCache) + { + return nullptr; + } + auto cachePtr = mCache->peek(k); if (cachePtr) { mCacheHitMeter.Mark(); + // The entry is shared; the copy of the pointer is taken under + // the lock. return *cachePtr; } diff --git a/src/bucket/LiveBucketIndex.h b/src/bucket/LiveBucketIndex.h index d12eb46431..2ac5ea6d6a 100644 --- a/src/bucket/LiveBucketIndex.h +++ b/src/bucket/LiveBucketIndex.h @@ -113,6 +113,11 @@ class LiveBucketIndex : public NonMovableOrCopyable std::vector const& inMemoryState, BucketMetadata const& metadata); + // An index that was built elsewhere (shard by shard, see + // ShardedTopBucket). + LiveBucketIndex(BucketManager& bm, + std::unique_ptr inMemoryIndex); + // Initializes the random eviction cache if it has not already been // initialized. The random eviction cache itself has an entry limit, but we // expose a memory limit in the validator config. To account for this, we diff --git a/src/bucket/ShardedTopBucket.cpp b/src/bucket/ShardedTopBucket.cpp new file mode 100644 index 0000000000..51a38420c3 --- /dev/null +++ b/src/bucket/ShardedTopBucket.cpp @@ -0,0 +1,1060 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "bucket/ShardedTopBucket.h" +#include "bucket/BucketInputIterator.h" +#include "bucket/BucketManager.h" +#include "bucket/BucketUtils.h" +#include "bucket/InMemoryIndex.h" +#include "bucket/LedgerCmp.h" +#include "bucket/LiveBucketIndex.h" +#include "crypto/Hex.h" +#include "crypto/Random.h" +#include "history/HistoryArchive.h" +#include "ledger/LedgerHashUtils.h" +#include "ledger/LedgerTypeUtils.h" +#include "main/Application.h" +#include "medida/timer.h" +#include "util/Fs.h" +#include "util/GlobalChecks.h" +#include "util/Logging.h" +#include "util/MetricsRegistry.h" +#include "util/ProtocolVersion.h" +#include "util/XDRStream.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace stellar +{ + +namespace +{ +uint32_t const GROUP_SOROBAN = 0; +uint32_t const GROUP_TTL = 1; +uint32_t const GROUP_CLASSIC = 2; +uint32_t const NUM_GROUPS = 3; + +// The first 8 of 32 random-looking bytes of a key, mixed with a value that +// only this process knows: whoever chooses keys (account ids, contract code) +// cannot aim them at one shard. +uint64_t +load64(unsigned char const* p) +{ + static uint64_t const salt = []() { + auto bytes = randomBytes(sizeof(uint64_t)); + uint64_t v; + std::memcpy(&v, bytes.data(), sizeof(v)); + return v; + }(); + uint64_t v; + std::memcpy(&v, p, sizeof(v)); + v ^= salt; + // (the finalizer of splitmix64) + v = (v ^ (v >> 30)) * 0xbf58476d1ce4e5b9ULL; + v = (v ^ (v >> 27)) * 0x94d049bb133111ebULL; + return v ^ (v >> 31); +} + +uint32_t +groupOf(LedgerEntryType t) +{ + switch (t) + { + case CONTRACT_DATA: + case CONTRACT_CODE: + return GROUP_SOROBAN; + case TTL: + return GROUP_TTL; + default: + return GROUP_CLASSIC; + } +} + +// The same as `ShardedTopBucket::shardOf` of the entry's key, without +// building the key for the three types that carry a random 32 bytes. +size_t +shardOfEntry(LedgerEntry const& e, uint32_t n) +{ + switch (e.data.type()) + { + case ACCOUNT: + if (e.data.account().accountID.type() == PUBLIC_KEY_TYPE_ED25519) + { + return GROUP_CLASSIC * n + + load64(e.data.account().accountID.ed25519().data()) % n; + } + break; + case TTL: + return GROUP_TTL * n + load64(e.data.ttl().keyHash.data()) % n; + case CONTRACT_CODE: + return GROUP_SOROBAN * n + + load64(e.data.contractCode().hash.data()) % n; + default: + break; + } + return ShardedTopBucket::shardOf(LedgerEntryKey(e), n); +} + +// One entry of a ledger's batch, as the cut hands it to its shard. +enum ItemKind : uint8_t +{ + ITEM_INIT, + ITEM_LIVE, + ITEM_DEAD +}; +struct Item +{ + ItemKind kind; + void const* ptr; +}; + +// The 4 bytes that stand in front of every record of an XDR file (see +// XDROutputFileStream::writeOne), then the record. +void +appendRecord(std::string& out, char const* data, size_t size) +{ + releaseAssert(size < 0x80000000u); + char mark[4]; + mark[0] = static_cast((size >> 24) & 0xFF) | '\x80'; + mark[1] = static_cast((size >> 16) & 0xFF); + mark[2] = static_cast((size >> 8) & 0xFF); + mark[3] = static_cast(size & 0xFF); + out.append(mark, 4); + out.append(data, size); +} + +LtHash16 +hashOfRecords(ShardedTopBucket::Shard const& shard) +{ + LtHash16 h; + for (size_t i = 0; i < shard.entries.size(); ++i) + { + h.add(ByteSlice(shard.records.data() + shard.offsets[i], + shard.offsets[i + 1] - shard.offsets[i])); + } + return h; +} + +// Writes the file of the bucket whose entries are those of `shards`: the +// metadata entry, then every entry in the order of its key. No two shards +// hold the same key, so this is the merge of the shards. +void +writeBucketFile( + std::vector> const& shards, + xdr::opaque_vec<> const* metaRecord, std::string const& filename, + asio::io_context& ctx, bool doFsync) +{ + ZoneScoped; + XDROutputFileStream out(ctx, doFsync); + out.open(filename); + + size_t const FLUSH_AT = 1 << 20; + std::string buf; + buf.reserve(FLUSH_AT + (64 << 10)); + if (metaRecord) + { + appendRecord(buf, reinterpret_cast(metaRecord->data()), + metaRecord->size()); + } + + struct Cursor + { + ShardedTopBucket::Shard const* shard; + size_t i; + }; + BucketEntryIdCmp idCmp; + // The smallest key on top. + auto later = [&idCmp](Cursor const& a, Cursor const& b) { + return idCmp(b.shard->entries[b.i], a.shard->entries[a.i]); + }; + std::priority_queue, decltype(later)> heap( + later); + for (auto const& shard : shards) + { + if (!shard->entries.empty()) + { + heap.push(Cursor{shard.get(), 0}); + } + } + while (!heap.empty()) + { + Cursor c = heap.top(); + heap.pop(); + auto const& s = *c.shard; + appendRecord(buf, s.records.data() + s.offsets[c.i], + s.offsets[c.i + 1] - s.offsets[c.i]); + if (buf.size() >= FLUSH_AT) + { + out.writeBytes(buf.data(), buf.size()); + buf.clear(); + } + if (++c.i < s.entries.size()) + { + heap.push(c); + } + } + if (!buf.empty()) + { + out.writeBytes(buf.data(), buf.size()); + } + out.close(); +} + +// Makes the names of the files created in `dir` durable. +void +syncDir(std::string const& dir) +{ + ZoneScoped; + int fd = ::open(dir.c_str(), O_RDONLY); + if (fd == -1) + { + throw std::runtime_error(fmt::format( + FMT_STRING("could not open {}: {}"), dir, strerror(errno))); + } + int rc = ::fsync(fd); + int err = errno; + ::close(fd); + if (rc == -1) + { + throw std::runtime_error(fmt::format( + FMT_STRING("could not sync {}: {}"), dir, strerror(err))); + } +} + +// A ledger's shard file is "shard----.xdr". +bool +isShardFile(std::string const& name) +{ + return name.size() > 10 && name.compare(0, 6, "shard-") == 0 && + name.compare(name.size() - 4, 4, ".xdr") == 0; +} + +struct ShardFileName +{ + uint32_t ledger{0}; + std::string prevHex; + std::string instance; +}; + +std::optional +parseShardFileName(std::string const& name) +{ + if (!isShardFile(name)) + { + return std::nullopt; + } + std::vector parts; + size_t begin = 0; + for (size_t i = 0; i <= name.size(); ++i) + { + if (i == name.size() || name[i] == '-') + { + parts.emplace_back(name.substr(begin, i - begin)); + begin = i + 1; + } + } + if (parts.size() != 5 || parts[2].size() != 64 || parts[1].empty() || + parts[1].size() > 10 || + parts[1].find_first_not_of("0123456789") != std::string::npos) + { + return std::nullopt; + } + ShardFileName res; + res.ledger = static_cast(std::stoull(parts[1])); + res.prevHex = parts[2]; + res.instance = parts[3]; + return res; +} + +#ifdef BUILD_TESTS +std::atomic gLoseFileForTesting{false}; +#endif +} + +#ifdef BUILD_TESTS +void +ShardedTopBucket::loseFileForTesting(bool lose) +{ + gLoseFileForTesting = lose; +} +#endif + +void +ShardedTopBucket::Shard::append(BucketEntry&& entry) +{ + auto rec = xdr::xdr_to_opaque(entry); + records.append(reinterpret_cast(rec.data()), rec.size()); + releaseAssert(records.size() <= std::numeric_limits::max()); + offsets.push_back(static_cast(records.size())); + entries.emplace_back(std::move(entry)); +} + +ShardedTopBucket::ShardedTopBucket(Application& app, uint32_t shardsPerGroup) + : mShardsPerGroup(shardsPerGroup) + , mNumShards(static_cast(NUM_GROUPS) * shardsPerGroup) + , mShardDir(app.getBucketManager().getBucketDir() + "/topshards") + , mInstance(binToHex(randomBytes(4))) + , mExecutor(/*pinWorkers=*/false) + , mCutTimer(app.getMetrics().NewTimer({"bucket", "top-shard", "cut"})) + , mShardsTimer(app.getMetrics().NewTimer({"bucket", "top-shard", "shards"})) + , mFinishTimer(app.getMetrics().NewTimer({"bucket", "top-shard", "finish"})) + , mFileTimer(app.getMetrics().NewTimer({"bucket", "top-shard", "file"})) + , mFileWaitTimer( + app.getMetrics().NewTimer({"bucket", "top-shard", "file-wait"})) +{ + releaseAssert(shardsPerGroup > 0); + for (size_t i = 0; i < mNumShards; ++i) + { + mShards.emplace_back(std::make_shared()); + } + mExecutor.setPreferredTaskCount(app.getConfig().BATCH_EXECUTOR_TASKS); + bool const isNew = !fs::exists(mShardDir); + fs::mkpath(mShardDir); + if (isNew && !app.getConfig().DISABLE_XDR_FSYNC) + { + syncDir(app.getBucketManager().getBucketDir()); + } +} + +ShardedTopBucket::~ShardedTopBucket() +{ + // Let the file's writer finish. It uses the clock's io context, which + // outlives the application, and nothing else of the application, parts + // of which (the metrics, the bucket manager's config) are gone by now. + // The bucket manager waits for every such writer before it removes its + // directories (BucketManager::waitForTopBucketFile); this is the last + // resort. + if (mFileWriter.valid()) + { + mFileWriter.wait(); + } +} + +size_t +ShardedTopBucket::shardOf(LedgerKey const& key, uint32_t n) +{ + uint64_t h; + switch (key.type()) + { + case ACCOUNT: + if (key.account().accountID.type() == PUBLIC_KEY_TYPE_ED25519) + { + h = load64(key.account().accountID.ed25519().data()); + } + else + { + h = std::hash()(key); + } + break; + case TTL: + h = load64(key.ttl().keyHash.data()); + break; + case CONTRACT_CODE: + h = load64(key.contractCode().hash.data()); + break; + default: + h = std::hash()(key); + break; + } + return static_cast(groupOf(key.type())) * n + h % n; +} + +Hash +ShardedTopBucket::bucketHash(LtHash16 const& h) +{ + return h.isZero() ? Hash() : h.digest(); +} + +bool +ShardedTopBucket::hasMetaEntry(uint32_t protocolVersion) +{ + return protocolVersionStartsFrom( + protocolVersion, + LiveBucket::FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY); +} + +BucketEntry +ShardedTopBucket::metaEntry(uint32_t protocolVersion) +{ + BucketMetadata meta; + meta.ledgerVersion = protocolVersion; + if (protocolVersionStartsFrom( + protocolVersion, + LiveBucket::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION)) + { + meta.ext.v(1); + meta.ext.bucketListType() = BucketListType::LIVE; + } + BucketEntry bme; + bme.type(METAENTRY); + bme.metaEntry() = meta; + return bme; +} + +LtHash16 +ShardedTopBucket::hashOfFile(std::string const& filename) +{ + ZoneScoped; + // The file must be THE file of its records: the metadata entry first, + // the entries in the strict order of their keys, every record in its one + // XDR form. The commutative hash alone does not tell a bucket file from + // a file with the same records in another order, and whoever reads the + // file (a merge, the index) relies on the order. + // + // The file is read twice at once, record by record (it may be any size: + // catchup hands every live bucket that is not named by its SHA-256 to + // this): once decoded, once as bytes to hold the re-encoded record + // against. + std::ifstream raw(filename, std::ios::binary); + if (!raw) + { + throw std::runtime_error( + fmt::format(FMT_STRING("could not open {}"), filename)); + } + LtHash16 h; + XDRInputFileStream in; + in.open(filename); + BucketEntry be; + BucketEntry prev; + bool first = true; + bool havePrev = false; + std::string expected; + std::string found; + BucketEntryIdCmp idCmp; + while (in && in.readOne(be)) + { + auto rec = xdr::xdr_to_opaque(be); + expected.clear(); + appendRecord(expected, reinterpret_cast(rec.data()), + rec.size()); + found.resize(expected.size()); + raw.read(found.data(), static_cast(found.size())); + if (static_cast(raw.gcount()) != found.size() || + found != expected) + { + throw std::runtime_error(fmt::format( + FMT_STRING("{}: a record is not in its canonical form"), + filename)); + } + if (be.type() == METAENTRY) + { + if (!first) + { + throw std::runtime_error(fmt::format( + FMT_STRING("{}: a metadata entry that is not first"), + filename)); + } + } + else + { + if (havePrev && !idCmp(prev, be)) + { + throw std::runtime_error(fmt::format( + FMT_STRING("{}: entries out of order"), filename)); + } + prev = be; + havePrev = true; + } + first = false; + h.add(ByteSlice(rec.data(), rec.size())); + } + if (raw.peek() != std::ifstream::traits_type::eof()) + { + throw std::runtime_error(fmt::format( + FMT_STRING("{}: bytes after the last record"), filename)); + } + return h; +} + +void +ShardedTopBucket::waitForFile() +{ + if (mFileReady.valid()) + { + mFileReady.get(); + } + if (mFileWriter.valid()) + { + // The writer reports its time here and not by itself: it may outlive + // the application's metrics (see the destructor). + mFileTimer.Update(mFileWriter.get()); + } +} + +void +ShardedTopBucket::rebuildFrom(std::shared_ptr const& curr) +{ + ZoneScoped; + std::vector shards(mNumShards); + LtHash16 total; + if (!curr->isEmpty()) + { + LiveBucketInputIterator it(curr); + if (it.seenMetadata()) + { + BucketEntry bme; + bme.type(METAENTRY); + bme.metaEntry() = it.getMetadata(); + auto rec = xdr::xdr_to_opaque(bme); + total.add(ByteSlice(rec.data(), rec.size())); + } + for (; it; ++it) + { + BucketEntry e = *it; + auto& shard = + shards.at(shardOf(getBucketLedgerKey(e), mShardsPerGroup)); + shard.append(std::move(e)); + } + for (auto& shard : shards) + { + shard.hash = hashOfRecords(shard); + total += shard.hash; + } + } + if (bucketHash(total) != curr->getHash()) + { + throw std::runtime_error(fmt::format( + FMT_STRING("level 0's bucket {} does not have the commutative " + "hash of its records ({}): it was written with " + "EXPERIMENTAL_SHARDED_TOP_BUCKET off"), + binToHex(curr->getHash()), binToHex(bucketHash(total)))); + } + mShards.clear(); + for (auto& shard : shards) + { + mShards.emplace_back(std::make_shared(std::move(shard))); + } + mCurrHash = curr->getHash(); + mFileReady = {}; +} + +std::shared_ptr +ShardedTopBucket::addBatch(Application& app, uint32_t ledgerSeq, + uint32_t protocolVersion, + std::shared_ptr const& curr, + LedgerEntryRefs initEntries, + LedgerEntryRefs liveEntries, + LedgerKeyRefs deadEntries, bool countMergeEvents, + bool doFsync) +{ + ZoneScoped; + auto& bm = app.getBucketManager(); + + // The file of the bucket made a ledger ago is long written. Being sure of + // it here means that at most one level-0 file is ever missing on disk. + { + auto timer = mFileWaitTimer.TimeScope(); + waitForFile(); + } + // What goes out of use in this call (the shards of the bucket that + // spilled, of the bucket that is merged into a new one): tens of thousands + // of entries, which the file's writer frees, off this thread. + std::vector>> garbage; + garbage.emplace_back(mShards); + if (curr->getHash() != mCurrHash) + { + // A bucket this object did not make: the empty one after a spill, or + // one that came from disk. + rebuildFrom(curr); + } + auto prevShards = mShards; + // What a restart needs to know of this ledger's shard files: the bucket + // they were added to. + std::string const prevHex = binToHex(curr->getHash()); + + bool const useInit = protocolVersionStartsFrom( + protocolVersion, + LiveBucket::FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY); + uint32_t const n = mShardsPerGroup; + size_t const total = + initEntries.size() + liveEntries.size() + deadEntries.size(); + + // 1. Cut the batch into the shards. Every task takes a stretch of the + // batch and sorts its entries into its own list per shard. + size_t const cutTasks = std::max( + 1, std::min(mExecutor.preferredTaskCount(), total / 512)); + std::vector>> cut( + cutTasks, std::vector>(mNumShards)); + { + auto timer = mCutTimer.TimeScope(); + mExecutor.executeBatchOverRanges( + total, cutTasks, [&](size_t begin, size_t end, size_t task) { + auto& mine = cut.at(task); + size_t const initEnd = initEntries.size(); + size_t const liveEnd = initEnd + liveEntries.size(); + for (size_t i = begin; i < end; ++i) + { + if (i < initEnd) + { + LedgerEntry const& e = initEntries[i].get(); + mine[shardOfEntry(e, n)].push_back(Item{ITEM_INIT, &e}); + } + else if (i < liveEnd) + { + LedgerEntry const& e = liveEntries[i - initEnd].get(); + mine[shardOfEntry(e, n)].push_back(Item{ITEM_LIVE, &e}); + } + else + { + LedgerKey const& k = deadEntries[i - liveEnd].get(); + mine[shardOf(k, n)].push_back(Item{ITEM_DEAD, &k}); + } + } + }); + } + + // 2. Every shard on its own: sort, write and sync the ledger's entries, + // merge them with the shard's entries of the previous ledger, hash, + // and prepare the shard's part of the index. + struct Output + { + std::shared_ptr shard; + InMemoryIndexShard index; + MergeCounters mc; + std::string file; + }; + std::vector outputs(mNumShards); + auto& ctx = app.getClock().getIOContext(); + { + auto timer = mShardsTimer.TimeScope(); + size_t const shardTasks = std::max( + 1, std::min(mExecutor.preferredTaskCount(), mNumShards)); + // A task takes the next shard that no task has taken: shards differ + // in size, and there may be fewer or more of them than tasks. + std::atomic nextShard{0}; + mExecutor.executeBatchOverRanges( + shardTasks, shardTasks, [&](size_t, size_t, size_t) { + BucketEntryIdCmp idCmp; + for (size_t s = nextShard.fetch_add(1); s < mNumShards; + s = nextShard.fetch_add(1)) + { + auto& out = outputs[s]; + size_t count = 0; + for (auto const& task : cut) + { + count += task[s].size(); + } + std::vector fresh; + fresh.reserve(count); + for (auto const& task : cut) + { + for (auto const& item : task[s]) + { + auto& be = fresh.emplace_back(); + if (item.kind == ITEM_DEAD) + { + be.type(DEADENTRY); + be.deadEntry() = + *static_cast(item.ptr); + } + else + { + be.type(item.kind == ITEM_INIT && useInit + ? INITENTRY + : LIVEENTRY); + be.liveEntry() = + *static_cast(item.ptr); + } + if (countMergeEvents) + { + LiveBucket::countNewEntryType(out.mc, be); + } + } + } + std::sort(fresh.begin(), fresh.end(), idCmp); + releaseAssertOrThrow( + std::adjacent_find(fresh.begin(), fresh.end(), + [&idCmp](BucketEntry const& a, + BucketEntry const& b) { + return !idCmp(a, b); + }) == fresh.end()); + + auto freshShard = std::make_shared(); + freshShard->entries.reserve(fresh.size()); + freshShard->offsets.reserve(fresh.size() + 1); + for (auto& be : fresh) + { + freshShard->append(std::move(be)); + } + + // The ledger's entries of this shard on disk, synced: + // from here on the ledger does not depend on the + // bucket's file. + if (!freshShard->entries.empty()) + { + out.file = fmt::format( + FMT_STRING("{}/shard-{}-{}-{}-{}.xdr"), mShardDir, + ledgerSeq, prevHex, mInstance, s); + std::string fileBytes; + fileBytes.reserve(freshShard->records.size() + + 4 * freshShard->entries.size()); + for (size_t i = 0; i < freshShard->entries.size(); ++i) + { + appendRecord(fileBytes, + freshShard->records.data() + + freshShard->offsets[i], + freshShard->offsets[i + 1] - + freshShard->offsets[i]); + } + XDROutputFileStream shardOut(ctx, doFsync); + shardOut.open(out.file); + shardOut.writeBytes(fileBytes.data(), fileBytes.size()); + shardOut.close(); + } + + auto const& prev = *prevShards[s]; + if (prev.entries.empty()) + { + freshShard->hash = hashOfRecords(*freshShard); + out.shard = freshShard; + } + else + { + // (Also when the ledger has nothing for the shard: + // the merge with nothing is a copy.) + auto mergedEntries = LiveBucket::mergeEntriesInMemory( + bm, protocolVersion, prev.entries, + freshShard->entries, out.mc); + auto merged = std::make_shared(); + merged->entries.reserve(mergedEntries.size()); + merged->offsets.reserve(mergedEntries.size() + 1); + for (auto& be : mergedEntries) + { + merged->append(std::move(be)); + } + merged->hash = hashOfRecords(*merged); + out.shard = merged; + } + + auto const& shard = *out.shard; + for (size_t i = 0; i < shard.entries.size(); ++i) + { + out.index.add(shard.entries[i], + shard.offsets[i + 1] - shard.offsets[i]); + } + } + }); + } + + // 3. Add the shards up: the bucket's hash and its index. + auto timer = mFinishTimer.TimeScope(); + LtHash16 hashState; + size_t fileSize = 0; + size_t entryCount = 0; + std::vector> shards; + shards.reserve(mNumShards); + std::vector indexShards; + indexShards.reserve(mNumShards); + std::vector shardFiles; + MergeCounters mc; + for (auto& out : outputs) + { + hashState += out.shard->hash; + fileSize += out.shard->records.size() + 4 * out.shard->entries.size(); + entryCount += out.shard->entries.size(); + shards.emplace_back(out.shard); + indexShards.emplace_back(std::move(out.index)); + mc += out.mc; + if (!out.file.empty()) + { + shardFiles.emplace_back(std::move(out.file)); + } + } + if (countMergeEvents) + { + bm.incrMergeCounters(mc); + } + if (doFsync && !shardFiles.empty()) + { + syncDir(mShardDir); + } + + bool const hasMeta = hasMetaEntry(protocolVersion); + BucketEntry const bme = metaEntry(protocolVersion); + auto metaRecord = + std::make_shared const>(xdr::xdr_to_opaque(bme)); + if (hasMeta) + { + hashState.add(ByteSlice(metaRecord->data(), metaRecord->size())); + fileSize += metaRecord->size() + 4; + } + + mShards = shards; + + if (!hasMeta && entryCount == 0) + { + // What a bucket without a metadata entry and without entries is: + // no bucket. + mCurrHash = Hash(); + mFileReady = {}; + return std::make_shared(); + } + + Hash const hash = bucketHash(hashState); + auto index = std::make_shared( + bm, std::make_unique( + std::move(indexShards), + [n](LedgerKey const& k) { return shardOf(k, n); }, + bme.metaEntry(), hasMeta)); + + auto promise = std::make_shared>(); + std::shared_future ready = promise->get_future().share(); + bool created = false; + auto bucket = bm.adoptLiveBucketAheadOfItsFile( + hash, std::move(index), fileSize, ready, + // (What the bucket's file says: nothing, without a metadata entry.) + hasMeta ? protocolVersion : 0, created); + mCurrHash = hash; + + if (!created) + { + // The bucket manager has a bucket with these records already, with + // its file there or on its way. + promise->set_value(); + for (auto const& f : shardFiles) + { + std::remove(f.c_str()); + } + mFileReady = {}; + return bucket; + } + + // 4. The bucket's file, off this thread. + std::string tmpName = LiveBucket::randomBucketName(bm.getTmpDir()); + std::string canonicalName = bm.bucketFilename(hash); + // (Not the bucket manager's rename: the writer may run while the + // application is taken down.) + std::string const bucketDir = bm.getBucketDir(); + mFileReady = ready; + garbage.emplace_back(std::move(prevShards)); + mFileWriter = std::async( + std::launch::async, [promise, shards, metaRecord, hasMeta, tmpName, + canonicalName, shardFiles, bucketDir, &ctx, + doFsync, garbage = std::move(garbage)]() mutable { + auto const start = std::chrono::steady_clock::now(); + try + { +#ifdef BUILD_TESTS + if (gLoseFileForTesting) + { + // As if the node stopped here: the ledger's shard files + // are there, the bucket's file is not. + promise->set_value(); + return std::chrono::nanoseconds(0); + } +#endif + writeBucketFile(shards, hasMeta ? metaRecord.get() : nullptr, + tmpName, ctx, doFsync); + bool const renamed = + doFsync + ? fs::durableRename(tmpName, canonicalName, bucketDir) + : std::rename(tmpName.c_str(), canonicalName.c_str()) == + 0; + if (!renamed) + { + throw std::runtime_error( + fmt::format(FMT_STRING("could not rename {} to {}: {}"), + tmpName, canonicalName, strerror(errno))); + } + // The file holds what the ledger's shard files held. + for (auto const& f : shardFiles) + { + std::remove(f.c_str()); + } + promise->set_value(); + } + catch (...) + { + promise->set_exception(std::current_exception()); + } + auto const elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + // Here, and not where the next writer is started. + garbage.clear(); + shards.clear(); + return elapsed; + }); + if (shardFiles.empty()) + { + // A new bucket without a new entry (the batch was empty and the + // metadata entry changed): there are no shard files that a restart + // could rebuild it from, so its file is written before the ledger + // goes on. + waitForFile(); + } + return bucket; +} + +void +ShardedTopBucket::removeShardFiles(std::string const& shardDir) +{ + if (!fs::exists(shardDir)) + { + return; + } + for (auto const& f : fs::findfiles(shardDir, isShardFile)) + { + std::remove((shardDir + "/" + f).c_str()); + } +} + +void +ShardedTopBucket::prepareForRestart(Application& app, + HistoryArchiveState const& has, + uint32_t protocolVersion) +{ + ZoneScoped; + auto& bm = app.getBucketManager(); + std::string const shardDir = bm.getBucketDir() + "/topshards"; + Hash const hash = hexToBin256(has.currentBuckets.at(0).curr); + if (auto b = bm.getBucketIfExists(hash)) + { + // Made by this process: its file is there or on its way. + b->waitForFile(); + } + if (isZero(hash) || fs::exists(bm.bucketFilename(hash))) + { + // A level 0 that was written without the setting is found here and + // not in the middle of a ledger. + for (auto const& hex : + {has.currentBuckets.at(0).curr, has.currentBuckets.at(0).snap}) + { + Hash const h = hexToBin256(hex); + if (isZero(h) || !fs::exists(bm.bucketFilename(h))) + { + continue; + } + if (bucketHash(hashOfFile(bm.bucketFilename(h))) != h) + { + throw std::runtime_error(fmt::format( + FMT_STRING( + "EXPERIMENTAL_SHARDED_TOP_BUCKET is set, but level 0's " + "bucket {} does not have the commutative hash of its " + "records: this node's state was written without the " + "setting"), + hex)); + } + } + removeShardFiles(shardDir); + return; + } + + // The node stopped between the ledger's commit and the end of the + // bucket's file. The ledger's entries are in its shard files. + std::map, std::vector> + candidates; + if (fs::exists(shardDir)) + { + for (auto const& f : fs::findfiles(shardDir, isShardFile)) + { + auto parsed = parseShardFileName(f); + if (parsed && parsed->ledger == has.currentLedger) + { + candidates[{parsed->prevHex, parsed->instance}].emplace_back( + shardDir + "/" + f); + } + } + } + bool const doFsync = !app.getConfig().DISABLE_XDR_FSYNC; + bool const hasMeta = hasMetaEntry(protocolVersion); + BucketEntry const bme = metaEntry(protocolVersion); + auto const metaRecord = xdr::xdr_to_opaque(bme); + BucketEntryIdCmp idCmp; + for (auto const& [id, files] : candidates) + { + std::vector fresh; + for (auto const& f : files) + { + XDRInputFileStream in; + in.open(f); + BucketEntry be; + while (in && in.readOne(be)) + { + fresh.emplace_back(be); + } + } + std::sort(fresh.begin(), fresh.end(), idCmp); + + std::vector entries; + Hash const prev = hexToBin256(id.first); + if (isZero(prev)) + { + entries = std::move(fresh); + } + else + { + auto prevBucket = bm.getBucketByHash(prev); + if (!prevBucket) + { + CLOG_WARNING(Bucket, + "Shard files of ledger {} were added to bucket " + "{}, which is not there", + has.currentLedger, id.first); + continue; + } + std::vector old; + for (LiveBucketInputIterator it(prevBucket); it; ++it) + { + old.emplace_back(*it); + } + MergeCounters mc; + entries = LiveBucket::mergeEntriesInMemory(bm, protocolVersion, old, + fresh, mc); + } + + auto shard = std::make_shared(); + for (auto& be : entries) + { + shard->append(std::move(be)); + } + LtHash16 h = hashOfRecords(*shard); + if (hasMeta) + { + h.add(ByteSlice(metaRecord.data(), metaRecord.size())); + } + if (bucketHash(h) != hash) + { + CLOG_WARNING(Bucket, + "Shard files of ledger {} (instance {}) do not make " + "bucket {}", + has.currentLedger, id.second, binToHex(hash)); + continue; + } + std::string tmpName = LiveBucket::randomBucketName(bm.getTmpDir()); + writeBucketFile({shard}, hasMeta ? &metaRecord : nullptr, tmpName, + app.getClock().getIOContext(), doFsync); + if (!bm.renameBucketDirFile(tmpName, bm.bucketFilename(hash))) + { + throw std::runtime_error( + fmt::format(FMT_STRING("could not rename {} to {}: {}"), + tmpName, bm.bucketFilename(hash), strerror(errno))); + } + CLOG_INFO(Bucket, + "Rebuilt level 0's bucket {} of ledger {} from {} shard " + "files", + binToHex(hash), has.currentLedger, files.size()); + removeShardFiles(shardDir); + return; + } + // Not rebuilt: the caller's check for missing bucket files reports it. + CLOG_ERROR(Bucket, + "Level 0's bucket {} of ledger {} is missing and could not be " + "rebuilt from shard files", + binToHex(hash), has.currentLedger); +} +} diff --git a/src/bucket/ShardedTopBucket.h b/src/bucket/ShardedTopBucket.h new file mode 100644 index 0000000000..ecc8ea5063 --- /dev/null +++ b/src/bucket/ShardedTopBucket.h @@ -0,0 +1,181 @@ +#pragma once + +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "bucket/LiveBucket.h" +#include "crypto/LtHash.h" +#include "ledger/LedgerEntryRefs.h" +#include "util/BatchExecutor.h" +#include "util/NonCopyable.h" +#include "xdr/Stellar-ledger-entries.h" +#include "xdr/Stellar-ledger.h" +#include +#include +#include +#include +#include + +namespace medida +{ +class Timer; +} + +namespace stellar +{ + +class Application; +class BucketManager; +struct HistoryArchiveState; + +// Level 0 of the live bucket list, written as shards +// (EXPERIMENTAL_SHARDED_TOP_BUCKET). +// +// What a level-0 bucket IS does not change: the merge of the batches of the +// one or two ledgers it holds, sorted by key, behind one metadata entry. What +// changes is its hash and how it gets to disk. +// +// * The hash. Not the SHA-256 of the bucket's file but the LtHash16 of its +// records (the XDR of every bucket entry of the file, the metadata entry +// among them), condensed with SHA-256 where a 32-byte hash is expected +// (`bucketHash`). It does not depend on the order of the records or on how +// they are split into parts, so the parts can be hashed apart and added +// up. Levels 1 and deeper keep SHA-256. +// +// * Shards. A shard is a set of keys that is the same in every ledger: the +// shard of an entry is a function of its key (`shardOf`). There are three +// groups: contract data and code, TTL entries, everything else (classic +// entries); a group has N shards, by a hash of the key. How many shards a +// node uses changes no hash. +// +// * A ledger's batch. The entries are cut into the shards, and every shard, +// on its own thread, sorts its entries, writes them to a file of its own +// and syncs it (that is what makes the ledger durable), merges them with +// what the shard holds of the previous ledger (with the merge code of every +// bucket merge) and hashes the result. The shards' hashes are added up, and +// the new level-0 bucket exists: its hash, and its index for lookups. +// +// * The bucket's file. It is written afterwards by a thread of its own, the +// shards' entries interleaved by key: byte for byte the file that level 0 +// has without the setting. It is what spills into level 1 and what is +// published to history archives. Whoever needs it waits for it +// (`BucketBase::getFilename`). +// +// The bucket list is touched by its owning thread only, as without the +// setting; this class is not thread safe, and `addBatch` is called by that +// thread. +class ShardedTopBucket : public NonMovableOrCopyable +{ + public: + // One shard's part of the bucket. + struct Shard + { + // Sorted by key. + std::vector entries; + // The entries' XDR, one after the other, as the bucket file holds + // them (without the 4 bytes in front of each). + std::string records; + // Where entry i's XDR begins in `records`; one more at the end. + std::vector offsets{0}; + // Of the records. + LtHash16 hash; + + void append(BucketEntry&& entry); + }; + + ShardedTopBucket(Application& app, uint32_t shardsPerGroup); + ~ShardedTopBucket(); + + uint32_t + shardsPerGroup() const + { + return mShardsPerGroup; + } + + // Adds the entries of ledger `ledgerSeq` to `curr`, level 0's current + // bucket (empty after a spill), and returns the new current bucket. Its + // file may not be on disk yet. + std::shared_ptr + addBatch(Application& app, uint32_t ledgerSeq, uint32_t protocolVersion, + std::shared_ptr const& curr, + LedgerEntryRefs initEntries, LedgerEntryRefs liveEntries, + LedgerKeyRefs deadEntries, bool countMergeEvents, bool doFsync); + + // Blocks until the file of the bucket `addBatch` returned last is on + // disk. + void waitForFile(); + + // The shard of a key with `shardsPerGroup` shards per group: the group's + // number (0 contract data and code, 1 TTL, 2 the rest) times + // `shardsPerGroup`, plus a hash of the key modulo `shardsPerGroup`. + static size_t shardOf(LedgerKey const& key, uint32_t shardsPerGroup); + + // The hash of a level-0 bucket whose records have the LtHash `h`: the + // zero hash for no records (the empty bucket), the digest otherwise. + static Hash bucketHash(LtHash16 const& h); + + // The LtHash of the records of a bucket file. Throws unless the file is + // the canonical file of its records (the metadata entry first, entries + // in the strict order of their keys, canonical XDR, nothing else). + static LtHash16 hashOfFile(std::string const& filename); + + // At startup, before the bucket files of the last closed ledger are + // checked: makes sure that level 0's current bucket of `has` has its + // file. If the node stopped after the ledger's commit and before the + // file was written, the file is rebuilt from the ledger's shard files + // and the bucket they were added to. Removes all shard files left. + static void prepareForRestart(Application& app, + HistoryArchiveState const& has, + uint32_t protocolVersion); + + // The metadata entry of a live bucket of that protocol version, and + // whether such a bucket has one. + static BucketEntry metaEntry(uint32_t protocolVersion); + static bool hasMetaEntry(uint32_t protocolVersion); + +#ifdef BUILD_TESTS + std::vector> const& + getShardsForTesting() const + { + return mShards; + } + // While set, in every application of the process: a bucket's file is not + // written and the ledger's shard files stay, as on a node that stops + // before the file's writer is done. + static void loseFileForTesting(bool lose); +#endif + + private: + uint32_t const mShardsPerGroup; + size_t const mNumShards; + + // The bucket the shards describe. + Hash mCurrHash{}; + // Never changed once made: a ledger's tasks read the previous ledger's + // shards while the file's writer may still be reading them too. + std::vector> mShards; + + // The writer of that bucket's file, and what tells that it is done. + std::future mFileWriter; + std::shared_future mFileReady; + + // Where a ledger's shard files go, and what tells this object's files + // from another's in the same directory. + std::string mShardDir; + std::string mInstance; + BatchExecutor mExecutor; + + medida::Timer& mCutTimer; + medida::Timer& mShardsTimer; + medida::Timer& mFinishTimer; + medida::Timer& mFileTimer; + medida::Timer& mFileWaitTimer; + + static void removeShardFiles(std::string const& shardDir); + + // Takes the shards from `curr`, a bucket this object did not make (after + // a restart, or a bucket list assumed from a history archive state). + void rebuildFrom(std::shared_ptr const& curr); +}; +} diff --git a/src/bucket/test/ShardedTopBucketTests.cpp b/src/bucket/test/ShardedTopBucketTests.cpp new file mode 100644 index 0000000000..358598e648 --- /dev/null +++ b/src/bucket/test/ShardedTopBucketTests.cpp @@ -0,0 +1,706 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// Tests of level 0 of the live bucket list written as shards under the +// commutative hash (EXPERIMENTAL_SHARDED_TOP_BUCKET, +// bucket/ShardedTopBucket.h). + +#include "bucket/BucketInputIterator.h" +#include "bucket/BucketManager.h" +#include "bucket/LiveBucket.h" +#include "bucket/LiveBucketIndex.h" +#include "bucket/LiveBucketList.h" +#include "bucket/ShardedTopBucket.h" +#include "bucket/test/BucketTestUtils.h" +#include "crypto/Hex.h" +#include "crypto/LtHash.h" +#include "crypto/SHA.h" +#include "herder/Herder.h" +#include "ledger/LedgerManager.h" +#include "ledger/LedgerTypeUtils.h" +#include "ledger/test/LedgerTestUtils.h" +#include "main/Application.h" +#include "main/Config.h" +#include "medida/counter.h" +#include "medida/meter.h" +#include "medida/timer.h" +#include "simulation/LoadGenerator.h" +#include "simulation/Simulation.h" +#include "simulation/Topologies.h" +#include "test/Catch2.h" +#include "test/TestUtils.h" +#include "test/test.h" +#include "util/Fs.h" +#include "util/Math.h" +#include "util/MetricsRegistry.h" +#include "util/Timer.h" +#include "util/XDRStream.h" +#include +#include +#include +#include +#include +#include + +using namespace stellar; + +namespace +{ +std::string +fileBytes(std::filesystem::path const& name) +{ + std::ifstream in(name, std::ios::binary); + REQUIRE(in.good()); + std::stringstream ss; + ss << in.rdbuf(); + return ss.str(); +} + +// Batches that follow the rules a ledger follows: an entry is created (INIT) +// only where no live entry is, changed (LIVE) or deleted (DEAD) only where +// one is; a key occurs once per ledger. Keys come back after a deletion, and +// are deleted or changed in the ledger after their creation, so that every +// case of a merge of two ledgers at level 0 occurs. +struct BatchMaker +{ + std::map mLive; + std::vector mDeleted; + std::vector mCreatedLastLedger; + // The types of the entries that are created, changed and deleted freely. + std::unordered_set mTypes{ + ACCOUNT, TRUSTLINE, OFFER, DATA, CONTRACT_DATA, CONTRACT_CODE, TTL}; + // For a ledger manager, which keeps contract data with its TTL: so many + // new contract data entries per ledger, each with its TTL entry (while + // mTypes holds the classic types only). + size_t mContractDataWithTTL{0}; + + void + make(uint32_t ledger, size_t size, std::vector& init, + std::vector& live, std::vector& dead) + { + init.clear(); + live.clear(); + dead.clear(); + std::set touched; + auto& rng = getGlobalRandomEngine(); + + // Delete or change half of what the previous ledger created. + for (size_t i = 0; i < mCreatedLastLedger.size(); i += 2) + { + auto const& key = mCreatedLastLedger[i]; + auto it = mLive.find(key); + if (it == mLive.end() || !touched.insert(key).second) + { + continue; + } + if (i % 4 == 0) + { + dead.emplace_back(key); + mDeleted.emplace_back(it->second); + mLive.erase(it); + } + else + { + it->second.lastModifiedLedgerSeq = ledger; + live.emplace_back(it->second); + } + } + mCreatedLastLedger.clear(); + + // Bring back some of what was deleted (also in the ledger before). + size_t back = std::min(mDeleted.size(), size / 8 + 1); + for (size_t i = 0; i < back; ++i) + { + auto e = mDeleted.back(); + mDeleted.pop_back(); + auto key = LedgerEntryKey(e); + if (mLive.count(key) || !touched.insert(key).second) + { + continue; + } + e.lastModifiedLedgerSeq = ledger; + mLive[key] = e; + init.emplace_back(e); + mCreatedLastLedger.emplace_back(key); + } + + // New entries of every group. + for (size_t i = 0; i < size; ++i) + { + auto e = LedgerTestUtils::generateValidLedgerEntryWithTypes(mTypes); + auto key = LedgerEntryKey(e); + if (mLive.count(key) || !touched.insert(key).second) + { + continue; + } + e.lastModifiedLedgerSeq = ledger; + mLive[key] = e; + init.emplace_back(e); + mCreatedLastLedger.emplace_back(key); + } + for (size_t i = 0; i < mContractDataWithTTL; ++i) + { + auto e = LedgerTestUtils::generateValidLedgerEntryWithTypes( + {CONTRACT_DATA}); + if (!touched.insert(LedgerEntryKey(e)).second) + { + continue; + } + e.lastModifiedLedgerSeq = ledger; + LedgerEntry ttl; + ttl.lastModifiedLedgerSeq = ledger; + ttl.data.type(TTL); + ttl.data.ttl().keyHash = getTTLKey(e).ttl().keyHash; + ttl.data.ttl().liveUntilLedgerSeq = ledger + 10'000; + init.emplace_back(e); + init.emplace_back(ttl); + } + + // Change and delete some older entries. + std::vector keys; + for (auto const& [key, _] : mLive) + { + if (!touched.count(key)) + { + keys.emplace_back(key); + } + } + stellar::shuffle(keys.begin(), keys.end(), rng); + for (size_t i = 0; i < keys.size() && i < size / 2; ++i) + { + auto it = mLive.find(keys[i]); + touched.insert(keys[i]); + if (i % 3 == 0) + { + dead.emplace_back(keys[i]); + mDeleted.emplace_back(it->second); + mLive.erase(it); + } + else + { + it->second.lastModifiedLedgerSeq = ledger; + live.emplace_back(it->second); + } + } + } +}; + +Application::pointer +makeApp(VirtualClock& clock, int instance, uint32_t shardsPerGroup) +{ + Config cfg(getTestConfig(instance, Config::TESTDB_BUCKET_DB_PERSISTENT)); + cfg.EXPERIMENTAL_SHARDED_TOP_BUCKET = shardsPerGroup; + return createTestApplication(clock, cfg); +} +} + +namespace +{ +void +sameBucketsUnderAnotherHash(uint32_t version) +{ + INFO("protocol " << version); + VirtualClock clock; + // The first application has the setting off: its bucket list is what + // every other one must agree with, apart from the hash of level 0. + std::vector const shardCounts{0, 1, 2, 5, 16}; + std::vector apps; + std::vector> lists; + for (size_t i = 0; i < shardCounts.size(); ++i) + { + apps.emplace_back(makeApp(clock, static_cast(i), shardCounts[i])); + lists.emplace_back(std::make_unique()); + } + BatchMaker maker; + std::vector init, live; + std::vector dead; + size_t sawTwoLedgerBuckets = 0; + for (uint32_t ledger = 1; ledger <= 70; ++ledger) + { + // Now and then a ledger without entries. + maker.make(ledger, ledger % 11 == 0 ? 0 : 40, init, live, dead); + if (ledger % 23 == 0) + { + init.clear(); + live.clear(); + dead.clear(); + } + for (size_t i = 0; i < apps.size(); ++i) + { + lists[i]->addBatch(*apps[i], ledger, version, init, live, dead); + } + for (auto& list : lists) + { + list->resolveAllFutures(); + } + + auto const& off = *lists[0]; + auto offCurr = off.getLevel(0).getCurr(); + auto offSnap = off.getLevel(0).getSnap(); + // What the hash of a level-0 bucket is with the setting: the LtHash + // of the records of the file the bucket has without it. + auto expectHash = [](std::shared_ptr const& b) { + return b->isEmpty() ? Hash() + : ShardedTopBucket::bucketHash( + ShardedTopBucket::hashOfFile( + b->getFilename().string())); + }; + Hash const currHash = expectHash(offCurr); + Hash const snapHash = expectHash(offSnap); + if (!offCurr->isEmpty()) + { + REQUIRE(currHash != offCurr->getHash()); + } + + for (size_t i = 1; i < lists.size(); ++i) + { + INFO("ledger " << ledger << ", " << shardCounts[i] + << " shards per group"); + auto const& on = *lists[i]; + auto onCurr = on.getLevel(0).getCurr(); + auto onSnap = on.getLevel(0).getSnap(); + // Level 0: the commutative hash, whatever the shard count, and + // the same file, byte for byte. + REQUIRE(onCurr->getHash() == currHash); + REQUIRE(onSnap->getHash() == snapHash); + REQUIRE(onCurr->isEmpty() == offCurr->isEmpty()); + if (!onCurr->isEmpty()) + { + REQUIRE(fileBytes(onCurr->getFilename()) == + fileBytes(offCurr->getFilename())); + REQUIRE(onCurr->getSize() == offCurr->getSize()); + REQUIRE(onCurr->getBucketVersion() == + offCurr->getBucketVersion()); + REQUIRE(onCurr->getBucketEntryCounters() == + offCurr->getBucketEntryCounters()); + } + if (!onSnap->isEmpty()) + { + REQUIRE(fileBytes(onSnap->getFilename()) == + fileBytes(offSnap->getFilename())); + } + // Levels 1 and deeper: the same buckets under the same SHA-256. + for (uint32_t level = 1; level < LiveBucketList::kNumLevels; + ++level) + { + REQUIRE(on.getLevel(level).getCurr()->getHash() == + off.getLevel(level).getCurr()->getHash()); + REQUIRE(on.getLevel(level).getSnap()->getHash() == + off.getLevel(level).getSnap()->getHash()); + REQUIRE(on.getLevel(level).getHash() == + off.getLevel(level).getHash()); + } + // The bucket list hash: the same for every shard count, and not + // the one without the setting (level 0 is hashed differently). + REQUIRE(on.getHash() == lists[1]->getHash()); + if (!offCurr->isEmpty() || !offSnap->isEmpty()) + { + REQUIRE(on.getHash() != off.getHash()); + } + + // Every entry of the ledger is found through the bucket's index, + // as the bucket without the setting has it. + if (!onCurr->isEmpty()) + { + auto const& index = onCurr->getIndexForTesting(); + auto const& offIndex = offCurr->getIndexForTesting(); + auto check = [&](LedgerKey const& key) { + auto got = index.lookup(key); + auto want = offIndex.lookup(key); + REQUIRE(got.getState() == want.getState()); + if (want.getState() == IndexReturnState::CACHE_HIT) + { + REQUIRE(*got.cacheHit() == *want.cacheHit()); + } + }; + for (auto const& e : init) + { + check(LedgerEntryKey(e)); + } + for (auto const& e : live) + { + check(LedgerEntryKey(e)); + } + for (auto const& k : dead) + { + check(k); + } + // And a key that is in no bucket. + check(LedgerEntryKey( + LedgerTestUtils::generateValidLedgerEntryWithTypes( + {ACCOUNT, CONTRACT_DATA, TTL}))); + for (auto type : {ACCOUNT, TRUSTLINE, OFFER, DATA, + CONTRACT_DATA, CONTRACT_CODE, TTL}) + { + REQUIRE(index.getRangeForType(type) == + offIndex.getRangeForType(type)); + } + } + } + if (ledger % 2 == 1 && ledger > 1 && !offCurr->isEmpty()) + { + ++sawTwoLedgerBuckets; + } + } + REQUIRE(sawTwoLedgerBuckets > 20); +} +} + +TEST_CASE("sharded level 0 is the same buckets under another hash", + "[bucket][bucketlist][sharded-top-bucket]") +{ + // The current protocol, and the last one without INIT and metadata + // entries and the first one with them (a bucket without a metadata entry + // and without entries is no bucket; a merge of a ledger that creates an + // entry and a ledger that deletes it keeps or drops the tombstone). + auto const init = static_cast( + LiveBucket::FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY); + for (uint32_t version : + {getTestConfig().LEDGER_PROTOCOL_VERSION, init - 1, init}) + { + sameBucketsUnderAnotherHash(version); + } +} + +TEST_CASE("the commutative hash is taken of a canonical bucket file only", + "[bucket][sharded-top-bucket]") +{ + VirtualClock clock; + auto app = makeApp(clock, 0, 2); + auto version = app->getConfig().LEDGER_PROTOCOL_VERSION; + LiveBucketList list; + BatchMaker maker; + std::vector init, live; + std::vector dead; + maker.make(1, 40, init, live, dead); + list.addBatch(*app, 1, version, init, live, dead); + auto bucket = list.getLevel(0).getCurr(); + std::string const good = bucket->getFilename().string(); + REQUIRE(ShardedTopBucket::bucketHash(ShardedTopBucket::hashOfFile(good)) == + bucket->getHash()); + + std::vector entries; + { + XDRInputFileStream in; + in.open(good); + BucketEntry be; + while (in && in.readOne(be)) + { + entries.emplace_back(be); + } + } + REQUIRE(entries.size() > 10); + REQUIRE(entries.front().type() == METAENTRY); + + auto write = [&](std::vector const& es, char const* name, + std::string const& tail = "") { + std::string file = + app->getBucketManager().getTmpDir() + "/" + name + ".xdr"; + XDROutputFileStream out(clock.getIOContext(), false); + out.open(file); + for (auto const& e : es) + { + out.writeOne(e); + } + if (!tail.empty()) + { + out.writeBytes(tail.data(), tail.size()); + } + out.close(); + return file; + }; + auto unordered = [](std::vector const& es) { + LtHash16 h; + for (auto const& e : es) + { + auto rec = xdr::xdr_to_opaque(e); + h.add(ByteSlice(rec.data(), rec.size())); + } + return ShardedTopBucket::bucketHash(h); + }; + + // The same file written again is taken. + REQUIRE(ShardedTopBucket::bucketHash(ShardedTopBucket::hashOfFile( + write(entries, "same"))) == bucket->getHash()); + + // The same records in another order have the same commutative hash, and + // are not the bucket's file. + auto swapped = entries; + std::swap(swapped[3], swapped[4]); + REQUIRE(unordered(swapped) == bucket->getHash()); + REQUIRE_THROWS_WITH(ShardedTopBucket::hashOfFile(write(swapped, "swapped")), + Catch::Matchers::Contains("out of order")); + + auto metaLast = entries; + std::rotate(metaLast.begin(), metaLast.begin() + 1, metaLast.end()); + REQUIRE(unordered(metaLast) == bucket->getHash()); + REQUIRE_THROWS_WITH( + ShardedTopBucket::hashOfFile(write(metaLast, "meta-last")), + Catch::Matchers::Contains("not first")); + + auto twice = entries; + twice.insert(twice.begin() + 5, entries[5]); + REQUIRE_THROWS_WITH(ShardedTopBucket::hashOfFile(write(twice, "twice")), + Catch::Matchers::Contains("out of order")); + + REQUIRE_THROWS(ShardedTopBucket::hashOfFile( + write(entries, "tail", std::string(3, '\0')))); + + // Two metadata entries in front; one key twice, alive and then dead. + auto twoMeta = entries; + twoMeta.insert(twoMeta.begin(), entries.front()); + REQUIRE_THROWS_WITH( + ShardedTopBucket::hashOfFile(write(twoMeta, "two-meta")), + Catch::Matchers::Contains("not first")); + auto liveThenDead = entries; + for (size_t i = 1; i < liveThenDead.size(); ++i) + { + if (liveThenDead[i].type() != DEADENTRY) + { + BucketEntry tombstone; + tombstone.type(DEADENTRY); + tombstone.deadEntry() = LedgerEntryKey(liveThenDead[i].liveEntry()); + liveThenDead.insert(liveThenDead.begin() + i + 1, tombstone); + break; + } + } + REQUIRE(liveThenDead.size() == entries.size() + 1); + REQUIRE_THROWS_WITH( + ShardedTopBucket::hashOfFile(write(liveThenDead, "live-then-dead")), + Catch::Matchers::Contains("out of order")); +} + +TEST_CASE("the setting over a level 0 written without it is refused at startup", + "[bucket][sharded-top-bucket]") +{ + using BucketTestUtils::BucketTestApplication; + Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)); + cfg.EXPERIMENTAL_SHARDED_TOP_BUCKET = 0; + { + VirtualClock clock; + auto app = createTestApplication(clock, cfg); + BatchMaker maker; + maker.mTypes = {ACCOUNT, TRUSTLINE, DATA}; + std::vector init, live; + std::vector dead; + for (uint32_t i = 0; i < 3; ++i) + { + maker.make(app->getLedgerManager().getLastClosedLedgerNum() + 1, 20, + init, live, dead); + app->getLedgerManager().setNextLedgerEntryBatchForBucketTesting( + init, live, dead); + BucketTestUtils::closeLedger(*app); + } + } + // The same state, with the setting off and then on. + { + VirtualClock clock; + REQUIRE_NOTHROW( + createTestApplication(clock, cfg, false)); + } + cfg.EXPERIMENTAL_SHARDED_TOP_BUCKET = 4; + { + VirtualClock clock; + REQUIRE_THROWS_WITH( + createTestApplication(clock, cfg, false), + Catch::Matchers::Contains("written without the setting")); + } +} + +TEST_CASE("level 0 is rebuilt from the shard files after a stop", + "[bucket][bucketlist][sharded-top-bucket]") +{ + using BucketTestUtils::BucketTestApplication; + // The ledger after which the node stops: one whose batch went into an + // empty level 0 (it spilled), and one whose batch was merged with the + // ledger before. + for (uint32_t stopAt : {12u, 13u}) + { + INFO("the node stops after ledger " << stopAt); + Config cfgA(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)); + cfgA.EXPERIMENTAL_SHARDED_TOP_BUCKET = 3; + // A node that stops there has not dropped the bucket of the ledger + // before: that happens once the file is written. + cfgA.DISABLE_BUCKET_GC = true; + // The control: the same ledgers without a stop. + Config cfgB(getTestConfig(1, Config::TESTDB_BUCKET_DB_PERSISTENT)); + cfgB.EXPERIMENTAL_SHARDED_TOP_BUCKET = 5; + + auto clockA = std::make_unique(); + VirtualClock clockB; + auto appA = createTestApplication(*clockA, cfgA); + auto appB = createTestApplication(clockB, cfgB); + + BatchMaker maker; + // (No offers: those the ledger manager also keeps in SQL, where + // entries put straight into the bucket list are not.) + maker.mTypes = {ACCOUNT, TRUSTLINE, DATA}; + maker.mContractDataWithTTL = 10; + std::vector init, live; + std::vector dead; + auto close = [&](BucketTestApplication& app) { + app.getLedgerManager().setNextLedgerEntryBatchForBucketTesting( + init, live, dead); + BucketTestUtils::closeLedger(app); + }; + auto bucketListHash = [](Application& app) { + return app.getLedgerManager() + .getLastClosedLedgerHeader() + .header.bucketListHash; + }; + auto shardFiles = [](Application& app) { + auto dir = app.getBucketManager().getBucketDir() + "/topshards"; + return fs::findfiles( + dir, [](std::string const& name) { return name[0] == 's'; }); + }; + + uint32_t ledger = appA->getLedgerManager().getLastClosedLedgerNum(); + REQUIRE(ledger == appB->getLedgerManager().getLastClosedLedgerNum()); + while (++ledger < stopAt) + { + maker.make(ledger, 40, init, live, dead); + close(*appA); + close(*appB); + REQUIRE(bucketListHash(*appA) == bucketListHash(*appB)); + } + + maker.make(stopAt, 40, init, live, dead); + ShardedTopBucket::loseFileForTesting(true); + close(*appA); + ShardedTopBucket::loseFileForTesting(false); + close(*appB); + REQUIRE(appA->getLedgerManager().getLastClosedLedgerNum() == stopAt); + REQUIRE(bucketListHash(*appA) == bucketListHash(*appB)); + + Hash const lost = appA->getBucketManager() + .getLiveBucketList() + .getLevel(0) + .getCurr() + ->getHash(); + std::string const lostFile = fmt::format( + "{}/bucket-{}.xdr", appA->getBucketManager().getBucketDir(), + binToHex(lost)); + REQUIRE(!isZero(lost)); + REQUIRE(!fs::exists(lostFile)); + REQUIRE(!shardFiles(*appA).empty()); + + // The restart. + appA.reset(); + clockA = std::make_unique(); + appA = + createTestApplication(*clockA, cfgA, false); + REQUIRE(appA->getLedgerManager().getLastClosedLedgerNum() == stopAt); + REQUIRE(fs::exists(lostFile)); + REQUIRE(shardFiles(*appA).empty()); + REQUIRE(ShardedTopBucket::bucketHash( + ShardedTopBucket::hashOfFile(lostFile)) == lost); + auto currB = + appB->getBucketManager().getLiveBucketList().getLevel(0).getCurr(); + REQUIRE(currB->getHash() == lost); + REQUIRE(fileBytes(lostFile) == fileBytes(currB->getFilename())); + REQUIRE(bucketListHash(*appA) == bucketListHash(*appB)); + + // And on from there, through the next spills. + for (uint32_t i = 0; i < 9; ++i) + { + maker.make(++ledger, 40, init, live, dead); + close(*appA); + close(*appB); + REQUIRE(bucketListHash(*appA) == bucketListHash(*appB)); + } + REQUIRE(shardFiles(*appB).size() <= 15); + } +} + +TEST_CASE("nodes with different shard counts close the same ledgers", + "[bucket][sharded-top-bucket][simulation]") +{ + auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE); + // How many shards a node uses is its own business. + std::vector const shardCounts{1, 2, 4, 7}; + // Threshold 1: every node votes, so every node computed the hash. + auto simulation = Topologies::core(4, 1, networkID, [&](int i) { + auto cfg = getTestConfig(i, Config::TESTDB_BUCKET_DB_PERSISTENT); + cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = 100; + cfg.GENESIS_TEST_ACCOUNT_COUNT = 100; + cfg.EXPERIMENTAL_SHARDED_TOP_BUCKET = + shardCounts.at(static_cast(i) % shardCounts.size()); + return cfg; + }); + simulation->startAllNodes(); + auto nodes = simulation->getNodes(); + std::set used; + for (auto const& n : nodes) + { + used.insert(n->getConfig().EXPERIMENTAL_SHARDED_TOP_BUCKET); + } + REQUIRE(used.size() == shardCounts.size()); + + uint32_t const desiredTxRate = 1; + uint32_t const ledgerWideLimit = static_cast( + desiredTxRate * simulation->getExpectedLedgerCloseTime().count() * 2); + upgradeSorobanNetworkConfig( + [&](SorobanNetworkConfig& cfg) { + setSorobanNetworkConfigForTest(cfg); + cfg.mLedgerMaxTxCount = ledgerWideLimit; + }, + simulation); + + // Classic entries from one node, contract code and TTL entries from + // another: all three groups of shards get entries. + auto& loadGen = nodes[0]->getLoadGenerator(); + auto& loadGenDone = + nodes[0]->getMetrics().NewMeter({"loadgen", "run", "complete"}, "run"); + auto& secondLoadGen = nodes[1]->getLoadGenerator(); + auto& secondLoadGenDone = + nodes[1]->getMetrics().NewMeter({"loadgen", "run", "complete"}, "run"); + auto count = loadGenDone.count(); + auto secondCount = secondLoadGenDone.count(); + uint32_t const txCount = 30; + loadGen.generateLoad(GeneratedLoadConfig::txLoad( + LoadGenMode::SOROBAN_UPLOAD, 50, txCount, desiredTxRate, 0)); + secondLoadGen.generateLoad(GeneratedLoadConfig::txLoad( + LoadGenMode::PAY, 50, txCount, desiredTxRate, 50)); + simulation->crankUntil( + [&]() { + return loadGenDone.count() > count && + secondLoadGenDone.count() > secondCount; + }, + 200 * simulation->getExpectedLedgerCloseTime(), false); + REQUIRE(nodes[0] + ->getMetrics() + .NewMeter({"loadgen", "run", "failed"}, "run") + .count() == 0); + REQUIRE(nodes[1] + ->getMetrics() + .NewMeter({"loadgen", "run", "failed"}, "run") + .count() == 0); + + uint32_t const target = + nodes[0]->getLedgerManager().getLastClosedLedgerNum() + 2; + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(target, 1); }, + 50 * simulation->getExpectedLedgerCloseTime(), false); + + // The same ledger at every node: the header holds the bucket list hash. + // And the ledgers were not empty. + uint32_t const seq = + std::min({nodes[0]->getLedgerManager().getLastClosedLedgerNum(), + nodes[1]->getLedgerManager().getLastClosedLedgerNum(), + nodes[2]->getLedgerManager().getLastClosedLedgerNum(), + nodes[3]->getLedgerManager().getLastClosedLedgerNum()}); + REQUIRE(seq >= target - 1); + for (auto const& n : nodes) + { + auto& shardTimer = + n->getMetrics().NewTimer({"bucket", "top-shard", "shards"}); + REQUIRE(shardTimer.count() >= seq - 1); + auto& applied = + n->getMetrics().NewCounter({"ledger", "apply", "success"}); + REQUIRE(applied.count() >= 2 * txCount); + } + // (haveAllExternalized compares the ledgers' hashes: a node that + // computed another bucket list hash would have stopped at its first + // non-empty ledger.) +} diff --git a/src/bucket/test/TopBucketBenchTests.cpp b/src/bucket/test/TopBucketBenchTests.cpp new file mode 100644 index 0000000000..c762bcc777 --- /dev/null +++ b/src/bucket/test/TopBucketBenchTests.cpp @@ -0,0 +1,342 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// A benchmark of what the bucket list does with one ledger's entries at +// level 0, with batches shaped like the native-asset payment load of the +// perfnet runs: per payment one changed account, one new contract data entry +// (a balance) and one new TTL entry. Hidden: run it by name or tag. + +#include "bucket/BucketManager.h" +#include "bucket/BucketOutputIterator.h" +#include "bucket/LiveBucket.h" +#include "bucket/LiveBucketIndex.h" +#include "bucket/LiveBucketList.h" +#include "bucket/test/BucketTestUtils.h" +#include "crypto/SHA.h" +#include "ledger/InMemorySorobanState.h" +#include "ledger/LedgerEntryRefs.h" +#include "ledger/LedgerManager.h" +#include "ledger/LedgerTypeUtils.h" +#include "ledger/NetworkConfig.h" +#include "main/Application.h" +#include "main/Config.h" +#include "medida/timer.h" +#include "test/Catch2.h" +#include "test/TestUtils.h" +#include "test/test.h" +#include "util/Logging.h" +#include "util/MetricsRegistry.h" +#include "util/Timer.h" +#include +#include +#include + +using namespace stellar; + +namespace TopBucketBench +{ +using Clock = std::chrono::steady_clock; + +double +msSince(Clock::time_point t) +{ + return std::chrono::duration(Clock::now() - t).count(); +} + +// The entries of `payments` payments of ledger `ledger`: an account that +// exists (LIVE), and a balance entry and its TTL that do not (INIT). +void +makeBatch(uint32_t ledger, uint32_t payments, std::vector& init, + std::vector& live) +{ + init.clear(); + live.clear(); + init.reserve(2 * payments); + live.reserve(payments); + Hash contract = sha256("native asset contract"); + for (uint32_t i = 0; i < payments; ++i) + { + auto seed = sha256(fmt::format("payment {} {}", ledger, i)); + + LedgerEntry account; + account.lastModifiedLedgerSeq = ledger; + account.data.type(ACCOUNT); + auto& ae = account.data.account(); + // Accounts repeat from ledger to ledger (the load's source accounts). + ae.accountID.ed25519() = sha256(fmt::format("account {}", i)); + ae.balance = 1'000'000'000 + ledger; + ae.seqNum = (static_cast(ledger) << 32) + i; + ae.thresholds[0] = 1; + ae.ext.v(1); + ae.ext.v1().ext.v(2); + ae.ext.v1().ext.v2().ext.v(3); + ae.ext.v1().ext.v2().ext.v3().seqLedger = ledger; + ae.ext.v1().ext.v2().ext.v3().seqTime = 1'700'000'000 + ledger; + live.emplace_back(std::move(account)); + + LedgerEntry balance; + balance.lastModifiedLedgerSeq = ledger; + balance.data.type(CONTRACT_DATA); + auto& cd = balance.data.contractData(); + cd.contract.type(SC_ADDRESS_TYPE_CONTRACT); + cd.contract.contractId() = contract; + cd.durability = PERSISTENT; + // Key: ["Balance", address] + cd.key.type(SCV_VEC); + cd.key.vec().activate(); + auto& keyVec = *cd.key.vec(); + keyVec.resize(2); + keyVec[0].type(SCV_SYMBOL); + keyVec[0].sym() = "Balance"; + keyVec[1].type(SCV_ADDRESS); + keyVec[1].address().type(SC_ADDRESS_TYPE_CONTRACT); + keyVec[1].address().contractId() = seed; + // Value: {amount, authorized, clawback} + cd.val.type(SCV_MAP); + cd.val.map().activate(); + auto& m = *cd.val.map(); + m.resize(3); + m[0].key.type(SCV_SYMBOL); + m[0].key.sym() = "amount"; + m[0].val.type(SCV_I128); + m[0].val.i128().lo = 1 + i; + m[1].key.type(SCV_SYMBOL); + m[1].key.sym() = "authorized"; + m[1].val.type(SCV_BOOL); + m[1].val.b() = true; + m[2].key.type(SCV_SYMBOL); + m[2].key.sym() = "clawback"; + m[2].val.type(SCV_BOOL); + m[2].val.b() = false; + + LedgerEntry ttl; + ttl.lastModifiedLedgerSeq = ledger; + ttl.data.type(TTL); + ttl.data.ttl().keyHash = getTTLKey(balance).ttl().keyHash; + ttl.data.ttl().liveUntilLedgerSeq = ledger + 500'000; + + init.emplace_back(std::move(balance)); + init.emplace_back(std::move(ttl)); + } +} +} + +TEST_CASE("top bucket add by step with payment-shaped batches", + "[.][bucketbench]") +{ + using namespace TopBucketBench; + VirtualClock clock; + Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)); + // As a validator: sync the files. + cfg.DISABLE_XDR_FSYNC = false; + auto app = createTestApplication(clock, cfg); + auto& bm = app->getBucketManager(); + auto version = app->getConfig().LEDGER_PROTOCOL_VERSION; + + for (uint32_t payments : {6400u, 10000u}) + { + std::vector init1, live1, init2, live2; + makeBatch(2, payments, init1, live1); + makeBatch(3, payments, init2, live2); + std::vector dead; + + size_t xdrBytes = 0; + for (auto const* v : {&init1, &live1}) + { + for (auto const& e : *v) + { + xdrBytes += xdr::xdr_size(e); + } + } + + // The steps apart, on the second ledger's shape (a bucket of two + // ledgers is written at every other ledger). + auto t = Clock::now(); + auto entries1 = + LiveBucket::convertToBucketEntry(true, init1, live1, dead); + double convert = msSince(t); + + BucketMetadata meta; + meta.ledgerVersion = version; + meta.ext.v(1); + meta.ext.bucketListType() = BucketListType::LIVE; + + t = Clock::now(); + auto index = + std::make_shared(bm, entries1, meta); + double indexMs = msSince(t); + + t = Clock::now(); + MergeCounters mc; + LiveBucketOutputIterator out(bm.getTmpDir(), true, meta, mc, + clock.getIOContext(), true); + for (auto const& e : entries1) + { + out.put(e); + } + double put = msSince(t); + t = Clock::now(); + auto b1 = out.getBucket(bm, nullptr, nullptr, index); + double close = msSince(t); + + // Serializing and hashing alone, without a file. + t = Clock::now(); + SHA256 hasher; + size_t bytes = 0; + for (auto const& e : entries1) + { + auto buf = xdr::xdr_to_opaque(e); + hasher.add(buf); + bytes += buf.size(); + } + auto h = hasher.finish(); + double serialize = msSince(t); + (void)h; + + t = Clock::now(); + std::vector copy(entries1); + double copyMs = msSince(t); + + // The whole add, as the bucket list does it, over six ledgers. + LiveBucketList bl; + std::vector adds; + for (uint32_t ledger = 2; ledger < 8; ++ledger) + { + std::vector init, live; + makeBatch(ledger, payments, init, live); + t = Clock::now(); + bl.addBatch(*app, ledger, version, init, live, dead); + adds.push_back(msSince(t)); + } + + LOG_INFO(DEFAULT_LOG, + "TOPBUCKET {} payments: {} entries, {:.2f} MB of XDR | " + "convert and sort {:.1f} ms | copy of the entries {:.1f} ms " + "| in-memory index {:.1f} ms | output: put (serialize, hash, " + "write) {:.1f} ms, close (sync, adopt) {:.1f} ms | serialize " + "and SHA-256 alone {:.1f} ms ({} bytes) | addBatch at " + "ledgers 2..7: {:.1f} {:.1f} {:.1f} {:.1f} {:.1f} {:.1f} ms", + payments, entries1.size(), xdrBytes / 1e6, convert, copyMs, + indexMs, put, close, serialize, bytes, adds[0], adds[1], + adds[2], adds[3], adds[4], adds[5]); + REQUIRE(!b1->isEmpty()); + } +} + +TEST_CASE("top bucket add as shards with payment-shaped batches", + "[.][bucketbench]") +{ + using namespace TopBucketBench; + for (uint32_t shardsPerGroup : {0u, 4u, 8u, 16u, 32u}) + { + VirtualClock clock; + Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)); + // TOPBUCKET_NO_FSYNC=1 shows what the syncs cost. + cfg.DISABLE_XDR_FSYNC = getenv("TOPBUCKET_NO_FSYNC") != nullptr; + cfg.EXPERIMENTAL_SHARDED_TOP_BUCKET = shardsPerGroup; + auto app = createTestApplication(clock, cfg); + auto version = app->getConfig().LEDGER_PROTOCOL_VERSION; + for (uint32_t payments : {6400u, 10000u}) + { + LiveBucketList bl; + std::vector dead; + std::vector adds; + // Ledgers 2 to 21. A level-0 spill at every even ledger; the + // first ledgers warm the threads up and are left out. + for (uint32_t ledger = 2; ledger < 22; ++ledger) + { + std::vector init, live; + makeBatch(ledger, payments, init, live); + auto t = Clock::now(); + bl.addBatch(*app, ledger, version, init, live, dead); + adds.push_back(msSince(t)); + // As a validator between two closes: let the background + // work (the bucket's file, the level-1 merge) get on. + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + } + double odd = 0, even = 0; + size_t nOdd = 0, nEven = 0; + for (size_t i = 4; i < adds.size(); ++i) + { + // adds[i] is ledger i + 2: an even ledger adds to an empty + // level 0 (it spilled), an odd one to one ledger. + if ((i + 2) % 2 == 0) + { + even += adds[i]; + ++nEven; + } + else + { + odd += adds[i]; + ++nOdd; + } + } + auto timer = [&](char const* name) { + auto& t = + app->getMetrics().NewTimer({"bucket", "top-shard", name}); + return t.count() ? t.sum() / t.count() : 0.0; + }; + LOG_INFO(DEFAULT_LOG, + "TOPSHARD {} shards per group, {} payments ({} " + "entries): addBatch mean {:.1f} ms after a spill, {:.1f} " + "ms onto one ledger | cut {:.1f} shards {:.1f} finish " + "{:.1f} file wait {:.2f} ms; the file (background) " + "{:.1f} ms", + shardsPerGroup, payments, 3 * payments, even / nEven, + odd / nOdd, timer("cut"), timer("shards"), timer("finish"), + timer("file-wait"), timer("file")); + } + } +} + +// What the ledger commit runs beside the bucket list add (and waits for): the +// update of the in-memory Soroban state with the same batch. Its cost grows +// with the state: the maps it inserts into are large. +TEST_CASE("in-memory soroban state update with payment-shaped batches", + "[.][bucketbench]") +{ + using namespace TopBucketBench; + VirtualClock clock; + Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)); + auto app = createTestApplication(clock, cfg); + auto& lm = app->getLedgerManager(); + std::optional sorobanConfig( + lm.getLastClosedSorobanNetworkConfig()); + auto& metrics = lm.getSorobanMetrics(); + + InMemorySorobanState state; + LedgerHeader lh; + lh.ledgerVersion = app->getConfig().LEDGER_PROTOCOL_VERSION; + std::vector dead; + uint32_t const payments = 10000; + uint32_t const ledgers = 260; + std::vector times; + for (uint32_t ledger = 1; ledger <= ledgers; ++ledger) + { + std::vector init, live; + makeBatch(ledger, payments, init, live); + lh.ledgerSeq = ledger; + auto initRefs = toRefs(init); + auto liveRefs = toRefs(live); + auto deadRefs = toRefs(dead); + auto t = Clock::now(); + state.updateState(initRefs, liveRefs, deadRefs, lh, sorobanConfig, + metrics); + times.push_back(msSince(t)); + } + auto mean = [&](size_t from, size_t to) { + double sum = 0; + for (size_t i = from; i < to; ++i) + { + sum += times[i]; + } + return sum / static_cast(to - from); + }; + LOG_INFO(DEFAULT_LOG, + "SOROBANSTATE {} payments per ledger (as many new contract data " + "entries with their TTL): updateState mean {:.1f} ms at ledgers " + "1..20, {:.1f} ms at 121..140 (1.2 million entries), {:.1f} ms " + "at 241..260 (2.4 million entries)", + payments, mean(0, 20), mean(120, 140), mean(240, 260)); +} diff --git a/src/crypto/LtHash.cpp b/src/crypto/LtHash.cpp new file mode 100644 index 0000000000..b0d4e05cc1 --- /dev/null +++ b/src/crypto/LtHash.cpp @@ -0,0 +1,143 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "crypto/LtHash.h" +#include "crypto/SHA.h" +#include "util/GlobalChecks.h" +#include +#include + +namespace stellar +{ + +namespace +{ +char const LTHASH_DOMAIN[] = "stellar lthash16 v1"; +} + +LtHash16::LtHash16() +{ + mElements.fill(0); +} + +void +LtHash16::expand(ByteSlice const& member, std::array& out) +{ + static_assert(crypto_stream_chacha20_ietf_KEYBYTES == 32, + "unexpected ChaCha20 key size"); + SHA256 keyHasher; + keyHasher.add(ByteSlice(LTHASH_DOMAIN, sizeof(LTHASH_DOMAIN) - 1)); + keyHasher.add(member); + uint256 key = keyHasher.finish(); + + unsigned char nonce[crypto_stream_chacha20_ietf_NONCEBYTES] = {0}; + std::array stream; + int res = crypto_stream_chacha20_ietf(stream.data(), stream.size(), nonce, + key.data()); + releaseAssert(res == 0); + for (size_t i = 0; i < ELEMENTS; ++i) + { + out[i] = static_cast(stream[2 * i]) | + static_cast(static_cast(stream[2 * i + 1]) + << 8); + } +} + +void +LtHash16::add(ByteSlice const& member) +{ + std::array e; + expand(member, e); + for (size_t i = 0; i < ELEMENTS; ++i) + { + mElements[i] = static_cast(mElements[i] + e[i]); + } +} + +void +LtHash16::remove(ByteSlice const& member) +{ + std::array e; + expand(member, e); + for (size_t i = 0; i < ELEMENTS; ++i) + { + mElements[i] = static_cast(mElements[i] - e[i]); + } +} + +void +LtHash16::add(LtHash16 const& other) +{ + for (size_t i = 0; i < ELEMENTS; ++i) + { + mElements[i] = static_cast(mElements[i] + other.mElements[i]); + } +} + +void +LtHash16::remove(LtHash16 const& other) +{ + for (size_t i = 0; i < ELEMENTS; ++i) + { + mElements[i] = static_cast(mElements[i] - other.mElements[i]); + } +} + +bool +LtHash16::isZero() const +{ + for (auto e : mElements) + { + if (e != 0) + { + return false; + } + } + return true; +} + +bool +LtHash16::operator==(LtHash16 const& other) const +{ + return mElements == other.mElements; +} + +std::array +LtHash16::toBytes() const +{ + std::array bytes; + for (size_t i = 0; i < ELEMENTS; ++i) + { + bytes[2 * i] = static_cast(mElements[i] & 0xff); + bytes[2 * i + 1] = static_cast(mElements[i] >> 8); + } + return bytes; +} + +LtHash16 +LtHash16::fromBytes(std::array const& bytes) +{ + LtHash16 res; + for (size_t i = 0; i < ELEMENTS; ++i) + { + res.mElements[i] = + static_cast(bytes[2 * i]) | + static_cast(static_cast(bytes[2 * i + 1]) << 8); + } + return res; +} + +uint256 +LtHash16::digest() const +{ + // Under a tag of its own: a digest is used where the SHA-256 of a file + // is used too (bucket hashes), and is never to be one. + static std::string const TAG = "stellar lthash16 digest v1"; + auto bytes = toBytes(); + SHA256 hasher; + hasher.add(ByteSlice(TAG.data(), TAG.size())); + hasher.add(ByteSlice(bytes.data(), bytes.size())); + return hasher.finish(); +} +} diff --git a/src/crypto/LtHash.h b/src/crypto/LtHash.h new file mode 100644 index 0000000000..cdcd2af49f --- /dev/null +++ b/src/crypto/LtHash.h @@ -0,0 +1,90 @@ +#pragma once + +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "crypto/ByteSlice.h" +#include "xdr/Stellar-types.h" +#include +#include + +namespace stellar +{ + +// A commutative ("homomorphic", multiset) hash: the hash of a collection of +// byte strings that does not depend on the order in which they are added, and +// from which a member can be taken away again. +// +// The construction is LtHash16 with 1024 elements (Bellare and Micciancio's +// lattice hash; the parameters of Lewi, Kim, Maykov and Weis, "Securing +// Update Propagation with Homomorphic Hashing", 2019). Every byte string is +// expanded to 1024 16-bit numbers; the state is the element-wise sum, modulo +// 2^16, of the expansions of the members. Two collections hash alike exactly +// when their sums agree, so +// +// hash(A union B) = hash(A) + hash(B) +// +// for collections with multiplicity, whatever the order and however the +// members are split between A and B. Finding two different collections with +// the same sum is a short-vector problem in a lattice of dimension 1024. +// +// The expansion of a byte string x is the ChaCha20 key stream (IETF variant, +// zero nonce) under the key SHA256("stellar lthash16 v1" || x), read as 1024 +// little-endian 16-bit numbers. +// +// The state is 2048 bytes. `digest` condenses it to 32 bytes with SHA-256 for +// use where a hash of that size is expected; sums are formed on states, never +// on digests. +class LtHash16 +{ + public: + static constexpr size_t ELEMENTS = 1024; + static constexpr size_t STATE_BYTES = 2 * ELEMENTS; + + // The hash of the empty collection: all zero. + LtHash16(); + + // Add or take away one member. + void add(ByteSlice const& member); + void remove(ByteSlice const& member); + + // Add or take away a whole collection, given by its hash. + void add(LtHash16 const& other); + void remove(LtHash16 const& other); + + LtHash16& + operator+=(LtHash16 const& other) + { + add(other); + return *this; + } + LtHash16& + operator-=(LtHash16 const& other) + { + remove(other); + return *this; + } + + bool isZero() const; + bool operator==(LtHash16 const& other) const; + bool + operator!=(LtHash16 const& other) const + { + return !(*this == other); + } + + // SHA-256 of the state, the elements as little-endian 16-bit numbers. + uint256 digest() const; + + // The state as bytes (little-endian elements), and back. + std::array toBytes() const; + static LtHash16 fromBytes(std::array const& bytes); + + private: + std::array mElements; + + static void expand(ByteSlice const& member, + std::array& out); +}; +} diff --git a/src/crypto/test/LtHashTests.cpp b/src/crypto/test/LtHashTests.cpp new file mode 100644 index 0000000000..b8838793e3 --- /dev/null +++ b/src/crypto/test/LtHashTests.cpp @@ -0,0 +1,182 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "crypto/Hex.h" +#include "crypto/LtHash.h" +#include "crypto/SHA.h" +#include "test/Catch2.h" +#include "util/Math.h" +#include +#include +#include +#include +#include + +using namespace stellar; + +namespace +{ +std::vector +members(size_t n, std::string const& tag) +{ + std::vector res; + for (size_t i = 0; i < n; ++i) + { + // Different lengths, the empty string among them. + res.emplace_back( + i == 0 ? std::string() + : fmt::format("{} {} {}", tag, i, std::string(i % 97, 'x'))); + } + return res; +} + +LtHash16 +hashOf(std::vector const& ms) +{ + LtHash16 h; + for (auto const& m : ms) + { + h.add(m); + } + return h; +} +} + +TEST_CASE("lthash does not depend on order or on how the members are split", + "[crypto][lthash]") +{ + auto ms = members(300, "member"); + auto whole = hashOf(ms); + REQUIRE(!whole.isZero()); + REQUIRE(LtHash16().isZero()); + + // Any order. + for (int round = 0; round < 5; ++round) + { + stellar::shuffle(ms.begin(), ms.end(), getGlobalRandomEngine()); + REQUIRE(hashOf(ms) == whole); + REQUIRE(hashOf(ms).digest() == whole.digest()); + } + + // Any split into parts, summed in any order. + for (size_t parts : {1u, 2u, 3u, 7u, 16u, 300u}) + { + std::vector partial(parts); + for (size_t i = 0; i < ms.size(); ++i) + { + partial[(i * 31 + 7) % parts].add(ms[i]); + } + stellar::shuffle(partial.begin(), partial.end(), + getGlobalRandomEngine()); + LtHash16 sum; + for (auto const& p : partial) + { + sum += p; + } + REQUIRE(sum == whole); + } +} + +TEST_CASE("lthash tells collections apart and counts repeated members", + "[crypto][lthash]") +{ + auto ms = members(50, "member"); + auto whole = hashOf(ms); + + // One member more, one less, one changed. + auto more = ms; + more.emplace_back("one more"); + REQUIRE(hashOf(more) != whole); + auto fewer = ms; + fewer.pop_back(); + REQUIRE(hashOf(fewer) != whole); + auto changed = ms; + changed[17] += "!"; + REQUIRE(hashOf(changed) != whole); + REQUIRE(hashOf(changed).digest() != whole.digest()); + + // A member twice is not the member once, and not nothing (as it would be + // under xor). + LtHash16 once, twice; + once.add(std::string("a")); + twice.add(std::string("a")); + twice.add(std::string("a")); + REQUIRE(twice != once); + REQUIRE(!twice.isZero()); + + // Concatenation is not confused with two members. + LtHash16 ab, a_b; + ab.add(std::string("ab")); + a_b.add(std::string("a")); + a_b.add(std::string("b")); + REQUIRE(ab != a_b); +} + +TEST_CASE("lthash members can be taken away again", "[crypto][lthash]") +{ + auto ms = members(100, "member"); + auto whole = hashOf(ms); + + // Replace a member: take the old one away, add the new one. + auto replaced = ms; + replaced[42] = "the new value"; + LtHash16 h = whole; + h.remove(ms[42]); + h.add(replaced[42]); + REQUIRE(h == hashOf(replaced)); + + // Take a whole part away. + std::vector first(ms.begin(), ms.begin() + 30); + std::vector rest(ms.begin() + 30, ms.end()); + LtHash16 g = whole; + g -= hashOf(first); + REQUIRE(g == hashOf(rest)); + g -= hashOf(rest); + REQUIRE(g.isZero()); + REQUIRE(g == LtHash16()); +} + +TEST_CASE("lthash state round-trips and has a fixed value", "[crypto][lthash]") +{ + auto whole = hashOf(members(10, "member")); + REQUIRE(LtHash16::fromBytes(whole.toBytes()) == whole); + + // The construction is part of what nodes must agree on: pin it. The + // digest of the empty collection is the SHA-256 of the digest's tag and + // 2048 zero bytes (and so not the SHA-256 of any state). + std::string tagged = "stellar lthash16 digest v1"; + tagged.append(LtHash16::STATE_BYTES, '\0'); + REQUIRE(LtHash16().digest() == + sha256(ByteSlice(tagged.data(), tagged.size()))); + REQUIRE(LtHash16().digest() != + sha256(ByteSlice(tagged.data() + 26, LtHash16::STATE_BYTES))); + LtHash16 abc; + abc.add(std::string("abc")); + INFO(binToHex(abc.digest())); + auto bytes = abc.toBytes(); + INFO(binToHex(ByteSlice(bytes.data(), 16))); + // (From the first run of this test with the digest under its tag.) + REQUIRE(binToHex(abc.digest()) == + "243136de057950006f0c715e341bf87d3f525bbabb461dca0834f5216b83e009"); + REQUIRE(binToHex(ByteSlice(bytes.data(), 16)) == + "0ecbf29822e9373da329597a8dcf31ad"); +} + +TEST_CASE("lthash cost per member", "[.][lthashbench]") +{ + std::vector ms; + for (size_t i = 0; i < 30000; ++i) + { + ms.emplace_back(fmt::format("{:0164}", i)); + } + auto start = std::chrono::steady_clock::now(); + auto h = hashOf(ms); + auto ns = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + WARN(fmt::format("lthash16: {} members of 164 bytes in {:.1f} ms, {:.0f} " + "ns per member", + ms.size(), ns / 1e6, double(ns) / ms.size())); + REQUIRE(!h.isZero()); +} diff --git a/src/herder/EarlyTxSetSelection.cpp b/src/herder/EarlyTxSetSelection.cpp new file mode 100644 index 0000000000..cd453225be --- /dev/null +++ b/src/herder/EarlyTxSetSelection.cpp @@ -0,0 +1,501 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "herder/EarlyTxSetSelection.h" +#include "crypto/ByteSlice.h" +#include "crypto/ShortHash.h" +#include "herder/TxSetUtils.h" +#include "main/AppConnector.h" +#include "main/Application.h" +#include "transactions/EventManager.h" +#include "transactions/MutableTransactionResult.h" +#include "transactions/TransactionFrameBase.h" +#include "transactions/TransactionUtils.h" +#include "util/GlobalChecks.h" +#include "util/Logging.h" +#include "util/MetricsRegistry.h" +#include "util/ProtocolVersion.h" +#include "xdrpp/marshal.h" + +#include +#include +#include +#include +#include + +namespace stellar +{ +namespace +{ +class PredictedLedgerView : public AbstractLedgerView +{ + public: + PredictedLedgerView(ImmutableLedgerDataPtr previous, + std::shared_ptr header, + std::shared_ptr changes, + MetricsRegistry& metrics) + : mPrevious(std::move(previous), metrics) + , mHeader(std::move(header)) + , mChanges(std::move(changes)) + { + releaseAssert(mHeader && mChanges); + } + + LedgerHeaderWrapper + getLedgerHeader() const override + { + return LedgerHeaderWrapper(mHeader); + } + + SorobanNetworkConfig const* + getSorobanNetworkConfig() const override + { + return mPrevious.getSorobanNetworkConfig(); + } + + LedgerEntryWrapper + getAccount(AccountID const& account) const override + { + return load(accountKey(account)); + } + + LedgerEntryWrapper + getAccount(LedgerHeaderWrapper const& header, + TransactionFrame const& tx) const override + { + return getAccount(tx.getSourceID()); + } + + LedgerEntryWrapper + getAccount(LedgerHeaderWrapper const& header, TransactionFrame const& tx, + AccountID const& accountID) const override + { + return getAccount(accountID); + } + + LedgerEntryWrapper + load(LedgerKey const& key) const override + { + if (key.type() != ACCOUNT) + { + // Nothing is predicted about other entries: the previous + // ledger's version could be stale. + throw std::logic_error( + "the predicted state of a ledger only holds account entries"); + } + auto entry = mPrevious.loadLiveEntry(key); + if (!entry) + { + return LedgerEntryWrapper(entry); + } + auto it = mChanges->find(key.account().accountID); + if (it == mChanges->end()) + { + return LedgerEntryWrapper(entry); + } + auto predicted = std::make_shared(*entry); + auto& account = predicted->data.account(); + account.seqNum = std::max(account.seqNum, it->second.mSeqNum); + account.balance -= std::min(account.balance, it->second.mFees); + return LedgerEntryWrapper( + std::shared_ptr(std::move(predicted))); + } + + private: + ApplyLedgerView mPrevious; + std::shared_ptr mHeader; + std::shared_ptr mChanges; +}; +} + +void +predictAccountChanges(TransactionFrameBase const& tx, + PredictedAccountChanges& changes) +{ + auto& source = changes[tx.getSourceID()]; + source.mSeqNum = std::max(source.mSeqNum, tx.getSeqNum()); + auto& feeSource = changes[tx.getFeeSourceID()]; + auto const fee = std::max(0, tx.getFullFee()); + feeSource.mFees = + feeSource.mFees > std::numeric_limits::max() - fee + ? std::numeric_limits::max() + : feeSource.mFees + fee; +} + +std::unique_ptr +makePredictedLedgerView(ImmutableLedgerDataPtr previous, + std::shared_ptr header, + std::shared_ptr changes, + MetricsRegistry& metrics) +{ + return std::make_unique( + std::move(previous), std::move(header), std::move(changes), metrics); +} + +void +forEachPulledTx(std::vector const& payload, + std::function const& f) +{ + if (payload.size() < 4) + { + return; + } + uint32_t count = 0; + std::memcpy(&count, payload.data(), 4); + size_t offset = 4; + for (uint32_t i = 0; i < count && offset + 4 <= payload.size(); ++i) + { + uint32_t length = 0; + std::memcpy(&length, payload.data() + offset, 4); + offset += 4; + if (length > payload.size() - offset) + { + break; + } + f(payload.data() + offset, length); + offset += length; + } +} + +uint64_t +hashPulledTx(uint8_t const* data, size_t length) +{ + // Keyed per process: nobody outside can make one transaction's bytes + // hash like another's. + return shortHash::computeHash(ByteSlice(data, length)); +} + +bool +decodePulledTx(uint8_t const* data, size_t length, TransactionEnvelope& tx) +{ + try + { + // XDR is read in 4-byte words from 4-byte aligned memory. Well + // formed entries are (XDR lengths are multiples of 4), so decode + // them in place; copy anything else to aligned memory first. + if ((reinterpret_cast(data) & 3) == 0) + { + xdr::xdr_get g(data, data + length); + xdr::xdr_argpack_archive(g, tx); + g.done(); + } + else + { + std::vector const copy(data, data + length); + xdr::xdr_from_opaque(copy, tx); + } + return true; + } + catch (std::exception const&) + { + return false; + } +} + +void +EarlyTxSetJob::prepare(Application& app) +{ + ZoneScoped; + releaseAssert(mPayload && mClosingTxSet && mPreviousLedger); + auto const* base = mPayload->data(); + forEachPulledTx(*mPayload, [&](uint8_t const* data, size_t length) { + mPulled.emplace_back(static_cast(data - base), length); + mSeen.insert(hashPulledTx(data, length)); + }); + + // What N does, as far as its transaction set says. + auto changes = std::make_shared(); + mClosingTxHashes.reserve(mClosingTxSet->sizeTxTotal()); + for (auto const& phase : + mClosingTxSet->createTransactionFrames(app.getNetworkID())) + { + for (auto const& tx : phase) + { + mClosingTxHashes.insert(tx->getFullHash()); + predictAccountChanges(*tx, *changes); + } + } + mPredicted = std::move(changes); + + // N's header as far as validation reads it: number, value (close time), + // and what only an upgrade changes (N has none). + auto const& previous = mPreviousLedger->getLastClosedLedgerHeader(); + auto header = std::make_shared(previous.header); + header->ledgerSeq = mClosingLedgerSeq; + header->previousLedgerHash = previous.hash; + header->scpValue = mClosingValue; + mPredictedHeader = std::move(header); +} + +std::shared_ptr +EarlyTxSetJob::select(Application& app) +{ + ZoneScoped; + auto& metrics = app.getMetrics(); + auto timer = + metrics.NewTimer({"herder", "txset", "early-job-select"}).TimeScope(); + + // Every candidate in mempool order, and what the tasks found. + std::vector candidates; + { + std::lock_guard lock(mMutex); + if (mStop.load()) + { + return nullptr; + } + size_t total = 0; + for (auto const& results : mTaskResults) + { + total += results.size(); + } + candidates.reserve(total); + for (auto& results : mTaskResults) + { + candidates.insert(candidates.end(), + std::make_move_iterator(results.begin()), + std::make_move_iterator(results.end())); + results.clear(); + } + } + std::sort(candidates.begin(), candidates.end(), + [](Candidate const& a, Candidate const& b) { + return a.mIndex < b.mIndex; + }); + // The results hold for the predicted state and for the close time this + // job assumes: they are used by this selection and by nothing else. + TxValidationCache results; + results.reserve(candidates.size()); + TxFrameList classicTxs; + TxFrameList sorobanTxs; + for (auto const& candidate : candidates) + { + results.emplace(candidate.mTx->getFullHash(), candidate.mResult); + (candidate.mTx->isSoroban() ? sorobanTxs : classicTxs) + .push_back(candidate.mTx); + } + TxSetUtils::keepOnePerSourceAccount(classicTxs, sorobanTxs); + + auto selection = std::make_shared(); + selection->mLedgerSeq = mClosingLedgerSeq + 1; + selection->mPreviousLedger = mPreviousLedger; + selection->mSelectionSeed = mSelectionSeed; + // The limits of the ledger before N. N carries no upgrades, so they are + // N's as well; the node compares them with the closed ledger's before it + // uses the selection. + selection->mLimits.mLclHeader = + mPreviousLedger->getLastClosedLedgerHeader().header; + selection->mLimits.mSorobanConfig = + mPreviousLedger->hasSorobanConfig() + ? &mPreviousLedger->getSorobanConfig() + : nullptr; + releaseAssert(selection->mLimits.mSorobanConfig); + + PerPhaseTransactionList txPhases; + txPhases.emplace_back(std::move(classicTxs)); + txPhases.emplace_back(std::move(sorobanTxs)); + PerPhaseTransactionList invalidTxPhases; + invalidTxPhases.resize(txPhases.size()); + + // Validity is read from the tasks' results. The context's view is the + // predicted state all the same, for anything the tasks did not cover. + LedgerHeaderHistoryEntry predicted; + predicted.header = *mPredictedHeader; + TxSetValidationContext const ctx{ + predicted, selection->mLimits.mSorobanConfig, + [this, &metrics]() { + return makePredictedLedgerView(mPreviousLedger, mPredictedHeader, + mPredicted, metrics); + }, + 1, &results}; + auto const offset = ApplyTimeOffset::fromSeconds(mAssumedCloseTimeOffset); + selection->mPhases = TxSetBuilder::selectPhases( + txPhases, app, selection->mLimits, ctx, offset, invalidTxPhases, {}, + "early", mSelectionSeed, &selection->mHadTxNotFittingLane); + if (mStop.load()) + { + return nullptr; + } + // The structure of the selection is validated when it is completed, on + // the phases as they are then (TxSetBuilder::writeSelection). + + UnorderedSet selected; + for (auto const& phase : selection->mPhases) + { + for (auto const& tx : phase) + { + selected.insert(tx.get()); + } + } + selection->mSelected = selected.size(); + selection->mLeftovers.reserve(candidates.size() - selected.size()); + for (auto const& candidate : candidates) + { + if (selected.count(candidate.mTx.get()) == 0) + { + selection->mLeftovers.push_back(candidate.mTx); + } + } + selection->mSeen = std::move(mSeen); + selection->mSelectedAt = std::chrono::steady_clock::now(); + return selection; +} + +void +EarlyTxSetJob::runTask(Application& app, + std::shared_ptr const& job, size_t task) +{ + ZoneScopedN("early tx set selection"); +#ifdef BUILD_TESTS + while (job->mHoldForTesting && job->mHoldForTesting->load()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +#endif + { + std::lock_guard lock(job->mMutex); + --job->mPendingTasks; + if (job->mStop.load()) + { + if (job->mPendingTasks == 0 && job->mActiveTasks == 0) + { + job->mDone.store(true); + } + return; + } + ++job->mActiveTasks; + } + std::vector mine; + try + { + std::call_once(job->mPrepared, [&]() { + // Must not throw out of call_once. + try + { + job->prepare(app); + } + catch (std::exception const& e) + { + CLOG_WARNING(Herder, + "Early tx set selection could not read its " + "input: {}", + e.what()); + std::lock_guard lock(job->mMutex); + job->mStop.store(true); + } + }); + if (!job->mStop.load()) + { + auto& metrics = app.getMetrics(); + auto view = makePredictedLedgerView(job->mPreviousLedger, + job->mPredictedHeader, + job->mPredicted, metrics); + auto const& header = *job->mPredictedHeader; + // The frames keep a reference to the network ID: the + // application's outlives them. + Hash const& networkID = app.getNetworkID(); + auto& appConnector = app.getAppConnector(); + auto diagnostics = DiagnosticEventManager::createDisabled(); + bool const supportsSoroban = protocolVersionStartsFrom( + header.ledgerVersion, SOROBAN_PROTOCOL_VERSION); + std::optional validationLedgerSeq; + if (protocolVersionStartsFrom(header.ledgerVersion, + ProtocolVersion::V_19)) + { + validationLedgerSeq = job->mClosingLedgerSeq + 1; + } + auto const offset = job->mAssumedCloseTimeOffset; + auto const* base = job->mPayload->data(); + mine.reserve(job->mPulled.size() / job->mTasks + 1); + for (size_t i = task; i < job->mPulled.size(); i += job->mTasks) + { + if (job->mStop.load(std::memory_order_relaxed)) + { + break; + } + auto const& [at, length] = job->mPulled[i]; + TransactionEnvelope env; + if (!decodePulledTx(base + at, length, env)) + { + continue; + } + auto tx = TransactionFrameBase::makeTransactionFromWire( + networkID, env); + if (tx->isSoroban() && !supportsSoroban) + { + continue; + } + // N's own transactions are still in the mempool. + if (job->mClosingTxHashes.count(tx->getFullHash()) != 0) + { + continue; + } + TxValidationResult result; + result.mValid = + tx->checkValid(appConnector, *view, 0, offset, offset, + diagnostics, validationLedgerSeq) + ->isSuccess(); + if (result.mValid) + { + auto feeSource = view->getAccount(tx->getFeeSourceID()); + if (feeSource) + { + result.mFeeSourceBalance = + getAvailableBalance(header, feeSource.current()); + } + } + mine.push_back(Candidate{i, std::move(tx), result}); + } + } + } + catch (std::exception const& e) + { + // The ledger gets a set built the usual way. + CLOG_WARNING(Herder, "Early tx set selection failed: {}", e.what()); + mine.clear(); + std::lock_guard lock(job->mMutex); + job->mStop.store(true); + } + bool last = false; + { + std::lock_guard lock(job->mMutex); + if (!job->mStop.load()) + { + job->mTaskResults.at(task) = std::move(mine); + } + last = --job->mActiveTasks == 0 && job->mPendingTasks == 0; + } + if (!last) + { + return; + } + // Every task has left: nothing else touches the job's transactions. + std::shared_ptr selection; + if (!job->mStop.load()) + { + try + { + selection = job->select(app); + } + catch (std::exception const& e) + { + CLOG_WARNING(Herder, + "Could not select the TX set for ledger {} ahead of " + "time: {}", + job->mClosingLedgerSeq + 1, e.what()); + } + } + { + std::lock_guard lock(job->mMutex); + if (selection && !job->mStop.load()) + { + job->mFinishedAfterNs = + std::chrono::duration_cast( + selection->mSelectedAt - job->mStart) + .count(); + job->mSelection = std::move(selection); + } + } + job->mDone.store(true); +} +} diff --git a/src/herder/EarlyTxSetSelection.h b/src/herder/EarlyTxSetSelection.h new file mode 100644 index 0000000000..b91352c790 --- /dev/null +++ b/src/herder/EarlyTxSetSelection.h @@ -0,0 +1,178 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#pragma once + +// The selection of the next transaction set before the current ledger has +// closed. +// +// Ledger N is going through consensus; the last closed ledger is N-1. A node +// that expects to lead ledger N+1 knows N's transaction set as soon as it +// votes for it, and with it most of what N will do to the candidates of N+1: +// which transactions are used up, which source accounts move their sequence +// number, which fees are charged. An EarlyTxSetJob selects the set for N+1 on +// that prediction, on background threads, while N is still in its ballot. +// +// The prediction is an optimization only. When N has closed, the node +// completes the selection (TxSetBuilder::completeSelection): every one of its +// transactions is validated against N as it really closed, what is not valid +// is removed, and the set is topped up with what the job did not select or +// see. Nothing here decides what is valid; it decides what is worth +// validating later. + +#include "herder/TxSetFrame.h" +#include "ledger/ImmutableLedgerView.h" +#include "util/UnorderedMap.h" +#include "util/UnorderedSet.h" +#include "xdr/Stellar-ledger.h" + +#include +#include +#include +#include +#include +#include + +namespace stellar +{ +class Application; +class MetricsRegistry; + +// What a ledger is predicted to do to one account, read off the ledger's +// transaction set alone. +struct PredictedAccountChange +{ + // The highest sequence number a transaction of the set consumes for the + // account as its source (a transaction consumes its sequence number + // whether or not it succeeds), or 0. + SequenceNumber mSeqNum{0}; + // The fees the set's transactions bid in full with the account as their + // fee source: an upper bound of what the ledger charges it. + int64_t mFees{0}; +}; +using PredictedAccountChanges = UnorderedMap; + +// Adds what `tx`, as a transaction of the ledger being predicted, does to +// its source account and to its fee source. +void predictAccountChanges(TransactionFrameBase const& tx, + PredictedAccountChanges& changes); + +// A view of the state `previous` with `changes` applied to its accounts, for +// validating transactions on one thread. It answers for accounts only, and +// throws on any other kind of entry rather than answer with the previous +// ledger's version. +std::unique_ptr +makePredictedLedgerView(ImmutableLedgerDataPtr previous, + std::shared_ptr header, + std::shared_ptr changes, + MetricsRegistry& metrics); + +// Calls `f(data, length)` for every transaction of a GET_TOP_TXS response +// payload ([count:4]([len:4][TransactionEnvelope XDR:len])*), in order. A +// truncated payload ends the list. +void forEachPulledTx(std::vector const& payload, + std::function const& f); +// A hash of a pulled transaction's bytes, to recognize a transaction that +// was pulled before without decoding it. Equal bytes hash equally within a +// process; a collision makes a new transaction look like a known one. +uint64_t hashPulledTx(uint8_t const* data, size_t length); +// Decodes one pulled transaction into `tx`; false if it does not parse. +bool decodePulledTx(uint8_t const* data, size_t length, + TransactionEnvelope& tx); + +// What an EarlyTxSetJob produces. +struct EarlyTxSetSelection +{ + // The ledger the set is for (N+1). + uint32_t mLedgerSeq{0}; + // The selection: both phases, within the limits of N-1. + std::vector mPhases; + // Per phase and surge pricing lane, whether the selection left a + // transaction out for the lane's limits. + std::vector> mHadTxNotFittingLane; + // Every other candidate the job decoded (not in N, not selected), in + // mempool order: the first candidates of the top-up. + TxFrameList mLeftovers; + // hashPulledTx of everything the job pulled, N's transactions included. + UnorderedSet mSeen; + // The limits the selection was made within (those of N-1), and what + // keeps their Soroban configuration alive. + TxSetBuildLimits mLimits; + ImmutableLedgerDataPtr mPreviousLedger; + // Randomized the order of candidates with equal fee rates. + size_t mSelectionSeed{0}; + size_t mSelected{0}; + std::chrono::steady_clock::time_point mSelectedAt; +}; + +struct EarlyTxSetJob +{ + // --- Inputs, set on the main thread before the tasks are posted. --- + // The ledger in its ballot (N), the value this node votes for and that + // value's transaction set. + uint32_t mClosingLedgerSeq{0}; + StellarValue mClosingValue; + TxSetXDRFrameConstPtr mClosingTxSet; + // The last closed ledger (N-1). + ImmutableLedgerDataPtr mPreviousLedger; + // What the main thread pulled from the mempool. + std::shared_ptr const> mPayload; + // The close time the candidates are validated for, as an apply-time + // offset from N. + uint64_t mAssumedCloseTimeOffset{0}; + size_t mSelectionSeed{0}; + size_t mTasks{1}; + std::chrono::steady_clock::time_point mStart; + + // Set by whoever gives up on the job (the main thread, or a task that + // failed); tasks stop at their next transaction and nothing is handed + // over afterwards. + std::atomic mStop{false}; + + // --- Written once, by the first task (prepare). --- + std::once_flag mPrepared; + // The pulled transactions: where each is in the payload. + std::vector> mPulled; + UnorderedSet mSeen; + UnorderedSet mClosingTxHashes; + std::shared_ptr mPredicted; + std::shared_ptr mPredictedHeader; + + // --- Guarded by mMutex. --- + std::mutex mMutex; + size_t mPendingTasks{0}; + size_t mActiveTasks{0}; + struct Candidate + { + // Position in the pull (mempool order). + size_t mIndex{0}; + TransactionFrameBasePtr mTx; + TxValidationResult mResult; + }; + // What each task decoded and validated. + std::vector> mTaskResults; + // The result, once it exists and until it is taken. + std::shared_ptr mSelection; + +#ifdef BUILD_TESTS + // While this is set and true the job's tasks wait before they do + // anything: lets a test have the ledger close, or the next job start, + // while this one is still running. + std::shared_ptr> mHoldForTesting; +#endif + + // Set when the last task has left, with or without a result. + std::atomic mDone{false}; + // Nanoseconds from mStart until the selection was handed over, or -1. + std::atomic mFinishedAfterNs{-1}; + + // The body of one of the job's mTasks background tasks. + static void runTask(Application& app, + std::shared_ptr const& job, size_t task); + + private: + void prepare(Application& app); + std::shared_ptr select(Application& app); +}; +} diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp index 859b1d13f3..d6dca274d0 100644 --- a/src/herder/HerderImpl.cpp +++ b/src/herder/HerderImpl.cpp @@ -317,6 +317,7 @@ HerderImpl::HerderImpl(Application& app) , mLastExternalize(app.getClock().now()) , mTriggerTimer(app) , mPrepareTxSetTimer(app) + , mEarlyTxSetTimer(app) , mOutOfSyncTimer(app) , mTxSetGarbageCollectTimer(app) , mCheckForDeadNodesTimer(app) @@ -435,6 +436,85 @@ HerderImpl::lostSync() updateTxFloodTargets(); } +void +HerderImpl::updateTxSetTree() +{ + auto const& cfg = mApp.getConfig(); + uint8_t const mode = cfg.txSetTreeMode(); + if (mode == 0 || mApp.isStopping()) + { + // Off: the overlay is never told, and pushes as before. + return; + } + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + uint32_t const next = lcl.header.ledgerSeq + 1; + // The nodes of the trees are the nodes of the leader schedule (the + // transitive quorum). + auto const& nodes = mHerderSCPDriver.getLeaderSchedule(next); + if (nodes.empty()) + { + return; + } + // Who may root a tree: the leaders of the slot's first rounds. A node + // relays only for those, so that no other validator can have its frames + // passed on. (The leader of a later round is not named: its overlay + // pushes its set to every peer instead.) + size_t const ROUNDS = 3; + std::vector nextLeaders; + for (uint32_t round = 1; round <= ROUNDS; ++round) + { + auto leader = mHerderSCPDriver.getRoundLeader(next, round); + if (leader && std::find(nextLeaders.begin(), nextLeaders.end(), + *leader) == nextLeaders.end()) + { + nextLeaders.emplace_back(*leader); + } + } + // The slot after: the node that closes the next ledger may keep leading + // (its term), or a new order starts, seeded with the last closed + // ledger's hash (see LeaderSchedule.h). + std::vector afterLeaders = nextLeaders; + auto const fresh = computeLeaderSchedule(lcl.hash, nodes); + for (size_t i = 0; i < std::min(ROUNDS, fresh.size()); ++i) + { + if (std::find(afterLeaders.begin(), afterLeaders.end(), fresh[i]) == + afterLeaders.end()) + { + afterLeaders.emplace_back(fresh[i]); + } + } + // The seeds are those of the leader schedules: the hash of the ledger + // before the last closed one for the next slot, and the last closed + // ledger's own for the slot after. + std::vector const slots{ + {next, lcl.header.previousLedgerHash, std::move(nextLeaders)}, + {next + 1, lcl.hash, std::move(afterLeaders)}}; + // For tests and fault runs: the validators with the smallest node IDs + // withhold what they should relay. The overlay is told with the top bit + // of the mode byte. + uint8_t withhold = 0; + if (auto const withholders = + cfg.EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING; + withholders > 0) + { + auto const& self = cfg.NODE_SEED.getPublicKey(); + size_t const smaller = + std::count_if(nodes.begin(), nodes.end(), + [&](NodeID const& node) { return node < self; }); + bool const listed = + std::find(nodes.begin(), nodes.end(), self) != nodes.end(); + if (listed && smaller < withholders) + { + withhold = 0x80; + } + } + mApp.getOverlayManager().setTxSetTree( + static_cast(mode | withhold), + static_cast( + std::min(cfg.EXPERIMENTAL_TX_SET_TREE_FANOUT, UINT8_MAX)), + slots, nodes); +} + std::optional> HerderImpl::computeTxFloodTargets() { @@ -464,8 +544,7 @@ HerderImpl::computeTxFloodTargets() // Every validator of the schedule, in an order that does not change // from slot to slot (the overlay is only told when the list // changes). - std::vector everyone = - mHerderSCPDriver.getLeaderSchedule(slot); + std::vector everyone = mHerderSCPDriver.getLeaderSchedule(slot); if (everyone.empty()) { return std::nullopt; @@ -509,9 +588,8 @@ HerderImpl::computeTxFloodTargets() { auto const next = computeLeaderSchedule( lcl.hash, mHerderSCPDriver.getLeaderSchedule(slot)); - if (!next.empty() && - std::find(targets.begin(), targets.end(), next.front()) == - targets.end()) + if (!next.empty() && std::find(targets.begin(), targets.end(), + next.front()) == targets.end()) { targets.emplace_back(next.front()); } @@ -738,11 +816,11 @@ HerderImpl::checkTxAdmission(TransactionFrameBasePtr const& tx, validationLedgerSeq = lcl.header.ledgerSeq + 1; } auto diagnostics = DiagnosticEventManager::createDisabled(); - auto result = tx->checkValid( - mApp.getAppConnector(), *ledgerView, 0, 0, - getUpperBoundCloseTimeOffset(mApp, - getApplyTime(lcl.header.scpValue)), - diagnostics, validationLedgerSeq); + auto result = + tx->checkValid(mApp.getAppConnector(), *ledgerView, 0, 0, + getUpperBoundCloseTimeOffset( + mApp, getApplyTime(lcl.header.scpValue)), + diagnostics, validationLedgerSeq); if (!result->isSuccess()) { CLOG_DEBUG(Herder, "Rejecting transaction {} from {}: {}", @@ -895,6 +973,7 @@ HerderImpl::shutdown() discardPreparedTxSet(); stopSignaturePrewarm(); stopProposalPrevalidation(nullptr); + stopEarlyTxSetSelection(); mTrackingTimer.cancel(); mOutOfSyncTimer.cancel(); mTriggerTimer.cancel(); @@ -1271,7 +1350,14 @@ HerderImpl::emitEnvelope(SCPEnvelope const& envelope) broadcast(envelope); + // Before the execution is handed over: its end starts the + // pre-validation of the next ledger's candidates, unless this node + // selects that ledger's set itself. + maybeScheduleEarlyTxSetSelection(envelope); maybeStartSpeculativeApply(envelope); + // A set this node built and has not seen checked: by now its own + // execution has it, or the check runs in the background. + resolvePendingTxSetSelfCheck(); } void @@ -1306,34 +1392,54 @@ HerderImpl::maybeStartSpeculativeApply(SCPEnvelope const& envelope) return; } + startSpeculativeApply(st.slotIndex, *value, nullptr); +} + +void +HerderImpl::startSpeculativeApply(uint64_t slotIndex, Value const& value, + TxSetXDRFrameConstPtr txSet) +{ + auto const& cfg = mApp.getConfig(); + if (!cfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY || + !cfg.backgroundLedgerApply()) + { + return; + } // Only the ledger right after the LCL can be executed, and only while // nothing else is being applied: the ledger apply thread executes it on // top of the state the LCL left. auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); - if (st.slotIndex != lcl.header.ledgerSeq + 1 || + if (slotIndex != lcl.header.ledgerSeq + 1 || !protocolValidatesTxSetsAtApply(lcl.header.ledgerVersion) || !isTracking() || !mLedgerManager.isSynced() || mLedgerManager.isApplying()) { return; } - if (mSpeculativeApplyValue && - mSpeculativeApplyValue->first == st.slotIndex && - mSpeculativeApplyValue->second == *value) + if (mSpeculativeApplyValue && mSpeculativeApplyValue->first == slotIndex && + mSpeculativeApplyValue->second == value) { // already handed over return; } StellarValue sv; - if (!toStellarValue(*value, sv) || isEmptyTxSetStellarValue(sv)) + if (!toStellarValue(value, sv) || isEmptyTxSetStellarValue(sv)) { // An empty-tx-set value takes no time to apply. return; } - auto txSetResult = mPendingEnvelopes.getTxSet(sv.txSetHash); - auto const* txSet = std::get_if(&txSetResult); - if (!txSet || !*txSet || (*txSet)->previousLedgerHash() != lcl.hash) + if (!txSet) + { + auto txSetResult = mPendingEnvelopes.getTxSet(sv.txSetHash); + if (auto const* received = + std::get_if(&txSetResult)) + { + txSet = *received; + } + } + if (!txSet || txSet->getContentsHash() != sv.txSetHash || + txSet->previousLedgerHash() != lcl.hash) { return; } @@ -1341,11 +1447,11 @@ HerderImpl::maybeStartSpeculativeApply(SCPEnvelope const& envelope) // Executing the ledger starts with validating its transaction set, which // verifies the signatures the prewarm has not reached itself, on all // cores; a prewarm that is still running would only compete with it. - if (mSignaturePrewarm && mSignaturePrewarm->ledgerSeq <= st.slotIndex) + if (mSignaturePrewarm && mSignaturePrewarm->ledgerSeq <= slotIndex) { stopSignaturePrewarm(); } - mSpeculativeApplyValue = std::make_pair(st.slotIndex, *value); + mSpeculativeApplyValue = std::make_pair(slotIndex, value); // A pre-validation that is still running was for a build on top of the // LCL that this node has not needed (it votes for another leader's // value): keep what it found, and leave the cores to the execution. @@ -1354,17 +1460,39 @@ HerderImpl::maybeStartSpeculativeApply(SCPEnvelope const& envelope) // leaves, to validate its candidates on while this ledger is in its // ballot. A ledger with upgrades is left out: they are applied when the // ledger commits, after that state. - LedgerManager::SpeculativeApplyExecuted onExecuted; - if (sv.upgrades.empty() && getSCP().isValidator() && + bool const prevalidate = + sv.upgrades.empty() && getSCP().isValidator() && !buildsLegacyTxSet(lcl.header.ledgerVersion) && - mayLeadAfter(st.slotIndex, sv, /*appliedTransactions=*/true)) + mayLeadAfter(slotIndex, sv, /*appliedTransactions=*/true); + // Executing a ledger starts with the full validation of its set. For a + // set this node built, that is the verdict every other node will reach + // when it applies the ledger: if it comes before the proposal (the + // leader executes a prepared set ahead of the trigger), an invalid set + // is not proposed. + bool const ownTxSet = mLastBuiltTxSet && + mLastBuiltTxSet->first == sv.txSetHash && + mLastBuiltTxSet->second == getConsensusTime(sv); + LedgerManager::SpeculativeApplyExecuted onExecuted; + if (prevalidate || ownTxSet) { - onExecuted = [this](std::shared_ptr state) { - startProposalPrevalidation(std::move(state)); + onExecuted = [this, prevalidate, ownTxSet, + ledgerSeq = static_cast(slotIndex), + txSetHash = sv.txSetHash]( + std::shared_ptr state) { + if (ownTxSet && state && state->mTxSetSkipped) + { + txSetSelfCheckFailed(txSetHash, ledgerSeq, + "this node's own execution of the " + "ledger found the set invalid"); + } + if (prevalidate) + { + startProposalPrevalidation(std::move(state)); + } }; } mLedgerManager.startSpeculativeApply( - LedgerCloseData(static_cast(st.slotIndex), *txSet, sv), + LedgerCloseData(static_cast(slotIndex), txSet, sv), std::move(onExecuted)); } @@ -1405,6 +1533,16 @@ HerderImpl::startProposalPrevalidation( { return; } + // The node first in line for the next ledger selects its set ahead of + // time instead (see EarlyTxSetJob): that job validates the same + // candidates, and the selection is validated again on the closed + // ledger, so results computed here would only serve a fallback build. + if (mEarlyTxSetPlan && mEarlyTxSetPlan->planned && + mEarlyTxSetPlan->slotIndex == state->mLedgerSeq && + mEarlyTxSetPlan->stellarValue == state->mValue) + { + return; + } stopProposalPrevalidation(nullptr); // The pre-validation verifies the same signatures, and more. if (mSignaturePrewarm && mSignaturePrewarm->ledgerSeq > state->mLedgerSeq) @@ -1461,8 +1599,11 @@ HerderImpl::startProposalPrevalidation( for (size_t task = 0; task < tasks; ++task) { mApp.postOnBackgroundThread( + // The network ID by reference to the application's: the + // transaction frames made below keep a reference to it, and + // outlive this task. [job, state, payload, windows, supportsSoroban, validationLedgerSeq, - networkID = mApp.getNetworkID(), + &networkID = mApp.getNetworkID(), &appConnector = mApp.getAppConnector(), &metrics, task, tasks]() { ZoneScopedN("proposal prevalidation"); { @@ -1644,13 +1785,19 @@ HerderImpl::expectedTxSetBuildDuration() const // Until this node has built a set of some size: what a build costs per // transaction on the slower of the machines measured (about 0.17 s for // 4,000 transactions with most candidates validated ahead of time). + return std::max(mLastTxSetBuildDuration, + expectedTxSetBuildDuration(mLastClosedTxSetSize)); +} + +std::chrono::milliseconds +HerderImpl::expectedTxSetBuildDuration(size_t txs) const +{ static std::chrono::nanoseconds const DEFAULT_COST_PER_TX{40'000}; auto const perTx = mTxSetBuildCostPerTx > std::chrono::nanoseconds::zero() ? mTxSetBuildCostPerTx : DEFAULT_COST_PER_TX; - auto const bySize = std::chrono::duration_cast( - perTx * static_cast(mLastClosedTxSetSize)); - return std::max(mLastTxSetBuildDuration, bySize); + return std::chrono::duration_cast( + perTx * static_cast(txs)); } std::chrono::milliseconds @@ -2271,6 +2418,21 @@ HerderImpl::lastClosedLedgerIncreased(bool latest, TxSetXDRFrameConstPtr txSet, { stopProposalPrevalidation(nullptr); } + mPendingTxSetSelfCheck.reset(); + // The same goes for a set selected ahead of time. + if (mEarlyTxSetPlan && + (mEarlyTxSetPlan->slotIndex != lcl.header.ledgerSeq || + !(mEarlyTxSetPlan->stellarValue == lcl.header.scpValue))) + { + if (mEarlyTxSetPlan->planned && + mEarlyTxSetPlan->slotIndex == lcl.header.ledgerSeq) + { + mApp.getMetrics() + .NewMeter({"herder", "txset", "early-discarded"}, "txset") + .Mark(); + } + stopEarlyTxSetSelection(); + } // Results computed on the previous ledger are of no use either, // whether or not this node builds on the new one. if (mValidationCache.lclHash != lcl.hash) @@ -2305,6 +2467,7 @@ HerderImpl::lastClosedLedgerIncreased(bool latest, TxSetXDRFrameConstPtr txSet, // The new ledger's hash seeds the next slot's leader schedule: the // transactions this node holds go to the new leaders now. updateTxFloodTargets(); + updateTxSetTree(); // Now that the new ledger is closed, process any already-buffered SCP // envelopes for the next slot. Posted to the main thread so control @@ -2557,42 +2720,8 @@ HerderImpl::setupTriggerNextLedger() }, &VirtualTimer::onFailureNoop); - // Prepare early only if the trigger is still ahead. A due trigger - // fires now and builds itself; both timers would fire at the same - // instant in an unspecified order (event sequence numbers are per - // timer), so the leader could build once to prepare and again at the - // trigger. - if (getSCP().isValidator() && !triggerDue) - { - if (mHerderSCPDriver.isLocalNodeRoundLeader(nextIndex, 1)) - { - // Validate against the close time we expect at the trigger, - // not the earlier wall-clock time at which we start work. - auto closeTime = std::max( - minCandidateCt, - ConsensusTime::fromSystemTime( - mApp.getClock().system_now() + - std::chrono::duration_cast< - VirtualClock::system_time_point::duration>( - triggerTime - now), - lcl.header.ledgerVersion)); - // Build shortly before the trigger rather than right away: - // the set then holds what arrived in the meantime, and is - // still built and on its way to the peers when consensus - // starts (see makeProposalValue). This also yields out of - // ledger close before touching its read view. - mPrepareTxSetTimer.expires_at(std::max( - now, triggerTime - - prepareTxSetLeadTime( - milliseconds, expectedTxSetBuildDuration()))); - mPrepareTxSetTimer.async_wait( - [this, nextIndex, closeTime]() { - prepareTxSet(static_cast(nextIndex), - closeTime); - }, - &VirtualTimer::onFailureNoop); - } - } + scheduleTxSetPreparation(nextIndex, triggerTime, triggerDue, now, + milliseconds, minCandidateCt); } #ifdef BUILD_TESTS @@ -2600,6 +2729,45 @@ HerderImpl::setupTriggerNextLedger() #endif } +void +HerderImpl::scheduleTxSetPreparation( + uint64_t nextIndex, VirtualClock::time_point triggerTime, bool triggerDue, + VirtualClock::time_point now, std::chrono::milliseconds expectedCloseTime, + ConsensusTime minCandidateCt) +{ + // Prepare early only if the trigger is still ahead. A due trigger fires + // now and builds itself; both timers would fire at the same instant in + // an unspecified order (event sequence numbers are per timer), so the + // leader could build once to prepare and again at the trigger. + if (!getSCP().isValidator() || triggerDue || + !mHerderSCPDriver.isLocalNodeRoundLeader(nextIndex, 1)) + { + return; + } + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + // Validate against the close time we expect at the trigger, not the + // earlier wall-clock time at which we start work. + auto closeTime = std::max( + minCandidateCt, ConsensusTime::fromSystemTime( + mApp.getClock().system_now() + + std::chrono::duration_cast< + VirtualClock::system_time_point::duration>( + triggerTime - now), + lcl.header.ledgerVersion)); + // Build shortly before the trigger rather than right away: the set then + // holds what arrived in the meantime, and is still built and on its way + // to the peers when consensus starts (see makeProposalValue). This also + // yields out of ledger close before touching its read view. + mPrepareTxSetTimer.expires_at(std::max( + now, triggerTime - prepareTxSetLeadTime(expectedCloseTime, + expectedTxSetBuildDuration()))); + mPrepareTxSetTimer.async_wait( + [this, nextIndex, closeTime]() { + prepareTxSet(static_cast(nextIndex), closeTime); + }, + &VirtualTimer::onFailureNoop); +} + // Returns the ledger index of the oldest ledger that it is safe to delete while // still keeping all the information needed to publish checkpoints static uint32_t @@ -2756,6 +2924,223 @@ HerderImpl::prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime) "close time {}", ledgerSeq, mPreparedTxSet->txSet->sizeTxTotal(), closeTime.toString()); + executePreparedTxSet(); +} + +void +HerderImpl::handOffTxSet(TxSetXDRFrameConstPtr const& txSet, uint32_t ledgerSeq, + bool pushToPeers) +{ + auto t = mApp.getMetrics() + .NewTimer({"herder", "txset", "build-handoff"}) + .TimeScope(); + // Legacy (pre-protocol 20) sets cannot be carried by the Rust overlay. + if (!txSet->isGeneralizedTxSet()) + { + return; + } + // Serialized straight into the IPC message, without copying the set's + // XDR first. The overlay's eager compression runs concurrently in Rust. + if (mApp.getOverlayManager().cacheTxSet( + txSet->getContentsHash(), txSet->getGeneralizedXDR(), ledgerSeq)) + { + mTxSetsStoredByOverlay.insert(txSet->getContentsHash()); + } + // Leader's push: this set is about to be proposed, so send it to the + // peers now. + if (pushToPeers) + { + pushTxSetToPeers(txSet, ledgerSeq); + } +} + +void +HerderImpl::executePreparedTxSet() +{ + if (!mPreparedTxSet || !mPreparedTxSet->txSet || + !mPreparedTxSet->applicableTxSet) + { + return; + } +#ifdef BUILD_TESTS + // The trigger proposes something else than the prepared set. + if (mProposedTxSetOverride || + mApp.getConfig().TESTING_PROPOSE_RANDOM_VALUES) + { + return; + } +#endif + auto const lcl = mLedgerManager.getLastClosedLedgerHeader(); + if (mPreparedTxSet->ledgerSeq != lcl.header.ledgerSeq + 1 || + mPreparedTxSet->previousLedgerHash != lcl.hash || + !getSCP().isValidator()) + { + return; + } + // The value makeProposalValue builds at the trigger, if nothing changes + // until then. + auto const value = makeStellarValue( + mPreparedTxSet->txSet->getContentsHash(), mPreparedTxSet->closeTime, + proposalUpgrades(lcl), mApp.getConfig().NODE_SEED); + startSpeculativeApply(mPreparedTxSet->ledgerSeq, + Value(xdr::xdr_to_opaque(value)), + mPreparedTxSet->txSet); +} + +void +HerderImpl::startTxSetSelfCheck(std::shared_ptr check, + uint32_t ledgerSeq) +{ + if (!check) + { + return; + } + if (executesVotedValues(mLedgerManager.getLastClosedLedgerHeader())) + { + // This node's own execution of the set reads it back and validates + // it in full, and its verdict is taken there. The check is kept + // until the node has emitted a statement: if the ledger was not + // handed to the apply thread with this set by then (see + // resolvePendingTxSetSelfCheck), it runs after all. + mPendingTxSetSelfCheck = std::make_pair(std::move(check), ledgerSeq); + return; + } + runTxSetSelfCheck(std::move(check), ledgerSeq); +} + +void +HerderImpl::runTxSetSelfCheck(std::shared_ptr check, + uint32_t ledgerSeq) +{ + mApp.postOnBackgroundThread( + [this, check = std::move(check), ledgerSeq, &app = mApp]() { + ZoneScopedN("tx set self-check"); + auto const failure = check->run(app); + if (!failure) + { + app.getMetrics() + .NewMeter({"herder", "txset", "self-check-passed"}, "txset") + .Mark(); + return; + } + app.postOnMainThread( + [this, check, ledgerSeq, reason = *failure]() { + txSetSelfCheckFailed(check->getTxSet()->getContentsHash(), + ledgerSeq, reason); + }, + "tx set self-check failed"); + }, + "Herder: tx set self-check"); +} + +void +HerderImpl::resolvePendingTxSetSelfCheck() +{ + if (!mPendingTxSetSelfCheck) + { + return; + } + auto pending = std::move(*mPendingTxSetSelfCheck); + mPendingTxSetSelfCheck.reset(); + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + if (pending.second != lcl.header.ledgerSeq + 1) + { + // For a ledger that is no longer the next one: nothing to check. + return; + } + // Handed to the apply thread with this set? + StellarValue executing; + if (mSpeculativeApplyValue && + mSpeculativeApplyValue->first == pending.second && + toStellarValue(mSpeculativeApplyValue->second, executing) && + executing.txSetHash == pending.first->getTxSet()->getContentsHash()) + { + return; + } + runTxSetSelfCheck(std::move(pending.first), pending.second); +} + +bool +HerderImpl::executesVotedValues(LedgerHeaderHistoryEntry const& lcl) const +{ + auto const& cfg = mApp.getConfig(); + return cfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY && + cfg.backgroundLedgerApply() && + protocolValidatesTxSetsAtApply(lcl.header.ledgerVersion); +} + +void +HerderImpl::txSetSelfCheckFailed(Hash const& txSetHash, uint32_t ledgerSeq, + std::string const& reason) +{ + releaseAssert(threadIsMain()); + CLOG_ERROR(Herder, + "TX set {} built for ledger {} failed the builder's " + "self-check: {}", + hexAbbrev(txSetHash), ledgerSeq, reason); + CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG); + mApp.getMetrics() + .NewMeter({"herder", "txset", "self-check-failed"}, "txset") + .Mark(); + // No more unchecked sets from this process. + mTxSetSelfCheckFailed = true; + if (mPreparedTxSet && mPreparedTxSet->txSet && + mPreparedTxSet->txSet->getContentsHash() == txSetHash) + { + // Not proposed yet: the trigger builds again, and checks what it + // builds before using it. + discardPreparedTxSet(); + } +} + +#ifdef BUILD_TESTS +void +HerderImpl::overrideBuiltTxSetForTesting( + TxSetXDRFrameConstPtr& txSet, + std::shared_ptr& selfCheck) +{ + if (!mBuiltTxSetOverrideForTesting) + { + return; + } + txSet = mBuiltTxSetOverrideForTesting(txSet); + if (selfCheck) + { + selfCheck = selfCheck->withTxSetForTesting(txSet); + } +} +#endif + +xdr::xvector +HerderImpl::proposalUpgrades(LedgerHeaderHistoryEntry const& lcl) +{ + auto newUpgrades = emptyUpgradeSteps; + + // see if we need to include some upgrades + std::vector upgrades; + { + auto ledgerView = mApp.getLedgerManager().getLCLView(); + upgrades = mUpgrades.createUpgradesFor(lcl.header, *ledgerView, + mApp.getConfig()); + } + for (auto const& upgrade : upgrades) + { + Value v(xdr::xdr_to_opaque(upgrade)); + if (v.size() >= UpgradeType::max_size()) + { + CLOG_ERROR( + Herder, + "HerderImpl::triggerNextLedger exceeded size for upgrade " + "step (got {} ) for upgrade type {}", + v.size(), upgrade.type()); + CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG); + } + else + { + newUpgrades.emplace_back(v.begin(), v.end()); + } + } + return newUpgrades; } void @@ -2787,6 +3172,64 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, : legacyTxSet ? std::string("validator (legacy tx set)") : std::string("first-round leader (early)"), ledgerSeq); + // This build validates every candidate itself. + stopSignaturePrewarm(); + auto& metrics = mApp.getMetrics(); + + // How long after this node's vote for the previous ledger its set for + // this one is needed: what the start of the next early selection is + // timed by (earlyTxSetStartDelay). + if (mFirstVote && mFirstVote->first + 1 == ledgerSeq && round <= 1) + { + // (the first build of the ledger, by the first round's leader: a + // rebuild at the trigger or a later round's build comes when it + // comes) + auto const votedAt = mFirstVote->second; + mFirstVote.reset(); + mLastVoteToBuild = + std::max(std::chrono::milliseconds(1), + std::chrono::duration_cast( + mApp.getClock().now() - votedAt)); + mVoteToBuild = mVoteToBuild == std::chrono::milliseconds::zero() + ? mLastVoteToBuild + : (3 * mVoteToBuild + mLastVoteToBuild) / 4; + } + + auto finish = [&](PreparedTxSet&& prepared, bool built) { + auto const buildDuration = + std::chrono::steady_clock::now() - txSetBuildStart; + mSCPMetrics.mTxSetBuild.Update( + std::chrono::duration_cast( + buildDuration)); + if (!built) + { + // A completed selection says nothing about what a build costs, + // which is what a build's lead time has to cover. + return std::move(prepared); + } + // The next build is scheduled far enough ahead of its trigger to fit + // one like this (see prepareTxSetLeadTime). + mLastTxSetBuildDuration = + std::chrono::duration_cast( + buildDuration); + if (prepared.txSet->sizeTxTotal() >= 100) + { + mTxSetBuildCostPerTx = + std::chrono::duration_cast( + buildDuration) / + static_cast(prepared.txSet->sizeTxTotal()); + } + return std::move(prepared); + }; + + // A set this node selected while the previous ledger was in its ballot + // only has to be completed. + if (auto completed = + completeEarlyTxSet(ledgerSeq, closeTime, pushToPeers, lcl)) + { + return finish(std::move(*completed), /*built=*/false); + } + // Apply-time offset: see the comment in triggerNextLedger auto const closeTimeOffset = closeTime.toApplyTime() - getApplyTime(lcl.header.scpValue); @@ -2796,77 +3239,121 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, // Build TX set from Rust overlay's mempool (not local TransactionQueue) // The Rust overlay maintains the mempool via TX flooding PerPhaseTransactionList txPhases; + bool const supportsSoroban = protocolVersionStartsFrom( + lcl.header.ledgerVersion, SOROBAN_PROTOCOL_VERSION); + auto* const validationCache = validationCacheFor(lcl); - // Get TXs from Rust overlay - auto& overlayMgr = mApp.getOverlayManager(); - // This build validates every candidate itself. - stopSignaturePrewarm(); - auto& metrics = mApp.getMetrics(); - auto txEnvelopes = [&]() { - auto t = - metrics.NewTimer({"herder", "txset", "build-pull"}).TimeScope(); - return pullTopTransactions(maxTxSetCandidates() * 2); - }(); - auto framesTimer = - metrics.NewTimer({"herder", "txset", "build-frames"}).TimeScope(); - - CLOG_INFO(Herder, "Got {} transactions from Rust overlay mempool", - txEnvelopes.size()); + // Signature cache lookups of this build, the validation for a refill + // included. + uint64_t sigCacheHits = 0; + uint64_t sigCacheMisses = 0; + PubKeyUtils::getVerifySigCacheCounts(sigCacheHits, sigCacheMisses); - // Convert TransactionEnvelopes to TransactionFrameBasePtrs and place them - // into the phase expected by TxSetFrame. + // The pull is the capacity of the set plus a margin. If validation + // takes so much out of a phase that it ends below its capacity while + // the mempool has more to give, the build pulls again, twice as much, + // and goes on with what it has not seen yet. TxFrameList classicTxs; TxFrameList sorobanTxs; - Hash const& networkID = mApp.getNetworkID(); - bool const supportsSoroban = protocolVersionStartsFrom( - lcl.header.ledgerVersion, SOROBAN_PROTOCOL_VERSION); - for (auto const& env : txEnvelopes) - { - auto txFrame = - TransactionFrameBase::makeTransactionFromWire(networkID, env); - if (txFrame->isSoroban()) + UnorderedSet seen; + bool mempoolExhausted = false; + size_t pulled = 0; + { + std::array const capacity{ + mLedgerManager.getLastMaxTxSetSizeOps(), + supportsSoroban && + mLedgerManager.hasLastClosedSorobanNetworkConfig() + ? static_cast( + mLedgerManager.getLastClosedSorobanNetworkConfig() + .ledgerMaxTxCount()) + : size_t(0)}; + // Per phase: how many candidates were pulled, and how much of the + // phase's capacity those of them fill that are valid and the first + // of their source account (a set holds one transaction per + // account). + std::array candidates{0, 0}; + std::array usable{0, 0}; + std::array filled{0, 0}; + UnorderedSet sources; + UnorderedMap feeTotals; + for (size_t pull = 0; pull < MAX_TX_SET_PULLS; ++pull) { - if (supportsSoroban) + bool full = false; + auto const fresh = + pullNewCandidates(txSetPullCount() << pull, seen, full); + pulled += fresh.size(); + std::array freshPhases; + for (auto const& tx : fresh) { - sorobanTxs.push_back(txFrame); + freshPhases[tx->isSoroban() ? 1 : 0].push_back(tx); } - else + classicTxs.insert(classicTxs.end(), freshPhases[0].begin(), + freshPhases[0].end()); + sorobanTxs.insert(sorobanTxs.end(), freshPhases[1].begin(), + freshPhases[1].end()); + if (!full) { - CLOG_DEBUG(Herder, - "Ignoring Soroban transaction before Soroban " - "protocol support"); + mempoolExhausted = true; + break; } - } - else - { - classicTxs.push_back(txFrame); - } - } - // The mempool is fee-ordered and sequence-number-oblivious, so it can - // hand us several transactions from one source account (e.g. a chained - // pair). A tx set may only contain one tx per source account, so keep - // the lowest sequence number per account and let the others wait for a - // later ledger. - auto onePerSourceAccount = [](TxFrameList& txs) { - std::unordered_map firstBySource; - TxFrameList kept; - for (auto const& tx : txs) - { - auto [it, inserted] = - firstBySource.emplace(tx->getSourceID(), kept.size()); - if (inserted) + if (pull + 1 == MAX_TX_SET_PULLS) { - kept.push_back(tx); + break; } - else if (tx->getSeqNum() < kept[it->second]->getSeqNum()) + // The mempool may hold more. More is needed if a phase lost + // candidates (to validation, or to the one-per-account rule) + // and what is left does not fill it: validate what is new for + // the phases that are not filled yet (the selection below finds + // the results in the cache). A phase that lost none got all the + // mempool's top holds for it. A phase that lost some goes on + // as long as each pull brings it something new: a stale + // transaction of a kind the mempool holds little of costs one + // deeper pull, not all of them. + auto refillTimer = + metrics.NewTimer({"herder", "txset", "build-refill-check"}) + .TimeScope(); + auto ctx = TxSetValidationContext::forLCL(mApp); + ctx.mValidationCache = validationCache; + bool shortPhase = false; + for (size_t p = 0; p < freshPhases.size(); ++p) { - kept[it->second] = tx; + if (filled[p] >= capacity[p]) + { + continue; + } + TxFrameList notValid; + auto const ok = TxSetUtils::trimInvalid( + freshPhases[p], mApp, ctx, feeTotals, + closeTimeOffset.seconds(), closeTimeOffset.seconds(), + notValid); + candidates[p] += freshPhases[p].size(); + for (auto const& tx : ok) + { + if (sources.insert(tx->getSourceID()).second) + { + ++usable[p]; + filled[p] += p == 0 ? tx->getNumOperations() : 1; + } + } + shortPhase = + shortPhase || + (usable[p] < candidates[p] && filled[p] < capacity[p] && + (pull == 0 || !freshPhases[p].empty())); + } + if (!shortPhase) + { + break; } + metrics.NewMeter({"herder", "txset", "refill-pull"}, "pull").Mark(); } - txs = std::move(kept); - }; - onePerSourceAccount(classicTxs); - onePerSourceAccount(sorobanTxs); + } + auto framesTimer = + metrics.NewTimer({"herder", "txset", "build-frames"}).TimeScope(); + + CLOG_INFO(Herder, "Got {} transactions from Rust overlay mempool", pulled); + + // One transaction per source account. + TxSetUtils::keepOnePerSourceAccount(classicTxs, sorobanTxs); txPhases.emplace_back(std::move(classicTxs)); if (supportsSoroban) @@ -2882,9 +3369,15 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, invalidTxPhases.resize(txPhases.size()); framesTimer.Stop(); - uint64_t sigCacheHits = 0; - uint64_t sigCacheMisses = 0; - PubKeyUtils::getVerifySigCacheCounts(sigCacheHits, sigCacheMisses); + // The builder's check of its own set does not hold up the set: it runs + // in the background (startTxSetSelfCheck), unless one has failed before. + std::shared_ptr selfCheck; + // Only where every node validates the set when it applies the ledger: + // before that protocol the followers validate before they vote, and + // this node records its own set as valid for them (cacheValidTxSet). + bool const deferSelfCheck = + !legacyTxSet && !mTxSetSelfCheckFailed && + protocolValidatesTxSetsAtApply(lcl.header.ledgerVersion); std::tie(proposedSet, applicableProposedSet) = makeTxSetFromTransactions( txPhases, mApp, closeTimeOffset, invalidTxPhases #ifdef BUILD_TESTS @@ -2893,36 +3386,35 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, #endif , [&](TxSetXDRFrameConstPtr const& txSet) { - auto t = metrics.NewTimer({"herder", "txset", "build-handoff"}) - .TimeScope(); - // Hand the final XDR to the overlay before the builder's remaining - // validation. Its eager compression runs concurrently in Rust. - if (txSet->isGeneralizedTxSet()) - { - // Serialized straight into the IPC message, without copying - // the set's XDR first. - if (overlayMgr.cacheTxSet(txSet->getContentsHash(), - txSet->getGeneralizedXDR(), - ledgerSeq)) - { - mTxSetsStoredByOverlay.insert(txSet->getContentsHash()); - } - // Leader's push: this set is about to be proposed, so send - // it to the peers now, while the builder's remaining - // validation and the proposal's persistence still run. If - // that validation fails nothing is proposed, and peers drop - // the unused set. - if (pushToPeers) - { - pushTxSetToPeers(txSet, ledgerSeq); - } - } + // Hand the final XDR to the overlay, and with `pushToPeers` to + // the peers, before the builder's check of the set. If that + // check fails before the proposal, nothing is proposed, and + // peers drop the unused set. + handOffTxSet(txSet, ledgerSeq, pushToPeers); }, - targets, validationCacheFor(lcl)); + targets, validationCache, deferSelfCheck ? &selfCheck : nullptr); + if (applicableProposedSet) + { + recordTxSetFill(*applicableProposedSet, mempoolExhausted); + } +#ifdef BUILD_TESTS + overrideBuiltTxSetForTesting(proposedSet, selfCheck); +#endif + mLastBuiltTxSet = std::make_pair(proposedSet->getContentsHash(), closeTime); + startTxSetSelfCheck(selfCheck, ledgerSeq); markSignatureCacheLookups(mApp, "build", sigCacheHits, sigCacheMisses); CLOG_INFO(Herder, "Proposed TX set has {} transactions", proposedSet->sizeTxTotal()); + return finish(PreparedTxSet{lcl.hash, ledgerSeq, closeTime, proposedSet, + std::move(applicableProposedSet), + invalidTxsToRemove(invalidTxPhases)}, + /*built=*/true); +} + +std::vector +HerderImpl::invalidTxsToRemove(PerPhaseTransactionList const& invalidTxs) +{ // The mempool does no stateful validation, so it would keep handing us the // transactions that just failed validation (stale sequence number, can't // pay fee, expired, ...) on every nomination, crowding out valid ones. @@ -2930,10 +3422,10 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, // a *future* sequence number: those are chained behind a pending // transaction from the same account and become valid once it applies. std::vector invalidTxHashes; - if (!invalidTxPhases.empty()) + if (!invalidTxs.empty()) { auto ledgerView = mLedgerManager.getLCLView(); - for (auto const& phase : invalidTxPhases) + for (auto const& phase : invalidTxs) { for (auto const& tx : phase) { @@ -2952,28 +3444,565 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, } } } + return invalidTxHashes; +} + +size_t +HerderImpl::txSetPullCount() const +{ + size_t const capacity = maxTxSetCandidates(); + size_t const margin = + (capacity * mApp.getConfig().EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT + + 99) / + 100; + return std::max(1, capacity + margin); +} - auto const buildDuration = - std::chrono::steady_clock::now() - txSetBuildStart; - mSCPMetrics.mTxSetBuild.Update( - std::chrono::duration_cast(buildDuration)); - // The next build is scheduled far enough ahead of its trigger to fit one - // like this (see prepareTxSetLeadTime). - mLastTxSetBuildDuration = - std::chrono::duration_cast(buildDuration); - if (proposedSet->sizeTxTotal() >= 100) +TxFrameList +HerderImpl::pullNewCandidates(size_t count, UnorderedSet& seen, + bool& full) +{ + auto& metrics = mApp.getMetrics(); + auto const payload = [&]() { + auto t = + metrics.NewTimer({"herder", "txset", "build-pull"}).TimeScope(); + return pullTopTransactionsPayload(count); + }(); + auto framesTimer = + metrics.NewTimer({"herder", "txset", "build-frames"}).TimeScope(); + uint32_t pulled = 0; + if (payload.size() >= 4) { - mTxSetBuildCostPerTx = - std::chrono::duration_cast( - buildDuration) / - static_cast(proposedSet->sizeTxTotal()); + std::memcpy(&pulled, payload.data(), 4); + } + full = pulled >= count; + TxFrameList fresh; + Hash const& networkID = mApp.getNetworkID(); + bool const supportsSoroban = protocolVersionStartsFrom( + mLedgerManager.getLastClosedLedgerHeader().header.ledgerVersion, + SOROBAN_PROTOCOL_VERSION); + forEachPulledTx(payload, [&](uint8_t const* data, size_t length) { + if (!seen.insert(hashPulledTx(data, length)).second) + { + return; + } + TransactionEnvelope env; + if (!decodePulledTx(data, length, env)) + { + CLOG_WARNING(Herder, "Failed to parse a pulled transaction"); + return; + } + auto tx = TransactionFrameBase::makeTransactionFromWire(networkID, env); + if (tx->isSoroban() && !supportsSoroban) + { + CLOG_DEBUG(Herder, "Ignoring Soroban transaction before Soroban " + "protocol support"); + return; + } + fresh.push_back(std::move(tx)); + }); + return fresh; +} + +void +HerderImpl::recordTxSetFill(ApplicableTxSetFrame const& txSet, + bool mempoolExhausted) +{ + auto& metrics = mApp.getMetrics(); + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + // A phase below its capacity although the mempool had more to give: + // the build stopped looking (its pulls are bounded), or what it found + // did not fit. A phase without transactions is left out: the mempool's + // top held no candidate of its kind, whatever else the mempool holds. + auto record = [&](char const* fill, char const* isShort, size_t size, + size_t capacity) { + if (capacity == 0) + { + return; + } + metrics.NewHistogram({"herder", "txset", fill}) + .Update(static_cast( + std::min(100, size * 100 / capacity))); + if (size != 0 && size < capacity && !mempoolExhausted) + { + metrics.NewMeter({"herder", "txset", isShort}, "txset").Mark(); + } + }; + record("fill-classic", "short-classic", txSet.sizeOp(TxSetPhase::CLASSIC), + mLedgerManager.getLastMaxTxSetSizeOps()); + if (txSet.numPhases() > static_cast(TxSetPhase::SOROBAN) && + protocolVersionStartsFrom(lcl.header.ledgerVersion, + SOROBAN_PROTOCOL_VERSION) && + mLedgerManager.hasLastClosedSorobanNetworkConfig()) + { + record("fill-soroban", "short-soroban", + txSet.sizeTx(TxSetPhase::SOROBAN), + mLedgerManager.getLastClosedSorobanNetworkConfig() + .ledgerMaxTxCount()); + } +} + +std::chrono::milliseconds +HerderImpl::earlyTxSetStartDelay() const +{ + using std::chrono::milliseconds; +#ifdef BUILD_TESTS + if (mEarlyTxSetStartDelayForTesting) + { + return *mEarlyTxSetStartDelayForTesting; + } +#endif + if (mVoteToBuild == milliseconds::zero() || + mEarlyTxSetJobDuration == milliseconds::zero()) + { + // Nothing measured yet: start at once. + return milliseconds::zero(); + } + // The earlier of what the build did lately and what it did last time, + // less a selection that takes as long as the slower of the two, with a + // quarter to spare, and 30 ms for the hand-over. + auto const untilBuild = std::min(mVoteToBuild, mLastVoteToBuild); + auto const job = + std::max(mEarlyTxSetJobDuration, mLastEarlyTxSetJobDuration); + auto const delay = untilBuild - job - job / 4 - milliseconds(30); + return std::max(milliseconds::zero(), delay); +} + +void +HerderImpl::stopEarlyTxSetSelection() +{ + mEarlyTxSetTimer.cancel(); + if (mEarlyTxSet) + { + std::lock_guard lock(mEarlyTxSet->mMutex); + mEarlyTxSet->mStop.store(true); + } + mEarlyTxSet.reset(); + mEarlyTxSetPlan.reset(); +} + +void +HerderImpl::maybeScheduleEarlyTxSetSelection(SCPEnvelope const& envelope) +{ + auto const& cfg = mApp.getConfig(); + if (!cfg.EXPERIMENTAL_EARLY_TX_SET_SELECTION || !getSCP().isValidator()) + { + return; + } +#ifdef BUILD_TESTS + // The bucket list isn't populated in this mode: there is no snapshot of + // the last closed ledger to read from other threads. + if (cfg.MODE_USES_IN_MEMORY_LEDGER) + { + return; + } +#endif + // The value this node is working on: what it votes to prepare, or to + // commit. + auto const& st = envelope.statement; + Value const* value = nullptr; + switch (st.pledges.type()) + { + case SCP_ST_PREPARE: + value = &st.pledges.prepare().ballot.value; + break; + case SCP_ST_CONFIRM: + value = &st.pledges.confirm().ballot.value; + break; + default: + return; } + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + if (st.slotIndex != lcl.header.ledgerSeq + 1 || + !protocolValidatesTxSetsAtApply(lcl.header.ledgerVersion) || + buildsLegacyTxSet(lcl.header.ledgerVersion) || !isTracking() || + !mLedgerManager.isSynced()) + { + return; + } + if (!mFirstVote || mFirstVote->first != st.slotIndex) + { + mFirstVote = std::make_pair(st.slotIndex, mApp.getClock().now()); + } + if (mEarlyTxSetPlan && mEarlyTxSetPlan->slotIndex == st.slotIndex && + mEarlyTxSetPlan->value == *value) + { + // Looked at already. + return; + } + // The first vote for this ledger, or a vote for another value: whatever + // was planned or running was for a value this node no longer votes for. + stopEarlyTxSetSelection(); + EarlyTxSetPlan plan; + plan.slotIndex = st.slotIndex; + plan.value = *value; + mEarlyTxSetPlan = plan; + + // No selection on a ledger with upgrades: they change what the next set + // is validated against and the limits it is selected within. None after + // an empty set either: nothing is known about the next ledger then that + // a build would not know. + StellarValue sv; + if (!toStellarValue(*value, sv) || isEmptyTxSetStellarValue(sv) || + !sv.upgrades.empty()) + { + return; + } + auto txSetResult = mPendingEnvelopes.getTxSet(sv.txSetHash); + auto const* received = std::get_if(&txSetResult); + if (!received || !*received) + { + // Not in hand yet: look again at the next statement for the value. + mEarlyTxSetPlan.reset(); + return; + } + if (!(*received)->isGeneralizedTxSet() || + (*received)->previousLedgerHash() != lcl.hash || + (*received)->sizeTxTotal() == 0) + { + return; + } + // Only the node first in line for the next ledger. + auto const schedule = mHerderSCPDriver.getExpectedLeaderSchedule( + st.slotIndex + 1, sv, /*appliedTransactions=*/true); + if (schedule.empty() || !(schedule[0] == getSCP().getLocalNodeID())) + { + return; + } + mEarlyTxSetPlan->stellarValue = sv; + mEarlyTxSetPlan->planned = true; + mApp.getMetrics() + .NewMeter({"herder", "txset", "early-planned"}, "txset") + .Mark(); + // Through the timer even without a delay: this is called while SCP + // emits an envelope, and the start pulls the mempool. + mEarlyTxSetTimer.expires_from_now(earlyTxSetStartDelay()); + mEarlyTxSetTimer.async_wait( + [this, slotIndex = st.slotIndex, value = *value, txSet = *received]() { + startEarlyTxSetSelection(slotIndex, value, txSet); + }, + &VirtualTimer::onFailureNoop); +} + +void +HerderImpl::startEarlyTxSetSelection(uint64_t slotIndex, Value const& value, + TxSetXDRFrameConstPtr const& txSet) +{ + ZoneScoped; + releaseAssert(threadIsMain()); + if (mApp.isStopping()) + { + return; + } + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + // Only while the ledger is still the next one to close and this node + // still votes for the value. + if (!mEarlyTxSetPlan || !mEarlyTxSetPlan->planned || mEarlyTxSet || + mEarlyTxSetPlan->slotIndex != slotIndex || + !(mEarlyTxSetPlan->value == value) || + slotIndex != lcl.header.ledgerSeq + 1 || + txSet->previousLedgerHash() != lcl.hash || !isTracking()) + { + return; + } + auto& metrics = mApp.getMetrics(); + auto job = std::make_shared(); + job->mStart = std::chrono::steady_clock::now(); + job->mClosingLedgerSeq = static_cast(slotIndex); + job->mClosingValue = mEarlyTxSetPlan->stellarValue; + job->mClosingTxSet = txSet; + job->mPreviousLedger = mLedgerManager.getLastClosedLedgerState(); + if (job->mPreviousLedger->getLastClosedLedgerHeader().hash != lcl.hash) + { + return; + } + // The closing ledger's transactions are still in the mempool: look past + // that many more. + job->mPayload = [&]() { + auto timer = + metrics.NewTimer({"herder", "txset", "early-pull"}).TimeScope(); + return std::make_shared const>( + pullTopTransactionsPayload(txSetPullCount() + + txSet->sizeTxTotal())); + }(); + { + // The candidates are validated for the close time the next trigger + // is expected to have: the target after this ledger's close time, + // or now if that has passed. The completion validates them again + // for the close time the set really gets. + auto const version = lcl.header.ledgerVersion; + auto const closingCloseTime = getConsensusTime(job->mClosingValue); + auto const expectedTrigger = std::max( + closingCloseTime.toSystemTime() + + mLedgerManager.getExpectedLedgerCloseTime(), + mApp.getClock().system_now()); + auto const assumedCloseTime = + std::max(ConsensusTime::fromSystemTime(expectedTrigger, version), + closingCloseTime.next(version)); + job->mAssumedCloseTimeOffset = + (assumedCloseTime.toApplyTime() - getApplyTime(job->mClosingValue)) + .seconds(); + } + { + // What orders candidates with equal fee rates must not be + // predictable by who submits them. The global random engine is not + // used: whether a job runs depends on timing, and a draw here would + // shift every later draw of this node. A hash keyed with a secret + // of this node is as unpredictable to others. + if (!mTieBreakSecret) + { + mTieBreakSecret = + sha256(mApp.getConfig().NODE_SEED.getStrKeySeed().value + + " tx set tie-break"); + } + SHA256 hasher; + hasher.add(*mTieBreakSecret); + hasher.add(xdr::xdr_to_opaque(job->mClosingLedgerSeq, + job->mClosingValue.txSetHash)); + auto const digest = hasher.finish(); + static_assert(sizeof(size_t) <= sizeof(digest)); + std::memcpy(&job->mSelectionSeed, digest.data(), sizeof(size_t)); + } + // As many background tasks as the batch executor would use, but leave + // worker threads for long background jobs such as bucket merges. + size_t const workers = static_cast(mApp.getConfig().WORKER_THREADS); + job->mTasks = std::max( + 1, std::min(mApp.getBatchExecutor().preferredTaskCount(), + workers > 2 ? workers - 2 : 1)); + job->mPendingTasks = job->mTasks; + job->mTaskResults.resize(job->mTasks); +#ifdef BUILD_TESTS + job->mHoldForTesting = mEarlyTxSetHoldForTesting; +#endif + mEarlyTxSet = job; + metrics.NewMeter({"herder", "txset", "early-started"}, "txset").Mark(); + CLOG_DEBUG(Herder, + "Selecting TX set for ledger {} early, while ledger {} is in " + "its ballot: {} tasks", + slotIndex + 1, slotIndex, job->mTasks); + for (size_t task = 0; task < job->mTasks; ++task) + { + mApp.postOnBackgroundThread( + [job, task, &app = mApp]() { + EarlyTxSetJob::runTask(app, job, task); + }, + "Herder: early tx set selection"); + } +} + +std::optional +HerderImpl::completeEarlyTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, + bool pushToPeers, + LedgerHeaderHistoryEntry const& lcl) +{ + ZoneScoped; + releaseAssert(threadIsMain()); + auto& metrics = mApp.getMetrics(); + auto const plan = mEarlyTxSetPlan; + auto const job = mEarlyTxSet; + // Whatever comes of it, the plan ends here: it was for this build. + stopEarlyTxSetSelection(); + if (!plan || !plan->planned || mLedgerManager.isApplying()) + { + return std::nullopt; + } + auto mark = [&](char const* name) { + metrics.NewMeter({"herder", "txset", name}, "txset").Mark(); + }; + auto discard = [&](char const* why) { + mark("early-discarded"); + CLOG_INFO(Herder, + "Not using the TX set selected early for ledger {}: {}", + ledgerSeq, why); + return std::nullopt; + }; + // The selection was made for one ledger closing with one value. + if (plan->slotIndex != lcl.header.ledgerSeq || + ledgerSeq != lcl.header.ledgerSeq + 1 || + !(plan->stellarValue == lcl.header.scpValue) || + !lcl.header.scpValue.upgrades.empty() || + buildsLegacyTxSet(lcl.header.ledgerVersion)) + { + return discard("the ledger did not close with the value it is for"); + } + if (!job) + { + // The ledger closed before the selection was due to start. + mark("early-not-started"); + return std::nullopt; + } + std::shared_ptr early; + { + std::lock_guard lock(job->mMutex); + early = std::move(job->mSelection); + } + if (!early) + { + // Not finished (or failed): the usual build. Nobody waits for the + // job. It needed more time than it had: the next one starts earlier + // by half of that again (earlyTxSetStartDelay). + mark("early-not-ready"); + if (!job->mDone.load()) + { + auto const had = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - job->mStart); + mLastEarlyTxSetJobDuration = + std::min(std::chrono::milliseconds(2000), + std::max(mLastEarlyTxSetJobDuration, had + had / 2)); + mEarlyTxSetJobDuration = + std::max(mEarlyTxSetJobDuration, mLastEarlyTxSetJobDuration); + } + return std::nullopt; + } + { + auto const took = + std::chrono::nanoseconds(job->mFinishedAfterNs.load()); + metrics.NewTimer({"herder", "txset", "early-job"}).Update(took); + mLastEarlyTxSetJobDuration = std::max( + std::chrono::milliseconds(1), + std::chrono::duration_cast(took)); + mEarlyTxSetJobDuration = + mEarlyTxSetJobDuration == std::chrono::milliseconds::zero() + ? mLastEarlyTxSetJobDuration + : (3 * mEarlyTxSetJobDuration + mLastEarlyTxSetJobDuration) / 4; + } + if (early->mLedgerSeq != ledgerSeq) + { + return discard("it is not for this ledger"); + } + // Nothing but an upgrade changes the limits a set is selected within, + // and the closed value had none. Checked all the same. + auto const limits = TxSetBuildLimits::forLCL(mApp); + if (!limits.sameAs(early->mLimits)) + { + return discard("the limits of the ledger changed"); + } + + auto timer = + metrics.NewTimer({"herder", "txset", "early-complete"}).TimeScope(); + auto const closeTimeOffset = + closeTime.toApplyTime() - getApplyTime(lcl.header.scpValue); + // The closed ledger, read as a build reads it. No validation cache: + // every transaction of the selection is validated here and now. + auto ctx = TxSetValidationContext::forLCL(mApp); + PerPhaseTransactionList invalidTxPhases; + invalidTxPhases.resize(early->mPhases.size()); + TxSetCompletionStats stats; + + // Further candidates for the top-up: what the job decoded and did not + // select, then what the mempool holds that the job did not see, in + // pulls of growing size while they come back full. + auto seen = std::move(early->mSeen); + bool leftoversGiven = false; + bool mempoolExhausted = false; + size_t pulls = 0; + TxSetCandidateSource const moreCandidates = + [&]() -> std::optional { + if (!leftoversGiven) + { + leftoversGiven = true; + return std::make_optional( + TxSetCandidateBatch{std::move(early->mLeftovers), false}); + } + // One pull per call, twice as much each time, handed over also when + // it holds nothing that was not seen before: a pull full of known + // transactions says nothing about what lies behind them, and the + // completion decides whether a phase is worth a deeper one. + if (mempoolExhausted || pulls >= MAX_TX_SET_PULLS) + { + return std::nullopt; + } + bool full = false; + auto fresh = pullNewCandidates(txSetPullCount() << pulls, seen, full); + if (pulls > 0) + { + metrics.NewMeter({"herder", "txset", "refill-pull"}, "pull").Mark(); + } + ++pulls; + mempoolExhausted = !full; + return std::make_optional(TxSetCandidateBatch{std::move(fresh), true}); + }; + + TxSetXDRFrameConstPtr txSet; + ApplicableTxSetFrameConstPtr applicableTxSet; + std::shared_ptr selfCheck; + try + { +#ifdef BUILD_TESTS + if (mSkipEarlyTxSetFinalPassForTesting) + { + // What the node would propose if it trusted the selection. + std::tie(txSet, applicableTxSet) = TxSetBuilder::writeSelection( + early->mPhases, mApp, ctx, closeTimeOffset, + [&](TxSetXDRFrameConstPtr const& written) { + handOffTxSet(written, ledgerSeq, pushToPeers); + }, + &selfCheck); + } + else +#endif + { + auto const phases = TxSetBuilder::completeSelection( + early->mPhases, early->mHadTxNotFittingLane, moreCandidates, + mApp, limits, ctx, closeTimeOffset, invalidTxPhases, stats, + early->mSelectionSeed); + // As in a build: the structure of the selection is validated + // before the set is written and handed over, the written set + // in the background unless such a check has failed before. + std::tie(txSet, applicableTxSet) = TxSetBuilder::writeSelection( + phases, mApp, ctx, closeTimeOffset, + [&](TxSetXDRFrameConstPtr const& written) { + handOffTxSet(written, ledgerSeq, pushToPeers); + }, + mTxSetSelfCheckFailed ? nullptr : &selfCheck); + } + } + catch (std::exception const& e) + { + // The completion made something that is not a valid set. Nothing of + // it is proposed: the build that follows starts from the mempool, + // and from now on this node checks every set in line. + CLOG_ERROR(Herder, + "Completing the TX set selected early for ledger {} " + "failed: {}", + ledgerSeq, e.what()); + CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG); + mark("early-failed"); + mTxSetSelfCheckFailed = true; + return std::nullopt; + } + timer.Stop(); + // (no pull: every phase was full with what the job had left) + recordTxSetFill(*applicableTxSet, mempoolExhausted || pulls == 0); +#ifdef BUILD_TESTS + overrideBuiltTxSetForTesting(txSet, selfCheck); +#endif + mLastBuiltTxSet = std::make_pair(txSet->getContentsHash(), closeTime); + startTxSetSelfCheck(selfCheck, ledgerSeq); + // The validation results of an earlier build on this ledger stay + // available to a later one (a rebuild at the trigger, a later round's). + validationCacheFor(lcl); + + mark("early-completed"); + metrics.NewMeter({"herder", "txset", "early-removed"}, "tx") + .Mark(stats.mRemoved); + metrics.NewMeter({"herder", "txset", "early-added"}, "tx") + .Mark(stats.mAdded); + metrics.NewTimer({"herder", "txset", "early-lead"}) + .Update(std::chrono::steady_clock::now() - early->mSelectedAt); + CLOG_INFO(Herder, + "Completed the TX set selected early for ledger {}: {} " + "selected, {} not valid on ledger {}, {} of {} further " + "candidates added ({} pulls), {} transactions", + ledgerSeq, stats.mSelected, stats.mRemoved, lcl.header.ledgerSeq, + stats.mAdded, stats.mCandidates, pulls, txSet->sizeTxTotal()); return PreparedTxSet{lcl.hash, ledgerSeq, closeTime, - proposedSet, - std::move(applicableProposedSet), - std::move(invalidTxHashes)}; + txSet, + std::move(applicableTxSet), + invalidTxsToRemove(invalidTxPhases)}; } // called to take a position during the next round @@ -3156,6 +4185,10 @@ std::vector HerderImpl::pullTopTransactionsPayload(size_t count) { #ifdef BUILD_TESTS + if (mGetTopTransactionsPayloadForTesting) + { + return mGetTopTransactionsPayloadForTesting(count); + } if (mGetTopTransactionsForTesting) { // Encode the test mempool's answer as the overlay does. @@ -3475,32 +4508,7 @@ HerderImpl::makeProposalValue(uint32_t ledgerSeqToTrigger) return nullptr; } - auto newUpgrades = emptyUpgradeSteps; - - // see if we need to include some upgrades - std::vector upgrades; - { - auto ledgerView = mApp.getLedgerManager().getLCLView(); - upgrades = mUpgrades.createUpgradesFor(lcl.header, *ledgerView, - mApp.getConfig()); - } - for (auto const& upgrade : upgrades) - { - Value v(xdr::xdr_to_opaque(upgrade)); - if (v.size() >= UpgradeType::max_size()) - { - CLOG_ERROR( - Herder, - "HerderImpl::triggerNextLedger exceeded size for upgrade " - "step (got {} ) for upgrade type {}", - v.size(), upgrade.type()); - CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG); - } - else - { - newUpgrades.emplace_back(v.begin(), v.end()); - } - } + auto const newUpgrades = proposalUpgrades(lcl); #ifdef BUILD_TESTS if (mApp.getConfig().TESTING_PROPOSE_RANDOM_VALUES && diff --git a/src/herder/HerderImpl.h b/src/herder/HerderImpl.h index c4f1b93695..c5a1753f83 100644 --- a/src/herder/HerderImpl.h +++ b/src/herder/HerderImpl.h @@ -4,6 +4,7 @@ #pragma once +#include "herder/EarlyTxSetSelection.h" #include "herder/Herder.h" #include "herder/HerderSCPDriver.h" #include "herder/LedgerCloseData.h" @@ -86,6 +87,14 @@ class HerderImpl : public Herder // Recomputes the targets and tells the overlay if they changed. Called // when a ledger closes, when SCP enters a round, and when sync is lost. void updateTxFloodTargets(); + + // Tell the overlay the spanning trees of the next two slots + // (EXPERIMENTAL_TX_SET_TREE): the nodes of the leader schedule and the + // seed of each slot. The tree of a slot and leader is a function of + // those, as the schedule is, so every node derives the same one. Sent a + // slot ahead because a leader may push the set of slot N+1 the moment N + // closes, before a follower has closed N. + void updateTxSetTree(); #ifdef BUILD_TESTS // What the overlay was last told (nullopt = every peer). std::optional> const& @@ -152,6 +161,13 @@ class HerderImpl : public Herder // apply thread for speculative execution (see // LedgerManager::startSpeculativeApply), unless it was already. void maybeStartSpeculativeApply(SCPEnvelope const& envelope); + // Hands the ledger `slotIndex` with `value` to the apply thread for + // speculative execution if it is the ledger after the LCL, nothing is + // being applied and it was not handed over already. `txSet` is the + // value's transaction set if the caller holds it; otherwise it is looked + // up among the sets this node has received. + void startSpeculativeApply(uint64_t slotIndex, Value const& value, + TxSetXDRFrameConstPtr txSet); #ifdef BUILD_TESTS TxSubmitStatus recvTransaction(TransactionFrameBasePtr tx, @@ -188,8 +204,30 @@ class HerderImpl : public Herder // still run through their production paths. std::function(size_t)> mGetTopTransactionsForTesting; + // The same for callers that take the pull as the overlay encodes it + // (pullTopTransactionsPayload); when not set, the envelopes of + // mGetTopTransactionsForTesting are encoded. + std::function(size_t)> + mGetTopTransactionsPayloadForTesting; friend class LeaderBuildTestAccess; friend class ProposalPrevalidationTestAccess; + friend class EarlyTxSetTestAccess; + // When set, replaces the delay between this node's vote for a ledger + // and the start of its early selection for the next one + // (earlyTxSetStartDelay). + std::optional mEarlyTxSetStartDelayForTesting; + // When set, the validity pass over an early selection on the closed + // ledger is skipped: for the tests that show what that pass is for. + bool mSkipEarlyTxSetFinalPassForTesting{false}; + // Handed to every early selection job that starts (see + // EarlyTxSetJob::mHoldForTesting). + std::shared_ptr> mEarlyTxSetHoldForTesting; + + // When set, a set this node just built as a leader is replaced by what + // this returns, for the proposal and for the builder's self-check alike: + // lets a test make the node build a broken set. + std::function + mBuiltTxSetOverrideForTesting; std::optional mMaxClassicTxSize; std::optional mMaxTxSizeOverride; @@ -345,6 +383,8 @@ class HerderImpl : public Herder // rose (every node's first term under load starts that way), and a lead // time sized for it would make the first ledger of the term late. std::chrono::milliseconds expectedTxSetBuildDuration() const; + // What a build of a set of `txs` transactions costs at this node's rate. + std::chrono::milliseconds expectedTxSetBuildDuration(size_t txs) const; std::optional mPreparedTxSet; // `pushToPeers`: this is the set the node is about to propose as a round // leader, so the overlay sends it to every peer as soon as its XDR exists @@ -359,6 +399,82 @@ class HerderImpl : public Herder void prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime); void discardPreparedTxSet(); ValueWrapperPtr makeProposalValue(uint32_t ledgerSeq); + // The upgrades a value proposed now on top of `lcl` carries. + xdr::xvector + proposalUpgrades(LedgerHeaderHistoryEntry const& lcl); + // Hands a set this node built to the overlay, which keeps it for peers + // that ask and, with `pushToPeers`, sends it to every peer at once. + void handOffTxSet(TxSetXDRFrameConstPtr const& txSet, uint32_t ledgerSeq, + bool pushToPeers); + + // The leader executes its own set as soon as it can. + // + // A prepared set is proposed unchanged at the trigger, with the close + // time it was prepared for. So the value this node will propose is known + // as soon as the set is: the node hands the ledger to the apply thread + // right then (startSpeculativeApply) instead of when it emits its first + // ballot statement at the trigger, and the execution runs beside the + // set's dissemination and the wait for the trigger. If the trigger ends + // up proposing another value (the prepared set was dropped, the upgrades + // to propose changed), the statement it emits replaces the execution, as + // any change of vote does. Called whenever mPreparedTxSet has been set. + void executePreparedTxSet(); + + // The builder's self-check (see TxSetSelfCheck) of a set built by this + // node as a leader does not hold up the push of the set or its proposal. + // It re-reads the written set and validates the structure of what it + // read. A node that executes its own value speculatively does exactly + // that, and more, on its apply thread as soon as it proposes (or as soon + // as the set is prepared, see executePreparedTxSet), and the verdict of + // that execution on a set this node built takes the same path as a + // failed self-check (see startSpeculativeApply): there the check would + // be done twice, so it is left to the execution. Otherwise it runs on a + // background thread. Either way, if it fails the set may already be + // proposed: see txSetSelfCheckFailed. + void startTxSetSelfCheck(std::shared_ptr check, + uint32_t ledgerSeq); + // Whether this node hands the ledger after `lcl` to its apply thread + // when it votes for a value (see maybeStartSpeculativeApply). + bool executesVotedValues(LedgerHeaderHistoryEntry const& lcl) const; + // Runs the check on a background thread. + void runTxSetSelfCheck(std::shared_ptr check, + uint32_t ledgerSeq); + // A check that was left to the node's own execution of the set, and the + // ledger the set is for, until it is known whether that execution + // happens. Called when the node has emitted a statement: the check is + // dropped if the apply thread was handed the ledger with this set, and + // run in the background otherwise (the node was not tracking, was + // applying, or voted for another value). + void resolvePendingTxSetSelfCheck(); + std::optional, uint32_t>> + mPendingTxSetSelfCheck; + // Called on the main thread when a self-check failed, or when this + // node's own execution of a set it built found the set invalid. The + // failure is logged and counted. A set that has not been proposed yet is + // dropped, + // so that the trigger builds another one. A proposed set cannot be + // recalled; if it is invalid, every node finds that at apply and closes + // the ledger without applying it. From now on this node checks every set + // before it uses it (mTxSetSelfCheckFailed), where a failure throws. + void txSetSelfCheckFailed(Hash const& txSetHash, uint32_t ledgerSeq, + std::string const& reason); + bool mTxSetSelfCheckFailed{false}; + // The last set this node built itself (by a build, or by completing a + // set it selected early), and the close time it built it for. When the + // node's own apply thread executes that set with that close time and + // finds it invalid, that is a verdict on this node's builder (see + // startSpeculativeApply). With another close time it is not: another + // leader may propose this node's set for a time it was not built for. + std::optional> mLastBuiltTxSet; + // Keys the tie-break seed of early selections; derived from the node + // seed on first use. + std::optional mTieBreakSecret; +#ifdef BUILD_TESTS + // Applies mBuiltTxSetOverrideForTesting to a set that was just built. + void overrideBuiltTxSetForTesting( + TxSetXDRFrameConstPtr& txSet, + std::shared_ptr& selfCheck); +#endif // Whether a ledger closing on an LCL with this protocol uses a legacy // (non-generalized) tx set, which every validator builds locally. static bool buildsLegacyTxSet(uint32_t lclLedgerVersion); @@ -429,6 +545,96 @@ class HerderImpl : public Herder // Nanoseconds from `start` until the last task finished, or -1. std::atomic finishedAfterNs{-1}; }; + // The part of setupTriggerNextLedger that arms the timer for the + // first-round leader's build a lead time ahead of the trigger at + // `triggerTime`; nothing if the trigger is due (it builds itself). + void scheduleTxSetPreparation(uint64_t nextIndex, + VirtualClock::time_point triggerTime, + bool triggerDue, VirtualClock::time_point now, + std::chrono::milliseconds expectedCloseTime, + ConsensusTime minCandidateCt); + // Early selection of the next transaction set (see + // herder/EarlyTxSetSelection.h). + // + // When this node votes for a value for ledger N and expects to lead + // round 1 of N+1 if N closes with it, it plans a background selection of + // the set for N+1 (maybeScheduleEarlyTxSetSelection), started as late + // as it can be and still finish before the set is needed + // (earlyTxSetStartDelay): the candidates of N+1 arrive while N is in its + // ballot. The selection is made on N-1's state and a prediction of what + // N's set does to it. + // + // The build of N+1 (buildTxSet) uses the selection if the job has + // finished and N closed with exactly the value the job was for + // (completeEarlyTxSet): every transaction of the selection is validated + // against N as it closed, what is not valid is removed, and the set is + // topped up from what the job left and from the mempool. Otherwise the + // set is built as always. + struct EarlyTxSetPlan + { + // The ledger in its ballot (N) and the value this node votes for. + uint64_t slotIndex{0}; + Value value; + StellarValue stellarValue; + // False if the vote was looked at and no selection is to be made + // for it (this node is not first in line, the value has upgrades, + // ...). + bool planned{false}; + }; + std::optional mEarlyTxSetPlan; + // The running or finished selection of the plan. + std::shared_ptr mEarlyTxSet; + // Called with every statement this node emits. + void maybeScheduleEarlyTxSetSelection(SCPEnvelope const& envelope); + void startEarlyTxSetSelection(uint64_t slotIndex, Value const& value, + TxSetXDRFrameConstPtr const& txSet); + // Drops the plan, its timer and its job. + void stopEarlyTxSetSelection(); + // How long after its vote for a ledger this node starts the selection + // for the next one: so that it is expected to finish a little before + // the build that uses it. Zero until the node has measured both (how + // long after its vote it builds, and how long a selection takes). + std::chrono::milliseconds earlyTxSetStartDelay() const; + // When this node first voted for a value of the ledger after the LCL. + std::optional> mFirstVote; + // From that vote to the start of the build of the ledger after: a + // running average and the last sample; zero before the first. + std::chrono::milliseconds mVoteToBuild{0}; + std::chrono::milliseconds mLastVoteToBuild{0}; + // How long a selection job takes (pull to result), likewise. + std::chrono::milliseconds mEarlyTxSetJobDuration{0}; + std::chrono::milliseconds mLastEarlyTxSetJobDuration{0}; + // Completes the early selection for `ledgerSeq` on top of `lcl`, as + // buildTxSet builds a set; nullopt if there is none that can be used + // (the caller then builds). Always ends the plan. + std::optional + completeEarlyTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, + bool pushToPeers, LedgerHeaderHistoryEntry const& lcl); + + // How many transactions one pull of the mempool asks for: the capacity + // of a set plus EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT. A set that is + // short after validation pulls again, twice as much each time, at most + // MAX_TX_SET_PULLS times. + size_t txSetPullCount() const; + static constexpr size_t MAX_TX_SET_PULLS{3}; + // Pulls the top `count` transactions of the mempool and returns those + // not in `seen` (by hashPulledTx; they are added to it), decoded, in + // mempool order. `full` tells whether the mempool had `count` + // transactions to give, i.e. may hold more. + TxFrameList pullNewCandidates(size_t count, UnorderedSet& seen, + bool& full); + // The transactions of `invalidTxs` (found invalid on the LCL by a build + // or a completion) to remove from the mempool: all but those with a + // future sequence number, which wait for a pending transaction of their + // account. + std::vector + invalidTxsToRemove(PerPhaseTransactionList const& invalidTxs); + // Records a set this node is about to use: its size against the + // capacity of the phases, and whether it is short although the mempool + // had more to give. + void recordTxSetFill(ApplicableTxSetFrame const& txSet, + bool mempoolExhausted); + std::shared_ptr mProposalPrevalidation; // Whether the local node would lead round 1 or 2 of the ledger after // `slotIndex` if `slotIndex` closes with `value`. @@ -548,6 +754,8 @@ class HerderImpl : public Herder VirtualTimer mTriggerTimer; VirtualTimer mPrepareTxSetTimer; + // Fires when the planned early selection is to start (mEarlyTxSetPlan). + VirtualTimer mEarlyTxSetTimer; // What updateTxFloodTargets last told the overlay (nullopt = everyone, // which is also what the overlay starts with). diff --git a/src/herder/ParallelTxSetBuilder.cpp b/src/herder/ParallelTxSetBuilder.cpp index bbb9d552e2..9f91b03131 100644 --- a/src/herder/ParallelTxSetBuilder.cpp +++ b/src/herder/ParallelTxSetBuilder.cpp @@ -705,7 +705,8 @@ buildSurgePricedParallelSorobanPhase( TxFrameList const& txFrames, Config const& cfg, SorobanNetworkConfig const& sorobanCfg, std::shared_ptr laneConfig, - std::vector& hadTxNotFittingLane, uint32_t ledgerVersion) + std::vector& hadTxNotFittingLane, uint32_t ledgerVersion, + std::optional seed) { ZoneScoped; // We prefer the transaction sets that are well utilized, but we also want @@ -720,9 +721,11 @@ buildSurgePricedParallelSorobanPhase( // Sort transactions in decreasing inclusion fee order. Look up each // transaction's fee bid and operation count once, not per comparison. - TxFeeComparator txComparator( - /* isGreater */ true, - stellar::rand_uniform(0, std::numeric_limits::max())); + size_t const tieBreakSeed = + seed ? *seed + : stellar::rand_uniform( + 0, std::numeric_limits::max()); + TxFeeComparator txComparator(/* isGreater */ true, tieBreakSeed); std::vector> feeBids; feeBids.reserve(txFrames.size()); for (auto const& tx : txFrames) @@ -809,4 +812,202 @@ buildSurgePricedParallelSorobanPhase( return std::move(bestResult.mStages); } +ParallelPhaseExtender::ParallelPhaseExtender( + TxStageFrameList stages, SorobanNetworkConfig const& sorobanCfg, + Resource const& laneLimit, uint32_t ledgerVersion) + : mStages(std::move(stages)) + , mLaneLimit(laneLimit) + , mTotal(Resource::makeEmptySoroban()) + , mLedgerVersion(ledgerVersion) + , mMaxClusters(sorobanCfg.ledgerMaxDependentTxClusters()) + , mMaxInstructions(sorobanCfg.ledgerMaxInstructions()) +{ + ZoneScoped; + mState.resize(mStages.size()); + for (size_t s = 0; s < mStages.size(); ++s) + { + auto& state = mState[s]; + auto const& stage = mStages[s]; + state.mClusterInstructions.resize(stage.size(), 0); + for (size_t c = 0; c < stage.size(); ++c) + { + for (auto const& tx : stage[c]) + { + state.mClusterInstructions[c] += + tx->sorobanResources().instructions; + index(state, static_cast(c), *tx); + mTotal += tx->getResources(/* useByteLimitInClassic */ false, + mLedgerVersion); + ++mTxCount; + } + state.mMaxClusterInstructions = std::max( + state.mMaxClusterInstructions, state.mClusterInstructions[c]); + } + mSequentialInstructions += state.mMaxClusterInstructions; + } +} + +void +ParallelPhaseExtender::index(StageState& stage, uint32_t cluster, + TransactionFrameBase const& tx) +{ + std::hash keyHasher; + auto const& footprint = tx.sorobanResources().footprint; + for (auto const& key : footprint.readWrite) + { + stage.mWriters[keyHasher(key)] = cluster; + } + for (auto const& key : footprint.readOnly) + { + auto& readers = stage.mReaders[keyHasher(key)]; + if (std::find(readers.begin(), readers.end(), cluster) == readers.end()) + { + readers.push_back(cluster); + } + } +} + +std::optional> +ParallelPhaseExtender::conflictingCluster(StageState const& stage, + TransactionFrameBase const& tx) const +{ + // Keys are compared by hash, as the builder of a phase does: a collision + // makes a conflict of two transactions that have none, which only costs + // the transaction its place. + std::hash keyHasher; + std::optional conflict; + // Returns false once a second cluster conflicts. + auto add = [&conflict](uint32_t cluster) { + if (conflict && *conflict != cluster) + { + return false; + } + conflict = cluster; + return true; + }; + auto const& footprint = tx.sorobanResources().footprint; + for (auto const& key : footprint.readWrite) + { + auto const h = keyHasher(key); + if (auto it = stage.mWriters.find(h); it != stage.mWriters.end()) + { + if (!add(it->second)) + { + return std::nullopt; + } + } + if (auto it = stage.mReaders.find(h); it != stage.mReaders.end()) + { + for (auto cluster : it->second) + { + if (!add(cluster)) + { + return std::nullopt; + } + } + } + } + for (auto const& key : footprint.readOnly) + { + if (auto it = stage.mWriters.find(keyHasher(key)); + it != stage.mWriters.end()) + { + if (!add(it->second)) + { + return std::nullopt; + } + } + } + return std::make_optional(conflict); +} + +bool +ParallelPhaseExtender::withinInstructionLimit( + StageState const& stage, uint64_t clusterInstructions) const +{ + // What TxSetPhaseFrame::checkValidSoroban checks: stages run one after + // another, each as long as its largest cluster. + auto const stageMax = + std::max(stage.mMaxClusterInstructions, clusterInstructions); + return mSequentialInstructions - stage.mMaxClusterInstructions + stageMax <= + mMaxInstructions; +} + +bool +ParallelPhaseExtender::tryAdd(TransactionFrameBasePtr const& tx) +{ + releaseAssert(tx->isSoroban()); + auto const resources = + tx->getResources(/* useByteLimitInClassic */ false, mLedgerVersion); + if (!mTotal.canAdd(resources) || anyGreater(mTotal + resources, mLaneLimit)) + { + return false; + } + uint64_t const instructions = tx->sorobanResources().instructions; + if (mStages.empty()) + { + // The first transaction of the phase: one stage, one cluster, if + // that is allowed. (No stage is left behind without a + // transaction.) + if (mMaxClusters == 0 || instructions > mMaxInstructions) + { + return false; + } + mStages.emplace_back(); + mState.emplace_back(); + } + for (size_t s = 0; s < mStages.size(); ++s) + { + auto& state = mState[s]; + auto& stage = mStages[s]; + auto const conflict = conflictingCluster(state, *tx); + if (!conflict) + { + // It would join two clusters of this stage. + continue; + } + std::optional cluster = *conflict; + if (!cluster) + { + // Free to go anywhere: the emptiest cluster, counting one that + // does not exist yet. + if (stage.size() < mMaxClusters) + { + cluster = static_cast(stage.size()); + } + else + { + auto const least = + std::min_element(state.mClusterInstructions.begin(), + state.mClusterInstructions.end()); + cluster = static_cast( + least - state.mClusterInstructions.begin()); + } + } + bool const isNew = *cluster == stage.size(); + uint64_t const clusterInstructions = + (isNew ? 0 : state.mClusterInstructions[*cluster]) + instructions; + if (!withinInstructionLimit(state, clusterInstructions)) + { + continue; + } + if (isNew) + { + stage.emplace_back(); + state.mClusterInstructions.push_back(0); + } + stage[*cluster].push_back(tx); + state.mClusterInstructions[*cluster] = clusterInstructions; + mSequentialInstructions -= state.mMaxClusterInstructions; + state.mMaxClusterInstructions = + std::max(state.mMaxClusterInstructions, clusterInstructions); + mSequentialInstructions += state.mMaxClusterInstructions; + index(state, *cluster, *tx); + mTotal += resources; + ++mTxCount; + return true; + } + return false; +} + } // namespace stellar diff --git a/src/herder/ParallelTxSetBuilder.h b/src/herder/ParallelTxSetBuilder.h index 9ea3feabc2..bdc7f20ab3 100644 --- a/src/herder/ParallelTxSetBuilder.h +++ b/src/herder/ParallelTxSetBuilder.h @@ -24,6 +24,87 @@ TxStageFrameList buildSurgePricedParallelSorobanPhase( TxFrameList const& txFrames, Config const& cfg, SorobanNetworkConfig const& sorobanCfg, std::shared_ptr laneConfig, - std::vector& hadTxNotFittingLane, uint32_t ledgerVersion); + std::vector& hadTxNotFittingLane, uint32_t ledgerVersion, + // Randomizes the order of transactions with equal fee rates. Drawn from + // the global random engine (main thread only) unless provided. + std::optional seed = std::nullopt); + +// Adds transactions to a parallel Soroban phase that has been selected +// already, without selecting the phase again. +// +// A leader selects the set for the next ledger before the current one has +// closed, and completes it at the close: what turned out invalid is removed +// (a phase stays valid when transactions are taken out of it), and what +// arrived since is added. Selecting the whole phase again +// (`buildSurgePricedParallelSorobanPhase`) costs as much as the first time; +// this adds one transaction at a time to the stages that exist. +// +// A transaction goes into the first stage where it conflicts with at most +// one cluster: into that cluster, or, without a conflict, into the cluster +// with the fewest instructions (a new one while the stage has fewer than +// `ledgerMaxDependentTxClusters`). It is only added if the result is within +// the limits a set is validated against: the lane's resources, and the +// ledger's instruction limit for the sum over the stages of each stage's +// largest cluster. Clusters are never merged and stages are never added (a +// phase without stages gets one): a transaction that would need either is not +// added. The result is checked like any selection +// (`TxSetBuilder::checkSelection`) by whoever uses it. +class ParallelPhaseExtender +{ + public: + // `stages` is the phase so far; it may be empty. `laneLimit` is the + // limit of the phase's (only) surge pricing lane. + ParallelPhaseExtender(TxStageFrameList stages, + SorobanNetworkConfig const& sorobanCfg, + Resource const& laneLimit, uint32_t ledgerVersion); + + // Adds `tx` if it fits; returns whether it did. + bool tryAdd(TransactionFrameBasePtr const& tx); + + size_t + sizeTx() const + { + return mTxCount; + } + + TxStageFrameList const& + getStages() const + { + return mStages; + } + + private: + struct StageState + { + std::vector mClusterInstructions; + uint64_t mMaxClusterInstructions{0}; + // Hash of a footprint key -> the cluster that writes it. One cluster + // at most: two clusters of a stage never share a written key. + std::unordered_map mWriters; + // Hash of a footprint key -> the clusters that only read it. + std::unordered_map> mReaders; + }; + + void index(StageState& stage, uint32_t cluster, + TransactionFrameBase const& tx); + // The clusters of `stage` that `tx` conflicts with: none, one, or (as + // nullopt) more than one. + std::optional> + conflictingCluster(StageState const& stage, + TransactionFrameBase const& tx) const; + bool withinInstructionLimit(StageState const& stage, + uint64_t clusterInstructions) const; + + TxStageFrameList mStages; + std::vector mState; + Resource mLaneLimit; + Resource mTotal; + uint32_t const mLedgerVersion; + uint32_t const mMaxClusters; + uint64_t const mMaxInstructions; + // Sum over the stages of their largest cluster's instructions. + uint64_t mSequentialInstructions{0}; + size_t mTxCount{0}; +}; } // namespace stellar diff --git a/src/herder/SurgePricingUtils.cpp b/src/herder/SurgePricingUtils.cpp index edec65d71c..2b8c83920c 100644 --- a/src/herder/SurgePricingUtils.cpp +++ b/src/herder/SurgePricingUtils.cpp @@ -146,13 +146,17 @@ std::vector SurgePricingPriorityQueue::getMostTopTxsWithinLimits( std::vector const& txs, std::shared_ptr laneConfig, - std::vector& hadTxNotFittingLane, uint32_t ledgerVersion) + std::vector& hadTxNotFittingLane, uint32_t ledgerVersion, + std::optional seed) { ZoneScoped; - SurgePricingPriorityQueue queue( - /* isHighestPriority */ true, laneConfig, - stellar::rand_uniform(0, std::numeric_limits::max())); + size_t const tieBreakSeed = + seed ? *seed + : stellar::rand_uniform( + 0, std::numeric_limits::max()); + SurgePricingPriorityQueue queue(/* isHighestPriority */ true, laneConfig, + tieBreakSeed); bool allFit = true; auto res = queue.countTxsResources(txs, ledgerVersion); diff --git a/src/herder/SurgePricingUtils.h b/src/herder/SurgePricingUtils.h index f6e7fb3b6f..95ca192df9 100644 --- a/src/herder/SurgePricingUtils.h +++ b/src/herder/SurgePricingUtils.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include "transactions/TransactionFrameBase.h" @@ -153,10 +154,14 @@ class SurgePricingPriorityQueue // `hadTxNotFittingLane` is an output parameter that for every lane will // identify whether there was a transaction that didn't fit into that lane's // limit. + // `seed` randomizes the order of transactions with equal fee rates. It + // is drawn from the global random engine (main thread only) unless the + // caller provides one, which a caller on another thread has to. static std::vector getMostTopTxsWithinLimits( std::vector const& txs, std::shared_ptr laneConfig, - std::vector& hadTxNotFittingLane, uint32_t ledgerVersion); + std::vector& hadTxNotFittingLane, uint32_t ledgerVersion, + std::optional seed = std::nullopt); // Returns total amount of resources in all the transactions in this queue. Resource totalResources() const; diff --git a/src/herder/TxSetFrame.cpp b/src/herder/TxSetFrame.cpp index de090b70ce..cc93a99333 100644 --- a/src/herder/TxSetFrame.cpp +++ b/src/herder/TxSetFrame.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -496,31 +497,27 @@ computeLaneBaseFee(TxSetPhase phase, LedgerHeader const& ledgerHeader, } std::shared_ptr -createSurgePricingLangeConfig(TxSetPhase phase, Application& app) +createSurgePricingLaneConfig(TxSetPhase phase, Config const& cfg, + TxSetBuildLimits const& limits) { ZoneScoped; - releaseAssert(threadIsMain()); - releaseAssert(!app.getLedgerManager().isApplying()); - auto const& lclHeader = - app.getLedgerManager().getLastClosedLedgerHeader().header; + auto const& lclHeader = limits.mLclHeader; std::vector hadTxNotFittingLane; std::shared_ptr surgePricingLaneConfig; if (phase == TxSetPhase::CLASSIC) { - auto maxOps = Resource( - {static_cast( - app.getLedgerManager().getLastMaxTxSetSizeOps()), - static_cast(app.getConfig().getClassicByteAllowance())}); + auto maxOps = + Resource({limits.maxTxSetSizeOps(), + static_cast(cfg.getClassicByteAllowance())}); std::optional dexOpsLimit; - if (app.getConfig().MAX_DEX_TX_OPERATIONS_IN_TX_SET) + if (cfg.MAX_DEX_TX_OPERATIONS_IN_TX_SET) { // DEX operations limit implies that DEX transactions should // compete with each other in in a separate fee lane, which // is only possible with generalized tx set. - dexOpsLimit = - Resource({*app.getConfig().MAX_DEX_TX_OPERATIONS_IN_TX_SET, - MAX_CLASSIC_BYTE_ALLOWANCE}); + dexOpsLimit = Resource({*cfg.MAX_DEX_TX_OPERATIONS_IN_TX_SET, + MAX_CLASSIC_BYTE_ALLOWANCE}); } surgePricingLaneConfig = @@ -529,9 +526,9 @@ createSurgePricingLangeConfig(TxSetPhase phase, Application& app) else { releaseAssert(phase == TxSetPhase::SOROBAN); + releaseAssert(limits.mSorobanConfig); - auto limits = app.getLedgerManager().maxLedgerResources( - /* isSoroban */ true); + auto limitsRes = limits.mSorobanConfig->maxLedgerResources(); // When building Soroban tx sets with parallel execution support, // instructions are accounted for by the build logic, not by the surge // pricing config, so we need to relax the instruction limit in surge @@ -539,17 +536,17 @@ createSurgePricingLangeConfig(TxSetPhase phase, Application& app) if (protocolVersionStartsFrom(lclHeader.ledgerVersion, PARALLEL_SOROBAN_PHASE_PROTOCOL_VERSION)) { - limits.setVal(Resource::Type::INSTRUCTIONS, - std::numeric_limits::max()); + limitsRes.setVal(Resource::Type::INSTRUCTIONS, + std::numeric_limits::max()); } - auto byteLimit = std::min( - static_cast(app.getConfig().getSorobanByteAllowance()), - limits.getVal(Resource::Type::TX_BYTE_SIZE)); - limits.setVal(Resource::Type::TX_BYTE_SIZE, byteLimit); + auto byteLimit = + std::min(static_cast(cfg.getSorobanByteAllowance()), + limitsRes.getVal(Resource::Type::TX_BYTE_SIZE)); + limitsRes.setVal(Resource::Type::TX_BYTE_SIZE, byteLimit); surgePricingLaneConfig = - std::make_shared(limits); + std::make_shared(limitsRes); } return surgePricingLaneConfig; } @@ -558,16 +555,91 @@ TxFrameList buildSurgePricedSequentialPhase( TxFrameList const& txs, std::shared_ptr surgePricingLaneConfig, - std::vector& hadTxNotFittingLane, uint32_t ledgerVersion) + std::vector& hadTxNotFittingLane, uint32_t ledgerVersion, + std::optional seed) { ZoneScoped; return SurgePricingPriorityQueue::getMostTopTxsWithinLimits( - txs, surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion); + txs, surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion, seed); } +// The base fee every transaction of a selected phase is included at: per +// surge pricing lane, the ledger's base fee, or the lowest fee rate included +// if transactions of the lane (or of the generic lane) were left out. +std::shared_ptr +computeInclusionFeeMap( + TxSetPhase phase, + std::variant const& includedTxs, + SurgePricingLaneConfig const& surgePricingLaneConfig, + LedgerHeader const& lclHeader, std::vector const& hadTxNotFittingLane) +{ + auto visitIncludedTxs = + [&includedTxs]( + std::function visitor) { + std::visit( + [&visitor](auto const& txs) { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + for (auto const& tx : txs) + { + visitor(tx); + } + } + else if constexpr (std::is_same_v) + { + for (auto const& stage : txs) + { + for (auto const& thread : stage) + { + for (auto const& tx : thread) + { + visitor(tx); + } + } + } + } + else + { + // This can't be just `false` as if an assertion is not + // dependent on template argument, it will be + // unconditionally triggered. + static_assert(!std::is_same_v, + "Non-exhaustive visitor"); + } + }, + includedTxs); + }; + + std::vector lowestLaneFee; + size_t laneCount = surgePricingLaneConfig.getLaneLimits().size(); + releaseAssert(hadTxNotFittingLane.size() == laneCount); + lowestLaneFee.resize(laneCount, std::numeric_limits::max()); + visitIncludedTxs( + [&lowestLaneFee, &surgePricingLaneConfig, &lclHeader](auto const& tx) { + size_t lane = surgePricingLaneConfig.getLane(*tx); + auto perOpFee = computePerOpFee(*tx, lclHeader.ledgerVersion); + lowestLaneFee[lane] = std::min(lowestLaneFee[lane], perOpFee); + }); + auto laneBaseFee = + computeLaneBaseFee(phase, lclHeader, surgePricingLaneConfig, + lowestLaneFee, hadTxNotFittingLane); + auto inclusionFeeMapPtr = std::make_shared(); + auto& inclusionFeeMap = *inclusionFeeMapPtr; + visitIncludedTxs([&inclusionFeeMap, &laneBaseFee, + &surgePricingLaneConfig](auto const& tx) { + inclusionFeeMap[tx] = laneBaseFee[surgePricingLaneConfig.getLane(*tx)]; + }); + return inclusionFeeMapPtr; +} + +// `hadTxNotFittingLaneOut`, if given, receives per surge pricing lane of the +// phase whether a transaction was left out for the lane's limits. std::pair, std::shared_ptr> -applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app +applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Config const& cfg, + TxSetBuildLimits const& limits, std::optional seed, + std::vector* hadTxNotFittingLaneOut #ifdef BUILD_TESTS , bool enforceTxsApplyOrder, @@ -576,10 +648,10 @@ applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app ) { ZoneScoped; - auto surgePricingLaneConfig = createSurgePricingLangeConfig(phase, app); + auto surgePricingLaneConfig = + createSurgePricingLaneConfig(phase, cfg, limits); std::vector hadTxNotFittingLane; - uint32_t ledgerVersion = - app.getLedgerManager().getLastClosedLedgerHeader().header.ledgerVersion; + uint32_t ledgerVersion = limits.mLclHeader.ledgerVersion; bool isParallelSoroban = phase == TxSetPhase::SOROBAN && protocolVersionStartsFrom(ledgerVersion, @@ -623,10 +695,10 @@ applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app else { #endif + releaseAssert(limits.mSorobanConfig); includedTxs = buildSurgePricedParallelSorobanPhase( - txs, app.getConfig(), - app.getLedgerManager().getLastClosedSorobanNetworkConfig(), - surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion); + txs, cfg, *limits.mSorobanConfig, surgePricingLaneConfig, + hadTxNotFittingLane, ledgerVersion, seed); #ifdef BUILD_TESTS } #endif @@ -634,69 +706,17 @@ applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app else { includedTxs = buildSurgePricedSequentialPhase( - txs, surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion); + txs, surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion, + seed); } - auto visitIncludedTxs = - [&includedTxs]( - std::function visitor) { - std::visit( - [&visitor](auto const& txs) { - using T = std::decay_t; - if constexpr (std::is_same_v) - { - for (auto const& tx : txs) - { - visitor(tx); - } - } - else if constexpr (std::is_same_v) - { - for (auto const& stage : txs) - { - for (auto const& thread : stage) - { - for (auto const& tx : thread) - { - visitor(tx); - } - } - } - } - else - { - // This can't be just `false` as if an assertion is not - // dependent on template argument, it will be - // unconditionally triggered. - static_assert(!std::is_same_v, - "Non-exhaustive visitor"); - } - }, - includedTxs); - }; - - std::vector lowestLaneFee; - auto const& lclHeader = - app.getLedgerManager().getLastClosedLedgerHeader().header; - - size_t laneCount = surgePricingLaneConfig->getLaneLimits().size(); - lowestLaneFee.resize(laneCount, std::numeric_limits::max()); - visitIncludedTxs( - [&lowestLaneFee, &surgePricingLaneConfig, &lclHeader](auto const& tx) { - size_t lane = surgePricingLaneConfig->getLane(*tx); - auto perOpFee = computePerOpFee(*tx, lclHeader.ledgerVersion); - lowestLaneFee[lane] = std::min(lowestLaneFee[lane], perOpFee); - }); - auto laneBaseFee = - computeLaneBaseFee(phase, lclHeader, *surgePricingLaneConfig, - lowestLaneFee, hadTxNotFittingLane); - auto inclusionFeeMapPtr = std::make_shared(); - auto& inclusionFeeMap = *inclusionFeeMapPtr; - visitIncludedTxs([&inclusionFeeMap, &laneBaseFee, - &surgePricingLaneConfig](auto const& tx) { - inclusionFeeMap[tx] = laneBaseFee[surgePricingLaneConfig->getLane(*tx)]; - }); - + auto inclusionFeeMapPtr = + computeInclusionFeeMap(phase, includedTxs, *surgePricingLaneConfig, + limits.mLclHeader, hadTxNotFittingLane); + if (hadTxNotFittingLaneOut) + { + *hadTxNotFittingLaneOut = hadTxNotFittingLane; + } return std::make_pair(includedTxs, inclusionFeeMapPtr); } @@ -882,16 +902,264 @@ makeTxSetFromTransactions( , TxSetReadyCallback const& onTxSetReady, std::vector const& validTxTargets, - TxValidationCache* validationCache) + TxValidationCache* validationCache, + std::shared_ptr* deferredSelfCheck) +{ + releaseAssert(threadIsMain()); + releaseAssert(!app.getLedgerManager().isApplying()); + releaseAssert(txPhases.size() == invalidTxs.size()); + + // Everything is read from the LCL: validity through the context, the + // limits of the set from its header and Soroban configuration. + auto ctx = TxSetValidationContext::forLCL(app); + ctx.mValidationCache = validationCache; + auto const validatedPhases = TxSetBuilder::selectPhases( + txPhases, app, TxSetBuildLimits::forLCL(app), ctx, closeTimeOffset, + invalidTxs, validTxTargets, "build", std::nullopt, nullptr +#ifdef BUILD_TESTS + , + enforceTxsApplyOrder, parallelSorobanOrder, + disableTxValidationForLegacyScenario +#endif + ); + + return TxSetBuilder::writeSelection(validatedPhases, app, ctx, + closeTimeOffset, onTxSetReady, + deferredSelfCheck +#ifdef BUILD_TESTS + , + enforceTxsApplyOrder +#endif + ); +} + +std::pair +TxSetBuilder::writeSelection( + std::vector const& validatedPhases, Application& app, + TxSetValidationContext const& ctx, ApplyTimeOffset closeTimeOffset, + TxSetReadyCallback const& onTxSetReady, + std::shared_ptr* deferredSelfCheck +#ifdef BUILD_TESTS + , + bool enforceTxsApplyOrder +#endif +) { releaseAssert(threadIsMain()); releaseAssert(!app.getLedgerManager().isApplying()); + auto const& lclHeader = app.getLedgerManager().getLastClosedLedgerHeader(); + bool const selfCheckIsDeferred = deferredSelfCheck +#ifdef BUILD_TESTS + && !enforceTxsApplyOrder +#endif + ; + // The check of the written set comes after the set is in use. What the + // selection itself can get wrong (two transactions of one account, fees, + // limits, conflicting clusters) is checked before anything leaves this + // function, as it always was: on another thread while this one writes + // the set (both only read the selection), and waited for before the set + // is handed to anyone. + std::future selectionCheck; + if (selfCheckIsDeferred) + { + bool const isGeneralized = protocolVersionStartsFrom( + lclHeader.header.ledgerVersion, SOROBAN_PROTOCOL_VERSION); + // The two threads read the same frames. That is safe because the + // check reads no lazily computed member of a frame, and the write + // only reads full hashes, which every selected frame computed when + // it was validated. + auto check = [&validatedPhases, &app, &ctx, isGeneralized, + closeTimeOffset]() { + return TxSetBuilder::checkSelection(validatedPhases, app, ctx, + isGeneralized, closeTimeOffset); + }; + try + { + selectionCheck = std::async(std::launch::async, check); + } + catch (std::system_error const&) + { + // No thread to be had: check here. + std::promise done; + done.set_value(check()); + selectionCheck = done.get_future(); + } + } + auto xdrTimer = + app.getMetrics().NewTimer({"herder", "txset", "build-xdr"}).TimeScope(); + // Preliminary applicable frame - we don't know the contents hash yet, but + // we also don't return this. + std::unique_ptr preliminaryApplicableTxSet( + new ApplicableTxSetFrame(app, lclHeader, validatedPhases, + std::nullopt)); + + // Do the roundtrip through XDR to ensure we never build an incorrect tx set + // for nomination. + auto outputTxSet = preliminaryApplicableTxSet->toWireTxSetFrame(); +#ifdef BUILD_TESTS + if (enforceTxsApplyOrder) + { + // Fill in the contents hash if we're skipping the normal roundtrip + // and validation flow. + preliminaryApplicableTxSet->mContentsHash = + outputTxSet->getContentsHash(); + return std::make_pair(outputTxSet, + std::move(preliminaryApplicableTxSet)); + } +#endif + + xdrTimer.Stop(); + if (selectionCheck.valid()) + { + auto const result = selectionCheck.get(); + if (result != TxSetValidationResult::VALID) + { + throw std::runtime_error(fmt::format( + FMT_STRING( + "Created invalid tx set frame, validation result: {:s}"), + toString(result))); + } + } + // The final XDR is available now. Start proposal encoding while the + // roundtrip and final validity checks below run; only the local builder + // supplies this callback. The callback must not nominate the set. + if (onTxSetReady) + { + onTxSetReady(outputTxSet); + } + + if (deferredSelfCheck) + { + // What is cheap to check of the written set is checked here: the + // shape of its XDR (no empty component, stage or cluster, ...), and + // that it holds as many transactions as the selection. + if (outputTxSet->isGeneralizedTxSet() && + !validateTxSetXDRStructure(outputTxSet->getGeneralizedXDR(), + lclHeader.header.ledgerVersion)) + { + throw std::runtime_error( + "Couldn't prepare created tx set frame for apply"); + } + size_t selected = 0; + for (auto const& phase : validatedPhases) + { + selected += phase.sizeTx(); + } + if (outputTxSet->sizeTxTotal() != selected) + { + throw std::runtime_error("Created invalid tx set frame - shape is " + "mismatched after writing."); + } + // The caller runs the check (see TxSetSelfCheck) and goes on with + // the set as selected, which is what the check compares the XDR + // with. + *deferredSelfCheck = TxSetBuilder::makeSelfCheck( + app, outputTxSet, lclHeader, validatedPhases, closeTimeOffset); + preliminaryApplicableTxSet->mContentsHash = + outputTxSet->getContentsHash(); + return std::make_pair(outputTxSet, + std::move(preliminaryApplicableTxSet)); + } + + std::vector phaseSizes; + for (auto const& phase : validatedPhases) + { + phaseSizes.push_back(phase.sizeTx()); + } + ApplicableTxSetFrameConstPtr outputApplicableTxSet; + // We already trimmed invalid transactions in an earlier call to + // `trimInvalid`, so the check skips transaction validation. + auto const failure = TxSetBuilder::checkBuiltTxSet( + app, *outputTxSet, TxSetValidationContext::forLCL(app), phaseSizes, + closeTimeOffset.seconds(), &outputApplicableTxSet); + if (failure) + { + throw std::runtime_error(*failure); + } + + return std::make_pair(outputTxSet, std::move(outputApplicableTxSet)); +} + +uint32_t +TxSetBuildLimits::maxTxSetSizeOps() const +{ + // As LedgerManager::getLastMaxTxSetSizeOps. + auto const n = mLclHeader.maxTxSetSize; + return protocolVersionStartsFrom(mLclHeader.ledgerVersion, + ProtocolVersion::V_11) + ? n + : (n * MAX_OPS_PER_TX); +} + +bool +TxSetBuildLimits::sameAs(TxSetBuildLimits const& other) const +{ + auto const& a = mLclHeader; + auto const& b = other.mLclHeader; + if (a.ledgerVersion != b.ledgerVersion || a.baseFee != b.baseFee || + a.baseReserve != b.baseReserve || a.maxTxSetSize != b.maxTxSetSize) + { + return false; + } + if ((mSorobanConfig == nullptr) != (other.mSorobanConfig == nullptr)) + { + return false; + } + if (!mSorobanConfig) + { + return true; + } + auto const& s = *mSorobanConfig; + auto const& t = *other.mSorobanConfig; + return s.maxLedgerResources() == t.maxLedgerResources() && + s.ledgerMaxInstructions() == t.ledgerMaxInstructions() && + s.ledgerMaxDependentTxClusters() == + t.ledgerMaxDependentTxClusters() && + s.ledgerMaxTxCount() == t.ledgerMaxTxCount(); +} + +TxSetBuildLimits +TxSetBuildLimits::forLCL(Application& app) +{ + releaseAssert(threadIsMain()); + auto& lm = app.getLedgerManager(); + TxSetBuildLimits limits; + limits.mLclHeader = lm.getLastClosedLedgerHeader().header; + if (protocolVersionStartsFrom(limits.mLclHeader.ledgerVersion, + SOROBAN_PROTOCOL_VERSION)) + { + limits.mSorobanConfig = &lm.getLastClosedSorobanNetworkConfig(); + } + return limits; +} + +std::vector +TxSetBuilder::selectPhases( + PerPhaseTransactionList const& txPhases, Application& app, + TxSetBuildLimits const& limits, TxSetValidationContext const& ctx, + ApplyTimeOffset closeTimeOffset, PerPhaseTransactionList& invalidTxs, + std::vector const& validTxTargets, std::string const& timerPrefix, + std::optional tieBreakSeed, + std::vector>* hadTxNotFittingLane +#ifdef BUILD_TESTS + , + bool enforceTxsApplyOrder, + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + bool disableTxValidationForLegacyScenario +#endif +) +{ releaseAssert(txPhases.size() == invalidTxs.size()); + if (hadTxNotFittingLane) + { + hadTxNotFittingLane->assign(txPhases.size(), {}); + } releaseAssert(validTxTargets.empty() || validTxTargets.size() == txPhases.size()); releaseAssert(txPhases.size() <= static_cast(TxSetPhase::PHASE_COUNT)); + auto const& cfg = app.getConfig(); std::vector validatedPhases; UnorderedMap accountFeeMap; for (size_t i = 0; i < txPhases.size(); ++i) @@ -911,38 +1179,38 @@ makeTxSetFromTransactions( #ifdef BUILD_TESTS if (disableTxValidationForLegacyScenario) { - REQUIRE(protocolVersionIsBefore(app.getLedgerManager() - .getLastClosedLedgerHeader() - .header.ledgerVersion, + REQUIRE(protocolVersionIsBefore(limits.mLclHeader.ledgerVersion, ProtocolVersion::V_20)); validatedTxs = phaseTxs; } else { #endif - auto trimTimer = app.getMetrics() - .NewTimer({"herder", "txset", "build-trim"}) - .TimeScope(); + auto trimTimer = + app.getMetrics() + .NewTimer({"herder", "txset", timerPrefix + "-trim"}) + .TimeScope(); if (validTxTargets.empty()) { - validatedTxs = TxSetUtils::trimInvalid( - phaseTxs, app, accountFeeMap, closeTimeOffset.seconds(), - closeTimeOffset.seconds(), invalid, validationCache); + validatedTxs = + TxSetUtils::trimInvalid(phaseTxs, app, ctx, accountFeeMap, + closeTimeOffset.seconds(), + closeTimeOffset.seconds(), invalid); } else { // Transactions of limited lanes (DEX) can't fill the phase, // so only generic lane ones count toward the target. - auto laneConfig = createSurgePricingLangeConfig( - static_cast(i), app); + auto laneConfig = createSurgePricingLaneConfig( + static_cast(i), cfg, limits); validatedTxs = TxSetUtils::trimInvalidPrefix( phaseTxs, validTxTargets[i], [&laneConfig](TransactionFrameBase const& tx) { return laneConfig->getLane(tx) == SurgePricingPriorityQueue::GENERIC_LANE; }, - app, accountFeeMap, closeTimeOffset.seconds(), - closeTimeOffset.seconds(), invalid, validationCache); + app, ctx, accountFeeMap, closeTimeOffset.seconds(), + closeTimeOffset.seconds(), invalid); } #ifdef BUILD_TESTS // In tests we shouldn't be really trying to close ledgers with @@ -963,16 +1231,19 @@ makeTxSetFromTransactions( } #endif auto phaseType = static_cast(i); - auto selectTimer = app.getMetrics() - .NewTimer({"herder", "txset", "build-select"}) - .TimeScope(); - auto [includedTxs, inclusionFeeMapBinding] = - applySurgePricing(phaseType, validatedTxs, app + auto selectTimer = + app.getMetrics() + .NewTimer({"herder", "txset", timerPrefix + "-select"}) + .TimeScope(); + auto [includedTxs, inclusionFeeMapBinding] = applySurgePricing( + phaseType, validatedTxs, cfg, limits, + tieBreakSeed ? std::make_optional(*tieBreakSeed + i) : std::nullopt, + hadTxNotFittingLane ? &(*hadTxNotFittingLane)[i] : nullptr #ifdef BUILD_TESTS - , - enforceTxsApplyOrder, parallelSorobanOrder + , + enforceTxsApplyOrder, parallelSorobanOrder #endif - ); + ); auto inclusionFeeMap = inclusionFeeMapBinding; std::visit( [&validatedPhases, phaseType, inclusionFeeMap](auto&& txs) { @@ -998,88 +1269,526 @@ makeTxSetFromTransactions( }, includedTxs); } + return validatedPhases; +} - auto const& lclHeader = app.getLedgerManager().getLastClosedLedgerHeader(); - auto xdrTimer = - app.getMetrics().NewTimer({"herder", "txset", "build-xdr"}).TimeScope(); - // Preliminary applicable frame - we don't know the contents hash yet, but - // we also don't return this. - std::unique_ptr preliminaryApplicableTxSet( - new ApplicableTxSetFrame(app, lclHeader, validatedPhases, - std::nullopt)); +TxSetValidationResult +TxSetBuilder::checkSelection(std::vector const& phases, + Application& app, + TxSetValidationContext const& ctx, + bool isGeneralized, + ApplyTimeOffset closeTimeOffset) +{ + ZoneScoped; + auto timer = app.getMetrics() + .NewTimer({"herder", "txset", "build-selection-check"}) + .TimeScope(); + // The transactions were validated by the selection. + return ApplicableTxSetFrame::checkPhasesValid( + app, ctx, phases, isGeneralized, closeTimeOffset.seconds(), + closeTimeOffset.seconds(), /* txsAreValidated */ true); +} - // Do the roundtrip through XDR to ensure we never build an incorrect tx set - // for nomination. - auto outputTxSet = preliminaryApplicableTxSet->toWireTxSetFrame(); -#ifdef BUILD_TESTS - if (enforceTxsApplyOrder) +namespace +{ +// Removes `gone` from a phase's transactions, and the clusters and stages +// that are left empty with them. +template +TxStageFrameList +withoutTxs(TxStageFrameList const& stages, Gone const& gone) +{ + TxStageFrameList res; + for (auto const& stage : stages) { - // Fill in the contents hash if we're skipping the normal roundtrip - // and validation flow. - preliminaryApplicableTxSet->mContentsHash = - outputTxSet->getContentsHash(); - return std::make_pair(outputTxSet, - std::move(preliminaryApplicableTxSet)); + TxStageFrame keptStage; + for (auto const& cluster : stage) + { + TxClusterFrame keptCluster; + for (auto const& tx : cluster) + { + if (gone.count(tx.get()) == 0) + { + keptCluster.push_back(tx); + } + } + if (!keptCluster.empty()) + { + keptStage.emplace_back(std::move(keptCluster)); + } + } + if (!keptStage.empty()) + { + res.emplace_back(std::move(keptStage)); + } } + return res; +} +} + +std::vector +TxSetBuilder::completeSelection( + std::vector const& selected, + std::vector> const& hadTxNotFittingLane, + TxSetCandidateSource const& moreCandidates, Application& app, + TxSetBuildLimits const& limits, TxSetValidationContext const& ctx, + ApplyTimeOffset closeTimeOffset, PerPhaseTransactionList& invalidTxs, + TxSetCompletionStats& stats, std::optional tieBreakSeed) +{ + ZoneScoped; + size_t const phaseCount = selected.size(); + releaseAssert(phaseCount == static_cast(TxSetPhase::PHASE_COUNT)); + releaseAssert(hadTxNotFittingLane.size() == phaseCount); + releaseAssert(invalidTxs.size() == phaseCount); + releaseAssert(limits.mSorobanConfig); + auto& metrics = app.getMetrics(); + auto const& cfg = app.getConfig(); + auto const offset = closeTimeOffset.seconds(); + auto const ledgerVersion = limits.mLclHeader.ledgerVersion; + + struct PhaseState + { + std::shared_ptr mLaneConfig; + bool mParallel{false}; + // What is selected so far, in one of the two shapes. + TxFrameList mSequential; + TxStageFrameList mStages; + std::optional mExtender; + // What the selection reported, per surge pricing lane. + std::vector mHadTxNotFittingLane; + // The same for the completion: a valid candidate was left out. + std::vector mLeftOutLanes; + bool mLeftOut{false}; + // A sequential phase was selected again. + bool mReselected{false}; + size_t mRemoved{0}; + size_t mAdded{0}; + // Candidates were looked at for this phase during the completion. + bool mExamined{false}; + // The phase lost transactions or candidates: to the validity pass, + // or because a candidate was invalid or of an account that is in + // the set already. (Not: a valid candidate that did not fit.) + bool mLost{false}; + // Batches from the mempool the phase has looked at. + size_t mMempoolBatches{0}; + // Nothing more is looked for (see the top-up below). + bool mFull{false}; + bool mStarved{false}; + }; + std::vector states(phaseCount); + // One fee-source total for the whole set, as in a build. + UnorderedMap accountFeeMap; + // One transaction per source account, across the phases. + UnorderedSet sources; + + auto atCapacity = [&](size_t p) { + auto const& state = states[p]; + auto const limit = state.mLaneConfig->getLaneLimits().at( + SurgePricingPriorityQueue::GENERIC_LANE); + int64_t ops = 0; + if (state.mExtender) + { + ops = static_cast(state.mExtender->sizeTx()); + } + else if (state.mParallel) + { + for (auto const& stage : state.mStages) + { + for (auto const& cluster : stage) + { + ops += static_cast(cluster.size()); + } + } + } + else + { + ops = static_cast(countOps(state.mSequential)); + } + return ops >= limit.getVal(Resource::Type::OPERATIONS); + }; + + // 1. The validity pass. Whatever the selection was made on, every one of + // its transactions is validated here against the ledger the set goes on + // top of, by the function that validates a set a node receives; what + // fails leaves the selection. + { + auto timer = metrics.NewTimer({"herder", "txset", "early-final-pass"}) + .TimeScope(); + for (size_t p = 0; p < phaseCount; ++p) + { + auto const& phase = selected[p]; + auto& state = states[p]; + state.mLaneConfig = createSurgePricingLaneConfig( + static_cast(p), cfg, limits); + state.mParallel = phase.isParallel(); + state.mHadTxNotFittingLane = hadTxNotFittingLane[p]; + releaseAssert(state.mHadTxNotFittingLane.size() == + state.mLaneConfig->getLaneLimits().size()); + state.mLeftOutLanes.assign(state.mHadTxNotFittingLane.size(), + false); + stats.mSelected += phase.sizeTx(); + auto invalid = TxSetUtils::getInvalidTxListWithErrors( + phase, app, ctx, accountFeeMap, offset, offset) + .first; + UnorderedSet gone; + for (auto const& tx : invalid) + { + gone.insert(tx.get()); + } + state.mRemoved = gone.size(); + stats.mRemoved += gone.size(); + if (state.mParallel) + { + state.mStages = withoutTxs(phase.getParallelStages(), gone); + } + else + { + for (auto const& tx : phase.getSequentialTxs()) + { + if (gone.count(tx.get()) == 0) + { + state.mSequential.push_back(tx); + } + } + } + for (auto const& tx : phase) + { + if (gone.count(tx.get()) == 0) + { + sources.insert(tx->getSourceID()); + } + } + invalidTxs[p] = std::move(invalid); + // Only a phase at its capacity is full from the start. That the + // selection left a transaction out says little: one that did + // not fit where it had to go (a cluster, a lane of its own) + // leaves room for others. + state.mFull = atCapacity(p); + } + } + + // 2. The top-up: while a phase has room, take further candidates, each + // validated like the selection's own. + // + // The rule is a build's: a build looks at the mempool's top (the + // capacity plus the margin) and looks deeper only for a phase that is + // short and lost candidates. So a phase stops asking when + // - it is at its capacity; or + // - it has seen the mempool's top once and lost nothing (no + // transaction removed by the validity pass, no candidate that was + // invalid or of an account already in the set): it got all that the + // top holds for it; or + // - it lost something, went on to a deeper pull, and that pull held + // nothing new of its kind. + // That valid candidates did not fit closes nothing: a transaction that + // could not go where it had to (a full cluster, a lane of its own) + // leaves room for others, whatever their number. + for (size_t p = 0; p < phaseCount; ++p) + { + states[p].mLost = states[p].mRemoved != 0; + } + // Selects a sequential phase again, from what it holds and `extra`; + // returns how many of `extra` are in. + auto reselect = [&](size_t p, TxFrameList const& extra) { + auto& state = states[p]; + TxFrameList all = state.mSequential; + all.insert(all.end(), extra.begin(), extra.end()); + std::vector flags; + auto [included, feeMap] = applySurgePricing( + static_cast(p), all, cfg, limits, + tieBreakSeed ? std::make_optional(*tieBreakSeed + p) : std::nullopt, + &flags +#ifdef BUILD_TESTS + , + false, {} #endif + ); + state.mSequential = std::get(included); + state.mReselected = true; + releaseAssert(flags.size() == state.mLeftOutLanes.size()); + for (size_t lane = 0; lane < flags.size(); ++lane) + { + state.mLeftOutLanes[lane] = + state.mLeftOutLanes[lane] || flags[lane]; + } + UnorderedSet in; + for (auto const& tx : state.mSequential) + { + in.insert(tx.get()); + } + size_t added = 0; + for (auto const& tx : extra) + { + if (in.count(tx.get()) != 0) + { + sources.insert(tx->getSourceID()); + ++added; + } + } + return added; + }; + auto wantsMore = [&]() { + return std::any_of( + states.begin(), states.end(), + [](PhaseState const& s) { return !s.mFull && !s.mStarved; }); + }; + { + auto timer = + metrics.NewTimer({"herder", "txset", "early-top-up"}).TimeScope(); + while (wantsMore()) + { + auto batch = moreCandidates(); + if (!batch) + { + break; + } + stats.mCandidates += batch->mTxs.size(); + TxFrameList classicTxs; + TxFrameList sorobanTxs; + std::vector offered(phaseCount, 0); + for (auto const& tx : batch->mTxs) + { + size_t const p = tx->isSoroban() ? 1 : 0; + if (states[p].mFull) + { + continue; + } + ++offered[p]; + if (sources.count(tx->getSourceID()) != 0) + { + continue; + } + (p == 1 ? sorobanTxs : classicTxs).push_back(tx); + } + TxSetUtils::keepOnePerSourceAccount(classicTxs, sorobanTxs); + for (size_t p = 0; p < phaseCount; ++p) + { + auto& state = states[p]; + if (state.mFull) + { + continue; + } + // The selection's own verdict on what it left out is void + // from here on: the candidates are judged on this ledger. + state.mExamined = true; + auto const& candidates = p == 1 ? sorobanTxs : classicTxs; + size_t valid = 0; + size_t added = 0; + if (!candidates.empty()) + { + TxFrameList invalid; + auto const validTxs = TxSetUtils::trimInvalid( + candidates, app, ctx, accountFeeMap, offset, offset, + invalid); + invalidTxs[p].insert(invalidTxs[p].end(), invalid.begin(), + invalid.end()); + valid = validTxs.size(); + if (valid != 0 && state.mParallel) + { + if (!state.mExtender) + { + state.mExtender.emplace( + std::move(state.mStages), + *limits.mSorobanConfig, + state.mLaneConfig->getLaneLimits().at( + SurgePricingPriorityQueue::GENERIC_LANE), + ledgerVersion); + } + for (auto const& tx : validTxs) + { + if (state.mExtender->tryAdd(tx)) + { + sources.insert(tx->getSourceID()); + ++added; + } + } + } + else if (valid != 0) + { + // A sequential phase is cheap to select: again, by + // the standard surge pricing. + added = reselect(p, validTxs); + } + } + state.mAdded += added; + // A valid candidate that is not in: the phase's + // transactions competed for its room. + state.mLeftOut = state.mLeftOut || added < valid; + state.mLost = state.mLost || valid < offered[p]; + if (atCapacity(p)) + { + state.mFull = true; + } + else if (batch->mFromMempool) + { + ++state.mMempoolBatches; + state.mStarved = + !state.mLost || + (state.mMempoolBatches > 1 && offered[p] == 0); + } + } + } + } - xdrTimer.Stop(); - // The final XDR is available now. Start proposal encoding while the - // roundtrip and final validity checks below run; only the local builder - // supplies this callback. The callback must not nominate the set. - if (onTxSetReady) + // 3. The phases as they are now. Base fees by the rule of a build: + // transactions of a lane compete on fees only if a valid one was left + // out. For a phase that looked at candidates here, that is what + // happened here; otherwise it is what the selection reported. + std::vector res; + for (size_t p = 0; p < phaseCount; ++p) { - onTxSetReady(outputTxSet); + auto& state = states[p]; + auto const phaseType = static_cast(p); + stats.mAdded += state.mAdded; + std::vector flags = state.mHadTxNotFittingLane; + if (state.mExamined || state.mRemoved != 0) + { + flags = state.mLeftOutLanes; + if (state.mParallel) + { + flags.at(SurgePricingPriorityQueue::GENERIC_LANE) = + state.mLeftOut; + } + } + if (state.mRemoved == 0 && state.mAdded == 0 && !state.mReselected && + flags == state.mHadTxNotFittingLane) + { + // Untouched. + res.emplace_back(selected[p]); + continue; + } + if (state.mParallel) + { + auto stages = state.mExtender ? state.mExtender->getStages() + : std::move(state.mStages); + std::variant included = stages; + auto feeMap = + computeInclusionFeeMap(phaseType, included, *state.mLaneConfig, + limits.mLclHeader, flags); + res.emplace_back( + TxSetPhaseFrame(phaseType, std::move(stages), feeMap)); + } + else + { + if (!state.mReselected) + { + // Transactions were removed and none added: select what is + // left. + reselect(p, {}); + flags = state.mLeftOutLanes; + } + std::variant included = + state.mSequential; + auto feeMap = + computeInclusionFeeMap(phaseType, included, *state.mLaneConfig, + limits.mLclHeader, flags); + res.emplace_back( + TxSetPhaseFrame(phaseType, state.mSequential, feeMap)); + } } + return res; +} +std::shared_ptr +TxSetBuilder::makeSelfCheck(Application& app, + TxSetXDRFrameConstPtr const& txSet, + LedgerHeaderHistoryEntry const& lcl, + std::vector const& phases, + ApplyTimeOffset closeTimeOffset) +{ + releaseAssert(threadIsMain()); + std::shared_ptr check(new TxSetSelfCheck()); + check->mTxSet = txSet; + check->mLcl = lcl; + if (protocolVersionStartsFrom(lcl.header.ledgerVersion, + SOROBAN_PROTOCOL_VERSION)) + { + check->mSorobanConfig = std::make_shared( + app.getLedgerManager().getLastClosedSorobanNetworkConfig()); + } + for (auto const& phase : phases) + { + check->mPhaseSizes.push_back(phase.sizeTx()); + } + check->mCloseTimeOffset = closeTimeOffset.seconds(); + return check; +} + +std::optional +TxSetBuilder::checkBuiltTxSet(Application& app, TxSetXDRFrame const& txSet, + TxSetValidationContext const& ctx, + std::vector const& phaseSizes, + uint64_t closeTimeOffset, + ApplicableTxSetFrameConstPtr* roundTripped) +{ + ZoneScoped; auto roundtripTimer = app.getMetrics() .NewTimer({"herder", "txset", "build-roundtrip"}) .TimeScope(); - ApplicableTxSetFrameConstPtr outputApplicableTxSet = - outputTxSet->prepareForApply(app, lclHeader.header); + ApplicableTxSetFrameConstPtr applicable = + txSet.prepareForApply(app, ctx.mLcl.header); roundtripTimer.Stop(); - if (!outputApplicableTxSet) + if (!applicable) { - throw std::runtime_error( - "Couldn't prepare created tx set frame for apply"); + return "Couldn't prepare created tx set frame for apply"; } // Make sure no transactions were lost during the roundtrip and the output // tx set is valid. - bool valid = preliminaryApplicableTxSet->numPhases() == - outputApplicableTxSet->numPhases(); + bool valid = phaseSizes.size() == applicable->numPhases(); if (valid) { - for (size_t i = 0; i < preliminaryApplicableTxSet->numPhases(); ++i) + for (size_t i = 0; i < phaseSizes.size(); ++i) { - valid = valid && preliminaryApplicableTxSet->sizeTx( - static_cast(i)) == - outputApplicableTxSet->sizeTx( - static_cast(i)); + valid = valid && phaseSizes[i] == + applicable->sizeTx(static_cast(i)); } } if (!valid) { - throw std::runtime_error("Created invalid tx set frame - shape is " - "mismatched after roundtrip."); + return "Created invalid tx set frame - shape is mismatched after " + "roundtrip."; } - // We already trimmed invalid transactions in an earlier call to - // `trimInvalid`, so skip transaction validation here auto finalTimer = app.getMetrics() .NewTimer({"herder", "txset", "build-final-check"}) .TimeScope(); - auto validationResult = outputApplicableTxSet->checkValidInternalWithResult( - app, closeTimeOffset.seconds(), closeTimeOffset.seconds(), true); + auto validationResult = applicable->checkValidInternalWithResult( + app, ctx, closeTimeOffset, closeTimeOffset, true); finalTimer.Stop(); if (validationResult != TxSetValidationResult::VALID) { - throw std::runtime_error(fmt::format( + return fmt::format( FMT_STRING("Created invalid tx set frame, validation result: {:s}"), - toString(validationResult))); + toString(validationResult)); + } + if (roundTripped) + { + *roundTripped = std::move(applicable); } + return std::nullopt; +} - return std::make_pair(outputTxSet, std::move(outputApplicableTxSet)); +std::optional +TxSetSelfCheck::run(Application& app) const +{ + // The check does not validate transactions, so it never asks for a view + // of the ledger. + TxSetValidationContext const ctx{ + mLcl, mSorobanConfig.get(), + []() -> std::unique_ptr { + throw std::runtime_error( + "the tx set self-check does not read the ledger"); + }, + 1}; + try + { + return TxSetBuilder::checkBuiltTxSet(app, *mTxSet, ctx, mPhaseSizes, + mCloseTimeOffset, nullptr); + } + catch (std::exception const& e) + { + return fmt::format(FMT_STRING("tx set self-check threw: {}"), e.what()); + } } TxSetXDRFrameConstPtr @@ -2458,7 +3167,20 @@ ApplicableTxSetFrame::checkValidInternalWithResult( releaseAssert(mPhases.size() == 1); } - bool enforceUniqueSourceAccount = needGeneralizedTxSet; + return checkPhasesValid(app, ctx, mPhases, isGeneralizedTxSet(), + lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, txsAreValidated); +} + +TxSetValidationResult +ApplicableTxSetFrame::checkPhasesValid( + Application& app, TxSetValidationContext const& ctx, + std::vector const& phases, bool isGeneralized, + uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, + bool txsAreValidated) +{ + auto const& lcl = ctx.mLcl; + bool enforceUniqueSourceAccount = isGeneralized; #ifdef BUILD_TESTS if (app.getRunInOverlayOnlyMode()) { @@ -2470,7 +3192,7 @@ ApplicableTxSetFrame::checkValidInternalWithResult( // Ensure the tx set does not contain multiple txs per source // account std::unordered_set seenAccounts; - for (auto const& phase : mPhases) + for (auto const& phase : phases) { for (auto const& tx : phase) { @@ -2489,7 +3211,7 @@ ApplicableTxSetFrame::checkValidInternalWithResult( bool useCrossPhaseFeeMap = protocolVersionStartsFrom( lcl.header.ledgerVersion, ProtocolVersion::V_26); UnorderedMap accountFeeMap; - for (auto const& phase : mPhases) + for (auto const& phase : phases) { if (!useCrossPhaseFeeMap) { diff --git a/src/herder/TxSetFrame.h b/src/herder/TxSetFrame.h index ef074dceda..888ad9e318 100644 --- a/src/herder/TxSetFrame.h +++ b/src/herder/TxSetFrame.h @@ -27,6 +27,8 @@ class AbstractLedgerView; class SorobanNetworkConfig; class TxSetXDRFrame; class ApplicableTxSetFrame; +class TxSetBuilder; +class TxSetSelfCheck; using TxSetXDRFrameConstPtr = std::shared_ptr; using TxSetReadyCallback = std::function; using ApplicableTxSetFrameConstPtr = @@ -152,6 +154,30 @@ struct TxSetValidationContext static TxSetValidationContext forLCL(Application& app); }; +// The limits a transaction set is selected within: those of the ledger it +// goes on top of. Selection (surge pricing, the Soroban stages) only reads +// the ledger through this, so it can run for a ledger that is not the main +// thread's LCL, and on another thread. +struct TxSetBuildLimits +{ + // Header of the ledger the set applies on top of. Selection reads its + // protocol version, base fee and maximum tx set size; its hashes are not + // used. + LedgerHeader mLclHeader; + // Soroban network configuration of that ledger; null before Soroban. + // Whoever makes the limits keeps it alive. + SorobanNetworkConfig const* mSorobanConfig{nullptr}; + + // Capacity of the classic phase, in operations. + uint32_t maxTxSetSizeOps() const; + // Whether a set selected within `other` is selected within these limits + // too: everything selection reads is equal. + bool sameAs(TxSetBuildLimits const& other) const; + + // The limits of the LCL as seen by the main thread. Main thread only. + static TxSetBuildLimits forLCL(Application& app); +}; + using TxFrameList = std::vector; using PerPhaseTransactionList = std::vector; @@ -215,7 +241,16 @@ makeTxSetFromTransactions( std::vector const& validTxTargets = {}, // Results of earlier validations against the LCL (see // `TxValidationCache`); extended with this build's. - TxValidationCache* validationCache = nullptr); + TxValidationCache* validationCache = nullptr, + // The builder checks every set it makes: it reads the XDR back and + // validates the structure of the result (`TxSetSelfCheck`). Without + // `deferredSelfCheck` it does so before returning, and throws if the + // check fails. With it, only the structure of the selection is validated + // before returning (`TxSetBuilder::checkSelection`; a failure throws); + // the check of the written set is not run: it is returned here for the + // caller to run (on any thread), so that it does not hold up the use of + // the set. + std::shared_ptr* deferredSelfCheck = nullptr); #ifdef BUILD_TESTS std::pair @@ -480,7 +515,9 @@ class TxSetPhaseFrame , TxSetReadyCallback const& onTxSetReady, std::vector const& validTxTargets, - TxValidationCache* validationCache); + TxValidationCache* validationCache, + std::shared_ptr* deferredSelfCheck); + friend class TxSetBuilder; #ifdef BUILD_TESTS friend std::pair makeTxSetFromTransactions( @@ -674,7 +711,9 @@ class ApplicableTxSetFrame , TxSetReadyCallback const& onTxSetReady, std::vector const& validTxTargets, - TxValidationCache* validationCache); + TxValidationCache* validationCache, + std::shared_ptr* deferredSelfCheck); + friend class TxSetBuilder; #ifdef BUILD_TESTS friend std::pair makeTxSetFromTransactions( @@ -695,6 +734,17 @@ class ApplicableTxSetFrame ApplicableTxSetFrame(ApplicableTxSetFrame const&) = default; ApplicableTxSetFrame(ApplicableTxSetFrame&&) = default; + // Everything `checkValidInternalWithResult` checks about the phases of a + // set, i.e. all of it but the set's parent and kind: one transaction per + // source account across the phases, and each phase's fees, limits and + // (unless `txsAreValidated`) transactions. Used on its own for a + // selection that has no parent yet. + static TxSetValidationResult + checkPhasesValid(Application& app, TxSetValidationContext const& ctx, + std::vector const& phases, + bool isGeneralized, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, bool txsAreValidated); + void toXDR(TransactionSet& set) const; void toXDR(GeneralizedTransactionSet& generalizedTxSet) const; @@ -716,4 +766,205 @@ class ApplicableTxSetFrame std::optional mContentsHash; }; +// The check a builder makes of a set it just built, before the set is relied +// on: the XDR is read back into an applicable set, which must have the shape +// of the selection it was written from and pass the structural validation of +// a set on top of the ledger it was built for (limits, fees, order; the +// transactions themselves were validated by the selection). +// +// It holds copies of everything it reads, so it can run on any thread while +// the main thread goes on with the set. +class TxSetSelfCheck +{ + public: + // Runs the check. Returns why the set must not be used, or nullopt if it + // passed. `app` is only read for the network ID and the configuration. + std::optional run(Application& app) const; + + TxSetXDRFrameConstPtr const& + getTxSet() const + { + return mTxSet; + } + +#ifdef BUILD_TESTS + // The same check of another set: for tests that make a builder produce + // a broken one. + std::shared_ptr + withTxSetForTesting(TxSetXDRFrameConstPtr const& txSet) const + { + std::shared_ptr res(new TxSetSelfCheck(*this)); + res->mTxSet = txSet; + return res; + } +#endif + + private: + friend class TxSetBuilder; + TxSetSelfCheck() = default; + + TxSetXDRFrameConstPtr mTxSet; + // The ledger the set was built on top of. + LedgerHeaderHistoryEntry mLcl; + std::shared_ptr mSorobanConfig; + // Number of transactions per phase of the selection. + std::vector mPhaseSizes; + uint64_t mCloseTimeOffset{0}; +}; + +// What the completion of a selection that was made ahead of time did (see +// `TxSetBuilder::completeSelection`). +struct TxSetCompletionStats +{ + // Transactions of the selection as it came. + size_t mSelected{0}; + // Of them, removed because they are not valid on the closed ledger. + size_t mRemoved{0}; + // Further candidates that were looked at, and those of them added. + size_t mCandidates{0}; + size_t mAdded{0}; +}; + +// Where the completion of a selection gets further candidates from: each call +// returns the next batch (decoded, in mempool order, none handed out before), +// or nullopt when there is nothing more to get. +struct TxSetCandidateBatch +{ + TxFrameList mTxs; + // The batch is what a pull of the mempool's top held that was not seen + // before (possibly nothing). + bool mFromMempool{false}; +}; +using TxSetCandidateSource = + std::function()>; + +// The steps `makeTxSetFromTransactions` is made of, for callers that run them +// apart from each other. +class TxSetBuilder +{ + public: + // Trims the invalid transactions of every phase and applies surge + // pricing, as `makeTxSetFromTransactions` does, but against an arbitrary + // ledger: validity is read through `ctx`, limits from `limits`. Nothing + // of the main thread's or of ledger apply's state is read, so this may + // run on any thread (`ctx` then has to ask for a single validation task). + // `timerPrefix` names the stage timers + // (`herder.txset.-trim` and `-select`). + static std::vector selectPhases( + PerPhaseTransactionList const& txPhases, Application& app, + TxSetBuildLimits const& limits, TxSetValidationContext const& ctx, + ApplyTimeOffset closeTimeOffset, PerPhaseTransactionList& invalidTxs, + std::vector const& validTxTargets, + std::string const& timerPrefix, + // Randomizes the order of transactions with equal fee + // rates. Drawn from the global random engine (main thread + // only) unless provided. + std::optional tieBreakSeed, + // If given, receives for every phase, per surge pricing lane, + // whether a transaction was left out for the lane's limits. + std::vector>* hadTxNotFittingLane = nullptr +#ifdef BUILD_TESTS + , + bool enforceTxsApplyOrder = false, + txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}, + bool disableTxValidationForLegacyScenario = false +#endif + ); + + // The structural validation of a selection (ApplicableTxSetFrame's, less + // the parent hash): what a set made of `phases` would be rejected for, + // whatever its parent and its bytes. Cheap next to the check of the + // written set (TxSetSelfCheck): it reads the frames the selection + // already holds. Any thread. + static TxSetValidationResult + checkSelection(std::vector const& phases, Application& app, + TxSetValidationContext const& ctx, bool isGeneralized, + ApplyTimeOffset closeTimeOffset); + + // Completes a selection that was made before the ledger it goes on top + // of had closed, now that it has: `ctx` and `limits` describe that + // ledger (the LCL). + // + // 1. Every transaction of `selected` is validated against `ctx`, with + // the close time the set gets, by the function that validates the + // transactions of a set a node receives + // (`TxSetUtils::getInvalidTxListWithErrors`); those that fail are + // removed (and returned in `invalidTxs`). Nothing the selection was + // made on is relied upon: after this step every transaction left is + // valid on the LCL. + // 2. While a phase is below its capacity, further candidates are taken + // from `moreCandidates`, validated the same way, kept to one + // transaction per source account against what is selected, and added: + // to a parallel Soroban phase one by one (`ParallelPhaseExtender`), to + // a sequential phase by selecting it again. + // 3. The base fees are set by the rule of a build. + // + // `hadTxNotFittingLane` is what the selection reported + // (`selectPhases`). The result has not been checked for its structure: + // `writeSelection` does that. Generalized sets only. Main thread. + // + // When a phase stops asking for candidates is described at the top-up + // in the implementation: at its capacity; when valid candidates could + // not be added; after one look at the mempool's top, unless it lost + // candidates and that top was dense in its kind. + static std::vector completeSelection( + std::vector const& selected, + std::vector> const& hadTxNotFittingLane, + TxSetCandidateSource const& moreCandidates, Application& app, + TxSetBuildLimits const& limits, TxSetValidationContext const& ctx, + ApplyTimeOffset closeTimeOffset, PerPhaseTransactionList& invalidTxs, + TxSetCompletionStats& stats, std::optional tieBreakSeed); + + // The second half of `makeTxSetFromTransactions`: writes the set of a + // selection for the ledger after the LCL (which `ctx` describes) and + // checks it. With `deferredSelfCheck` the structure of the selection is + // validated before anything is written and the check of the written set + // is returned instead of run; without, the written set is read back and + // checked here. Throws if a check fails. Main thread. + static std::pair + writeSelection(std::vector const& phases, Application& app, + TxSetValidationContext const& ctx, + ApplyTimeOffset closeTimeOffset, + TxSetReadyCallback const& onTxSetReady, + std::shared_ptr* deferredSelfCheck +#ifdef BUILD_TESTS + , + bool enforceTxsApplyOrder = false +#endif + ); + + private: + friend std::pair + makeTxSetFromTransactions( + PerPhaseTransactionList const& txPhases, Application& app, + ApplyTimeOffset closeTimeOffset, + PerPhaseTransactionList& invalidTxsPerPhase +#ifdef BUILD_TESTS + , + bool enforceTxsApplyOrder, + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + bool disableTxValidationForLegacyScenario +#endif + , + TxSetReadyCallback const& onTxSetReady, + std::vector const& validTxTargets, + TxValidationCache* validationCache, + std::shared_ptr* deferredSelfCheck); + friend class TxSetSelfCheck; + + static std::shared_ptr + makeSelfCheck(Application& app, TxSetXDRFrameConstPtr const& txSet, + LedgerHeaderHistoryEntry const& lcl, + std::vector const& phases, + ApplyTimeOffset closeTimeOffset); + // The check itself. `roundTripped`, if given, receives the applicable + // set read back from the XDR. + static std::optional + checkBuiltTxSet(Application& app, TxSetXDRFrame const& txSet, + TxSetValidationContext const& ctx, + std::vector const& phaseSizes, + uint64_t closeTimeOffset, + ApplicableTxSetFrameConstPtr* roundTripped); +}; + } // namespace stellar diff --git a/src/herder/TxSetUtils.cpp b/src/herder/TxSetUtils.cpp index 00483cd049..55645e3684 100644 --- a/src/herder/TxSetUtils.cpp +++ b/src/herder/TxSetUtils.cpp @@ -383,6 +383,18 @@ TxSetUtils::trimInvalid(TxFrameList const& txs, Application& app, releaseAssert(threadIsMain()); auto ctx = TxSetValidationContext::forLCL(app); ctx.mValidationCache = validationCache; + return trimInvalid(txs, app, ctx, accountFeeMap, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, invalidTxs); +} + +TxFrameList +TxSetUtils::trimInvalid(TxFrameList const& txs, Application& app, + TxSetValidationContext const& ctx, + UnorderedMap& accountFeeMap, + uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + TxFrameList& invalidTxs) +{ invalidTxs = getInvalidTxListWithErrors(txs, app, ctx, accountFeeMap, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset) @@ -397,6 +409,23 @@ TxSetUtils::trimInvalidPrefix( Application& app, UnorderedMap& accountFeeMap, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, TxValidationCache* validationCache) +{ + releaseAssert(threadIsMain()); + auto ctx = TxSetValidationContext::forLCL(app); + ctx.mValidationCache = validationCache; + return trimInvalidPrefix(txs, targetValidTxs, countsTowardTarget, app, ctx, + accountFeeMap, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, invalidTxs); +} + +TxFrameList +TxSetUtils::trimInvalidPrefix( + TxFrameList const& txs, size_t targetValidTxs, + std::function const& countsTowardTarget, + Application& app, TxSetValidationContext const& ctx, + UnorderedMap& accountFeeMap, + uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, + TxFrameList& invalidTxs) { ZoneScoped; TxFrameList validTxs; @@ -420,8 +449,8 @@ TxSetUtils::trimInvalidPrefix( validated += count; TxFrameList batchInvalid; for (auto const& tx : trimInvalid( - batch, app, accountFeeMap, lowerBoundCloseTimeOffset, - upperBoundCloseTimeOffset, batchInvalid, validationCache)) + batch, app, ctx, accountFeeMap, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, batchInvalid)) { if (!countsTowardTarget || countsTowardTarget(*tx)) { @@ -435,4 +464,77 @@ TxSetUtils::trimInvalidPrefix( return validTxs; } +// The mempool is fee-ordered and sequence-number-oblivious, so it can hand +// out several transactions from one source account (e.g. a chained pair). A +// tx set may only contain one tx per source account, so keep the lowest +// sequence number per account and let the others wait for a later ledger. +void +TxSetUtils::keepOnePerSourceAccount(TxFrameList& txs) +{ + std::unordered_map firstBySource; + TxFrameList kept; + for (auto const& tx : txs) + { + auto [it, inserted] = + firstBySource.emplace(tx->getSourceID(), kept.size()); + if (inserted) + { + kept.push_back(tx); + } + else if (tx->getSeqNum() < kept[it->second]->getSeqNum()) + { + kept[it->second] = tx; + } + } + txs = std::move(kept); +} + +// One transaction per source account holds across the phases of a set, not +// only within each: of an account's classic and Soroban candidates, the one +// with the lower sequence number stays (the classic one if they are equal). +void +TxSetUtils::keepOnePerSourceAccount(TxFrameList& classicTxs, + TxFrameList& sorobanTxs) +{ + keepOnePerSourceAccount(classicTxs); + keepOnePerSourceAccount(sorobanTxs); + if (classicTxs.empty() || sorobanTxs.empty()) + { + return; + } + std::unordered_map classicSeq; + for (auto const& tx : classicTxs) + { + classicSeq.emplace(tx->getSourceID(), tx->getSeqNum()); + } + std::unordered_set dropFromClassic; + TxFrameList keptSoroban; + for (auto const& tx : sorobanTxs) + { + auto it = classicSeq.find(tx->getSourceID()); + if (it == classicSeq.end()) + { + keptSoroban.push_back(tx); + } + else if (tx->getSeqNum() < it->second) + { + dropFromClassic.insert(it->first); + keptSoroban.push_back(tx); + } + } + sorobanTxs = std::move(keptSoroban); + if (!dropFromClassic.empty()) + { + TxFrameList keptClassic; + for (auto const& tx : classicTxs) + { + if (dropFromClassic.count(tx->getSourceID()) == 0) + { + keptClassic.push_back(tx); + } + } + classicTxs = std::move(keptClassic); + } +} + } // namespace stellar diff --git a/src/herder/TxSetUtils.h b/src/herder/TxSetUtils.h index 09ef4dc7cc..960c5b318a 100644 --- a/src/herder/TxSetUtils.h +++ b/src/herder/TxSetUtils.h @@ -73,6 +73,14 @@ class TxSetUtils uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, TxValidationCache* validationCache = nullptr); + // Same, against the ledger state described by `ctx` (which also carries + // the validation cache, if any) instead of the main thread's LCL. + static TxFrameList + trimInvalid(TxFrameList const& txs, Application& app, + TxSetValidationContext const& ctx, + UnorderedMap& accountFeeMap, + uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs); // Like `trimInvalid`, for candidates in priority order when only the // first `targetValidTxs` valid ones that `countsTowardTarget` accepts are @@ -88,5 +96,26 @@ class TxSetUtils Application& app, UnorderedMap& accountFeeMap, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, TxValidationCache* validationCache = nullptr); + // Same, against the ledger state described by `ctx`. + static TxFrameList + trimInvalidPrefix(TxFrameList const& txs, size_t targetValidTxs, + std::function const& + countsTowardTarget, + Application& app, TxSetValidationContext const& ctx, + UnorderedMap& accountFeeMap, + uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + TxFrameList& invalidTxs); + + // The mempool is fee-ordered and sequence-number-oblivious, so it can + // hand out several transactions of one source account (e.g. a chained + // pair). A tx set may hold one transaction per source account: keeps + // the one with the lowest sequence number, in the place of the first. + static void keepOnePerSourceAccount(TxFrameList& txs); + // The same across the two phases of a set: of an account's classic and + // Soroban candidates, the one with the lower sequence number stays (the + // classic one if they are equal). + static void keepOnePerSourceAccount(TxFrameList& classicTxs, + TxFrameList& sorobanTxs); }; // class TxSetUtils } // namespace stellar diff --git a/src/herder/test/EarlyTxSetTests.cpp b/src/herder/test/EarlyTxSetTests.cpp new file mode 100644 index 0000000000..3375891278 --- /dev/null +++ b/src/herder/test/EarlyTxSetTests.cpp @@ -0,0 +1,2673 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// Tests of the selection of the next transaction set before the current +// ledger has closed (herder/EarlyTxSetSelection.h), of its completion on the +// closed ledger (TxSetBuilder::completeSelection, HerderImpl::buildTxSet), and +// of the pull of a build (capacity plus a margin, refilled when validation +// leaves the set short). +// +// Two properties are what these tests are for: +// a. whatever the overlap between ledger N and the candidates of N+1, the +// set a node proposes for N+1 passes the validation every node applies +// to it, against N as it really closed; +// b. the set is as full as the mempool allows. + +#include "crypto/SHA.h" +#include "herder/EarlyTxSetSelection.h" +#include "herder/Herder.h" +#include "herder/HerderImpl.h" +#include "herder/HerderUtils.h" +#include "herder/LedgerCloseData.h" +#include "herder/ParallelTxSetBuilder.h" +#include "herder/TxSetFrame.h" +#include "herder/Upgrades.h" +#include "herder/test/TestTxSetUtils.h" +#include "ledger/ImmutableLedgerView.h" +#include "ledger/LedgerManager.h" +#include "ledger/test/LedgerTestUtils.h" +#include "main/Application.h" +#include "medida/histogram.h" +#include "medida/meter.h" +#include "simulation/ApplyLoad.h" +#include "simulation/Simulation.h" +#include "simulation/Topologies.h" +#include "test/Catch2.h" +#include "test/TestAccount.h" +#include "test/TestUtils.h" +#include "test/TxTests.h" +#include "test/test.h" +#include "transactions/SignatureUtils.h" +#include "transactions/TransactionUtils.h" +#include "util/BatchExecutor.h" +#include "util/Math.h" +#include "util/MetricsRegistry.h" +#include "util/ProtocolVersion.h" +#include "util/XDROperators.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace stellar; +using namespace stellar::txtest; + +namespace stellar +{ +class EarlyTxSetTestAccess +{ + public: + // What the herder does with a statement this node emits. + static void + voted(HerderImpl& herder, SCPEnvelope const& envelope) + { + herder.maybeScheduleEarlyTxSetSelection(envelope); + } + + static bool + hasPlan(HerderImpl& herder) + { + return herder.mEarlyTxSetPlan.has_value(); + } + + static bool + planned(HerderImpl& herder) + { + return herder.mEarlyTxSetPlan && herder.mEarlyTxSetPlan->planned; + } + + static std::shared_ptr + job(HerderImpl& herder) + { + return herder.mEarlyTxSet; + } + + static auto + build(HerderImpl& herder, uint32_t ledgerSeq, ConsensusTime closeTime) + { + return herder.buildTxSet(ledgerSeq, closeTime, false); + } + + static size_t + pullCount(HerderImpl& herder) + { + return herder.txSetPullCount(); + } + + static void + setStartDelay(HerderImpl& herder, + std::optional delay) + { + herder.mEarlyTxSetStartDelayForTesting = delay; + } + + static std::chrono::milliseconds + startDelay(HerderImpl& herder) + { + return herder.earlyTxSetStartDelay(); + } + + static void + setTimings(HerderImpl& herder, std::chrono::milliseconds voteToBuild, + std::chrono::milliseconds lastVoteToBuild, + std::chrono::milliseconds job, std::chrono::milliseconds lastJob) + { + herder.mVoteToBuild = voteToBuild; + herder.mLastVoteToBuild = lastVoteToBuild; + herder.mEarlyTxSetJobDuration = job; + herder.mLastEarlyTxSetJobDuration = lastJob; + } + + // What the node takes a selection job to last (running average, last). + static std::pair + jobDuration(HerderImpl& herder) + { + return {herder.mEarlyTxSetJobDuration, + herder.mLastEarlyTxSetJobDuration}; + } + + static void + skipFinalPass(HerderImpl& herder, bool skip) + { + herder.mSkipEarlyTxSetFinalPassForTesting = skip; + } + + static void + hold(HerderImpl& herder, std::shared_ptr> hold) + { + herder.mEarlyTxSetHoldForTesting = std::move(hold); + } + + static bool + selfCheckLatched(HerderImpl& herder) + { + return herder.mTxSetSelfCheckFailed; + } +}; +} + +namespace +{ +using Access = EarlyTxSetTestAccess; + +uint32_t const GATE_PROTOCOL = + static_cast(APPLY_TIME_TX_SET_VALIDATION_PROTOCOL_VERSION); + +int64_t +txsetMeter(Application& app, std::string const& name, + std::string const& unit = "txset") +{ + return static_cast( + app.getMetrics().NewMeter({"herder", "txset", name}, unit).count()); +} + +int64_t +applyCounter(Application& app, std::string const& name) +{ + return app.getMetrics().NewCounter({"ledger", "apply", name}).count(); +} + +std::set +hashesOf(std::vector const& txs) +{ + std::set res; + for (auto const& tx : txs) + { + res.insert(tx->getFullHash()); + } + return res; +} + +std::set +hashesOf(ApplicableTxSetFrame const& txSet) +{ + std::set res; + for (auto const& phase : txSet.getPhases()) + { + for (auto const& tx : phase) + { + res.insert(tx->getFullHash()); + } + } + return res; +} + +// What a node got when it asked for its next set. +struct Built +{ + TxSetXDRFrameConstPtr mTxSet; + ApplicableTxSetFrameConstPtr mApplicable; + // The set is a completed early selection, not a build. + bool mEarly{false}; + std::vector mInvalid; + + std::set + hashes() const + { + return hashesOf(*mApplicable); + } + + bool + holds(TransactionFrameBasePtr const& tx) const + { + return hashes().count(tx->getFullHash()) != 0; + } +}; + +// A single validator (so it leads every ledger) with funded accounts. Its +// mempool is `mMempool`, pulled in that order; the test plays the network: +// it has the node vote for a set, closes the ledger with a set, and asks the +// node for its next set. +struct EarlyFixture +{ + VirtualClock mClock; + Config mCfg; + Application::pointer mApp; + std::vector mAccounts; + std::vector mMempool; + // What each pull of the mempool asked for. + std::vector mPulls; + + explicit EarlyFixture(size_t accounts = 8, + std::optional maxTxSetSize = std::nullopt, + std::function tweak = nullptr) + : mCfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)) + { + mCfg.LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + mCfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + if (maxTxSetSize) + { + mCfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = *maxTxSetSize; + } + // The tests close ledgers themselves. + mCfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = false; + if (tweak) + { + tweak(mCfg); + } + REQUIRE(mCfg.backgroundLedgerApply()); + mApp = createTestApplication(mClock, mCfg); + auto root = mApp->getRoot(); + auto const balance = + mApp->getLedgerManager().getLastMinBalance(2) + 1'000'000'000; + for (size_t i = 0; i < accounts; ++i) + { + mAccounts.emplace_back( + root->create(fmt::format("account-{}", i), balance)); + } + // An early selection starts as soon as the node has voted. + Access::setStartDelay(herder(), std::chrono::milliseconds(0)); + herder().mGetTopTransactionsForTesting = [this](size_t count) { + mPulls.push_back(count); + std::vector envelopes; + for (size_t i = 0; i < mMempool.size() && i < count; ++i) + { + envelopes.push_back(mMempool[i]->getEnvelope()); + } + return envelopes; + }; + } + + ~EarlyFixture() + { + herder().mGetTopTransactionsForTesting = nullptr; + } + + HerderImpl& + herder() + { + return static_cast(mApp->getHerder()); + } + + LedgerManager& + lm() + { + return mApp->getLedgerManager(); + } + + TestAccount& + acc(size_t i) + { + return mAccounts.at(i); + } + + uint32_t + nextSeq() + { + return lm().getLastClosedLedgerNum() + 1; + } + + ConsensusTime + nextCloseTime() + { + return ConsensusTime::fromApplyTime(ApplyTime::fromTimePoint( + lm().getLastClosedLedgerHeader().header.scpValue.closeTime + 1)); + } + + // The sequence number and balance of an account on the LCL. + SequenceNumber + seqOf(TestAccount& account) + { + auto entry = lm().getLCLView()->getAccount(account.getPublicKey()); + REQUIRE(entry); + return entry.current().data.account().seqNum; + } + + bool + exists(TestAccount& account) + { + return static_cast( + lm().getLCLView()->getAccount(account.getPublicKey())); + } + + // A payment of `account` to the root account with the `n`-th sequence + // number after the account's current one. + TransactionFrameBasePtr + pay(TestAccount& account, int n, int64_t amount = 1, uint32_t fee = 100) + { + return transactionFromOperations( + *mApp, account.getSecretKey(), seqOf(account) + n, + {payment(mApp->getRoot()->getPublicKey(), amount)}, fee); + } + + TransactionFrameBasePtr + tx(TestAccount& account, int n, std::vector const& ops, + uint32_t fee = 100) + { + return transactionFromOperations( + *mApp, account.getSecretKey(), seqOf(account) + n, ops, + fee * static_cast(ops.size())); + } + + // A Soroban transaction of `account` that conflicts with no other + // (`seed` picks the entry it writes). + TransactionFrameBasePtr + upload(TestAccount& account, int n, uint64_t seed, + uint32_t inclusionFee = 100, uint32_t instructions = 1'000'000) + { + SorobanResources resources; + resources.instructions = instructions; + resources.diskReadBytes = 1000; + resources.writeBytes = 2000; + return createUploadWasmTx(*mApp, account, inclusionFee, + DEFAULT_TEST_RESOURCE_FEE, resources, + std::nullopt, 0, std::nullopt, + seqOf(account) + n, std::make_optional(seed)); + } + + // A set of classic transactions on top of the LCL, not validated. + TxSetXDRFrameConstPtr + setOf(std::vector const& txs) + { + auto res = testtxset::makeNonValidatedGeneralizedTxSet( + {{std::make_pair(std::nullopt, txs)}, {}}, *mApp, + lm().getLastClosedLedgerHeader().hash); + REQUIRE(res.first); + return res.first; + } + + StellarValue + valueFor(TxSetXDRFrameConstPtr const& txSet, + xdr::xvector const& upgrades = emptyUpgradeSteps, + std::optional closeTime = std::nullopt) + { + return herder().makeStellarValue( + txSet->getContentsHash(), closeTime ? *closeTime : nextCloseTime(), + upgrades, mCfg.NODE_SEED); + } + + // Cranks until `done`, giving background threads time to post to the + // main thread. + template + void + crankUntil(F&& done) + { + auto const deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (!done()) + { + REQUIRE(std::chrono::steady_clock::now() < deadline); + mClock.crank(false); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + void + crankFor(std::chrono::milliseconds duration) + { + auto const deadline = std::chrono::steady_clock::now() + duration; + while (std::chrono::steady_clock::now() < deadline) + { + mClock.crank(false); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + // The node emits a vote for `closing` as the next ledger. + void + vote(TxSetXDRFrameConstPtr const& closing, + xdr::xvector const& upgrades = emptyUpgradeSteps) + { + // By the time a node votes, SCP has asked for the slot's leaders, + // which brings the tracked quorum (the node set) up to date. + REQUIRE(herder().getHerderSCPDriver().getLeaderSchedule(nextSeq()) == + std::vector{mCfg.NODE_SEED.getPublicKey()}); + herder().getPendingEnvelopes().putTxSet(closing->getContentsHash(), + nextSeq(), closing); + SCPEnvelope envelope; + envelope.statement.slotIndex = nextSeq(); + envelope.statement.pledges.type(SCP_ST_PREPARE); + envelope.statement.pledges.prepare().ballot.counter = 1; + envelope.statement.pledges.prepare().ballot.value = + xdr::xdr_to_opaque(valueFor(closing, upgrades)); + Access::voted(herder(), envelope); + } + + // Waits until the selection the vote planned has started. + std::shared_ptr + waitStarted() + { + REQUIRE(Access::planned(herder())); + crankUntil([&]() { return Access::job(herder()) != nullptr; }); + return Access::job(herder()); + } + + // Waits until it has finished; returns whether it selected a set. + bool + waitSelected() + { + auto job = waitStarted(); + crankUntil([&]() { return job->mDone.load(); }); + std::lock_guard lock(job->mMutex); + return job->mSelection != nullptr; + } + + // The node votes for `closing` and selects the set of the ledger after. + void + voteAndSelect(TxSetXDRFrameConstPtr const& closing) + { + vote(closing); + REQUIRE(waitSelected()); + } + + // The next ledger closes with `closing`. The overlay drops a closed + // set's transactions from the mempool; with `keepInMempool` it has not + // got to it yet. + void + close(TxSetXDRFrameConstPtr const& closing, bool keepInMempool = false, + xdr::xvector const& upgrades = emptyUpgradeSteps, + std::optional closeTime = std::nullopt) + { + auto const seq = nextSeq(); + mApp->getHerder().externalizeValue( + closing, seq, closeTime ? *closeTime : nextCloseTime(), upgrades); + REQUIRE(lm().getLastClosedLedgerNum() == seq); + if (!keepInMempool) + { + std::set closed; + for (auto const& hash : closing->getTransactionFullHashes()) + { + closed.insert(hash); + } + std::vector left; + for (auto const& tx : mMempool) + { + if (closed.count(tx->getFullHash()) == 0) + { + left.push_back(tx); + } + } + mMempool = std::move(left); + } + } + + // The node's set for the next ledger. + Built + build() + { + auto const completed = txsetMeter(*mApp, "early-completed"); + auto prepared = Access::build(herder(), nextSeq(), nextCloseTime()); + Built built; + built.mTxSet = prepared.txSet; + built.mApplicable = std::move(prepared.applicableTxSet); + built.mInvalid = prepared.invalidTxHashes; + built.mEarly = txsetMeter(*mApp, "early-completed") == completed + 1; + REQUIRE(built.mTxSet); + REQUIRE(built.mApplicable); + return built; + } + + // Property a: the set passes the validation every node applies to a + // set, in full (structure and every transaction), on the LCL. + TxSetValidationResult + validate(Built const& built) + { + // As a node that receives the set: read from its bytes. (build() + // asks for a close time one second after the LCL's.) + auto received = TxSetXDRFrame::makeFromWire( + GeneralizedTransactionSet(built.mTxSet->getGeneralizedXDR())); + REQUIRE(received->getContentsHash() == built.mTxSet->getContentsHash()); + auto applicable = received->prepareForApply( + *mApp, lm().getLastClosedLedgerHeader().header); + REQUIRE(applicable); + REQUIRE(hashesOf(*applicable) == built.hashes()); + auto const result = applicable->checkValidWithResult(*mApp, 1, 1); + // The set the node keeps says the same. + REQUIRE(built.mApplicable->checkValidWithResult(*mApp, 1, 1) == result); + return result; + } + + // The set a build from scratch selects out of `txs` on the LCL. + Built + buildFrom(std::vector const& txs) + { + auto mempool = mMempool; + mMempool = txs; + auto built = build(); + mMempool = std::move(mempool); + REQUIRE(!built.mEarly); + return built; + } + + // Closes the next ledger with a set this node proposed and checks that + // it was applied: the set was not skipped as invalid, and every one of + // its transactions consumed its sequence number. + void + closeAndCheckApplied(Built const& built) + { + REQUIRE(validate(built) == TxSetValidationResult::VALID); + auto const skipped = applyCounter(*mApp, "txset-skipped"); + std::vector> expected; + for (auto const& phase : built.mApplicable->getPhases()) + { + for (auto const& tx : phase) + { + expected.emplace_back(tx->getSourceID(), tx->getSeqNum()); + } + } + close(built.mTxSet); + REQUIRE(applyCounter(*mApp, "txset-skipped") == skipped); + auto view = lm().getLCLView(); + for (auto const& [account, seq] : expected) + { + auto entry = view->getAccount(account); + // (an account that merged itself away is gone) + if (entry) + { + REQUIRE(entry.current().data.account().seqNum >= seq); + } + } + } +}; + +} + +// --------------------------------------------------------------------------- +// The mechanism. +// --------------------------------------------------------------------------- + +TEST_CASE("the next tx set is selected while the current ledger is in its " + "ballot", + "[herder][txset][early-txset]") +{ + EarlyFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto closingA = f.pay(f.acc(0), 1); + auto nextA = f.pay(f.acc(0), 2); + auto plainB = f.pay(f.acc(1), 1); + auto closing = f.setOf({closingA}); + // The closing ledger's transaction is still in the mempool. + f.mMempool = {closingA, nextA, plainB}; + + auto const planned = txsetMeter(app, "early-planned"); + auto const started = txsetMeter(app, "early-started"); + f.vote(closing); + REQUIRE(Access::planned(herder)); + REQUIRE(txsetMeter(app, "early-planned") == planned + 1); + // Nothing is pulled before the selection starts. + REQUIRE(f.mPulls.empty()); + REQUIRE(f.waitSelected()); + REQUIRE(txsetMeter(app, "early-started") == started + 1); + // One pull: the capacity of a set plus the margin, plus the closing + // ledger's transactions. + REQUIRE(f.mPulls == std::vector{Access::pullCount(herder) + 1}); + + { + auto job = Access::job(herder); + std::lock_guard lock(job->mMutex); + auto const& selection = *job->mSelection; + REQUIRE(selection.mLedgerSeq == f.nextSeq() + 1); + std::set selected; + for (auto const& phase : selection.mPhases) + { + for (auto const& tx : phase) + { + selected.insert(tx->getFullHash()); + } + } + // A's next payment is only valid once the closing ledger has + // applied: the selection is made on what that ledger will do. + REQUIRE(selected == hashesOf({nextA, plainB})); + REQUIRE(selection.mLeftovers.empty()); + } + + SECTION("the ledger closes with the value: the selection is completed") + { + f.close(closing); + f.mPulls.clear(); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({nextA, plainB})); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + REQUIRE(built.mTxSet->previousLedgerHash() == + f.lm().getLastClosedLedgerHeader().hash); + REQUIRE(built.mInvalid.empty()); + // The set had room: the mempool was asked once for what arrived + // since (nothing). + REQUIRE(f.mPulls == std::vector{Access::pullCount(herder)}); + // The same set a build from scratch selects. + REQUIRE(f.buildFrom({nextA, plainB}).hashes() == built.hashes()); + f.closeAndCheckApplied(built); + // The plan is used up. + REQUIRE(!Access::hasPlan(herder)); + REQUIRE(!Access::job(herder)); + } + SECTION("a second build on the same ledger is a build") + { + f.close(closing); + REQUIRE(f.build().mEarly); + auto const again = f.build(); + REQUIRE(!again.mEarly); + REQUIRE(again.hashes() == hashesOf({nextA, plainB})); + } + SECTION("statements for the same value change nothing") + { + auto job = Access::job(herder); + f.vote(closing); + REQUIRE(Access::job(herder) == job); + REQUIRE(txsetMeter(app, "early-planned") == planned + 1); + } +} + +TEST_CASE("when a tx set is selected early", "[herder][txset][early-txset]") +{ + SECTION("not when the setting is off") + { + EarlyFixture f(2, std::nullopt, [](Config& cfg) { + cfg.EXPERIMENTAL_EARLY_TX_SET_SELECTION = false; + }); + auto closingA = f.pay(f.acc(0), 1); + auto nextA = f.pay(f.acc(0), 2); + f.mMempool = {closingA, nextA}; + auto closing = f.setOf({closingA}); + f.vote(closing); + REQUIRE(!Access::hasPlan(f.herder())); + f.close(closing); + auto const built = f.build(); + REQUIRE(!built.mEarly); + REQUIRE(built.hashes() == hashesOf({nextA})); + } + SECTION("not for a value with upgrades") + { + EarlyFixture f(2); + auto closingA = f.pay(f.acc(0), 1); + auto nextA = f.pay(f.acc(0), 2); + f.mMempool = {closingA, nextA}; + auto closing = f.setOf({closingA}); + xdr::xvector upgrades; + LedgerUpgrade upgrade(LEDGER_UPGRADE_BASE_FEE); + upgrade.newBaseFee() = 150; + upgrades.emplace_back(LedgerTestUtils::toUpgradeType(upgrade)); + f.vote(closing, upgrades); + // Looked at, and nothing planned. + REQUIRE(Access::hasPlan(f.herder())); + REQUIRE(!Access::planned(f.herder())); + f.crankFor(std::chrono::milliseconds(20)); + REQUIRE(!Access::job(f.herder())); + REQUIRE(f.mPulls.empty()); + } + SECTION("not for an empty set") + { + EarlyFixture f(2); + f.mMempool = {f.pay(f.acc(0), 1)}; + auto empty = + TxSetXDRFrame::makeEmpty(f.lm().getLastClosedLedgerHeader()); + f.vote(empty); + REQUIRE(!Access::planned(f.herder())); + } + SECTION("not for a ledger that is not the next one") + { + EarlyFixture f(2); + auto closingA = f.pay(f.acc(0), 1); + auto closing = f.setOf({closingA}); + SCPEnvelope envelope; + envelope.statement.slotIndex = f.nextSeq() + 1; + envelope.statement.pledges.type(SCP_ST_PREPARE); + envelope.statement.pledges.prepare().ballot.counter = 1; + envelope.statement.pledges.prepare().ballot.value = + xdr::xdr_to_opaque(f.valueFor(closing)); + Access::voted(f.herder(), envelope); + REQUIRE(!Access::hasPlan(f.herder())); + } + SECTION("the start is timed to end shortly before the set is needed") + { + using std::chrono::milliseconds; + EarlyFixture f(1); + auto& herder = f.herder(); + Access::setStartDelay(herder, std::nullopt); + // Nothing measured: at once. + Access::setTimings(herder, milliseconds(0), milliseconds(0), + milliseconds(0), milliseconds(0)); + REQUIRE(Access::startDelay(herder) == milliseconds(0)); + Access::setTimings(herder, milliseconds(800), milliseconds(800), + milliseconds(0), milliseconds(0)); + REQUIRE(Access::startDelay(herder) == milliseconds(0)); + // 800 ms from the vote to the build, a selection takes 200 ms: + // 800 - 200 - 50 - 30. + Access::setTimings(herder, milliseconds(800), milliseconds(800), + milliseconds(200), milliseconds(200)); + REQUIRE(Access::startDelay(herder) == milliseconds(520)); + // The earlier of the two build times, the longer of the two jobs. + Access::setTimings(herder, milliseconds(800), milliseconds(700), + milliseconds(200), milliseconds(240)); + REQUIRE(Access::startDelay(herder) == + milliseconds(700 - 240 - 60 - 30)); + // Never negative. + Access::setTimings(herder, milliseconds(100), milliseconds(100), + milliseconds(200), milliseconds(200)); + REQUIRE(Access::startDelay(herder) == milliseconds(0)); + } + SECTION("a selection that has not started when the ledger closes") + { + EarlyFixture f(2); + auto& app = *f.mApp; + auto closingA = f.pay(f.acc(0), 1); + auto nextA = f.pay(f.acc(0), 2); + f.mMempool = {closingA, nextA}; + auto closing = f.setOf({closingA}); + // The start is due long after the close. + Access::setStartDelay(f.herder(), std::chrono::milliseconds(60'000)); + f.vote(closing); + REQUIRE(Access::planned(f.herder())); + f.close(closing); + auto const notStarted = txsetMeter(app, "early-not-started"); + auto const built = f.build(); + REQUIRE(!built.mEarly); + REQUIRE(txsetMeter(app, "early-not-started") == notStarted + 1); + REQUIRE(built.hashes() == hashesOf({nextA})); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + } +} + +// --------------------------------------------------------------------------- +// Property a: no invalid set, whatever the overlap between N and N+1. +// --------------------------------------------------------------------------- + +TEST_CASE("a set selected early is valid on the closed ledger whatever the " + "previous set held", + "[herder][txset][early-txset]") +{ + EarlyFixture f(8); + auto& herder = f.herder(); + auto& a = f.acc(0); + auto& b = f.acc(1); + auto& c = f.acc(2); + auto& d = f.acc(3); + + SECTION("the candidates are exactly the previous set's transactions") + { + std::vector txs{f.pay(a, 1), f.pay(b, 1), + f.pay(c, 1)}; + auto closing = f.setOf(txs); + f.mMempool = txs; + f.voteAndSelect(closing); + SECTION("the overlay has dropped them at the close") + { + f.close(closing); + } + SECTION("the overlay still holds them at the close") + { + f.close(closing, /*keepInMempool=*/true); + } + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes().empty()); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + } + SECTION("the same accounts with their next sequence numbers") + { + std::vector first{f.pay(a, 1), f.pay(b, 1), + f.pay(c, 1)}; + std::vector second{f.pay(a, 2), f.pay(b, 2), + f.pay(c, 2)}; + auto closing = f.setOf(first); + f.mMempool = first; + f.mMempool.insert(f.mMempool.end(), second.begin(), second.end()); + f.voteAndSelect(closing); + f.close(closing, /*keepInMempool=*/true); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf(second)); + f.closeAndCheckApplied(built); + REQUIRE(f.seqOf(a) == second[0]->getSeqNum()); + } + SECTION("a transaction of the previous set fails when it is applied") + { + // The payment is more than the account has: the transaction is + // valid, fails, and consumes its sequence number all the same. + auto failing = f.pay(a, 1, a.getBalance() * 2); + auto nextA = f.pay(a, 2); + auto closing = f.setOf({failing}); + f.mMempool = {failing, nextA}; + f.voteAndSelect(closing); + auto const balance = a.getBalance(); + f.close(closing); + // It failed: only the fee is gone. + REQUIRE(a.getBalance() == balance - 100); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({nextA})); + f.closeAndCheckApplied(built); + } + SECTION("another transaction with a sequence number the previous set " + "used") + { + auto closingA = f.pay(a, 1, 1); + auto sameSeq = f.pay(a, 1, 2); + auto plainB = f.pay(b, 1); + auto closing = f.setOf({closingA}); + f.mMempool = {closingA, sameSeq, plainB}; + f.voteAndSelect(closing); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({plainB})); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + } + SECTION("the previous set drains an account") + { + // Everything but the reserve goes: A cannot pay another fee. The + // selection cannot know (it predicts fees, not payments). + auto const spare = a.getAvailableBalance() - 100; + auto drain = f.pay(a, 1, spare); + auto nextA = f.pay(a, 2); + auto plainB = f.pay(b, 1); + auto closing = f.setOf({drain}); + f.mMempool = {drain, nextA, plainB}; + f.voteAndSelect(closing); + { + auto job = Access::job(herder); + std::lock_guard lock(job->mMutex); + REQUIRE(job->mSelection->mSelected == 2); + } + f.close(closing); + REQUIRE(a.getAvailableBalance() == 0); + SECTION("the validity pass removes its next transaction") + { + auto const removed = txsetMeter(*f.mApp, "early-removed", "tx"); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({plainB})); + REQUIRE(txsetMeter(*f.mApp, "early-removed", "tx") == removed + 1); + // It could not pay its fee: out of the mempool. + REQUIRE(built.mInvalid == std::vector{nextA->getFullHash()}); + f.closeAndCheckApplied(built); + } + SECTION("without the validity pass the set would be invalid") + { + Access::skipFinalPass(herder, true); + auto const built = f.build(); + Access::skipFinalPass(herder, false); + REQUIRE(built.mEarly); + REQUIRE(built.holds(nextA)); + REQUIRE(f.validate(built) != TxSetValidationResult::VALID); + } + } + SECTION("the previous set merges an account away") + { + auto merge = f.tx(a, 1, {accountMerge(b.getPublicKey())}); + auto nextA = f.pay(a, 2); + auto plainC = f.pay(c, 1); + auto closing = f.setOf({merge}); + f.mMempool = {merge, nextA, plainC}; + f.voteAndSelect(closing); + f.close(closing); + REQUIRE(!f.exists(a)); + SECTION("the validity pass removes its next transaction") + { + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({plainC})); + f.closeAndCheckApplied(built); + } + SECTION("without the validity pass the set would be invalid") + { + Access::skipFinalPass(herder, true); + auto const built = f.build(); + Access::skipFinalPass(herder, false); + REQUIRE(built.holds(nextA)); + REQUIRE(f.validate(built) != TxSetValidationResult::VALID); + } + } + SECTION("the previous set changes an account's signers") + { + // The master key loses its weight to another key: what A signed + // with it no longer counts. + auto other = SecretKey::pseudoRandomForTesting(); + auto rekey = f.tx(a, 1, + {setOptions(setSigner(makeSigner(other, 1))), + setOptions(setMasterWeight(0))}); + auto nextA = f.pay(a, 2); + auto plainC = f.pay(c, 1); + auto closing = f.setOf({rekey}); + f.mMempool = {rekey, nextA, plainC}; + f.voteAndSelect(closing); + f.close(closing); + REQUIRE(f.seqOf(a) == rekey->getSeqNum()); + SECTION("the validity pass removes what the old key signed") + { + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({plainC})); + f.closeAndCheckApplied(built); + } + SECTION("without the validity pass the set would be invalid") + { + Access::skipFinalPass(herder, true); + auto const built = f.build(); + Access::skipFinalPass(herder, false); + REQUIRE(built.holds(nextA)); + REQUIRE(f.validate(built) != TxSetValidationResult::VALID); + } + } + SECTION("fee bumps") + { + // D pays for A's transactions. In the previous set it also sends + // away all it has but one more fee bump's worth. + auto innerClosing = f.pay(a, 1); + auto bumpClosing = feeBump(*f.mApp, d, innerClosing, 200); + auto const fees = bumpClosing->getFullFee(); + auto drainD = f.pay(d, 1, d.getAvailableBalance() - 100 - 2 * fees); + auto closing = f.setOf({bumpClosing, drainD}); + // Two fee bumps by D for the next ledger: it can pay for one. + auto bumpNextA = feeBump(*f.mApp, d, f.pay(a, 2), 200); + auto bumpB = feeBump(*f.mApp, d, f.pay(b, 1), 200); + auto plainC = f.pay(c, 1); + f.mMempool = {bumpClosing, drainD, bumpNextA, bumpB, plainC}; + f.voteAndSelect(closing); + f.close(closing); + REQUIRE(f.seqOf(a) == innerClosing->getSeqNum()); + REQUIRE(d.getAvailableBalance() == fees); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.holds(plainC)); + // One of D's two, whichever comes first, and not the other. + REQUIRE(built.holds(bumpNextA)); + REQUIRE(!built.holds(bumpB)); + f.closeAndCheckApplied(built); + } + SECTION("the previous set's own transactions were not applied") + { + // The set this node voted for is not valid (a sequence number too + // far ahead): the ledger closes with it and applies nothing. What + // the selection predicted of it is wrong for every account. + auto closingA = f.pay(a, 1); + auto bad = f.pay(c, 5); + auto nextA = f.pay(a, 2); + auto plainB = f.pay(b, 1); + auto closing = f.setOf({closingA, bad}); + f.mMempool = {closingA, bad, nextA, plainB}; + f.voteAndSelect(closing); + auto const skipped = applyCounter(*f.mApp, "txset-skipped"); + auto const seqA = f.seqOf(a); + f.close(closing); + REQUIRE(applyCounter(*f.mApp, "txset-skipped") == skipped + 1); + REQUIRE(f.seqOf(a) == seqA); + auto const built = f.build(); + REQUIRE(built.mEarly); + // A's second payment waits for its first. + REQUIRE(built.hashes() == hashesOf({plainB})); + f.closeAndCheckApplied(built); + } +} + +TEST_CASE("a set selected early is valid for any overlap with the previous " + "set", + "[herder][txset][early-txset]") +{ + // Per account, one of: nothing in the previous set; its next transaction + // in the previous set; and as candidates any of: the transaction after + // that, another one with the sequence number the previous set uses, one + // far ahead. In random order, for a random share of the accounts. + size_t const ACCOUNTS = 24; + EarlyFixture f(ACCOUNTS); + auto const seed = GENERATE(1u, 2u, 3u, 4u, 5u, 6u); + auto const overlapPercent = GENERATE(0, 30, 70, 100); + CAPTURE(seed, overlapPercent); + std::mt19937 gen(seed * 1000 + static_cast(overlapPercent)); + auto chance = [&](int percent) { + return stellar::uniform_int_distribution(0, 99)(gen) < percent; + }; + + std::vector previous; + std::vector candidates; + // What must be in the next set, and what must not. + std::vector valid; + std::vector invalid; + for (size_t i = 0; i < ACCOUNTS; ++i) + { + auto& account = f.acc(i); + bool const inPrevious = chance(overlapPercent); + if (inPrevious) + { + previous.push_back(f.pay(account, 1, 1)); + if (chance(60)) + { + valid.push_back(f.pay(account, 2)); + candidates.push_back(valid.back()); + } + if (chance(30)) + { + invalid.push_back(f.pay(account, 1, 7)); + candidates.push_back(invalid.back()); + } + } + else if (chance(70)) + { + valid.push_back(f.pay(account, 1)); + candidates.push_back(valid.back()); + } + if (chance(20)) + { + invalid.push_back(f.pay(account, 9)); + candidates.push_back(invalid.back()); + } + } + if (previous.empty()) + { + // (an empty set plans no selection) + return; + } + f.mMempool = previous; + f.mMempool.insert(f.mMempool.end(), candidates.begin(), candidates.end()); + stellar::shuffle(f.mMempool.begin(), f.mMempool.end(), gen); + + auto closing = f.setOf(previous); + f.voteAndSelect(closing); + f.close(closing, /*keepInMempool=*/chance(50)); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + auto const hashes = built.hashes(); + for (auto const& tx : previous) + { + REQUIRE(hashes.count(tx->getFullHash()) == 0); + } + for (auto const& tx : invalid) + { + REQUIRE(hashes.count(tx->getFullHash()) == 0); + } + // Nothing is left behind that a build would take: it is the set a build + // from scratch selects on the closed ledger once the closed set's + // transactions have left the mempool. (A candidate that reuses a + // sequence number shadows the account's next one in both: one + // transaction per account, the lowest sequence number first.) + std::vector stillPending; + { + auto const closed = hashesOf(previous); + for (auto const& tx : f.mMempool) + { + if (closed.count(tx->getFullHash()) == 0) + { + stillPending.push_back(tx); + } + } + } + REQUIRE(hashes == f.buildFrom(stillPending).hashes()); + std::set validHashes = hashesOf(valid); + for (auto const& hash : hashes) + { + REQUIRE(validHashes.count(hash) == 1); + } + f.closeAndCheckApplied(built); +} + +TEST_CASE("a set selected early is not used when the ledger closes " + "otherwise", + "[herder][txset][early-txset]") +{ + EarlyFixture f(4); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = f.acc(0); + auto& b = f.acc(1); + auto closingA = f.pay(a, 1); + auto nextA = f.pay(a, 2); + auto plainB = f.pay(b, 1); + auto closing = f.setOf({closingA}); + f.mMempool = {closingA, nextA, plainB}; + f.voteAndSelect(closing); + auto const discarded = txsetMeter(app, "early-discarded"); + + SECTION("another set closes the ledger") + { + f.close(f.setOf({plainB})); + REQUIRE(txsetMeter(app, "early-discarded") == discarded + 1); + REQUIRE(!Access::hasPlan(herder)); + auto const built = f.build(); + REQUIRE(!built.mEarly); + // A's first payment is still to come. + REQUIRE(built.hashes() == hashesOf({closingA})); + f.closeAndCheckApplied(built); + } + SECTION("the same set closes the ledger with another close time") + { + auto const later = + ConsensusTime::fromApplyTime(ApplyTime::fromTimePoint( + f.lm().getLastClosedLedgerHeader().header.scpValue.closeTime + + 3)); + f.close(closing, false, emptyUpgradeSteps, later); + REQUIRE(txsetMeter(app, "early-discarded") == discarded + 1); + auto const built = f.build(); + REQUIRE(!built.mEarly); + REQUIRE(built.hashes() == hashesOf({nextA, plainB})); + } + SECTION("the ledger closes with an upgrade that lowers the limits") + { + xdr::xvector upgrades; + LedgerUpgrade upgrade(LEDGER_UPGRADE_MAX_TX_SET_SIZE); + upgrade.newMaxTxSetSize() = 1; + upgrades.emplace_back(LedgerTestUtils::toUpgradeType(upgrade)); + f.close(closing, false, upgrades); + REQUIRE(f.lm().getLastClosedLedgerHeader().header.maxTxSetSize == 1); + REQUIRE(txsetMeter(app, "early-discarded") == discarded + 1); + auto const built = f.build(); + REQUIRE(!built.mEarly); + // The selection held two transactions; the ledger takes one. + REQUIRE(built.hashes().size() == 1); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); + } + SECTION("the limits of the closed ledger are not the ones it assumed") + { + // Cannot happen without an upgrade; checked all the same. + { + auto job = Access::job(herder); + std::lock_guard lock(job->mMutex); + job->mSelection->mLimits.mLclHeader.maxTxSetSize += 1; + } + f.close(closing); + auto const built = f.build(); + REQUIRE(!built.mEarly); + REQUIRE(txsetMeter(app, "early-discarded") == discarded + 1); + REQUIRE(built.hashes() == hashesOf({nextA, plainB})); + } + SECTION("the node votes for another value before the ledger closes") + { + // The round's leader timed out; the next one proposes another set. + auto first = Access::job(herder); + auto other = f.setOf({plainB}); + f.vote(other); + // The first selection is abandoned ... + REQUIRE(first->mStop.load()); + REQUIRE(Access::job(herder) != first); + // ... and one for the new value made. + REQUIRE(f.waitSelected()); + SECTION("the ledger closes with the new value") + { + f.close(other); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({closingA})); + f.closeAndCheckApplied(built); + } + SECTION("the ledger closes with the first value after all") + { + f.close(closing); + REQUIRE(txsetMeter(app, "early-discarded") == discarded + 1); + auto const built = f.build(); + REQUIRE(!built.mEarly); + REQUIRE(built.hashes() == hashesOf({nextA, plainB})); + f.closeAndCheckApplied(built); + } + } +} + +TEST_CASE("a selection that is still running when it is needed", + "[herder][txset][early-txset]") +{ + EarlyFixture f(4); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = f.acc(0); + auto& b = f.acc(1); + auto closingA = f.pay(a, 1); + auto nextA = f.pay(a, 2); + auto afterNextA = f.pay(a, 3); + auto plainB = f.pay(b, 1); + auto closing = f.setOf({closingA}); + f.mMempool = {closingA, nextA, plainB}; + + // The job's tasks wait until the test lets them go. + auto hold = std::make_shared>(true); + Access::hold(herder, hold); + f.vote(closing); + auto first = f.waitStarted(); + REQUIRE(!first->mDone.load()); + REQUIRE(Access::jobDuration(herder).first == std::chrono::milliseconds(0)); + f.close(closing); + f.crankFor(std::chrono::milliseconds(20)); + auto const notReady = txsetMeter(app, "early-not-ready"); + auto const built = f.build(); + // The ledger gets a set built the usual way; nobody waits for the job. + REQUIRE(!built.mEarly); + REQUIRE(txsetMeter(app, "early-not-ready") == notReady + 1); + // The job had 20 ms and more and was not done: the node now takes a + // selection to last half as long again, so the next one starts earlier. + REQUIRE(Access::jobDuration(herder).first >= std::chrono::milliseconds(30)); + REQUIRE(Access::jobDuration(herder).second >= + std::chrono::milliseconds(30)); + REQUIRE(built.hashes() == hashesOf({nextA, plainB})); + REQUIRE(first->mStop.load()); + REQUIRE(!Access::job(herder)); + + SECTION("it finishes after the build and hands nothing over") + { + hold->store(false); + f.crankUntil([&]() { return first->mDone.load(); }); + std::lock_guard lock(first->mMutex); + REQUIRE(!first->mSelection); + } + SECTION("it is still running when the next selection starts") + { + // The next ledger is in its ballot, with this node's set. + f.mMempool = {nextA, plainB, afterNextA}; + Access::hold(herder, nullptr); + f.vote(built.mTxSet); + REQUIRE(f.waitSelected()); + auto second = Access::job(herder); + REQUIRE(second != first); + // Only now does the first job get to run. + hold->store(false); + f.crankUntil([&]() { return first->mDone.load(); }); + { + std::lock_guard lock(first->mMutex); + REQUIRE(!first->mSelection); + } + REQUIRE(Access::job(herder) == second); + f.closeAndCheckApplied(built); + auto const next = f.build(); + REQUIRE(next.mEarly); + REQUIRE(next.hashes() == hashesOf({afterNextA})); + f.closeAndCheckApplied(next); + } + hold->store(false); +} + +TEST_CASE("the application shuts down while a selection runs", + "[herder][txset][early-txset]") +{ + auto f = std::make_unique(4); + auto closingA = f->pay(f->acc(0), 1); + f->mMempool = {closingA, f->pay(f->acc(0), 2), f->pay(f->acc(1), 1)}; + auto hold = std::make_shared>(true); + Access::hold(f->herder(), hold); + f->vote(f->setOf({closingA})); + auto job = f->waitStarted(); + REQUIRE(!job->mDone.load()); + // The tasks are let go while the application is going down: they find + // the job stopped, or finish it; either way nothing is left running + // when the application is gone. + std::thread release([hold]() { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + hold->store(false); + }); + f.reset(); + release.join(); + REQUIRE(job->mDone.load()); +} + +TEST_CASE("candidates whose validity depends on the close time in a set " + "selected early", + "[herder][txset][early-txset]") +{ + EarlyFixture f(4); + auto& app = *f.mApp; + auto& a = f.acc(0); + auto& b = f.acc(1); + auto& c = f.acc(2); + auto closingA = f.pay(a, 1); + auto closing = f.setOf({closingA}); + auto const lclCloseTime = + f.lm().getLastClosedLedgerHeader().header.scpValue.closeTime; + auto timeBounded = [&](TestAccount& account, TimePoint maxTime) { + PreconditionsV2 cond; + cond.timeBounds.activate().minTime = 0; + cond.timeBounds.activate().maxTime = maxTime; + return TransactionFrameBasePtr(transactionFromOperationsV1( + app, account.getSecretKey(), f.seqOf(account) + 1, + {payment(app.getRoot()->getPublicKey(), 1)}, 100, cond)); + }; + // The closing ledger gets close time +1 and the next +2 (seconds after + // the LCL's). B's transaction expires before the next ledger, C's lasts. + auto expiring = timeBounded(b, lclCloseTime + 1); + auto lasting = timeBounded(c, lclCloseTime + 1000); + REQUIRE(txValidityDependsOnCloseTime(expiring->getEnvelope())); + f.mMempool = {closingA, expiring, lasting}; + f.voteAndSelect(closing); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + // Whatever close time the selection assumed, the completed set is + // validated for the one it gets. + REQUIRE(built.hashes() == hashesOf({lasting})); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); +} + +// --------------------------------------------------------------------------- +// Property b: full sets. +// --------------------------------------------------------------------------- + +TEST_CASE("a set selected early is topped up when the ledger has closed", + "[herder][txset][early-txset]") +{ + EarlyFixture f(10, 6); + auto& app = *f.mApp; + auto& herder = f.herder(); + REQUIRE(f.lm().getLastMaxTxSetSizeOps() == 6); + auto closingA = f.pay(f.acc(0), 1); + auto closing = f.setOf({closingA}); + std::vector early{f.pay(f.acc(1), 1), + f.pay(f.acc(2), 1)}; + f.mMempool = {closingA, early[0], early[1]}; + f.voteAndSelect(closing); + auto const added = txsetMeter(app, "early-added", "tx"); + + SECTION("with what arrived after the selection's pull") + { + std::vector late{ + f.pay(f.acc(3), 1), f.pay(f.acc(4), 1), f.pay(f.acc(0), 2)}; + f.mMempool.insert(f.mMempool.end(), late.begin(), late.end()); + f.close(closing); + f.mPulls.clear(); + auto const built = f.build(); + REQUIRE(built.mEarly); + auto all = early; + all.insert(all.end(), late.begin(), late.end()); + REQUIRE(built.hashes() == hashesOf(all)); + REQUIRE(txsetMeter(app, "early-added", "tx") == added + 3); + REQUIRE(f.mPulls == std::vector{Access::pullCount(herder)}); + f.closeAndCheckApplied(built); + } + SECTION("up to its capacity, and no further") + { + std::vector late; + for (size_t i = 3; i < 10; ++i) + { + late.push_back(f.pay(f.acc(i), 1)); + } + f.mMempool.insert(f.mMempool.end(), late.begin(), late.end()); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes().size() == 6); + REQUIRE(built.holds(early[0])); + REQUIRE(built.holds(early[1])); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); + } + SECTION("a new transaction of an account that is in the set already " + "waits") + { + auto second = f.pay(f.acc(1), 2); + auto plain = f.pay(f.acc(5), 1); + f.mMempool.push_back(second); + f.mMempool.push_back(plain); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({early[0], early[1], plain})); + f.closeAndCheckApplied(built); + } + SECTION("a full phase asks for nothing more") + { + // Another selection, with more candidates than the set takes. + f.close(closing); + f.closeAndCheckApplied(f.build()); + auto closingB = f.pay(f.acc(1), 1); + auto closing2 = f.setOf({closingB}); + f.mMempool = {closingB}; + for (size_t i = 2; i < 10; ++i) + { + f.mMempool.push_back(f.pay(f.acc(i), 1)); + } + f.voteAndSelect(closing2); + f.close(closing2); + f.mPulls.clear(); + auto const added = txsetMeter(app, "early-added", "tx"); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes().size() == 6); + REQUIRE(txsetMeter(app, "early-added", "tx") == added); + // One look at the mempool's top, for the Soroban phase, which has + // room and never had a candidate: nothing there for it, so no + // deeper pull. + REQUIRE(f.mPulls == std::vector{Access::pullCount(herder)}); + f.closeAndCheckApplied(built); + } +} + +TEST_CASE("a set selected early is refilled when transactions of it turn " + "out invalid", + "[herder][txset][early-txset]") +{ + // A set takes 10 transactions; the mempool holds 40 candidates beside + // the closing ledger's. The closing ledger drains the accounts of some + // of them, which only the closed ledger shows. + size_t const CAPACITY = 10; + size_t const CANDIDATES = 40; + EarlyFixture f(CANDIDATES + 1, CAPACITY); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto const drainedPercent = GENERATE(0, 10, 30, 60, 100); + CAPTURE(drainedPercent); + + // Candidate i comes from account i; the closing set drains the first + // `drained` of those accounts. + size_t const drained = CANDIDATES * drainedPercent / 100; + std::vector previous; + std::vector candidates; + for (size_t i = 0; i < CANDIDATES; ++i) + { + auto& account = f.acc(i); + if (i < drained) + { + previous.push_back( + f.pay(account, 1, account.getAvailableBalance() - 100)); + candidates.push_back(f.pay(account, 2)); + } + else + { + candidates.push_back(f.pay(account, 1)); + } + } + if (previous.empty()) + { + previous.push_back(f.pay(f.acc(CANDIDATES), 1)); + } + f.mMempool = previous; + f.mMempool.insert(f.mMempool.end(), candidates.begin(), candidates.end()); + auto closing = f.setOf(previous); + f.voteAndSelect(closing); + f.close(closing); + + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + auto const hashes = built.hashes(); + for (size_t i = 0; i < drained; ++i) + { + REQUIRE(hashes.count(candidates[i]->getFullHash()) == 0); + } + // Full whenever enough valid candidates exist. + size_t const validCandidates = CANDIDATES - drained; + REQUIRE(hashes.size() == std::min(CAPACITY, validCandidates)); + auto& fill = + app.getMetrics().NewHistogram({"herder", "txset", "fill-classic"}); + REQUIRE(fill.count() >= 1); + f.closeAndCheckApplied(built); +} + +TEST_CASE("a build pulls the capacity of a set plus a margin and refills", + "[herder][txset][early-txset]") +{ + size_t const CAPACITY = 20; + auto const marginPercent = GENERATE(0u, 10u, 50u); + CAPTURE(marginPercent); + size_t const ACCOUNTS = 120; + EarlyFixture f(ACCOUNTS, CAPACITY, [&](Config& cfg) { + cfg.EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT = marginPercent; + // Builds only. + cfg.EXPERIMENTAL_EARLY_TX_SET_SELECTION = false; + }); + auto& app = *f.mApp; + auto& herder = f.herder(); + // The capacity of both phases together, and the margin on top. + size_t const total = + CAPACITY + + f.lm().getLastClosedSorobanNetworkConfig().ledgerMaxTxCount(); + size_t const pull = total + (total * marginPercent + 99) / 100; + REQUIRE(Access::pullCount(herder) == pull); + + SECTION("one pull when the mempool holds less than it asks for") + { + for (size_t i = 0; i < 30; ++i) + { + f.mMempool.push_back(f.pay(f.acc(i), 1)); + } + auto const built = f.build(); + REQUIRE(f.mPulls == std::vector{pull}); + REQUIRE(built.hashes().size() == CAPACITY); + } + SECTION("candidates lost to the one-per-account rule are refilled too") + { + // The top of the mempool is one account's transactions with one + // sequence number, each valid by itself; a set takes one of them. + for (size_t i = 0; i < pull; ++i) + { + f.mMempool.push_back( + f.pay(f.acc(0), 1, static_cast(i + 1))); + } + for (size_t i = 1; i < 60; ++i) + { + f.mMempool.push_back(f.pay(f.acc(i), 1)); + } + auto const built = f.build(); + REQUIRE(f.mPulls.size() == 2); + REQUIRE(built.hashes().size() == CAPACITY); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + } + SECTION("X% of the candidates are invalid") + { + // The mempool holds more than three pulls can reach; every k-th + // candidate is invalid (a sequence number of the past). + auto const invalidPercent = GENERATE(0, 5, 20, 50, 90); + CAPTURE(invalidPercent); + auto const refills = txsetMeter(app, "refill-pull", "pull"); + std::mt19937 gen(42); + size_t valid = 0; + std::vector spent; + // One spent sequence number per account to make invalid ones from. + for (size_t i = 0; i < ACCOUNTS; ++i) + { + bool const bad = stellar::uniform_int_distribution(0, 99)( + gen) < invalidPercent; + if (bad) + { + f.mMempool.push_back(f.pay(f.acc(i), 0)); + } + else + { + f.mMempool.push_back(f.pay(f.acc(i), 1)); + ++valid; + } + } + auto const built = f.build(); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + // Count the valid candidates within reach of the pulls made. + size_t const reached = std::min(f.mPulls.back(), ACCOUNTS); + size_t validReached = 0; + for (size_t i = 0; i < reached; ++i) + { + if (f.mMempool[i]->getSeqNum() == f.seqOf(f.acc(i)) + 1) + { + ++validReached; + } + } + REQUIRE(built.hashes().size() == + std::min(CAPACITY, validReached)); + // Each pull asks for twice the one before. + for (size_t i = 0; i < f.mPulls.size(); ++i) + { + REQUIRE(f.mPulls[i] == pull << i); + } + REQUIRE(f.mPulls.size() <= 3); + REQUIRE(txsetMeter(app, "refill-pull", "pull") == + refills + static_cast(f.mPulls.size()) - 1); + // The set is full whenever the mempool holds enough valid + // transactions within three pulls. + if (valid >= CAPACITY && ACCOUNTS <= (pull << 2)) + { + REQUIRE(built.hashes().size() == CAPACITY); + } + } +} + +TEST_CASE("a phase that lost candidates is refilled from below a mempool top " + "of the other phase", + "[herder][txset][early-txset]") +{ + // The mempool has one fee order for both phases. A phase that lost + // candidates looks deeper also when the top holds little of its kind: + // what it needs may lie below the other phase's transactions. + size_t const CAPACITY = 10; + size_t const SOROBAN_CAPACITY = 10; + SECTION("a completion") + { + EarlyFixture f(60, CAPACITY); + auto& app = *f.mApp; + auto& herder = f.herder(); + overrideSorobanNetworkConfigForTest(app); + modifySorobanNetworkConfig(app, [&](SorobanNetworkConfig& cfg) { + cfg.mLedgerMaxTxCount = SOROBAN_CAPACITY; + }); + size_t const pull = Access::pullCount(herder); + REQUIRE(pull >= CAPACITY + SOROBAN_CAPACITY); + // The closing set drains accounts 0 and 1. The selection holds + // their next payments and two more: four classic candidates, all + // there are. + std::vector previous; + std::vector candidates; + for (size_t i = 0; i < 2; ++i) + { + auto& account = f.acc(i); + previous.push_back( + f.pay(account, 1, account.getAvailableBalance() - 100)); + candidates.push_back(f.pay(account, 2)); + } + candidates.push_back(f.pay(f.acc(2), 1)); + candidates.push_back(f.pay(f.acc(3), 1)); + f.mMempool = previous; + f.mMempool.insert(f.mMempool.end(), candidates.begin(), + candidates.end()); + auto closing = f.setOf(previous); + f.voteAndSelect(closing); + f.close(closing); + // During the ballot: more Soroban transactions than a pull holds, + // and below them classic ones. + for (size_t i = 0; i < pull + 4; ++i) + { + f.mMempool.push_back(f.upload(f.acc(4 + i), 1, 100 + i)); + } + std::vector below; + for (size_t i = 0; i < 8; ++i) + { + below.push_back(f.pay(f.acc(4 + pull + 4 + i), 1)); + } + f.mMempool.insert(f.mMempool.end(), below.begin(), below.end()); + auto const refills = txsetMeter(app, "refill-pull", "pull"); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + REQUIRE(!built.holds(candidates[0])); + REQUIRE(!built.holds(candidates[1])); + REQUIRE(built.holds(candidates[2])); + REQUIRE(built.holds(candidates[3])); + // The two it lost are made up for from below. + REQUIRE(txsetMeter(app, "refill-pull", "pull") > refills); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::CLASSIC) > 2); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == + SOROBAN_CAPACITY); + f.closeAndCheckApplied(built); + } + SECTION("a build") + { + EarlyFixture f(120, CAPACITY, [&](Config& cfg) { + cfg.EXPERIMENTAL_EARLY_TX_SET_SELECTION = false; + }); + auto& app = *f.mApp; + auto& herder = f.herder(); + overrideSorobanNetworkConfigForTest(app); + modifySorobanNetworkConfig(app, [&](SorobanNetworkConfig& cfg) { + cfg.mLedgerMaxTxCount = SOROBAN_CAPACITY; + }); + size_t const pull = Access::pullCount(herder); + // The top: six classic candidates, four of them with a spent + // sequence number, then Soroban transactions to the end of the + // pull. Below: valid classic ones. + for (size_t i = 0; i < 6; ++i) + { + f.mMempool.push_back(f.pay(f.acc(i), i < 4 ? 0 : 1)); + } + for (size_t i = 6; i < pull; ++i) + { + f.mMempool.push_back(f.upload(f.acc(i), 1, 100 + i)); + } + for (size_t i = 0; i < 12; ++i) + { + f.mMempool.push_back(f.pay(f.acc(pull + i), 1)); + } + auto const built = f.build(); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + REQUIRE(f.mPulls.size() == 2); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::CLASSIC) == CAPACITY); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == + SOROBAN_CAPACITY); + } +} + +TEST_CASE("what the top-up may not add to a set selected early", + "[herder][txset][early-txset]") +{ + EarlyFixture f(8); + auto& app = *f.mApp; + auto& a = f.acc(0); + auto& b = f.acc(1); + auto& c = f.acc(2); + auto& d = f.acc(3); + auto closingA = f.pay(a, 1); + auto closing = f.setOf({closingA}); + + SECTION("a transaction its fee source cannot pay for on top of the " + "selected ones") + { + // D pays the fees of other accounts' transactions. The closing + // ledger leaves it with what two fee bumps cost (the selection + // cannot know: it predicts fees, not payments). One bump by D is + // selected; two more arrive later. D can pay for one of them. + auto bumpB = feeBump(app, d, f.pay(b, 1), 200); + auto const fees = bumpB->getFullFee(); + auto drainD = f.pay(d, 1, d.getAvailableBalance() - 100 - 2 * fees); + auto closing2 = f.setOf({closingA, drainD}); + auto bumpLate1 = feeBump(app, d, f.pay(c, 1), 200); + auto bumpLate2 = feeBump(app, d, f.pay(f.acc(4), 1), 200); + auto plain = f.pay(f.acc(5), 1); + f.mMempool = {closingA, drainD, bumpB}; + f.voteAndSelect(closing2); + f.close(closing2); + REQUIRE(d.getAvailableBalance() == 2 * fees); + f.mMempool.push_back(bumpLate1); + f.mMempool.push_back(bumpLate2); + f.mMempool.push_back(plain); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.holds(bumpB)); + REQUIRE(built.holds(plain)); + // The selected bump counts against D's balance when the late ones + // are validated: one more fits, the next does not. + REQUIRE(built.holds(bumpLate1)); + REQUIRE(!built.holds(bumpLate2)); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); + } + SECTION("a second transaction of an account that has one in the set") + { + // Another transaction of B with the sequence number of the selected + // one: each is valid on the closed ledger, together they are not a + // set. + auto selectedB = f.pay(b, 1, 1); + auto otherB = f.pay(b, 1, 2); + auto plain = f.pay(c, 1); + f.mMempool = {closingA, selectedB}; + f.voteAndSelect(closing); + f.close(closing); + f.mMempool.push_back(otherB); + f.mMempool.push_back(plain); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({selectedB, plain})); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + REQUIRE(!Access::selfCheckLatched(f.herder())); + f.closeAndCheckApplied(built); + } +} + +TEST_CASE("the base fee of a set selected early follows what was left out", + "[herder][txset][early-txset]") +{ + // A set takes 4 transactions. Fees: 500, 400, 300, 200 selected; 150 + // left over. + EarlyFixture f(8, 4); + auto& app = *f.mApp; + auto closingA = f.pay(f.acc(0), 1); + auto closing = f.setOf({closingA}); + auto t500 = f.pay(f.acc(1), 1, 1, 500); + auto t400 = f.pay(f.acc(2), 1, 1, 400); + auto t300 = f.pay(f.acc(3), 1, 1, 300); + auto t150 = f.pay(f.acc(5), 1, 1, 150); + auto baseFeeOf = [&](Built const& built, + TransactionFrameBasePtr const& tx) { + for (auto const& candidate : + built.mApplicable->getPhase(TxSetPhase::CLASSIC)) + { + if (candidate->getFullHash() == tx->getFullHash()) + { + auto fee = built.mApplicable->getTxBaseFee(candidate); + REQUIRE(fee); + return *fee; + } + } + REQUIRE(false); + return int64_t(0); + }; + auto const ledgerBaseFee = + f.lm().getLastClosedLedgerHeader().header.baseFee; + + SECTION("untouched: the lowest fee selected, because one was left out") + { + auto t200 = f.pay(f.acc(4), 1, 1, 200); + f.mMempool = {closingA, t500, t400, t300, t200, t150}; + f.voteAndSelect(closing); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({t500, t400, t300, t200})); + REQUIRE(baseFeeOf(built, t500) == 200); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + } + SECTION("one removed and replaced by the left-over: its fee, nothing " + "is left out any more") + { + // The account of the 200 transaction is drained by the closing + // ledger. + auto& drained = f.acc(4); + auto drain = f.pay(drained, 1, drained.getAvailableBalance() - 100); + auto t200 = f.pay(drained, 2, 1, 200); + auto closing2 = f.setOf({drain}); + f.mMempool = {drain, t500, t400, t300, t200, t150}; + f.voteAndSelect(closing2); + f.close(closing2); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({t500, t400, t300, t150})); + // Everything valid is in: nobody competed for the room. + REQUIRE(baseFeeOf(built, t500) == ledgerBaseFee); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + // The same as a build from scratch. + auto const fresh = f.buildFrom({t500, t400, t300, t200, t150}); + REQUIRE(fresh.hashes() == built.hashes()); + REQUIRE(baseFeeOf(fresh, t500) == ledgerBaseFee); + f.closeAndCheckApplied(built); + } + SECTION("more arrive than fit: the lowest fee that got in") + { + f.mMempool = {closingA, t500, t400}; + f.voteAndSelect(closing); + f.close(closing); + auto t350 = f.pay(f.acc(4), 1, 1, 350); + auto t250 = f.pay(f.acc(6), 1, 1, 250); + f.mMempool = {t500, t400, t350, t300, t250, t150}; + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({t500, t400, t350, t300})); + REQUIRE(baseFeeOf(built, t500) == 300); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); + } +} + +// --------------------------------------------------------------------------- +// Soroban: the parallel phase is extended in place. +// --------------------------------------------------------------------------- + +TEST_CASE("a parallel Soroban phase is extended without selecting it again", + "[herder][txset][early-txset][soroban]") +{ + EarlyFixture f(12); + auto& app = *f.mApp; + // Limits under which the transactions of this test are valid. + overrideSorobanNetworkConfigForTest(app); + auto const lcl = f.lm().getLastClosedLedgerHeader(); + auto const version = lcl.header.ledgerVersion; + SorobanNetworkConfig config = f.lm().getLastClosedSorobanNetworkConfig(); + // No limit but the ones each section sets. + auto laneLimit = config.maxLedgerResources(); + for (auto type : + {Resource::Type::OPERATIONS, Resource::Type::INSTRUCTIONS, + Resource::Type::TX_BYTE_SIZE, Resource::Type::DISK_READ_BYTES, + Resource::Type::WRITE_BYTES, Resource::Type::READ_LEDGER_ENTRIES, + Resource::Type::WRITE_LEDGER_ENTRIES}) + { + laneLimit.setVal(type, std::numeric_limits::max() / 4); + } + + // The stages an extender made, as a set: validated like any set. + auto validity = [&](TxStageFrameList const& stages, + SorobanNetworkConfig const& cfg) { + auto set = testtxset::makeNonValidatedGeneralizedTxSet( + {}, std::make_optional(lcl.header.baseFee), stages, app, + lcl.hash); + REQUIRE(set.second); + TxSetValidationContext ctx = TxSetValidationContext::forLCL(app); + ctx.mSorobanConfig = &cfg; + return set.second->checkValidWithResult(app, ctx, 0, 0); + }; + auto count = [](TxStageFrameList const& stages) { + size_t n = 0; + for (auto const& stage : stages) + { + for (auto const& cluster : stage) + { + n += cluster.size(); + } + } + return n; + }; + + SECTION("independent transactions spread over the clusters") + { + config.mLedgerMaxDependentTxClusters = 4; + config.mLedgerMaxInstructions = 3'000'000; + ParallelPhaseExtender extender({}, config, laneLimit, version); + // 4 clusters of 3,000,000 instructions take 12 transactions of + // 1,000,000. + for (size_t i = 0; i < 12; ++i) + { + REQUIRE(extender.tryAdd(f.upload(f.acc(i), 1, 100 + i))); + } + REQUIRE(extender.sizeTx() == 12); + REQUIRE(extender.getStages().size() == 1); + REQUIRE(extender.getStages()[0].size() == 4); + for (auto const& cluster : extender.getStages()[0]) + { + REQUIRE(cluster.size() == 3); + } + REQUIRE(validity(extender.getStages(), config) == + TxSetValidationResult::VALID); + // The thirteenth does not fit the instruction limit. + REQUIRE(!extender.tryAdd(f.upload(f.acc(0), 2, 999))); + REQUIRE(extender.sizeTx() == 12); + } + SECTION("a transaction joins the cluster it conflicts with") + { + config.mLedgerMaxDependentTxClusters = 4; + config.mLedgerMaxInstructions = 10'000'000; + ParallelPhaseExtender extender({}, config, laneLimit, version); + // Same seed: the same entry written. + REQUIRE(extender.tryAdd(f.upload(f.acc(0), 1, 7))); + REQUIRE(extender.tryAdd(f.upload(f.acc(1), 1, 8))); + REQUIRE(extender.tryAdd(f.upload(f.acc(2), 1, 7))); + auto const& stage = extender.getStages().at(0); + REQUIRE(stage.size() == 2); + REQUIRE(stage[0].size() == 2); + REQUIRE(stage[1].size() == 1); + REQUIRE(validity(extender.getStages(), config) == + TxSetValidationResult::VALID); + } + SECTION("a transaction that would join two clusters is left out") + { + config.mLedgerMaxDependentTxClusters = 4; + config.mLedgerMaxInstructions = 10'000'000; + auto first = f.upload(f.acc(0), 1, 7); + auto second = f.upload(f.acc(1), 1, 8); + // Writes what the two others write. + SorobanResources resources; + resources.instructions = 1'000'000; + resources.diskReadBytes = 1000; + resources.writeBytes = 4000; + resources.footprint.readWrite = + first->sorobanResources().footprint.readWrite; + resources.footprint.readWrite.push_back( + second->sorobanResources().footprint.readWrite.at(0)); + auto both = createUploadWasmTx(app, f.acc(2), 100, + DEFAULT_TEST_RESOURCE_FEE, resources); + ParallelPhaseExtender extender({}, config, laneLimit, version); + REQUIRE(extender.tryAdd(first)); + REQUIRE(extender.tryAdd(second)); + REQUIRE(!extender.tryAdd(both)); + REQUIRE(extender.sizeTx() == 2); + // Had it been added to either cluster, the phase would not be + // valid. + auto broken = extender.getStages(); + broken[0][0].push_back(both); + REQUIRE(validity(broken, config) == + TxSetValidationResult::TX_ORDERING_INVALID); + } + SECTION("a reader conflicts with the writer, not with other readers") + { + config.mLedgerMaxDependentTxClusters = 4; + config.mLedgerMaxInstructions = 10'000'000; + auto writer = f.upload(f.acc(0), 1, 7); + auto const key = writer->sorobanResources().footprint.readWrite.at(0); + auto reader = [&](TestAccount& account, uint64_t seed) { + SorobanResources resources; + resources.instructions = 1'000'000; + resources.diskReadBytes = 1000; + resources.writeBytes = 2000; + auto own = f.upload(account, 1, seed); + resources.footprint.readWrite = + own->sorobanResources().footprint.readWrite; + resources.footprint.readOnly = {key}; + return createUploadWasmTx(app, account, 100, + DEFAULT_TEST_RESOURCE_FEE, resources, + std::nullopt, 0, std::nullopt, + std::nullopt, std::make_optional(seed)); + }; + SECTION("readers first") + { + ParallelPhaseExtender extender({}, config, laneLimit, version); + REQUIRE(extender.tryAdd(reader(f.acc(1), 11))); + REQUIRE(extender.tryAdd(reader(f.acc(2), 12))); + // Two readers, two clusters: the writer would join both. + REQUIRE(extender.getStages().at(0).size() == 2); + REQUIRE(!extender.tryAdd(writer)); + REQUIRE(validity(extender.getStages(), config) == + TxSetValidationResult::VALID); + } + SECTION("the writer first") + { + ParallelPhaseExtender extender({}, config, laneLimit, version); + REQUIRE(extender.tryAdd(writer)); + REQUIRE(extender.tryAdd(reader(f.acc(1), 11))); + REQUIRE(extender.tryAdd(reader(f.acc(2), 12))); + // Both readers are with the writer. + REQUIRE(extender.getStages().at(0).size() == 1); + REQUIRE(extender.getStages()[0][0].size() == 3); + REQUIRE(validity(extender.getStages(), config) == + TxSetValidationResult::VALID); + } + } + SECTION("the lane's limits hold") + { + config.mLedgerMaxDependentTxClusters = 4; + config.mLedgerMaxInstructions = 100'000'000; + auto limit = laneLimit; + // Three transactions. + limit.setVal(Resource::Type::OPERATIONS, 3); + ParallelPhaseExtender extender({}, config, limit, version); + REQUIRE(extender.tryAdd(f.upload(f.acc(0), 1, 1))); + REQUIRE(extender.tryAdd(f.upload(f.acc(1), 1, 2))); + REQUIRE(extender.tryAdd(f.upload(f.acc(2), 1, 3))); + REQUIRE(!extender.tryAdd(f.upload(f.acc(3), 1, 4))); + REQUIRE(count(extender.getStages()) == 3); + } + SECTION("a transaction joins the cluster of an existing phase it " + "conflicts with") + { + config.mLedgerMaxDependentTxClusters = 4; + config.mLedgerMaxInstructions = 10'000'000; + // One stage, two clusters, as a selection made them. + TxStageFrameList stages{ + {{f.upload(f.acc(0), 1, 1)}, {f.upload(f.acc(1), 1, 2)}}}; + ParallelPhaseExtender extender(stages, config, laneLimit, version); + // Writes what the second cluster writes. + REQUIRE(extender.tryAdd(f.upload(f.acc(2), 1, 2))); + REQUIRE(extender.getStages().at(0).size() == 2); + REQUIRE(extender.getStages()[0][0].size() == 1); + REQUIRE(extender.getStages()[0][1].size() == 2); + REQUIRE(validity(extender.getStages(), config) == + TxSetValidationResult::VALID); + // In the other cluster the phase would not be valid. + auto broken = stages; + broken[0][0].push_back(f.upload(f.acc(2), 1, 2)); + REQUIRE(validity(broken, config) == + TxSetValidationResult::TX_ORDERING_INVALID); + } + SECTION("stages that exist are extended, and stay within the limit") + { + config.mLedgerMaxDependentTxClusters = 2; + config.mLedgerMaxInstructions = 4'000'000; + // Two stages of one cluster each: 2,000,000 + 1,000,000 sequential + // instructions of 4,000,000. + TxStageFrameList stages{ + {{f.upload(f.acc(0), 1, 1), f.upload(f.acc(1), 1, 2)}}, + {{f.upload(f.acc(2), 1, 3)}}}; + ParallelPhaseExtender extender(stages, config, laneLimit, version); + REQUIRE(extender.sizeTx() == 3); + // A new cluster beside the first stage's: no more sequential + // instructions. + REQUIRE(extender.tryAdd(f.upload(f.acc(3), 1, 4))); + REQUIRE(extender.tryAdd(f.upload(f.acc(4), 1, 5))); + REQUIRE(extender.getStages()[0].size() == 2); + // The first stage is at 2,000,000 in both clusters: the next goes + // where a stage can still grow. + size_t added = 0; + for (size_t i = 5; i < 12; ++i) + { + if (extender.tryAdd(f.upload(f.acc(i), 1, 10 + i))) + { + ++added; + } + } + REQUIRE(validity(extender.getStages(), config) == + TxSetValidationResult::VALID); + // 4,000,000 sequential instructions in two clusters per stage: + // 8 transactions at most. + REQUIRE(extender.sizeTx() == 8); + REQUIRE(added == 3); + } +} + +TEST_CASE("a set selected early holds Soroban transactions and is topped up " + "with them", + "[herder][txset][early-txset][soroban]") +{ + EarlyFixture f(14); + auto& app = *f.mApp; + auto& herder = f.herder(); + overrideSorobanNetworkConfigForTest(app); + modifySorobanNetworkConfig( + app, [](SorobanNetworkConfig& cfg) { cfg.mLedgerMaxTxCount = 8; }); + REQUIRE(f.lm().getLastClosedSorobanNetworkConfig().ledgerMaxTxCount() == 8); + auto closingA = f.pay(f.acc(0), 1); + auto closing = f.setOf({closingA}); + std::vector early{f.upload(f.acc(1), 1, 1), + f.upload(f.acc(2), 1, 2), + f.upload(f.acc(3), 1, 3)}; + auto classic = f.pay(f.acc(4), 1); + f.mMempool = {closingA, early[0], classic, early[1], early[2]}; + f.voteAndSelect(closing); + + SECTION("candidates that arrive later are added to the phase") + { + std::vector late{f.upload(f.acc(5), 1, 5), + f.upload(f.acc(6), 1, 6)}; + f.mMempool.insert(f.mMempool.end(), late.begin(), late.end()); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + auto all = early; + all.insert(all.end(), late.begin(), late.end()); + all.push_back(classic); + REQUIRE(built.hashes() == hashesOf(all)); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == 5); + f.closeAndCheckApplied(built); + } + SECTION("one that conflicts with a selected one joins its cluster") + { + // Writes the entry the first selected transaction writes. + auto conflicting = f.upload(f.acc(5), 1, 1); + f.mMempool.push_back(conflicting); + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.holds(conflicting)); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == 4); + REQUIRE(!Access::selfCheckLatched(herder)); + f.closeAndCheckApplied(built); + } + SECTION("up to the phase's capacity") + { + for (size_t i = 5; i < 14; ++i) + { + f.mMempool.push_back(f.upload(f.acc(i), 1, 10 + i)); + } + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == 8); + REQUIRE(built.holds(classic)); + f.closeAndCheckApplied(built); + } + SECTION("one that the closed ledger made invalid is replaced") + { + // Account 1 is drained by the closing ledger... which is not what + // closes: another set does, with the same account's payment. + f.close(closing); + auto const built = f.build(); + REQUIRE(built.mEarly); + f.closeAndCheckApplied(built); + // Next round: the closing set drains account 5, whose Soroban + // transaction the selection picks. + auto& drainedAccount = f.acc(5); + auto drain = f.pay(drainedAccount, 1, + drainedAccount.getAvailableBalance() - 100); + auto closing2 = f.setOf({drain}); + auto doomed = f.upload(drainedAccount, 2, 50); + std::vector others; + for (size_t i = 6; i < 14; ++i) + { + others.push_back(f.upload(f.acc(i), 1, 60 + i)); + } + // The selection is full with `doomed` and seven others; one more + // is left over. + f.mMempool = {drain, doomed}; + f.mMempool.insert(f.mMempool.end(), others.begin(), others.end()); + f.voteAndSelect(closing2); + { + auto job = Access::job(herder); + std::lock_guard lock(job->mMutex); + REQUIRE(job->mSelection->mSelected == 8); + REQUIRE(job->mSelection->mLeftovers.size() == 1); + } + f.close(closing2); + f.mPulls.clear(); + auto const next = f.build(); + REQUIRE(next.mEarly); + REQUIRE(!next.holds(doomed)); + // Full again, from what the selection had left over. The classic + // phase, which never had a candidate, looks at the mempool's top + // once. + REQUIRE(next.mApplicable->sizeTx(TxSetPhase::SOROBAN) == 8); + REQUIRE(next.hashes() == hashesOf(others)); + REQUIRE(f.mPulls == std::vector{Access::pullCount(herder)}); + f.closeAndCheckApplied(next); + } +} + +TEST_CASE("a phase the selection could not fill is filled at the close", + "[herder][txset][early-txset][soroban]") +{ + EarlyFixture f(14); + auto& app = *f.mApp; + auto& herder = f.herder(); + overrideSorobanNetworkConfigForTest(app); + auto closingA = f.pay(f.acc(0), 1); + auto closing = f.setOf({closingA}); + + SECTION("a phase without a single candidate when the selection was made") + { + modifySorobanNetworkConfig( + app, [](SorobanNetworkConfig& cfg) { cfg.mLedgerMaxTxCount = 8; }); + closingA = f.pay(f.acc(0), 1); + closing = f.setOf({closingA}); + // Classic candidates only. + auto classic = f.pay(f.acc(1), 1); + f.mMempool = {closingA, classic}; + f.voteAndSelect(closing); + f.close(closing); + // Soroban transactions arrive during the ballot. + std::vector late{f.upload(f.acc(2), 1, 2), + f.upload(f.acc(3), 1, 3)}; + f.mMempool.insert(f.mMempool.end(), late.begin(), late.end()); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.hashes() == hashesOf({classic, late[0], late[1]})); + f.closeAndCheckApplied(built); + } + SECTION("a selection that left out a transaction for its cluster's " + "limit") + { + // Two clusters of 10,000,000 instructions. Three transactions of + // 5,000,000 on one entry: two fill a cluster, the third has + // nowhere to go, while the other cluster is empty. + modifySorobanNetworkConfig(app, [](SorobanNetworkConfig& cfg) { + cfg.mLedgerMaxTxCount = 8; + cfg.mLedgerMaxDependentTxClusters = 2; + cfg.mTxMaxInstructions = 10'000'000; + cfg.mLedgerMaxInstructions = 10'000'000; + }); + closingA = f.pay(f.acc(0), 1); + closing = f.setOf({closingA}); + std::vector hot{ + f.upload(f.acc(1), 1, 7, 100, 5'000'000), + f.upload(f.acc(2), 1, 7, 100, 5'000'000), + f.upload(f.acc(3), 1, 7, 100, 5'000'000)}; + f.mMempool = {closingA, hot[0], hot[1], hot[2]}; + f.voteAndSelect(closing); + { + auto job = Access::job(herder); + std::lock_guard lock(job->mMutex); + REQUIRE(job->mSelection->mSelected == 2); + REQUIRE(job->mSelection->mHadTxNotFittingLane.at(1).at(0)); + } + f.close(closing); + // Two that write other entries arrive: they have room. + std::vector late{ + f.upload(f.acc(4), 1, 8, 100, 5'000'000), + f.upload(f.acc(5), 1, 9, 100, 5'000'000)}; + f.mMempool.insert(f.mMempool.end(), late.begin(), late.end()); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.holds(late[0])); + REQUIRE(built.holds(late[1])); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == 4); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); + } + SECTION("many left-overs with nowhere to go do not close the phase") + { + // As above, with ten transactions on the one entry: eight valid + // left-overs that fit nowhere. The phase is not full for that: two + // that write other entries still have the second cluster. + modifySorobanNetworkConfig(app, [](SorobanNetworkConfig& cfg) { + cfg.mLedgerMaxTxCount = 16; + cfg.mLedgerMaxDependentTxClusters = 2; + cfg.mTxMaxInstructions = 10'000'000; + cfg.mLedgerMaxInstructions = 10'000'000; + }); + closingA = f.pay(f.acc(0), 1); + closing = f.setOf({closingA}); + f.mMempool = {closingA}; + for (size_t i = 1; i <= 10; ++i) + { + f.mMempool.push_back(f.upload(f.acc(i), 1, 7, 100, 5'000'000)); + } + f.voteAndSelect(closing); + { + auto job = Access::job(herder); + std::lock_guard lock(job->mMutex); + REQUIRE(job->mSelection->mSelected == 2); + } + f.close(closing); + std::vector late{ + f.upload(f.acc(11), 1, 8, 100, 5'000'000), + f.upload(f.acc(12), 1, 9, 100, 5'000'000)}; + f.mMempool.insert(f.mMempool.end(), late.begin(), late.end()); + auto const built = f.build(); + REQUIRE(built.mEarly); + REQUIRE(built.holds(late[0])); + REQUIRE(built.holds(late[1])); + REQUIRE(built.mApplicable->sizeTx(TxSetPhase::SOROBAN) == 4); + REQUIRE(f.validate(built) == TxSetValidationResult::VALID); + f.closeAndCheckApplied(built); + } +} + +// --------------------------------------------------------------------------- +// The prediction. +// --------------------------------------------------------------------------- + +TEST_CASE("what the transaction set of a ledger predicts of the ledger", + "[herder][txset][early-txset]") +{ + EarlyFixture f(4); + auto& app = *f.mApp; + auto& a = f.acc(0); + auto& b = f.acc(1); + auto& c = f.acc(2); + auto payA = f.pay(a, 1, 5, 300); + auto bump = feeBump(app, c, f.pay(b, 1), 500); + PredictedAccountChanges changes; + predictAccountChanges(*payA, changes); + predictAccountChanges(*bump, changes); + REQUIRE(changes.size() == 3); + // A: its sequence number and its fee. + REQUIRE(changes[a.getPublicKey()].mSeqNum == payA->getSeqNum()); + REQUIRE(changes[a.getPublicKey()].mFees == 300); + // B: the sequence number of the inner transaction, no fee. + REQUIRE(changes[b.getPublicKey()].mSeqNum == f.seqOf(b) + 1); + REQUIRE(changes[b.getPublicKey()].mFees == 0); + // C: the fee bump's fee, no sequence number. + REQUIRE(changes[c.getPublicKey()].mSeqNum == 0); + REQUIRE(changes[c.getPublicKey()].mFees == bump->getFullFee()); + + auto const& lcl = f.lm().getLastClosedLedgerHeader(); + auto header = std::make_shared(lcl.header); + header->ledgerSeq += 1; + auto view = makePredictedLedgerView( + f.lm().getLastClosedLedgerState(), header, + std::make_shared(changes), + app.getMetrics()); + REQUIRE(view->getLedgerHeader().current().ledgerSeq == + lcl.header.ledgerSeq + 1); + auto lclView = f.lm().getLCLView(); + auto account = [&](AbstractLedgerView const& v, TestAccount& acc) { + auto entry = v.getAccount(acc.getPublicKey()); + REQUIRE(entry); + return entry.current().data.account(); + }; + REQUIRE(account(*view, a).seqNum == payA->getSeqNum()); + REQUIRE(account(*view, a).balance == account(*lclView, a).balance - 300); + REQUIRE(account(*view, b).seqNum == account(*lclView, b).seqNum + 1); + REQUIRE(account(*view, b).balance == account(*lclView, b).balance); + REQUIRE(account(*view, c).seqNum == account(*lclView, c).seqNum); + REQUIRE(account(*view, c).balance == + account(*lclView, c).balance - bump->getFullFee()); + // An account the set does not name is the last closed ledger's. + REQUIRE(account(*view, f.acc(3)) == account(*lclView, f.acc(3))); + // An account that does not exist does not appear. + REQUIRE( + !view->getAccount(SecretKey::pseudoRandomForTesting().getPublicKey())); + // Nothing is predicted about other entries. + LedgerKey code(CONTRACT_CODE); + REQUIRE_THROWS_AS(view->load(code), std::logic_error); +} + +// --------------------------------------------------------------------------- +// In a network. +// --------------------------------------------------------------------------- + +TEST_CASE("leaders select their next set early in a network", + "[herder][txset][early-txset][simulation]") +{ + Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE); + auto simulation = Topologies::core(4, 0.75, networkID, [](int i) { + auto cfg = getTestConfig(i, Config::TESTDB_BUCKET_DB_PERSISTENT); + cfg.LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + return cfg; + }); + auto nodes = simulation->getNodes(); + simulation->startAllNodes(); + + // Every node's mempool holds the root account's next payments: one fits + // each ledger (one transaction per source account), and the one after it + // only becomes valid when that ledger has applied. A leader that selects + // early gets the next one right only by predicting what the ledger in + // its ballot does. + auto const rootKey = nodes[0]->getRoot()->getSecretKey(); + auto const rootID = rootKey.getPublicKey(); + auto const firstSeq = nodes[0]->getRoot()->getLastSequenceNumber(); + std::vector payments; + for (int i = 1; i <= 400; ++i) + { + payments.push_back(transactionFromOperations(*nodes[0], rootKey, + firstSeq + i, + {payment(rootID, 1)}, 100) + ->getEnvelope()); + } + auto rootSeq = [&](Application& app) { + auto root = app.getLedgerManager().getLCLView()->getAccount(rootID); + REQUIRE(root); + return root.current().data.account().seqNum; + }; + for (auto const& node : nodes) + { + auto* app = node.get(); + auto& herder = static_cast(app->getHerder()); + // Virtual time runs ahead of the job's threads: start at the vote. + Access::setStartDelay(herder, std::chrono::milliseconds(0)); + herder.mGetTopTransactionsForTesting = [&, app](size_t) { + std::vector top; + auto const next = static_cast(rootSeq(*app) - firstSeq); + for (size_t i = next; i < next + 4 && i < payments.size(); ++i) + { + top.push_back(payments[i]); + } + return top; + }; + } + + auto& lm = nodes[1]->getLedgerManager(); + uint32_t const target = lm.getLastClosedLedgerNum() + 40; + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(target, 2); }, + 80 * simulation->getExpectedLedgerCloseTime(), false); + + int64_t planned = 0; + int64_t completed = 0; + int64_t notReady = 0; + int64_t failed = 0; + int64_t selfCheckFailed = 0; + for (auto const& node : nodes) + { + planned += txsetMeter(*node, "early-planned"); + completed += txsetMeter(*node, "early-completed"); + notReady += txsetMeter(*node, "early-not-ready") + + txsetMeter(*node, "early-not-started"); + failed += txsetMeter(*node, "early-failed"); + selfCheckFailed += txsetMeter(*node, "self-check-failed"); + // No leader proposed a set that apply found invalid. + REQUIRE(applyCounter(*node, "txset-skipped") == 0); + } + INFO(fmt::format("planned {}, completed {}, not ready {}", planned, + completed, notReady)); + REQUIRE(failed == 0); + REQUIRE(selfCheckFailed == 0); + REQUIRE(planned > 0); + // How often the selection is ready depends on the machine (the jobs run + // in real time, the network in virtual time). + REQUIRE(completed + notReady > 0); + // The ledgers carried the payments. + REQUIRE(rootSeq(*nodes[1]) - firstSeq >= 20); + for (auto const& node : nodes) + { + static_cast(node->getHerder()) + .mGetTopTransactionsForTesting = nullptr; + } +} + +// What the leader's path to its next set costs at perfnet's scale, with real +// SAC payments (never validated by Core before: the Rust overlay admitted +// them): a build as it is now, and the completion of a set that was selected +// while the previous ledger was in its ballot, with a share of the candidates +// arriving only after the selection's pull. +// Env: BENCH_CAP (transactions per set, default 8000), BENCH_LATE_PERCENT +// (share of a ledger's candidates that arrive after the selection's pull, +// default 30), BENCH_BACKLOG (1: the mempool holds twice the capacity at the +// build, as in a backlog; default 0), BENCH_REPS (default 2), BENCH_TASKS +// (validation tasks, default 16), BENCH_CLUSTERS (default 16). +TEST_CASE("early tx set completion costs", "[herder][.][benchmark]") +{ + auto envInt = [](char const* name, uint32_t dflt) { + auto v = std::getenv(name); + return v ? static_cast(std::stoul(v)) : dflt; + }; + uint32_t const cap = envInt("BENCH_CAP", 8000); + uint32_t const latePercent = envInt("BENCH_LATE_PERCENT", 30); + bool const backlog = envInt("BENCH_BACKLOG", 0) != 0; + uint32_t const reps = envInt("BENCH_REPS", 2); + uint32_t const tasks = envInt("BENCH_TASKS", 16); + uint32_t const clusters = envInt("BENCH_CLUSTERS", 16); + // Per repetition: a closing ledger and the next one's candidates (twice + // as many in a backlog), at 80% of the cap as on perfnet. + uint32_t const perLedger = cap * 8 / 10; + uint32_t const nextCount = backlog ? 2 * cap : perLedger; + uint32_t const perRep = perLedger + nextCount; + uint32_t const candidates = (reps + 1) * perRep; + + auto cfg = getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT); + cfg.APPLY_LOAD_MODE = ApplyLoadMode::BENCHMARK_MODEL_TX; + cfg.APPLY_LOAD_MODEL_TX = ApplyLoadModelTx::SAC; + cfg.USE_CONFIG_FOR_GENESIS = true; + cfg.LEDGER_PROTOCOL_VERSION = Config::CURRENT_LEDGER_PROTOCOL_VERSION; + cfg.IGNORE_MESSAGE_LIMITS_FOR_TESTING = true; + cfg.APPLY_LOAD_MAX_SOROBAN_TX_COUNT = cap; + cfg.APPLY_LOAD_BATCH_SAC_COUNT = 1; + cfg.APPLY_LOAD_LEDGER_MAX_DEPENDENT_TX_CLUSTERS = clusters; + cfg.APPLY_LOAD_CLASSIC_TXS_PER_LEDGER = 0; + cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = 100; + // ApplyLoad makes one account per candidate it is configured for. + cfg.SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER = candidates / cap + 1; + cfg.GENESIS_TEST_ACCOUNT_COUNT = + cap * cfg.SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER + 10; + cfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = false; + cfg.DISABLE_TX_META_FOR_TESTING = true; + cfg.METADATA_DEBUG_LEDGERS = 0; + cfg.WORKER_THREADS = static_cast(tasks) + 2; + VirtualClock clock(VirtualClock::REAL_TIME); + auto app = createTestApplication(clock, cfg); + auto& herder = static_cast(app->getHerder()); + auto& lm = app->getLedgerManager(); + auto& metrics = app->getMetrics(); + Access::setStartDelay(herder, std::chrono::milliseconds(0)); + + std::vector envelopes; + { + ApplyLoad al(*app); + envelopes = al.generateSacPaymentEnvelopesForTesting(candidates); + } + REQUIRE(envelopes.size() == candidates); + app->getBatchExecutor().setPreferredTaskCountForTesting(tasks); + REQUIRE(lm.getLastClosedSorobanNetworkConfig().ledgerMaxTxCount() == cap); + + // The mempool: a range of the envelopes, handed over as the overlay + // encodes a pull ([count:4]([len:4][xdr])*), encoded ahead of time. + std::vector> encoded; + encoded.reserve(envelopes.size()); + for (auto const& env : envelopes) + { + encoded.emplace_back(xdr::xdr_to_opaque(env)); + } + std::vector mempool; + herder.mGetTopTransactionsPayloadForTesting = [&](size_t count) { + uint32_t const n = + static_cast(std::min(count, mempool.size())); + std::vector payload(4); + std::memcpy(payload.data(), &n, 4); + for (uint32_t i = 0; i < n; ++i) + { + auto const& bytes = encoded[mempool[i]]; + uint32_t const len = static_cast(bytes.size()); + auto const at = payload.size(); + payload.resize(at + 4 + len); + std::memcpy(payload.data() + at, &len, 4); + std::memcpy(payload.data() + at + 4, bytes.data(), len); + } + return payload; + }; + auto range = [&](size_t from, size_t count) { + std::vector res(count); + std::iota(res.begin(), res.end(), from); + return res; + }; + auto nextCloseTime = [&]() { + auto const& lcl = lm.getLastClosedLedgerHeader(); + return getConsensusTime(lcl.header.scpValue) + .next(lcl.header.ledgerVersion); + }; + auto crankUntil = [&](std::function done) { + auto const deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(120); + while (!done()) + { + REQUIRE(std::chrono::steady_clock::now() < deadline); + clock.crank(false); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }; + std::vector const stages = { + "build-pull", "build-frames", "build-refill-check", + "build-trim", "build-select", "early-final-pass", + "early-top-up", "build-selection-check", "build-xdr", + "build-handoff", "early-complete", "build", + "early-pull", "early-select", "early-job"}; + auto clearStages = [&]() { + for (auto const& stage : stages) + { + metrics.NewTimer({"herder", "txset", stage}).Clear(); + } + }; + std::map>> samples; + auto record = [&](std::string const& what) { + for (auto const& stage : stages) + { + auto& timer = metrics.NewTimer({"herder", "txset", stage}); + if (timer.count() > 0) + { + samples[what][stage].push_back(timer.sum()); + } + } + }; + + std::cout << "=== leader's next set: cap " << cap << ", " << perLedger + << " per ledger, " << (backlog ? "backlog, " : "") << latePercent + << "% arriving after the selection's pull, " << tasks + << " validation tasks, " << clusters + << " clusters ===" << std::endl; + size_t included = 0; + size_t builtSize = 0; + for (uint32_t rep = 0; rep < reps; ++rep) + { + size_t const base = rep * perRep; + auto const closingTxs = range(base, perLedger); + auto const nextTxs = range(base + perLedger, nextCount); + size_t const earlyCount = nextCount * (100 - latePercent) / 100; + + // The closing ledger's set, built from the mempool: a plain build. + // In a backlog the mempool holds twice the cap behind it. + mempool = closingTxs; + if (backlog) + { + auto more = + range(base + perLedger, std::min(nextCount, 2 * cap)); + mempool.insert(mempool.end(), more.begin(), more.end()); + } + PubKeyUtils::clearVerifySigCache(); + clearStages(); + auto closing = Access::build(herder, lm.getLastClosedLedgerNum() + 1, + nextCloseTime()); + record("1. build, cold signature cache"); + clearStages(); + auto closing2 = Access::build(herder, lm.getLastClosedLedgerNum() + 1, + nextCloseTime()); + record("2. build, warm signature cache"); + builtSize = closing.txSet->sizeTxTotal(); + + // The node votes for it; its early selection for the ledger after + // sees the closing ledger's transactions and the early arrivals. + mempool = closingTxs; + mempool.insert(mempool.end(), nextTxs.begin(), + nextTxs.begin() + earlyCount); + auto const seq = lm.getLastClosedLedgerNum() + 1; + auto const closeTime = nextCloseTime(); + herder.getPendingEnvelopes().putTxSet(closing.txSet->getContentsHash(), + seq, closing.txSet); + SCPEnvelope envelope; + envelope.statement.slotIndex = seq; + envelope.statement.pledges.type(SCP_ST_PREPARE); + envelope.statement.pledges.prepare().ballot.counter = 1; + envelope.statement.pledges.prepare().ballot.value = xdr::xdr_to_opaque( + herder.makeStellarValue(closing.txSet->getContentsHash(), closeTime, + emptyUpgradeSteps, cfg.NODE_SEED)); + clearStages(); + Access::voted(herder, envelope); + REQUIRE(Access::planned(herder)); + crankUntil([&]() { return Access::job(herder) != nullptr; }); + auto job = Access::job(herder); + crankUntil([&]() { return job->mDone.load(); }); + samples["3. early selection (background, during the ballot)"] + ["job, wall time"] + .push_back(static_cast(job->mFinishedAfterNs) / 1e6); + record("3. early selection (background, during the ballot)"); + + // The ledger closes; the rest of the candidates has arrived. + app->getHerder().externalizeValue(closing.txSet, seq, closeTime, + emptyUpgradeSteps); + REQUIRE(lm.getLastClosedLedgerNum() == seq); + mempool = nextTxs; + clearStages(); + auto const completedBefore = txsetMeter(*app, "early-completed"); + auto next = Access::build(herder, lm.getLastClosedLedgerNum() + 1, + nextCloseTime()); + REQUIRE(txsetMeter(*app, "early-completed") == completedBefore + 1); + record("4. completion at the close"); + included = next.txSet->sizeTxTotal(); + REQUIRE(next.applicableTxSet->checkValid(*app, 1, 1)); + REQUIRE(included == std::min(cap, nextCount)); + } + std::cout << "plain build: " << builtSize + << " transactions; completed set: " << included << " transactions" + << std::endl; + for (auto& [what, byStage] : samples) + { + std::cout << what << std::endl; + for (auto& [stage, v] : byStage) + { + std::sort(v.begin(), v.end()); + std::cout << " " << stage << ": median " << v[v.size() / 2] + << " ms (min " << v.front() << ", max " << v.back() + << ", n=" << v.size() << ")" << std::endl; + } + } + herder.mGetTopTransactionsPayloadForTesting = nullptr; +} diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index 012af8cb13..0f915dca98 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -73,6 +73,15 @@ using namespace stellar::txbridge; using namespace stellar::txtest; using namespace historytestutils; +namespace stellar +{ +// LeaderBuildTestAccess::selectLocalFirstLeader (defined further down), for +// the fixtures above it: replaces the second validator of a two-validator +// quorum set by one that sorts behind the local node in the next slot's +// leader order. +void makeLocalNodeFirstLeader(HerderImpl& herder); +} + static TransactionTestFramePtr makeMultiPayment(stellar::TestAccount& destAccount, stellar::TestAccount& src, int nbOps, int64 paymentBase, uint32 extraFee, uint32 feeMult) @@ -2960,6 +2969,10 @@ testSCPDriver(uint32 protocolVersion, uint32_t maxTxSetSize, size_t expectedOps) SECTION("validateValue txSet cached") { auto& herder = static_cast(app->getHerder()); + // Which of this fixture's two validators leads the next slot depends + // on the local node's key, i.e. on the test instance number; the + // expectations below are for the node that builds and proposes. + makeLocalNodeFirstLeader(herder); auto seq = herder.trackingConsensusLedgerIndex() + 1; auto& cache = herder.getHerderSCPDriver().getTxSetValidityCache(); @@ -5387,6 +5400,12 @@ namespace stellar class LeaderBuildTestAccess { public: + static void + checkInLine(HerderImpl& herder) + { + herder.mTxSetSelfCheckFailed = true; + } + static VirtualClock::time_point triggerAnchor(HerderImpl& herder, uint64_t slot) { @@ -5544,6 +5563,12 @@ class LeaderBuildTestAccess FAIL("Could not select a first-round local leader"); } }; + +void +makeLocalNodeFirstLeader(HerderImpl& herder) +{ + LeaderBuildTestAccess::selectLocalFirstLeader(herder); +} } TEST_CASE("trigger fallback credits elapsed time up to the first ballot", @@ -6008,10 +6033,17 @@ TEST_CASE("prepare proposal before trigger", "[herder][early-proposal]") tx->addSignature(alice); auto extra = makeTx(bob, alice); mempool = {tx->getEnvelope(), extra->getEnvelope()}; - size_t pulls = 0; - herder.mGetTopTransactionsForTesting = [&](size_t) { - ++pulls; - return mempool; + herder.mGetTopTransactionsForTesting = [&](size_t) { return mempool; }; + // Builds of a tx set since here. (Not pulls of the mempool: the leader + // starts executing a prepared set at once, and that execution pulls + // candidates for the ledger after it.) + auto const buildsBefore = + app->getMetrics().NewTimer({"herder", "txset", "build"}).count(); + auto builds = [&]() { + return app->getMetrics() + .NewTimer({"herder", "txset", "build"}) + .count() - + buildsBefore; }; // Exercise the real post-close hook, including its heartbeat reset after // advancing the fixture clock past the upgrade helper's close time. @@ -6023,7 +6055,7 @@ TEST_CASE("prepare proposal before trigger", "[herder][early-proposal]") [&]() { return LeaderBuildTestAccess::prepared(herder) != nullptr; }, std::chrono::seconds(1)); REQUIRE(clock.now() < trigger); - REQUIRE(pulls == 1); + REQUIRE(builds() == 1); auto prepared = LeaderBuildTestAccess::prepared(herder); auto const preparedCloseTime = LeaderBuildTestAccess::closeTime(herder); REQUIRE(prepared); @@ -6042,7 +6074,7 @@ TEST_CASE("prepare proposal before trigger", "[herder][early-proposal]") testutil::crankUntil( app, [&]() { return herder.getSCP().getLeaderRound(seq) != 0; }, std::chrono::seconds(1)); - REQUIRE(pulls == 1); + REQUIRE(builds() == 1); REQUIRE(!LeaderBuildTestAccess::prepared(herder)); REQUIRE(std::get( herder.getTxSet(prepared->getContentsHash())) == prepared); @@ -6071,7 +6103,7 @@ TEST_CASE("prepare proposal before trigger", "[herder][early-proposal]") xdr::xdr_from_opaque(ballot.value, value); REQUIRE(value.txSetHash == prepared->getContentsHash()); REQUIRE(getConsensusTime(value) == preparedCloseTime); - REQUIRE(pulls == 1); + REQUIRE(builds() == 1); } SECTION("clock regression rejects the prepared future close time") { @@ -6079,7 +6111,7 @@ TEST_CASE("prepare proposal before trigger", "[herder][early-proposal]") // The fallback builds with today's valid close time, and must not // claim that our time-bound transaction is valid for that time. herder.triggerNextLedger(seq, true); - REQUIRE(pulls == 2); + REQUIRE(builds() == 2); REQUIRE(!LeaderBuildTestAccess::prepared(herder)); REQUIRE(std::get( herder.getTxSet(prepared->getContentsHash())) == nullptr); @@ -6089,17 +6121,17 @@ TEST_CASE("prepare proposal before trigger", "[herder][early-proposal]") herder.externalizeValue(TxSetXDRFrame::makeEmpty(lcl), seq, preparedCloseTime, {}, cfg.NODE_SEED); REQUIRE(LeaderBuildTestAccess::prepared(herder) != prepared); - auto const pullsAfterClose = pulls; - // A stale timer/caller cannot propose for the old slot or pull again. + auto const buildsAfterClose = builds(); + // A stale timer/caller cannot propose for the old slot or build again. herder.triggerNextLedger(seq, true); - REQUIRE(pulls == pullsAfterClose); + REQUIRE(builds() == buildsAfterClose); } SECTION("loss of sync discards preparation") { herder.lostSync(); REQUIRE(!LeaderBuildTestAccess::prepared(herder)); herder.triggerNextLedger(seq, true); - REQUIRE(pulls == 1); + REQUIRE(builds() == 1); } SECTION("shutdown cancels preparation") { @@ -6958,17 +6990,20 @@ TEST_CASE("the set prepared before the trigger is proposed unchanged", // More candidates than capacity, but they cannot fill the ledger. mempool = {bad->getEnvelope(), bad->getEnvelope(), bad->getEnvelope()}; } - size_t pulls = 0; - herder.mGetTopTransactionsForTesting = [&](size_t) { - ++pulls; - return mempool; + herder.mGetTopTransactionsForTesting = [&](size_t) { return mempool; }; + // Builds, not pulls of the mempool: the leader starts executing the + // prepared set at once, and that execution pulls candidates for the + // ledger after it. + auto builds = [&]() { + return app->getMetrics().NewTimer({"herder", "txset", "build"}).count(); }; + REQUIRE(builds() == 0); LeaderBuildTestAccess::schedule(herder); testutil::crankUntil( app, [&]() { return LeaderBuildTestAccess::prepared(herder) != nullptr; }, std::chrono::seconds(1)); - REQUIRE(pulls == 1); + REQUIRE(builds() == 1); auto const prepared = LeaderBuildTestAccess::prepared(herder); REQUIRE(prepared->sizeTxTotal() == (invalidFirst ? 0 : 1)); // It was built a short lead time before the trigger, not earlier. @@ -6982,7 +7017,7 @@ TEST_CASE("the set prepared before the trigger is proposed unchanged", app, [&]() { return herder.getSCP().getLeaderRound(seq) != 0; }, std::chrono::seconds(1)); // No second build: the proposal names the prepared set. - REQUIRE(pulls == 1); + REQUIRE(builds() == 1); auto messages = herder.getSCP().getLatestMessagesSend(seq); REQUIRE(!messages.empty()); REQUIRE(messages.front().statement.pledges.type() == SCP_ST_PREPARE); @@ -7458,9 +7493,9 @@ TEST_CASE("leader builds validate only the top of the mempool pull", mempool.push_back(tx->getEnvelope()); hashes.push_back(tx->getFullHash()); } - size_t requested = 0; + std::vector requested; herder.mGetTopTransactionsForTesting = [&](size_t count) { - requested = count; + requested.push_back(count); return std::vector( mempool.begin(), mempool.begin() + std::min(count, mempool.size())); }; @@ -7471,10 +7506,13 @@ TEST_CASE("leader builds validate only the top of the mempool pull", getConsensusTime(lcl.header.scpValue).next(lcl.header.ledgerVersion)); herder.mGetTopTransactionsForTesting = nullptr; - // The pull is unchanged: twice the capacity of both phases. - REQUIRE( - requested == - 2 * (10 + lm.getLastClosedSorobanNetworkConfig().ledgerMaxTxCount())); + // The pull is the capacity of both phases plus the margin of 10%. Five + // of its candidates are invalid, which leaves the set short while the + // mempool has more: the build pulls again, twice as much. + size_t const capacity = + 10 + lm.getLastClosedSorobanNetworkConfig().ledgerMaxTxCount(); + size_t const pull = capacity + (capacity * 10 + 99) / 100; + REQUIRE(requested == std::vector{pull, 2 * pull}); // The invalid top candidates were replaced: the set is full, with the // best valid ones. auto applicable = txSet->prepareForApply(*app, lcl.header); @@ -7486,7 +7524,7 @@ TEST_CASE("leader builds validate only the top of the mempool pull", } REQUIRE(included == std::set(hashes.begin() + 5, hashes.begin() + 15)); - // #40 is past the validated candidates (0-22): it stays in the mempool + // #40 is past the candidates the build pulled: it stays in the mempool // until it reaches them. REQUIRE(std::set(invalid.begin(), invalid.end()) == std::set(hashes.begin(), hashes.begin() + 5)); @@ -7566,7 +7604,29 @@ TEST_CASE("leader tx set build costs", "[herder][.][benchmark]") herder.mGetTopTransactionsForTesting = [&](size_t) { return OverlayIPC::parseTopTransactions(payload); }; + // A build takes the payload itself, and only as many transactions as + // it asks for. + std::vector ends{4}; + for (uint32_t i = 0; i < pull; ++i) + { + uint32_t len = 0; + std::memcpy(&len, payload.data() + ends.back(), 4); + ends.push_back(ends.back() + 4 + len); + } + herder.mGetTopTransactionsPayloadForTesting = [&](size_t n) { + uint32_t const given = static_cast(std::min(n, pull)); + std::vector res(payload.begin(), + payload.begin() + ends[given]); + std::memcpy(res.data(), &given, 4); + return res; + }; + // BENCH_INLINE_CHECK=1: the build checks its written set before it + // returns (as it does once a background check has failed). + if (envInt("BENCH_INLINE_CHECK", 0) != 0) + { + LeaderBuildTestAccess::checkInLine(herder); + } auto const lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto const seq = lcl.header.ledgerSeq + 1; auto const closeTime = @@ -7637,6 +7697,11 @@ TEST_CASE("leader tx set build costs", "[herder][.][benchmark]") // against perfnet's herder.txset.validate). auto const& lastSet = LeaderBuildTestAccess::build(herder, seq, closeTime); + if (envInt("BENCH_SETTLE_MS", 0) != 0) + { + std::this_thread::sleep_for( + std::chrono::milliseconds(envInt("BENCH_SETTLE_MS", 0))); + } for (uint32_t rep = 0; rep < reps; ++rep) { auto received = TxSetXDRFrame::makeFromWire( @@ -7647,6 +7712,10 @@ TEST_CASE("leader tx set build costs", "[herder][.][benchmark]") } auto const start = std::chrono::steady_clock::now(); auto applicable = received->prepareForApply(*app, lcl.header); + samples["received-set-prepare"].push_back( + std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count()); REQUIRE(applicable); REQUIRE(applicable->checkValid(*app, 0, 0)); samples["received-set-validate"].push_back( @@ -7733,5 +7802,6 @@ TEST_CASE("leader tx set build costs", "[herder][.][benchmark]") << ", n=" << v.size() << ")" << std::endl; } } + herder.mGetTopTransactionsPayloadForTesting = nullptr; herder.mGetTopTransactionsForTesting = nullptr; } diff --git a/src/herder/test/ProposalPrevalidationTests.cpp b/src/herder/test/ProposalPrevalidationTests.cpp index 0de8f5eead..454d4c5ea8 100644 --- a/src/herder/test/ProposalPrevalidationTests.cpp +++ b/src/herder/test/ProposalPrevalidationTests.cpp @@ -8,10 +8,17 @@ // next ledger validates its candidates against it in the background // (HerderImpl::startProposalPrevalidation), and the build of the next set // takes those results instead of validating again (TxValidationCache). +// +// And of what the leader does beyond that: it executes its own set as soon +// as it is prepared (HerderImpl::executePreparedTxSet), and checks the sets +// it builds in the background (TxSetSelfCheck). The selection of the next +// set ahead of time has its own tests (EarlyTxSetTests.cpp); here it only +// serves as one of the two ways a set gets prepared. #include "crypto/SHA.h" #include "herder/Herder.h" #include "herder/HerderImpl.h" +#include "herder/HerderUtils.h" #include "herder/LeaderSchedule.h" #include "herder/LedgerCloseData.h" #include "herder/TxSetFrame.h" @@ -140,6 +147,110 @@ class ProposalPrevalidationTestAccess { return herder.mLastClosedTxSetSize; } + + // What the herder does with a statement this node emits, as far as the + // early selection of the next set goes. + static void + voted(HerderImpl& herder, SCPEnvelope const& envelope) + { + herder.maybeScheduleEarlyTxSetSelection(envelope); + } + + static bool + earlySelectionPlanned(HerderImpl& herder) + { + return herder.mEarlyTxSetPlan && herder.mEarlyTxSetPlan->planned; + } + + // The running or finished early selection job, if any. + static std::shared_ptr + earlyJob(HerderImpl& herder) + { + return herder.mEarlyTxSet; + } + + // Prepares the set for `ledgerSeq` as the first-round leader does a lead + // time before the trigger; returns whether the set is a completed early + // selection (and not a build). + static bool + prepareEarly(HerderImpl& herder, uint32_t ledgerSeq, + ConsensusTime closeTime) + { + auto& completed = herder.mApp.getMetrics().NewMeter( + {"herder", "txset", "early-completed"}, "txset"); + auto const before = completed.count(); + herder.prepareTxSet(ledgerSeq, closeTime); + return herder.mPreparedTxSet && completed.count() == before + 1; + } + + // What the first-round leader does a lead time before the trigger. + static void + prepareByBuilding(HerderImpl& herder, uint32_t ledgerSeq, + ConsensusTime closeTime) + { + herder.prepareTxSet(ledgerSeq, closeTime); + } + + static TxSetXDRFrameConstPtr + prepared(HerderImpl& herder) + { + return herder.mPreparedTxSet ? herder.mPreparedTxSet->txSet : nullptr; + } + + static std::vector + preparedInvalidTxs(HerderImpl& herder) + { + REQUIRE(herder.mPreparedTxSet); + return herder.mPreparedTxSet->invalidTxHashes; + } + + // What the node proposes when the trigger fires. + static ValueWrapperPtr + propose(HerderImpl& herder, uint32_t ledgerSeq) + { + return herder.makeProposalValue(ledgerSeq); + } + + // The (slot, value) last handed to the apply thread by the herder. + static std::optional> const& + speculativeValue(HerderImpl& herder) + { + return herder.mSpeculativeApplyValue; + } + + // What the herder does when the node emits `envelope`. + static void + emitted(HerderImpl& herder, SCPEnvelope const& envelope) + { + herder.maybeStartSpeculativeApply(envelope); + } + + // What the herder does about a set it built once it has emitted a + // statement. + static void + statementEmitted(HerderImpl& herder) + { + herder.resolvePendingTxSetSelfCheck(); + } + + static bool + selfCheckHasFailed(HerderImpl& herder) + { + return herder.mTxSetSelfCheckFailed; + } + + // Whether a build is scheduled for a lead time before the trigger. + static bool + buildScheduled(HerderImpl& herder) + { + return herder.mPrepareTxSetTimer.seq() != 0; + } + + static void + discardPrepared(HerderImpl& herder) + { + herder.discardPreparedTxSet(); + } }; } @@ -237,14 +348,26 @@ struct PrevalidationFixture std::optional mE; // What the mempool holds; the Herder pulls it in this order. std::vector mMempool; - - PrevalidationFixture() + // How often the Herder pulled it. + size_t mPulls{0}; + + // With `herderSpeculates` the Herder hands ledgers to the apply thread + // itself, where it would in production; the tests still drive the + // speculation of the ledger they close first. + explicit PrevalidationFixture( + bool herderSpeculates = false, + std::optional maxTxSetSize = std::nullopt, + uint32_t protocol = GATE_PROTOCOL) : mCfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)) { - mCfg.LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; - mCfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + mCfg.LEDGER_PROTOCOL_VERSION = protocol; + mCfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = protocol; + if (maxTxSetSize) + { + mCfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = *maxTxSetSize; + } // The tests drive speculation themselves. - mCfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = false; + mCfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = herderSpeculates; REQUIRE(mCfg.backgroundLedgerApply()); mApp = createTestApplication(mClock, mCfg); auto root = mApp->getRoot(); @@ -255,7 +378,10 @@ struct PrevalidationFixture mC.emplace(root->create("C", balance)); mD.emplace(root->create("D", balance)); mE.emplace(root->create("E", balance)); + // An early selection starts as soon as the node has voted. + herder().mEarlyTxSetStartDelayForTesting = std::chrono::milliseconds(0); herder().mGetTopTransactionsForTesting = [this](size_t) { + ++mPulls; std::vector envelopes; for (auto const& tx : mMempool) { @@ -268,6 +394,74 @@ struct PrevalidationFixture ~PrevalidationFixture() { herder().mGetTopTransactionsForTesting = nullptr; + herder().mBuiltTxSetOverrideForTesting = nullptr; + } + + // Cranks until `done`, giving background threads time to post to the + // main thread. + template + void + crankUntil(F&& done) + { + auto const deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (!done()) + { + REQUIRE(std::chrono::steady_clock::now() < deadline); + mClock.crank(false); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + void + crankFor(std::chrono::milliseconds duration) + { + auto const deadline = std::chrono::steady_clock::now() + duration; + while (std::chrono::steady_clock::now() < deadline) + { + mClock.crank(false); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + // The node votes for `closing` as the next ledger and, being first in + // line for the one after, selects that one's set ahead of time: waits + // for the selection. + void + selectEarly(TxSetXDRFrameConstPtr const& closing, bool wait = true) + { + auto const value = valueFor(closing); + // By the time a node votes, SCP has asked for the slot's leaders, + // which brings the tracked quorum (the node set) up to date. + REQUIRE(herder().getHerderSCPDriver().getLeaderSchedule(nextSeq()) == + std::vector{mCfg.NODE_SEED.getPublicKey()}); + herder().getPendingEnvelopes().putTxSet(closing->getContentsHash(), + nextSeq(), closing); + SCPEnvelope vote; + vote.statement.slotIndex = nextSeq(); + vote.statement.pledges.type(SCP_ST_PREPARE); + vote.statement.pledges.prepare().ballot.counter = 1; + vote.statement.pledges.prepare().ballot.value = + xdr::xdr_to_opaque(value); + Access::voted(herder(), vote); + REQUIRE(Access::earlySelectionPlanned(herder())); + crankUntil([&]() { return Access::earlyJob(herder()) != nullptr; }); + if (!wait) + { + return; + } + auto job = Access::earlyJob(herder()); + crankUntil([&]() { return job->mDone.load(); }); + std::lock_guard lock(job->mMutex); + REQUIRE(job->mSelection); + } + + // The same, and then the ledger closes with `closing`. + void + closeWithEarlySelection(TxSetXDRFrameConstPtr const& closing) + { + selectEarly(closing); + externalize(closing); } HerderImpl& @@ -986,6 +1180,9 @@ TEST_CASE("leaders keep their term and pre-validate in a network", cfg.LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + // This test is about the pre-validation, which the node first in + // line skips when it selects the next set ahead of time. + cfg.EXPERIMENTAL_EARLY_TX_SET_SELECTION = false; return cfg; }); auto nodes = simulation->getNodes(); @@ -1144,3 +1341,912 @@ TEST_CASE("leaders keep their term and pre-validate in a network", .mGetTopTransactionsForTesting = nullptr; } } + +// --------------------------------------------------------------------------- +// Early selection: the set for ledger N+1 is selected while ledger N is in +// its ballot and completed when N has closed. +// --------------------------------------------------------------------------- + +namespace +{ +int64_t +txsetMeter(Application& app, std::string const& name) +{ + return meter(app, "txset", name, "txset"); +} + +StellarValue +stellarValueOf(ValueWrapperPtr const& value) +{ + REQUIRE(value); + StellarValue sv; + REQUIRE(toStellarValue(value->getValue(), sv)); + return sv; +} +} + +TEST_CASE("the leader executes its prepared set before the trigger", + "[herder][txset][speculative-apply][early-txset]") +{ + PrevalidationFixture f(/*herderSpeculates=*/true); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& lm = f.lm(); + auto& a = *f.mA; + auto& b = *f.mB; + auto const aSeq = a.getLastSequenceNumber(); + auto const bSeq = b.getLastSequenceNumber(); + auto closingA = paymentWithSeq(app, a, aSeq + 1); + auto nextA = paymentWithSeq(app, a, aSeq + 2); + auto plainB = paymentWithSeq(app, b, bSeq + 1); + auto closing = makeUnvalidatedTxSet(app, {closingA}); + f.mMempool = {closingA, nextA, plainB}; + f.closeWithEarlySelection(closing); + f.mMempool = {nextA, plainB}; + + auto const seq = f.nextSeq(); + auto const closeTime = f.nextCloseTime(); + auto const started = applyCounter(app, "speculation-started"); + auto const hits = applyCounter(app, "speculation-hit"); + auto const misses = applyCounter(app, "speculation-miss"); + + auto afterPreparing = [&]() { + auto prepared = Access::prepared(herder); + REQUIRE(prepared); + // The ledger is with the apply thread already, although the node + // has not proposed anything yet. + REQUIRE(applyCounter(app, "speculation-started") == started + 1); + while (lm.isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + REQUIRE(lm.getLastClosedLedgerNum() == seq - 1); + // It is the very value the trigger proposes ... + auto const proposal = Access::propose(herder, seq); + REQUIRE(proposal); + REQUIRE(Access::speculativeValue(herder)); + REQUIRE(Access::speculativeValue(herder)->first == seq); + REQUIRE(Access::speculativeValue(herder)->second == + proposal->getValue()); + REQUIRE(stellarValueOf(proposal).txSetHash == + prepared->getContentsHash()); + // ... so the vote the node then emits does not start it again. + SCPEnvelope vote; + vote.statement.slotIndex = seq; + vote.statement.pledges.type(SCP_ST_PREPARE); + vote.statement.pledges.prepare().ballot.counter = 1; + vote.statement.pledges.prepare().ballot.value = proposal->getValue(); + Access::emitted(herder, vote); + REQUIRE(applyCounter(app, "speculation-started") == started + 1); + + SECTION("its value externalizes: the execution is committed") + { + f.externalize(prepared); + REQUIRE(applyCounter(app, "speculation-hit") == hits + 1); + REQUIRE(applyCounter(app, "speculation-miss") == misses); + REQUIRE(a.getLastSequenceNumber() == aSeq + 2); + REQUIRE(b.getLastSequenceNumber() == bSeq + 1); + } + SECTION("another value externalizes: the execution is dropped") + { + f.externalize(makeUnvalidatedTxSet(app, {plainB})); + REQUIRE(applyCounter(app, "speculation-hit") == hits); + REQUIRE(applyCounter(app, "speculation-miss") == misses + 1); + REQUIRE(a.getLastSequenceNumber() == aSeq + 1); + REQUIRE(b.getLastSequenceNumber() == bSeq + 1); + } + SECTION("the node votes for another value: that one is executed") + { + auto other = makeUnvalidatedTxSet(app, {plainB}); + herder.getPendingEnvelopes().putTxSet(other->getContentsHash(), seq, + other); + auto const otherValue = f.valueFor(other); + vote.statement.pledges.prepare().ballot.value = + xdr::xdr_to_opaque(otherValue); + Access::emitted(herder, vote); + REQUIRE(applyCounter(app, "speculation-started") == started + 2); + while (lm.isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + f.externalize(other); + REQUIRE(applyCounter(app, "speculation-hit") == hits + 1); + REQUIRE(a.getLastSequenceNumber() == aSeq + 1); + REQUIRE(b.getLastSequenceNumber() == bSeq + 1); + } + }; + + SECTION("a set selected early") + { + REQUIRE(Access::prepareEarly(herder, seq, closeTime)); + afterPreparing(); + } + SECTION("a set built before the trigger") + { + Access::prepareByBuilding(herder, seq, closeTime); + afterPreparing(); + } +} + +TEST_CASE("a prepared set is not executed early when it cannot be", + "[herder][txset][speculative-apply][early-txset]") +{ + SECTION("speculative apply is off") + { + PrevalidationFixture f(/*herderSpeculates=*/false); + auto& app = *f.mApp; + auto& a = *f.mA; + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + auto const started = applyCounter(app, "speculation-started"); + Access::prepareByBuilding(f.herder(), f.nextSeq(), f.nextCloseTime()); + REQUIRE(Access::prepared(f.herder())); + REQUIRE(applyCounter(app, "speculation-started") == started); + REQUIRE(!Access::speculativeValue(f.herder())); + } + SECTION("the node is about to propose another set") + { + PrevalidationFixture f(/*herderSpeculates=*/true); + auto& app = *f.mApp; + auto& a = *f.mA; + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + f.herder().mProposedTxSetOverride = + [&](LedgerHeaderHistoryEntry const& lcl) { + return TxSetXDRFrame::makeEmpty(lcl); + }; + auto const started = applyCounter(app, "speculation-started"); + Access::prepareByBuilding(f.herder(), f.nextSeq(), f.nextCloseTime()); + REQUIRE(Access::prepared(f.herder())); + REQUIRE(applyCounter(app, "speculation-started") == started); + f.herder().mProposedTxSetOverride = nullptr; + } +} + +// --------------------------------------------------------------------------- +// The builder's self-check runs beside the use of the set. +// --------------------------------------------------------------------------- + +TEST_CASE("the builder checks the set it built", "[herder][txset][early-txset]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& a = *f.mA; + auto& b = *f.mB; + auto txA = paymentWithSeq(app, a, a.getLastSequenceNumber() + 1); + auto txB = paymentWithSeq(app, b, b.getLastSequenceNumber() + 1); + auto const lcl = f.lm().getLastClosedLedgerHeader(); + PerPhaseTransactionList phases{{txA, txB}, {}}; + PerPhaseTransactionList invalid(2); + + // Built without the check: the caller gets it to run. + std::shared_ptr check; + bool handedOut = false; + auto built = makeTxSetFromTransactions( + phases, app, ApplyTimeOffset::fromSeconds(1), invalid, false, {}, false, + [&](TxSetXDRFrameConstPtr const&) { handedOut = true; }, {}, nullptr, + &check); + REQUIRE(handedOut); + REQUIRE(built.first); + REQUIRE(built.second); + REQUIRE(check); + REQUIRE(check->getTxSet() == built.first); + REQUIRE(built.second->getContentsHash() == built.first->getContentsHash()); + REQUIRE(txHashes(built.second) == hashesOf({txA, txB})); + + SECTION("passes on the set that was built") + { + REQUIRE(!check->run(app)); + // the same set as a build that checks before it returns + auto checked = makeTxSetFromTransactions( + phases, app, ApplyTimeOffset::fromSeconds(1), invalid); + REQUIRE(checked.first->getContentsHash() == + built.first->getContentsHash()); + } + SECTION("runs on another thread") + { + std::optional failure{"not run"}; + std::thread t([&]() { failure = check->run(app); }); + t.join(); + REQUIRE(!failure); + } + SECTION("a set that lost a transaction") + { + auto broken = + check->withTxSetForTesting(makeUnvalidatedTxSet(app, {txA})); + auto const failure = broken->run(app); + REQUIRE(failure); + REQUIRE(failure->find("shape is mismatched") != std::string::npos); + } + SECTION("a set for another ledger") + { + // The same transactions on top of the ledger before. (Checked off + // the main thread, as the node does: on the main thread reading + // such a set back is an assertion failure.) + auto xdr = built.first->getGeneralizedXDR(); + xdr.v1TxSet().previousLedgerHash = lcl.header.previousLedgerHash; + auto other = check->withTxSetForTesting( + TxSetXDRFrame::makeFromWire(std::move(xdr))); + std::optional failure; + std::thread t([&]() { failure = other->run(app); }); + t.join(); + REQUIRE(failure); + REQUIRE(failure->find("PREVIOUS_LEDGER_HASH_MISMATCH") != + std::string::npos); + } + SECTION("a set with a base fee below the ledger's") + { + auto cheap = testtxset::makeNonValidatedGeneralizedTxSet( + {{std::make_pair(std::make_optional(1), + std::vector{txA, txB})}, + {}}, + app, lcl.hash); + REQUIRE(cheap.first); + REQUIRE(check->withTxSetForTesting(cheap.first)->run(app)); + } +} + +TEST_CASE("a set that fails its self-check after it was handed out", + "[herder][txset][early-txset]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto& c = *f.mC; + auto const aSeq = a.getLastSequenceNumber(); + auto const cSeq = c.getLastSequenceNumber(); + auto closingA = paymentWithSeq(app, a, aSeq + 1); + auto nextA = paymentWithSeq(app, a, aSeq + 2); + auto plainB = paymentWithSeq(app, b, b.getLastSequenceNumber() + 1); + auto closing = makeUnvalidatedTxSet(app, {closingA}); + f.mMempool = {closingA, nextA, plainB}; + f.closeWithEarlySelection(closing); + f.mMempool = {nextA, plainB}; + auto const seq = f.nextSeq(); + auto const closeTime = f.nextCloseTime(); + auto const failedBefore = txsetMeter(app, "self-check-failed"); + auto const passedBefore = txsetMeter(app, "self-check-passed"); + + // What the node "builds": a set that is not what it selected. The one + // transaction it holds is not valid either (a sequence number too far + // ahead), so ledger apply refuses it. + auto broken = makeUnvalidatedTxSet(app, {paymentWithSeq(app, c, cSeq + 5)}); + auto buildBroken = [&]() { + herder.mBuiltTxSetOverrideForTesting = + [&](TxSetXDRFrameConstPtr const&) { return broken; }; + }; + auto failed = [&]() { + return txsetMeter(app, "self-check-failed") == failedBefore + 1; + }; + + SECTION("a set that passes stays prepared") + { + REQUIRE(Access::prepareEarly(herder, seq, closeTime)); + auto prepared = Access::prepared(herder); + f.crankUntil([&]() { + return txsetMeter(app, "self-check-passed") == passedBefore + 1; + }); + REQUIRE(Access::prepared(herder) == prepared); + REQUIRE(!Access::selfCheckHasFailed(herder)); + REQUIRE(txsetMeter(app, "self-check-failed") == failedBefore); + } + SECTION("selected early, caught before the proposal") + { + buildBroken(); + REQUIRE(Access::prepareEarly(herder, seq, closeTime)); + // The set is out (prepared, pushed) before the check has run. + REQUIRE(Access::prepared(herder) == broken); + herder.mBuiltTxSetOverrideForTesting = nullptr; + f.crankUntil(failed); + // Dropped: the trigger builds another one, checked before use. + REQUIRE(!Access::prepared(herder)); + REQUIRE(Access::selfCheckHasFailed(herder)); + auto const pulls = f.mPulls; + auto const proposed = stellarValueOf(Access::propose(herder, seq)); + REQUIRE(f.mPulls == pulls + 1); + REQUIRE(proposed.txSetHash != broken->getContentsHash()); + auto txSet = herder.getTxSet(proposed.txSetHash); + auto const* frame = std::get_if(&txSet); + REQUIRE(frame); + REQUIRE(*frame); + auto applicable = (*frame)->prepareForApply( + app, f.lm().getLastClosedLedgerHeader().header); + REQUIRE(applicable); + REQUIRE(txHashes(applicable) == hashesOf({nextA, plainB})); + // That build ran its check before returning: none in the + // background. + f.crankFor(std::chrono::milliseconds(50)); + REQUIRE(txsetMeter(app, "self-check-passed") == passedBefore); + } + SECTION("built before the trigger, caught before the proposal") + { + buildBroken(); + Access::prepareByBuilding(herder, seq, closeTime); + REQUIRE(Access::prepared(herder) == broken); + herder.mBuiltTxSetOverrideForTesting = nullptr; + f.crankUntil(failed); + REQUIRE(!Access::prepared(herder)); + REQUIRE(Access::selfCheckHasFailed(herder)); + } + SECTION("caught after the proposal") + { + buildBroken(); + REQUIRE(Access::prepareEarly(herder, seq, closeTime)); + // The trigger fires before the check is back: the set is proposed. + auto const proposed = stellarValueOf(Access::propose(herder, seq)); + REQUIRE(proposed.txSetHash == broken->getContentsHash()); + herder.mBuiltTxSetOverrideForTesting = nullptr; + f.crankUntil(failed); + REQUIRE(Access::selfCheckHasFailed(herder)); + // It cannot be recalled. Every node validates it when it applies + // the ledger, this one included: the ledger closes without it. + auto const skipped = applyCounter(app, "txset-skipped"); + f.externalize(broken); + REQUIRE(applyCounter(app, "txset-skipped") == skipped + 1); + REQUIRE(c.getLastSequenceNumber() == cSeq); + // From now on a set selected early is checked in line when it is + // completed, like a build: no check in the background. + auto nextNextA = paymentWithSeq(app, a, aSeq + 3); + f.mMempool = {nextA, nextNextA, plainB}; + f.closeWithEarlySelection(makeUnvalidatedTxSet(app, {nextA})); + f.mMempool = {nextNextA, plainB}; + REQUIRE(Access::prepareEarly(herder, f.nextSeq(), f.nextCloseTime())); + REQUIRE(Access::prepared(herder)); + { + auto applicable = Access::prepared(herder)->prepareForApply( + app, f.lm().getLastClosedLedgerHeader().header); + REQUIRE(applicable); + REQUIRE(txHashes(applicable) == hashesOf({nextNextA, plainB})); + } + f.crankFor(std::chrono::milliseconds(50)); + REQUIRE(txsetMeter(app, "self-check-passed") == passedBefore); + REQUIRE(txsetMeter(app, "self-check-failed") == failedBefore + 1); + Access::discardPrepared(herder); + // The build that takes its place checks its own result. + f.mMempool = {nextNextA, plainB}; + REQUIRE(f.build() == hashesOf({nextNextA, plainB})); + } +} + +TEST_CASE("candidates of one account in both phases", + "[herder][txset][prevalidation][early-txset][soroban]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto const aSeq = a.getLastSequenceNumber(); + auto both = app.getRoot()->create( + "S", app.getLedgerManager().getLastMinBalance(2) + 1'000'000'000); + auto const sSeq = both.getLastSequenceNumber(); + SorobanResources resources; + resources.instructions = 1'000'000; + resources.diskReadBytes = 1000; + resources.writeBytes = 1000; + auto plainB = paymentWithSeq(app, b, b.getLastSequenceNumber() + 1); + auto classic = paymentWithSeq(app, both, sSeq + 1); + auto soroban = + createUploadWasmTx(app, both, 100, DEFAULT_TEST_RESOURCE_FEE, resources, + std::nullopt, 0, std::nullopt, sSeq + 1); + auto laterClassic = paymentWithSeq(app, both, sSeq + 2); + REQUIRE(soroban->getSeqNum() == classic->getSeqNum()); + + SECTION("a build keeps one of them") + { + SECTION("the same sequence number: the classic one") + { + f.mMempool = {soroban, classic, plainB}; + REQUIRE(f.build() == hashesOf({classic, plainB})); + } + SECTION("the Soroban one comes first in the account's sequence") + { + f.mMempool = {laterClassic, soroban, plainB}; + REQUIRE(f.build() == hashesOf({soroban, plainB})); + } + SECTION("the classic one comes first in the account's sequence") + { + auto laterSoroban = createUploadWasmTx( + app, both, 100, DEFAULT_TEST_RESOURCE_FEE, resources, + std::nullopt, 0, std::nullopt, sSeq + 2); + f.mMempool = {laterSoroban, classic, plainB}; + REQUIRE(f.build() == hashesOf({classic, plainB})); + } + } + SECTION("so does a set selected early") + { + auto closingA = paymentWithSeq(app, a, aSeq + 1); + f.mMempool = {closingA, soroban, classic, plainB}; + f.closeWithEarlySelection(makeUnvalidatedTxSet(app, {closingA})); + REQUIRE(Access::prepareEarly(herder, f.nextSeq(), f.nextCloseTime())); + auto applicable = Access::prepared(herder)->prepareForApply( + app, f.lm().getLastClosedLedgerHeader().header); + REQUIRE(applicable); + REQUIRE(txHashes(applicable) == hashesOf({classic, plainB})); + REQUIRE(applicable->checkValid(app, 1, 1)); + } +} + +TEST_CASE("the limits a tx set is selected within", + "[herder][txset][early-txset]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto const limits = TxSetBuildLimits::forLCL(app); + REQUIRE(limits.mSorobanConfig); + REQUIRE(limits.sameAs(limits)); + REQUIRE(limits.maxTxSetSizeOps() == f.lm().getLastMaxTxSetSizeOps()); + REQUIRE(limits.mLclHeader == f.lm().getLastClosedLedgerHeader().header); + + auto differs = [&](TxSetBuildLimits const& other) { + return !limits.sameAs(other) && !other.sameAs(limits); + }; + auto withHeader = [&](std::function change) { + auto other = limits; + change(other.mLclHeader); + return other; + }; + // What selection reads of the header ... + REQUIRE(differs(withHeader([](LedgerHeader& h) { h.ledgerVersion -= 1; }))); + REQUIRE(differs(withHeader([](LedgerHeader& h) { h.baseFee += 1; }))); + REQUIRE(differs(withHeader([](LedgerHeader& h) { h.baseReserve += 1; }))); + REQUIRE(differs(withHeader([](LedgerHeader& h) { h.maxTxSetSize += 1; }))); + // ... and what it does not: every ledger changes these. + REQUIRE(limits.sameAs(withHeader([](LedgerHeader& h) { + h.ledgerSeq += 1; + h.feePool += 100; + h.idPool += 1; + h.scpValue.closeTime += 1; + h.previousLedgerHash = sha256("another ledger"); + h.bucketListHash = sha256("another bucket list"); + h.txSetResultHash = sha256("other results"); + }))); + + // The Soroban limits of the ledger. + auto withoutSoroban = limits; + withoutSoroban.mSorobanConfig = nullptr; + REQUIRE(differs(withoutSoroban)); + auto withSoroban = [&](SorobanNetworkConfig& config, + std::function change) { + config = *limits.mSorobanConfig; + change(config); + auto other = limits; + other.mSorobanConfig = &config; + return other; + }; + SorobanNetworkConfig config = *limits.mSorobanConfig; + REQUIRE(limits.sameAs(withSoroban(config, [](SorobanNetworkConfig&) {}))); + REQUIRE(differs(withSoroban( + config, [](SorobanNetworkConfig& c) { c.mLedgerMaxTxCount += 1; }))); + REQUIRE(differs(withSoroban(config, [](SorobanNetworkConfig& c) { + c.mLedgerMaxInstructions += 1; + }))); + REQUIRE(differs(withSoroban(config, [](SorobanNetworkConfig& c) { + c.mLedgerMaxDependentTxClusters += 1; + }))); + REQUIRE(differs(withSoroban(config, [](SorobanNetworkConfig& c) { + c.mLedgerMaxTransactionsSizeBytes += 1; + }))); + REQUIRE(differs(withSoroban(config, [](SorobanNetworkConfig& c) { + c.mLedgerMaxDiskReadEntries += 1; + }))); +} + +TEST_CASE("the upgrades due are part of the value the leader executes early", + "[herder][txset][speculative-apply][early-txset]") +{ + PrevalidationFixture f(/*herderSpeculates=*/true); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + Upgrades::UpgradeParameters params; + params.mUpgradeTime = VirtualClock::system_time_point{}; + params.mBaseFee = 250; + herder.setUpgrades(params); + auto const seq = f.nextSeq(); + auto const started = applyCounter(app, "speculation-started"); + + Access::prepareByBuilding(herder, seq, f.nextCloseTime()); + REQUIRE(Access::prepared(herder)); + REQUIRE(applyCounter(app, "speculation-started") == started + 1); + while (f.lm().isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + auto const proposal = Access::propose(herder, seq); + auto const proposed = stellarValueOf(proposal); + REQUIRE(proposed.upgrades.size() == 1); + REQUIRE(Access::speculativeValue(herder)); + REQUIRE(Access::speculativeValue(herder)->second == proposal->getValue()); +} + +// Executing a ledger starts with the full validation of its set: the same +// check every other node runs when it applies the ledger. When the leader's +// own execution of its prepared set is done before the trigger, an invalid +// set is not proposed. +TEST_CASE("a prepared set that the leader finds invalid when executing it " + "is not proposed", + "[herder][txset][speculative-apply][early-txset]") +{ + PrevalidationFixture f(/*herderSpeculates=*/true); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto& c = *f.mC; + auto nextA = paymentWithSeq(app, a, a.getLastSequenceNumber() + 1); + auto plainB = paymentWithSeq(app, b, b.getLastSequenceNumber() + 1); + f.mMempool = {nextA, plainB}; + auto const seq = f.nextSeq(); + auto const failedBefore = txsetMeter(app, "self-check-failed"); + auto const passedBefore = txsetMeter(app, "self-check-passed"); + auto const skippedBefore = applyCounter(app, "txset-skipped"); + + // What the node "builds" has the shape of its selection (two classic + // transactions) and passes the structural check, but one of its + // transactions is not valid: only validating the transactions shows it. + auto invalid = makeUnvalidatedTxSet( + app, {nextA, paymentWithSeq(app, c, c.getLastSequenceNumber() + 5)}); + herder.mBuiltTxSetOverrideForTesting = [&](TxSetXDRFrameConstPtr const&) { + return invalid; + }; + Access::prepareByBuilding(herder, seq, f.nextCloseTime()); + herder.mBuiltTxSetOverrideForTesting = nullptr; + REQUIRE(Access::prepared(herder) == invalid); + + f.crankUntil([&]() { + return txsetMeter(app, "self-check-failed") == failedBefore + 1; + }); + // The verdict is the execution's: a node that executes its own value + // leaves the re-read of the written set to that execution, so no check + // ran in the background beside it. + f.crankFor(std::chrono::milliseconds(50)); + REQUIRE(txsetMeter(app, "self-check-passed") == passedBefore); + // The apply thread skipped the set, and the herder dropped it. + REQUIRE(applyCounter(app, "txset-skipped") == skippedBefore + 1); + REQUIRE(!Access::prepared(herder)); + REQUIRE(Access::selfCheckHasFailed(herder)); + REQUIRE(f.lm().getLastClosedLedgerNum() == seq - 1); + // The trigger builds a set of its own. + auto const proposed = stellarValueOf(Access::propose(herder, seq)); + REQUIRE(proposed.txSetHash != invalid->getContentsHash()); + auto txSet = herder.getTxSet(proposed.txSetHash); + auto const* frame = std::get_if(&txSet); + REQUIRE(frame); + REQUIRE(*frame); + auto applicable = (*frame)->prepareForApply( + app, f.lm().getLastClosedLedgerHeader().header); + REQUIRE(applicable); + REQUIRE(txHashes(applicable) == hashesOf({nextA, plainB})); +} + +// The check of the written set (TxSetSelfCheck) reads the set back and +// validates what it read. A node that executes the value it votes for does +// that on its apply thread, on the same bytes, as soon as it proposes: there +// the check is not run a second time in the background. +TEST_CASE("the check of a written set is left to the execution of the vote " + "where the node executes it", + "[herder][txset][speculative-apply][early-txset]") +{ + auto run = [](bool herderSpeculates) { + PrevalidationFixture f(herderSpeculates); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + auto const passedBefore = txsetMeter(app, "self-check-passed"); + auto const failedBefore = txsetMeter(app, "self-check-failed"); + auto const started = applyCounter(app, "speculation-started"); + Access::prepareByBuilding(herder, f.nextSeq(), f.nextCloseTime()); + REQUIRE(Access::prepared(herder)); + if (herderSpeculates) + { + // The apply thread has the ledger: it validates the set in + // full, starting with reading it back. + REQUIRE(applyCounter(app, "speculation-started") == started + 1); + while (f.lm().isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + // Also once the node has emitted its statement: the apply + // thread was handed this very set. + Access::statementEmitted(herder); + f.crankFor(std::chrono::milliseconds(50)); + REQUIRE(txsetMeter(app, "self-check-passed") == passedBefore); + } + else + { + REQUIRE(applyCounter(app, "speculation-started") == started); + f.crankUntil([&]() { + return txsetMeter(app, "self-check-passed") == passedBefore + 1; + }); + } + REQUIRE(txsetMeter(app, "self-check-failed") == failedBefore); + REQUIRE(Access::prepared(herder)); + REQUIRE(!Access::selfCheckHasFailed(herder)); + }; + SECTION("a node that executes its vote") + { + run(true); + } + SECTION("a node that does not") + { + run(false); + } + SECTION("a node that executes its votes, but not this set") + { + // The node built a set and then emits a statement without its + // apply thread having been handed that set (here: the trigger + // proposes another one; a node that is not tracking, or is still + // applying, gets there too): the check runs in the background + // after all. + PrevalidationFixture f(/*herderSpeculates=*/true); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + auto const passedBefore = txsetMeter(app, "self-check-passed"); + herder.mProposedTxSetOverride = + [&](LedgerHeaderHistoryEntry const& lcl) { + return TxSetXDRFrame::makeEmpty(lcl); + }; + // (with the override the prepared set is not executed early) + Access::prepareByBuilding(herder, f.nextSeq(), f.nextCloseTime()); + herder.mProposedTxSetOverride = nullptr; + REQUIRE(Access::prepared(herder)); + REQUIRE(!Access::speculativeValue(herder)); + Access::statementEmitted(herder); + f.crankUntil([&]() { + return txsetMeter(app, "self-check-passed") == passedBefore + 1; + }); + // Once only. + Access::statementEmitted(herder); + f.crankFor(std::chrono::milliseconds(50)); + REQUIRE(txsetMeter(app, "self-check-passed") == passedBefore + 1); + } +} + +TEST_CASE("the application shuts down while a set is being checked or " + "selected", + "[herder][txset][prevalidation][early-txset]") +{ + auto f = std::make_unique(); + auto& app = *f->mApp; + auto& herder = f->herder(); + auto& a = *f->mA; + auto const aSeq = a.getLastSequenceNumber(); + auto closingA = paymentWithSeq(app, a, aSeq + 1); + auto nextA = paymentWithSeq(app, a, aSeq + 2); + + SECTION("the self-check of a set built before the trigger") + { + f->mMempool = {closingA}; + Access::prepareByBuilding(herder, f->nextSeq(), f->nextCloseTime()); + REQUIRE(Access::prepared(herder)); + f.reset(); + } + SECTION("the selection") + { + f->mMempool = {closingA}; + for (int i = 0; i < 400; ++i) + { + f->mMempool.push_back(nextA); + } + // The job has started; its tasks may be anywhere. + f->selectEarly(makeUnvalidatedTxSet(app, {closingA}), /*wait=*/false); + f.reset(); + } + SECTION("the self-check of a set selected early") + { + f->mMempool = {closingA, nextA}; + f->closeWithEarlySelection(makeUnvalidatedTxSet(app, {closingA})); + REQUIRE(Access::prepareEarly(herder, f->nextSeq(), f->nextCloseTime())); + f.reset(); + } +} + +// --------------------------------------------------------------------------- +// When a set selected early is used. +// --------------------------------------------------------------------------- + +TEST_CASE("the verdict of the leader on its own set", + "[herder][txset][speculative-apply][early-txset]") +{ + PrevalidationFixture f(/*herderSpeculates=*/true); + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto& c = *f.mC; + auto const aSeq = a.getLastSequenceNumber(); + auto const failedBefore = txsetMeter(app, "self-check-failed"); + auto const skippedBefore = applyCounter(app, "txset-skipped"); + auto vote = [&](uint32_t seq, StellarValue const& value) { + SCPEnvelope envelope; + envelope.statement.slotIndex = seq; + envelope.statement.pledges.type(SCP_ST_PREPARE); + envelope.statement.pledges.prepare().ballot.counter = 1; + envelope.statement.pledges.prepare().ballot.value = + xdr::xdr_to_opaque(value); + Access::emitted(herder, envelope); + }; + auto executed = [&]() { + while (f.lm().isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + f.crankFor(std::chrono::milliseconds(20)); + }; + + SECTION("a set selected early that is invalid is dropped before the " + "proposal") + { + auto closingA = paymentWithSeq(app, a, aSeq + 1); + auto nextA = paymentWithSeq(app, a, aSeq + 2); + auto plainB = paymentWithSeq(app, b, b.getLastSequenceNumber() + 1); + f.mMempool = {closingA, nextA, plainB}; + f.closeWithEarlySelection(makeUnvalidatedTxSet(app, {closingA})); + // Same shape as the selection, one transaction not valid. + auto invalid = makeUnvalidatedTxSet( + app, + {nextA, paymentWithSeq(app, c, c.getLastSequenceNumber() + 5)}); + herder.mBuiltTxSetOverrideForTesting = + [&](TxSetXDRFrameConstPtr const&) { return invalid; }; + REQUIRE(Access::prepareEarly(herder, f.nextSeq(), f.nextCloseTime())); + herder.mBuiltTxSetOverrideForTesting = nullptr; + REQUIRE(Access::prepared(herder) == invalid); + f.crankUntil([&]() { + return txsetMeter(app, "self-check-failed") == failedBefore + 1; + }); + REQUIRE(applyCounter(app, "txset-skipped") == skippedBefore + 1); + REQUIRE(!Access::prepared(herder)); + REQUIRE(Access::selfCheckHasFailed(herder)); + } + SECTION("a verdict that comes after the proposal is recorded") + { + auto nextA = paymentWithSeq(app, a, aSeq + 1); + auto plainB = paymentWithSeq(app, b, b.getLastSequenceNumber() + 1); + f.mMempool = {nextA, plainB}; + auto invalid = makeUnvalidatedTxSet( + app, + {nextA, paymentWithSeq(app, c, c.getLastSequenceNumber() + 5)}); + herder.mBuiltTxSetOverrideForTesting = + [&](TxSetXDRFrameConstPtr const&) { return invalid; }; + auto const seq = f.nextSeq(); + Access::prepareByBuilding(herder, seq, f.nextCloseTime()); + herder.mBuiltTxSetOverrideForTesting = nullptr; + // The trigger fires before the execution has got anywhere. + auto const proposed = stellarValueOf(Access::propose(herder, seq)); + REQUIRE(proposed.txSetHash == invalid->getContentsHash()); + f.crankUntil([&]() { + return txsetMeter(app, "self-check-failed") == failedBefore + 1; + }); + REQUIRE(Access::selfCheckHasFailed(herder)); + // The ledger closes without applying the set, here as everywhere. + f.externalize(invalid); + REQUIRE(f.lm().getLastClosedLedgerNum() == seq); + REQUIRE(a.getLastSequenceNumber() == aSeq); + } + SECTION("an invalid set of another node says nothing about this node") + { + auto bad = paymentWithSeq(app, c, c.getLastSequenceNumber() + 5); + auto foreign = makeUnvalidatedTxSet(app, {bad}); + auto const seq = f.nextSeq(); + herder.getPendingEnvelopes().putTxSet(foreign->getContentsHash(), seq, + foreign); + vote(seq, f.valueFor(foreign)); + executed(); + REQUIRE(applyCounter(app, "txset-skipped") == skippedBefore + 1); + REQUIRE(txsetMeter(app, "self-check-failed") == failedBefore); + REQUIRE(!Access::selfCheckHasFailed(herder)); + } + SECTION("nor does this node's set under a close time it was not built " + "for") + { + // Valid for the close time the node built it for; another leader + // puts it forward for a later one, at which its time-bounded + // transaction has expired. + auto const builtFor = f.nextCloseTime(); + PreconditionsV2 cond; + cond.timeBounds.activate().maxTime = builtFor.toApplyTime().timePoint(); + auto bounded = transactionWithV2Precondition(app, b, 1, 100, cond); + f.mMempool = {bounded}; + auto const seq = f.nextSeq(); + Access::prepareByBuilding(herder, seq, builtFor); + auto own = Access::prepared(herder); + REQUIRE(own); + REQUIRE(own->sizeTxTotal() == 1); + executed(); + REQUIRE(applyCounter(app, "txset-skipped") == skippedBefore); + herder.getPendingEnvelopes().putTxSet(own->getContentsHash(), seq, own); + auto const later = ConsensusTime::fromApplyTime( + ApplyTime::fromTimePoint(builtFor.toApplyTime().timePoint() + 5)); + vote(seq, herder.makeStellarValue(own->getContentsHash(), later, + emptyUpgradeSteps, f.mCfg.NODE_SEED)); + executed(); + REQUIRE(applyCounter(app, "txset-skipped") == skippedBefore + 1); + REQUIRE(txsetMeter(app, "self-check-failed") == failedBefore); + REQUIRE(!Access::selfCheckHasFailed(herder)); + REQUIRE(Access::prepared(herder) == own); + } +} + +// What the selection itself can get wrong is checked before the set leaves +// the builder, also when the check of the written set comes later. +TEST_CASE("a build checks its selection before the set leaves it", + "[herder][txset][early-txset][soroban]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto both = app.getRoot()->create( + "S", app.getLedgerManager().getLastMinBalance(2) + 1'000'000'000); + auto const sSeq = both.getLastSequenceNumber(); + SorobanResources resources; + resources.instructions = 1'000'000; + resources.diskReadBytes = 1000; + resources.writeBytes = 1000; + TransactionFrameBasePtr classic = paymentWithSeq(app, both, sSeq + 1); + TransactionFrameBasePtr soroban = + createUploadWasmTx(app, both, 100, DEFAULT_TEST_RESOURCE_FEE, resources, + std::nullopt, 0, std::nullopt, sSeq + 1); + // Candidates as the herder must not hand them over: one account in both + // phases. + PerPhaseTransactionList phases{{classic}, {soroban}}; + PerPhaseTransactionList invalid(2); + bool handedOut = false; + auto onReady = [&](TxSetXDRFrameConstPtr const&) { handedOut = true; }; + + SECTION("with the check of the written set deferred") + { + std::shared_ptr check; + REQUIRE_THROWS_WITH( + makeTxSetFromTransactions( + phases, app, ApplyTimeOffset::fromSeconds(1), invalid, false, + {}, false, onReady, {}, nullptr, &check), + Catch::Matchers::Contains("MULTIPLE_TXS_PER_SOURCE_ACCOUNT")); + // Nothing was handed to the overlay or the peers. + REQUIRE(!handedOut); + REQUIRE(!check); + } + SECTION("with the check before returning, as always") + { + REQUIRE_THROWS_WITH( + makeTxSetFromTransactions(phases, app, + ApplyTimeOffset::fromSeconds(1), invalid, + false, {}, false, onReady), + Catch::Matchers::Contains("MULTIPLE_TXS_PER_SOURCE_ACCOUNT")); + } +} + +// Before tx sets are validated at apply, the other nodes validate a set +// before they vote for it, and the leader vouches for its own: there the +// builder checks the written set before it returns, as it always did. +TEST_CASE("the check of the written set is deferred only where sets are " + "validated at apply", + "[herder][txset][early-txset]") +{ + auto const deferred = GENERATE(false, true); + PrevalidationFixture f(/*herderSpeculates=*/false, std::nullopt, + deferred ? GATE_PROTOCOL : GATE_PROTOCOL - 1); + auto& app = *f.mApp; + auto& a = *f.mA; + REQUIRE(protocolValidatesTxSetsAtApply( + f.lm().getLastClosedLedgerHeader().header.ledgerVersion) == + deferred); + auto const passedBefore = txsetMeter(app, "self-check-passed"); + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + Access::prepareByBuilding(f.herder(), f.nextSeq(), f.nextCloseTime()); + REQUIRE(Access::prepared(f.herder())); + if (deferred) + { + f.crankUntil([&]() { + return txsetMeter(app, "self-check-passed") == passedBefore + 1; + }); + } + else + { + f.crankFor(std::chrono::milliseconds(50)); + REQUIRE(txsetMeter(app, "self-check-passed") == passedBefore); + } +} diff --git a/src/herder/test/TxSetTreeTests.cpp b/src/herder/test/TxSetTreeTests.cpp new file mode 100644 index 0000000000..9d2be05712 --- /dev/null +++ b/src/herder/test/TxSetTreeTests.cpp @@ -0,0 +1,373 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// Network-level tests of the spanning tree for the leader's tx set +// (EXPERIMENTAL_TX_SET_TREE): the leader writes its set to its children in a +// tree that every node derives from the nodes of the leader schedule and the +// slot's seed, and every node passes the set on to its own children; or, in +// "coded" mode, the leader cuts the set into erasure-coded pieces and every +// node gets its pieces from as many different senders. The logic is tested +// in the overlay (overlay/src/flood/txset_tree.rs, txset_coded.rs, +// overlay/src/libp2p_overlay/txset_tree_tests.rs, txset_coded_tests.rs); +// these tests run it under consensus, with Core telling every overlay the +// trees of the next slots. + +#include "crypto/SHA.h" +#include "herder/Herder.h" +#include "ledger/LedgerManager.h" +#include "main/Application.h" +#include "main/Config.h" +#include "overlay/RustOverlayManager.h" +#include "simulation/Simulation.h" +#include "simulation/Topologies.h" +#include "test/Catch2.h" +#include "test/test.h" +#include "util/MetricsRegistry.h" +#include + +using namespace stellar; + +namespace +{ +int const NODES = 7; + +// Network totals of the dissemination counters. +struct TreeCounts +{ + // Copies of a set written by its leader (tagged or not), by its leader + // with a header, and by relays. + int64_t mPushSent{0}; + int64_t mRootSent{0}; + int64_t mRelayed{0}; + // Tagged frames received, refused for relaying, received twice. + int64_t mReceived{0}; + int64_t mRefused{0}; + int64_t mDuplicate{0}; + // Requests put off because the tree should bring the set, and those + // that had to be sent after the wait. + int64_t mDeferred{0}; + int64_t mFallback{0}; + // Frames whose delay since the leader's send was recorded. + int64_t mDelayCount{0}; + // Core: sets delivered by the overlay, and requests repeated after the + // one-second retry interval. + int64_t mCoreFetches{0}; + int64_t mCoreRetries{0}; + // Relays not made by nodes that are set up to withhold them. + int64_t mWithheld{0}; + // Coded mode: pieces written by leaders and by relays, valid pieces + // received, pieces refused and pieces found wrong; sets rebuilt, with a + // parity piece, and rebuilds that gave no tx set. + int64_t mPiecesSent{0}; + int64_t mPiecesRelayed{0}; + int64_t mPiecesReceived{0}; + int64_t mPiecesRefused{0}; + int64_t mPiecesInvalid{0}; + int64_t mSetsRebuilt{0}; + int64_t mSetsRebuiltWithParity{0}; + int64_t mSetsFailed{0}; + int64_t mSetsInconsistent{0}; + + std::string + toString() const + { + return fmt::format( + "push sent {} (tagged {}), relayed {}, received {}, refused {}, " + "duplicate {}, deferred {}, fallback {}, delays {}; core fetches " + "{} retries {}; withheld {}; pieces sent {} relayed {} received " + "{} refused {} invalid {}; sets rebuilt {} (with parity {}) " + "failed {} inconsistent {}", + mPushSent, mRootSent, mRelayed, mReceived, mRefused, mDuplicate, + mDeferred, mFallback, mDelayCount, mCoreFetches, mCoreRetries, + mWithheld, mPiecesSent, mPiecesRelayed, mPiecesReceived, + mPiecesRefused, mPiecesInvalid, mSetsRebuilt, + mSetsRebuiltWithParity, mSetsFailed, mSetsInconsistent); + } +}; + +TreeCounts +countTree(std::vector const& nodes) +{ + TreeCounts res; + for (auto const& app : nodes) + { + app->getOverlayManager().syncOverlayMetrics(); + auto& metrics = app->getMetrics(); + auto tree = [&](char const* name, char const* unit = "txset") { + return static_cast( + metrics.NewMeter({"overlay", "txset-tree", name}, unit) + .count()); + }; + res.mPushSent += static_cast( + metrics.NewMeter({"overlay", "txset-push", "sent"}, "txset") + .count()); + res.mRootSent += tree("root-sent"); + res.mRelayed += tree("relayed"); + res.mReceived += tree("received"); + res.mRefused += tree("refused"); + res.mDuplicate += tree("duplicate"); + res.mDeferred += tree("fetch-deferred"); + res.mFallback += tree("fetch-fallback"); + res.mDelayCount += tree("delay-count"); + res.mWithheld += tree("withheld"); + auto coded = [&](char const* name, char const* unit) { + return static_cast( + metrics.NewMeter({"overlay", "txset-coded", name}, unit) + .count()); + }; + res.mPiecesSent += coded("pieces-sent", "piece"); + res.mPiecesRelayed += coded("pieces-relayed", "piece"); + res.mPiecesReceived += coded("pieces-received", "piece"); + res.mPiecesRefused += coded("pieces-refused", "piece"); + res.mPiecesInvalid += coded("pieces-invalid", "piece"); + res.mSetsRebuilt += coded("sets-rebuilt", "txset"); + res.mSetsRebuiltWithParity += + coded("sets-rebuilt-with-parity", "txset"); + res.mSetsFailed += coded("sets-failed", "txset"); + res.mSetsInconsistent += coded("sets-inconsistent", "txset"); + res.mCoreFetches += static_cast( + metrics.NewTimer({"scp", "fetch", "txset"}).count()); + res.mCoreRetries += static_cast( + metrics.NewCounter({"scp", "fetch", "txset-retry"}).count()); + } + return res; +} + +// A 7-node network with a 5-of-7 quorum, every node connected to every other +// one. With a fanout of 2 its trees have two levels: the leader, two relays, +// four leaves. +Simulation::pointer +makeNetwork(std::function const& adjust) +{ + return Topologies::core( + NODES, 0.7, sha256(getTestConfig().NETWORK_PASSPHRASE), [&](int i) { + auto cfg = getTestConfig(i); + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + cfg.EXPERIMENTAL_TX_SET_TREE_FANOUT = 2; + adjust(cfg, i); + return cfg; + }); +} + +// Starts the network, cranks until every node closed `lastLedger` and +// returns how many ledgers that were (from the first node's ledger at the +// start). +int64_t +closeLedgers(Simulation::pointer const& simulation, uint32_t lastLedger) +{ + simulation->startAllNodes(); + auto nodes = simulation->getNodes(); + auto const first = nodes[0]->getLedgerManager().getLastClosedLedgerNum(); + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(lastLedger, 3); }, + std::chrono::seconds(300), false); + REQUIRE(simulation->haveAllExternalized(lastLedger, 3)); + return static_cast(lastLedger) - first; +} +} + +TEST_CASE("tx set tree mode is read from the config", "[herder][txset-tree]") +{ + Config cfg; + REQUIRE(cfg.EXPERIMENTAL_TX_SET_TREE == "off"); + REQUIRE(cfg.EXPERIMENTAL_TX_SET_TREE_FANOUT == 0); + REQUIRE(cfg.txSetTreeMode() == 0); + cfg.EXPERIMENTAL_TX_SET_TREE = "star"; + REQUIRE(cfg.txSetTreeMode() == 1); + cfg.EXPERIMENTAL_TX_SET_TREE = "hash"; + REQUIRE(cfg.txSetTreeMode() == 2); + cfg.EXPERIMENTAL_TX_SET_TREE = "rtt"; + REQUIRE(cfg.txSetTreeMode() == 3); + cfg.EXPERIMENTAL_TX_SET_TREE = "coded"; + REQUIRE(cfg.txSetTreeMode() == 4); + REQUIRE(cfg.EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING == 0); +} + +TEST_CASE("the tx set of a leader travels down the hash tree", + "[herder][txset-tree][simulation]") +{ + auto simulation = makeNetwork( + [](Config& cfg, int) { cfg.EXPERIMENTAL_TX_SET_TREE = "hash"; }); + uint32_t const lastLedger = 16; + auto const ledgers = closeLedgers(simulation, lastLedger); + REQUIRE(ledgers >= 12); + + auto counts = countTree(simulation->getNodes()); + INFO(counts.toString()); + // Sets were relayed, by nodes that their parent in the tree sent them + // to: nothing was refused. + REQUIRE(counts.mRelayed > 0); + REQUIRE(counts.mRefused == 0); + // Once the nodes know each other's validator identity and the trees, + // a leader writes 2 copies of its set and its 2 children write 2 each; + // before that (the first ledgers) it writes to all 6 peers. So the + // leaders wrote far fewer copies than 6 per ledger, and most of the + // copies came through relays. + REQUIRE(counts.mPushSent <= 4 * ledgers); + REQUIRE(counts.mRelayed >= 2 * (ledgers - 6)); + // Every node still got every set from its overlay without the + // one-second retry. + REQUIRE(counts.mCoreFetches >= (NODES - 1) * (ledgers - 2)); + REQUIRE(counts.mCoreRetries == 0); + // The delay since the leader's send was recorded for the tagged frames. + REQUIRE(counts.mDelayCount == counts.mReceived); + REQUIRE(counts.mReceived > 0); +} + +TEST_CASE("star mode pushes to every peer with a header", + "[herder][txset-tree][simulation]") +{ + auto simulation = makeNetwork( + [](Config& cfg, int) { cfg.EXPERIMENTAL_TX_SET_TREE = "star"; }); + uint32_t const lastLedger = 12; + auto const ledgers = closeLedgers(simulation, lastLedger); + REQUIRE(ledgers >= 8); + + auto counts = countTree(simulation->getNodes()); + INFO(counts.toString()); + // Every leader wrote its set to its 6 peers; nothing was relayed, and + // nobody waited before asking. + REQUIRE(counts.mPushSent >= (NODES - 1) * (ledgers - 2)); + REQUIRE(counts.mRootSent > 0); + REQUIRE(counts.mRelayed == 0); + REQUIRE(counts.mRefused == 0); + REQUIRE(counts.mDeferred == 0); + REQUIRE(counts.mDelayCount == counts.mReceived); + REQUIRE(counts.mReceived > 0); + REQUIRE(counts.mCoreRetries == 0); +} + +TEST_CASE("without the setting nothing is tagged or relayed", + "[herder][txset-tree][simulation]") +{ + auto simulation = makeNetwork([](Config&, int) {}); + uint32_t const lastLedger = 10; + auto const ledgers = closeLedgers(simulation, lastLedger); + + auto counts = countTree(simulation->getNodes()); + INFO(counts.toString()); + REQUIRE(counts.mPushSent >= (NODES - 1) * (ledgers - 2)); + REQUIRE(counts.mRootSent == 0); + REQUIRE(counts.mReceived == 0); + REQUIRE(counts.mRelayed == 0); + REQUIRE(counts.mDeferred == 0); + REQUIRE(counts.mCoreRetries == 0); +} + +TEST_CASE("the leader lays the tree out by round-trip times", + "[herder][txset-tree][simulation]") +{ + auto simulation = makeNetwork( + [](Config& cfg, int) { cfg.EXPERIMENTAL_TX_SET_TREE = "rtt"; }); + uint32_t const lastLedger = 16; + auto const ledgers = closeLedgers(simulation, lastLedger); + REQUIRE(ledgers >= 12); + + auto counts = countTree(simulation->getNodes()); + INFO(counts.toString()); + // On loopback every peer is at the same distance, so the order is + // arbitrary; what matters here is that every node follows the order + // the leader sends along. + REQUIRE(counts.mRelayed >= 2 * (ledgers - 6)); + REQUIRE(counts.mRefused == 0); + REQUIRE(counts.mPushSent <= 4 * ledgers); + REQUIRE(counts.mCoreRetries == 0); +} + +TEST_CASE( + "a node that does not relay costs its subtree a wait and not a ledger", + "[herder][txset-tree][simulation]") +{ + // Node 0 neither relays nor is told about trees. When it is a relay of + // the leader's tree, its two children do not get the set through the + // tree: they ask for it after the wait, well before Core's one-second + // retry, and the network closes its ledgers. + auto simulation = makeNetwork([](Config& cfg, int i) { + cfg.EXPERIMENTAL_TX_SET_TREE = i == 0 ? "off" : "hash"; + }); + uint32_t const lastLedger = 30; + auto const ledgers = closeLedgers(simulation, lastLedger); + REQUIRE(ledgers >= 24); + + auto counts = countTree(simulation->getNodes()); + INFO(counts.toString()); + REQUIRE(counts.mRelayed > 0); + // Node 0 is one of the two relays in about a third of the trees it does + // not lead: over this many ledgers some subtree had to ask. + REQUIRE(counts.mFallback > 0); + REQUIRE(counts.mCoreRetries == 0); +} + +TEST_CASE("the tx set of a leader travels as coded pieces", + "[herder][txset-tree][txset-coded][simulation]") +{ + // With 6 other nodes a set is cut into 6 pieces of which 3 rebuild it; + // the leader writes one piece to each node, and each passes its piece + // on to the 5 others. + auto simulation = makeNetwork( + [](Config& cfg, int) { cfg.EXPERIMENTAL_TX_SET_TREE = "coded"; }); + uint32_t const lastLedger = 16; + auto const ledgers = closeLedgers(simulation, lastLedger); + REQUIRE(ledgers >= 12); + + auto counts = countTree(simulation->getNodes()); + INFO(counts.toString()); + // Once the nodes know each other's validator identity and the layouts, + // a leader writes 6 pieces per set and no whole copy; before that (the + // first ledgers) it writes the set to all 6 peers. + REQUIRE(counts.mPiecesSent >= 6 * (ledgers - 6)); + REQUIRE(counts.mPiecesRelayed >= 5 * 6 * (ledgers - 6)); + REQUIRE(counts.mPushSent <= 4 * ledgers); + // Every node rebuilt those sets from pieces, and no piece was wrong. + REQUIRE(counts.mSetsRebuilt >= (NODES - 1) * (ledgers - 6)); + REQUIRE(counts.mPiecesInvalid == 0); + REQUIRE(counts.mSetsFailed == 0); + REQUIRE(counts.mSetsInconsistent == 0); + REQUIRE(counts.mWithheld == 0); + // Every node still got every set from its overlay without the + // one-second retry, and the delay was recorded for every rebuilt set. + REQUIRE(counts.mCoreFetches >= (NODES - 1) * (ledgers - 2)); + REQUIRE(counts.mCoreRetries == 0); + REQUIRE(counts.mDelayCount == counts.mReceived); + REQUIRE(counts.mReceived >= counts.mSetsRebuilt); +} + +TEST_CASE("relays that withhold cost a tree a wait and coded pieces none", + "[herder][txset-tree][txset-coded][simulation]") +{ + // The validator with the smallest node ID passes nothing on. + // - In the hash tree (two relays, two children each) the children of + // that relay ask for the set after their wait, in every tree in which + // it is a relay. + // - In coded mode a node then holds at least 5 of its 6 pieces and + // needs 3: it rebuilds the set, mostly with a parity piece, and + // nobody has to ask. + auto run = [](std::string const& mode) { + auto simulation = makeNetwork([&](Config& cfg, int) { + cfg.EXPERIMENTAL_TX_SET_TREE = mode; + cfg.EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING = 1; + }); + uint32_t const lastLedger = 30; + auto const ledgers = closeLedgers(simulation, lastLedger); + REQUIRE(ledgers >= 24); + auto counts = countTree(simulation->getNodes()); + // The network closes its ledgers either way, without Core's + // one-second retry. + REQUIRE(counts.mCoreRetries == 0); + REQUIRE(counts.mWithheld > 0); + return counts; + }; + auto const tree = run("hash"); + auto const coded = run("coded"); + INFO("hash: " << tree.toString()); + INFO("coded: " << coded.toString()); + REQUIRE(tree.mFallback > 0); + REQUIRE(coded.mSetsRebuiltWithParity > 0); + REQUIRE(coded.mPiecesInvalid == 0); + REQUIRE(coded.mSetsFailed == 0); + // Coded: far fewer sets asked for than in the tree (none, unless two + // writes to one node failed for one set, which the first ledgers' + // stream set-up can do). + REQUIRE(coded.mFallback * 4 <= tree.mFallback); +} diff --git a/src/historywork/VerifyBucketWork.cpp b/src/historywork/VerifyBucketWork.cpp index 8a8e62038f..5baa1bb2b7 100644 --- a/src/historywork/VerifyBucketWork.cpp +++ b/src/historywork/VerifyBucketWork.cpp @@ -3,11 +3,14 @@ // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 #include "historywork/VerifyBucketWork.h" +#include "bucket/LiveBucket.h" #include "bucket/LiveBucketIndex.h" +#include "bucket/ShardedTopBucket.h" #include "crypto/Hex.h" #include "crypto/SHA.h" #include "history/HistoryArchive.h" #include "main/Application.h" +#include "main/Config.h" #include "main/ErrorMessages.h" #include "util/Fs.h" #include "util/Logging.h" @@ -17,10 +20,33 @@ #include #include +#include namespace stellar { +namespace +{ +// Whether `filename` is the canonical file of records with the commutative +// hash `hash` (see ShardedTopBucket::hashOfFile). A file that is not is +// reported by the caller as any other bucket that fails verification. +bool +hasCommutativeHash(std::string const& filename, uint256 const& hash) +{ + try + { + return ShardedTopBucket::bucketHash( + ShardedTopBucket::hashOfFile(filename)) == hash; + } + catch (std::exception const& e) + { + CLOG_WARNING(History, "Not a canonical level-0 bucket file: {}", + e.what()); + return false; + } +} +} + template VerifyBucketWork::VerifyBucketWork( Application& app, std::string const& bucketFile, uint256 const& hash, @@ -109,6 +135,17 @@ VerifyBucketWork::spawnVerifier() CLOG_DEBUG(History, "Verified hash ({}) for {}", hexAbbrev(hash), filename); } + else if (std::is_same_v && + app.getConfig().EXPERIMENTAL_SHARDED_TOP_BUCKET > 0 && + hasCommutativeHash(filename, hash)) + { + // A level-0 bucket of the live bucket list: named by the + // commutative hash of its records. (Which hash a bucket + // has follows from its level; the experiment takes + // either.) + CLOG_DEBUG(History, "Verified commutative hash ({}) for {}", + hexAbbrev(hash), filename); + } else { CLOG_WARNING(History, "FAILED verifying hash for {}", diff --git a/src/ledger/LedgerManager.h b/src/ledger/LedgerManager.h index 74d18659a7..ad551c68c1 100644 --- a/src/ledger/LedgerManager.h +++ b/src/ledger/LedgerManager.h @@ -276,6 +276,12 @@ class LedgerManager // instead. virtual std::unique_ptr getLCLView() const = 0; + // The immutable state of the last closed ledger itself, for a caller + // that reads it from other threads, each through a view of its own, and + // needs it to stay the same ledger while the LCL moves on. Callable from + // any thread but the apply thread. + virtual ImmutableLedgerDataPtr getLastClosedLedgerState() const = 0; + // Create a copy of the canonical ledger snapshot at the current moment in // apply flow. // The returned snapshot is *not* thread-safe, so every thread used during diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index abf426eaa4..fe588210f7 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -6,6 +6,7 @@ #include "bucket/BucketManager.h" #include "bucket/HotArchiveBucketList.h" #include "bucket/LiveBucketList.h" +#include "bucket/ShardedTopBucket.h" #include "catchup/AssumeStateWork.h" #include "crypto/Hex.h" #include "crypto/KeyUtils.h" @@ -619,6 +620,11 @@ LedgerManagerImpl::loadLastKnownLedgerInternal(bool skipBuildingFullState) releaseAssert(latestLedgerHeader.has_value()); + if (mApp.getConfig().EXPERIMENTAL_SHARDED_TOP_BUCKET > 0) + { + ShardedTopBucket::prepareForRestart(mApp, has, + latestLedgerHeader->ledgerVersion); + } auto missing = mApp.getBucketManager().checkForMissingBucketsFiles(has); auto pubmissing = mApp.getHistoryManager().getMissingBucketsReferencedByPublishQueue(); @@ -2975,6 +2981,15 @@ LedgerManagerImpl::getLCLView() const return std::make_unique(copyImmutableLedgerView()); } +ImmutableLedgerDataPtr +LedgerManagerImpl::getLastClosedLedgerState() const +{ + releaseAssert(!mApp.threadIsType(Application::ThreadType::APPLY)); + SharedLockShared guard(mLastClosedLedgerStateMutex); + releaseAssert(mLastClosedLedgerState); + return mLastClosedLedgerState; +} + ApplyLedgerView LedgerManagerImpl::copyApplyLedgerView() const { diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h index fbc81def65..fb4f59a5dc 100644 --- a/src/ledger/LedgerManagerImpl.h +++ b/src/ledger/LedgerManagerImpl.h @@ -731,6 +731,7 @@ class LedgerManagerImpl : public LedgerManager public: std::unique_ptr getLCLView() const override; + ImmutableLedgerDataPtr getLastClosedLedgerState() const override; ApplyLedgerView copyApplyLedgerView() const override; void syncWithLCLView( std::unique_ptr& ledgerView) const override; diff --git a/src/main/CommandHandler.cpp b/src/main/CommandHandler.cpp index 4582b9dc7e..4410eb3903 100644 --- a/src/main/CommandHandler.cpp +++ b/src/main/CommandHandler.cpp @@ -1161,6 +1161,10 @@ CommandHandler::generateLoad(std::string const& params, std::string& retStr) parseOptionalParam(map, "maxfeerate"); cfg.skipLowFeeTxs = parseOptionalParamOrDefault(map, "skiplowfeetxs", false); + // Ledgers to wait, after the last submission, for the load to be + // applied; 0 keeps the mode's default. + cfg.completionTimeoutLedgers = + parseOptionalParamOrDefault(map, "completiontimeout", 0); if (cfg.isSoroban()) { diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 50e994f443..26d1c694b0 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -47,6 +47,7 @@ static std::unordered_set const TESTING_ONLY_OPTIONS = { "GENESIS_TEST_ACCOUNT_COUNT", "UPDATE_SOROBAN_COSTS_DURING_PROTOCOL_UPGRADE_FOR_TESTING", "ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING", + "EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING", "ARTIFICIALLY_SET_CLOSE_TIME_FOR_TESTING", "ARTIFICIALLY_REPLAY_WITH_NEWEST_BUCKET_LOGIC_FOR_TESTING", "OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING", @@ -175,7 +176,13 @@ Config::Config() : NODE_SEED(SecretKey::random()) BACKGROUND_OVERLAY_PROCESSING = true; PARALLEL_LEDGER_APPLY = true; EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = true; + EXPERIMENTAL_EARLY_TX_SET_SELECTION = true; + EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT = 10; + EXPERIMENTAL_TX_SET_TREE = "off"; + EXPERIMENTAL_TX_SET_TREE_FANOUT = 0; + EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING = 0; EXPERIMENTAL_PARALLEL_TX_SET_DOWNLOAD = false; + EXPERIMENTAL_SHARDED_TOP_BUCKET = 0; DISABLE_SOROBAN_METRICS_FOR_TESTING = false; DISABLE_TX_META_FOR_TESTING = false; BACKGROUND_TX_SIG_VERIFICATION = true; @@ -1218,10 +1225,49 @@ Config::processConfig(std::shared_ptr t) [&]() { EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = readBool(item); }}, + {"EXPERIMENTAL_EARLY_TX_SET_SELECTION", + [&]() { + EXPERIMENTAL_EARLY_TX_SET_SELECTION = readBool(item); + }}, + {"EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT", + [&]() { + EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT = + readInt(item, 0, 1000); + }}, + {"EXPERIMENTAL_TX_SET_TREE", + [&]() { + EXPERIMENTAL_TX_SET_TREE = readString(item); + if (EXPERIMENTAL_TX_SET_TREE != "off" && + EXPERIMENTAL_TX_SET_TREE != "star" && + EXPERIMENTAL_TX_SET_TREE != "hash" && + EXPERIMENTAL_TX_SET_TREE != "rtt" && + EXPERIMENTAL_TX_SET_TREE != "coded") + { + throw std::invalid_argument( + "EXPERIMENTAL_TX_SET_TREE must be one of " + "\"off\", \"star\", \"hash\", \"rtt\", " + "\"coded\""); + } + }}, + {"EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING", + [&]() { + EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING = + readInt(item, 0, 1024); + }}, + {"EXPERIMENTAL_TX_SET_TREE_FANOUT", + [&]() { + EXPERIMENTAL_TX_SET_TREE_FANOUT = + readInt(item, 0, 255); + }}, {"EXPERIMENTAL_PARALLEL_TX_SET_DOWNLOAD", [&]() { EXPERIMENTAL_PARALLEL_TX_SET_DOWNLOAD = readBool(item); }}, + {"EXPERIMENTAL_SHARDED_TOP_BUCKET", + [&]() { + EXPERIMENTAL_SHARDED_TOP_BUCKET = + readInt(item, 0, 256); + }}, {"DISABLE_SOROBAN_METRICS_FOR_TESTING", [&]() { DISABLE_SOROBAN_METRICS_FOR_TESTING = readBool(item); @@ -1243,9 +1289,7 @@ Config::processConfig(std::shared_ptr t) readBool(item); }}, {"TX_FLOOD_TARGET_LEADERS", - [&]() { - TX_FLOOD_TARGET_LEADERS = readInt(item); - }}, + [&]() { TX_FLOOD_TARGET_LEADERS = readInt(item); }}, {"TX_FLOOD_LEGACY_ADVERTS", [&]() { TX_FLOOD_LEGACY_ADVERTS = readBool(item); }}, {"BATCH_EXECUTOR_TASKS", @@ -2151,6 +2195,14 @@ Config::processConfig(std::shared_ptr t) gIsProductionNetwork = NETWORK_PASSPHRASE == "Public Global Stellar Network ; September 2015"; + if (gIsProductionNetwork && EXPERIMENTAL_SHARDED_TOP_BUCKET > 0) + { + throw std::invalid_argument( + "EXPERIMENTAL_SHARDED_TOP_BUCKET changes the bucket list " + "hash of the ledger header and is not supported on the " + "public network"); + } + if (gIsProductionNetwork && usedExternalSecrets) { throw std::invalid_argument( @@ -2401,6 +2453,40 @@ Config::logBasicInfo() const "EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY=" "{}", EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY ? "true" : "false"); + LOG_INFO(DEFAULT_LOG, + "EXPERIMENTAL_EARLY_TX_SET_SELECTION={}, " + "EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT={}", + EXPERIMENTAL_EARLY_TX_SET_SELECTION ? "true" : "false", + EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT); + LOG_INFO(DEFAULT_LOG, + "EXPERIMENTAL_TX_SET_TREE={} (fanout {}, withholders for " + "testing {})", + EXPERIMENTAL_TX_SET_TREE, EXPERIMENTAL_TX_SET_TREE_FANOUT, + EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING); + LOG_INFO(DEFAULT_LOG, "EXPERIMENTAL_SHARDED_TOP_BUCKET={}", + EXPERIMENTAL_SHARDED_TOP_BUCKET); +} + +uint8_t +Config::txSetTreeMode() const +{ + if (EXPERIMENTAL_TX_SET_TREE == "star") + { + return 1; + } + if (EXPERIMENTAL_TX_SET_TREE == "hash") + { + return 2; + } + if (EXPERIMENTAL_TX_SET_TREE == "rtt") + { + return 3; + } + if (EXPERIMENTAL_TX_SET_TREE == "coded") + { + return 4; + } + return 0; } void diff --git a/src/main/Config.h b/src/main/Config.h index 623f760ddf..41e1a3e3c8 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -540,9 +540,71 @@ class Config : public std::enable_shared_from_this // APPLY_TIME_TX_SET_VALIDATION_PROTOCOL_VERSION on. bool EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY; + // The node that expects to lead the next ledger selects its transaction + // set while the current ledger is still in its ballot, from the current + // ledger's set and the last closed state, and completes it when the + // current ledger has closed: every selected transaction is validated + // against the closed ledger and the set is topped up from the mempool + // (see herder/EarlyTxSetSelection.h). Without it the set is built when + // the current ledger has closed. Only from + // APPLY_TIME_TX_SET_VALIDATION_PROTOCOL_VERSION on. + bool EXPERIMENTAL_EARLY_TX_SET_SELECTION; + + // How much more than a transaction set can hold a leader pulls from the + // mempool for a build, in percent of the set's capacity. A build whose + // set is short after validation pulls again, so the margin only decides + // how often that happens. + uint32_t EXPERIMENTAL_TX_SET_PULL_MARGIN_PERCENT; + + // How the round's leader gets its transaction set to the other nodes + // (experimental; see overlay/src/flood/txset_tree.rs): + // "off" - it writes the set to every peer (as before). + // "star" - the same, with a header that says when it was sent, so that + // the delivery can be measured like a tree's. + // "hash" - down a spanning tree that every node derives from the nodes + // of the leader schedule and the slot's seed: the leader writes + // the set to its children only, and every node passes it on to + // its own. + // "rtt" - down a tree the leader lays out by its round-trip times to + // its peers; the order travels with the set. + // "coded" - as erasure-coded pieces: the leader cuts the set into 16 + // pieces of which any 8 rebuild it (fewer in a small network), + // signs the root of their Merkle tree and sends every piece + // to its own relays; a node gets its pieces from as many + // different senders, so up to half of them can fail or lie + // without the node having to wait and ask. + // Every node of a network should use the same value: a node relays in + // "hash", "rtt" and "coded" only, and only what fits its own mode. + std::string EXPERIMENTAL_TX_SET_TREE; + + // Children per node in that tree. 0 = the smallest number that reaches + // every node in two hops (9 for 90 nodes). + uint32_t EXPERIMENTAL_TX_SET_TREE_FANOUT; + + // For tests and fault runs: the validators with the N smallest node IDs + // among the nodes of the leader schedule pass nothing on (tree frames or + // pieces). They still lead, receive and vote. 0 = nobody (default). + uint32_t EXPERIMENTAL_TX_SET_TREE_WITHHOLDERS_FOR_TESTING; + + // EXPERIMENTAL_TX_SET_TREE as the overlay's mode byte. + uint8_t txSetTreeMode() const; + // Allow downloading of transaction sets in parallel with SCP (experimental) bool EXPERIMENTAL_PARALLEL_TX_SET_DOWNLOAD; + // Experimental: level 0 of the live bucket list is hashed with a + // commutative hash (LtHash16 over the bucket's records) instead of the + // SHA-256 of its file, and a ledger's entries are written as shards, in + // parallel; the bucket's file is written afterwards, off the path of the + // ledger close (see bucket/ShardedTopBucket.h). 0 = off. N > 0: N shards + // for contract data and code, N for TTL entries and N for all other + // entries. + // + // This changes the bucket list hash in the ledger header: every node of + // a network must have it on, or every node off. The number of shards is + // each node's own choice and changes no hash. + uint32_t EXPERIMENTAL_SHARDED_TOP_BUCKET; + // Disable expensive Soroban metrics for performance testing bool DISABLE_SOROBAN_METRICS_FOR_TESTING; diff --git a/src/overlay/IPC.h b/src/overlay/IPC.h index c6de7e8704..db6b6b7540 100644 --- a/src/overlay/IPC.h +++ b/src/overlay/IPC.h @@ -96,6 +96,17 @@ enum class IPCMessageType : uint32_t /// everyone = 1: flood to all peers (the list is ignored). SET_TX_TARGETS = 16, + /// The spanning trees the leader's TX set travels down in the next + /// slots (see HerderImpl::updateTxSetTree and + /// overlay/src/flood/txset_tree.rs). + /// Payload: [mode:1][fanout:1][slots:1] + /// ([slot:4][seed:32][leaders:1][nodeID:32]...)... + /// [count:4][nodeID:32]... + /// mode: 0 off, 1 star, 2 hash, 3 rtt. fanout 0 = the smallest that + /// reaches every node in two hops. leaders of a slot: the nodes that may + /// lead it and so root a tree. + SET_TX_SET_TREE = 17, + // ═══ Overlay → Core (Critical Path) ═══ /// Received SCP envelope from network diff --git a/src/overlay/OverlayIPC.cpp b/src/overlay/OverlayIPC.cpp index c5f0a4666b..1eceb4ae97 100644 --- a/src/overlay/OverlayIPC.cpp +++ b/src/overlay/OverlayIPC.cpp @@ -905,6 +905,54 @@ OverlayIPC::setTxTargets(std::optional> const& targets) return mChannel->send(msg); } +bool +OverlayIPC::setTxSetTree(uint8_t mode, uint8_t fanout, + std::vector const& slots, + std::vector const& nodes) +{ + if (!mChannel || !mChannel->isConnected()) + { + return false; + } + releaseAssert(slots.size() <= UINT8_MAX); + IPCMessage msg; + msg.type = IPCMessageType::SET_TX_SET_TREE; + uint32_t const count = static_cast(nodes.size()); + size_t size = 3 + 4 + static_cast(count) * 32; + for (auto const& slot : slots) + { + // 255 leaders would read as "any node". + releaseAssert(slot.mLeaders.size() < UINT8_MAX); + size += 37 + slot.mLeaders.size() * 32; + } + msg.payload.resize(size); + auto* out = msg.payload.data(); + *out++ = mode; + *out++ = fanout; + *out++ = static_cast(slots.size()); + for (auto const& slot : slots) + { + std::memcpy(out, &slot.mSlot, 4); + std::memcpy(out + 4, slot.mSeed.data(), 32); + out[36] = static_cast(slot.mLeaders.size()); + out += 37; + for (auto const& leader : slot.mLeaders) + { + std::memcpy(out, leader.ed25519().data(), 32); + out += 32; + } + } + std::memcpy(out, &count, 4); + out += 4; + for (auto const& node : nodes) + { + std::memcpy(out, node.ed25519().data(), 32); + out += 32; + } + std::lock_guard lock(mSendMutex); + return mChannel->send(msg); +} + void OverlayIPC::setOnOverlayIdentity(OverlayIdentityCallback cb) { diff --git a/src/overlay/OverlayIPC.h b/src/overlay/OverlayIPC.h index c3bd5990c1..28dbb01dbe 100644 --- a/src/overlay/OverlayIPC.h +++ b/src/overlay/OverlayIPC.h @@ -238,6 +238,24 @@ class OverlayIPC */ bool setTxTargets(std::optional> const& targets); + // One slot of setTxSetTree: the seed of its trees and the nodes that may + // lead it (the leaders of its first rounds, as this node expects them). + struct TxSetTreeSlot + { + uint32_t mSlot; + Hash mSeed; + std::vector mLeaders; + }; + + /** + * Describe the spanning trees of the next slots to the overlay: the mode + * and fanout, each slot's seed and possible leaders, and the nodes of the + * trees. Returns whether it was sent. + */ + bool setTxSetTree(uint8_t mode, uint8_t fanout, + std::vector const& slots, + std::vector const& nodes); + /// Set callback for the overlay's peer id report void setOnOverlayIdentity(OverlayIdentityCallback cb); diff --git a/src/overlay/OverlayMetrics.cpp b/src/overlay/OverlayMetrics.cpp index 307068011e..ad7db82e08 100644 --- a/src/overlay/OverlayMetrics.cpp +++ b/src/overlay/OverlayMetrics.cpp @@ -69,6 +69,86 @@ OverlayMetrics::OverlayMetrics(Application& app) {"overlay", "txset-push", "dropped"}, "txset")) , mTxSetPushEvicted(app.getMetrics().NewMeter( {"overlay", "txset-push", "evicted"}, "txset")) + , mTxsetPushSentBytes(app.getMetrics().NewMeter( + {"overlay", "txset-push", "sent-bytes"}, "byte")) + , mTxsetTreeRootSent(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "root-sent"}, "txset")) + , mTxsetTreeRelayed(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "relayed"}, "txset")) + , mTxsetTreeRelayedBytes(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "relayed-bytes"}, "byte")) + , mTxsetTreeReceived(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "received"}, "txset")) + , mTxsetTreeDuplicate(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "duplicate"}, "txset")) + , mTxsetTreeRefused(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "refused"}, "txset")) + , mTxsetTreeUnreachable(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "unreachable"}, "peer")) + , mTxsetTreeChildBusy(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "child-busy"}, "peer")) + , mTxsetTreeRelayBusy(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "relay-busy"}, "peer")) + , mTxsetTreeRelayLimited(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "relay-limited"}, "txset")) + , mTxsetTreeFetchDeferred(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "fetch-deferred"}, "txset")) + , mTxsetTreeFetchFallback(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "fetch-fallback"}, "txset")) + , mTxsetTreeDelayMsSum(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-ms-sum"}, "ms")) + , mTxsetTreeDelayCount(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-count"}, "txset")) + , mTxsetTreeDelayHop1MsSum(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-hop1-ms-sum"}, "ms")) + , mTxsetTreeDelayHop1Count(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-hop1-count"}, "txset")) + , mTxsetTreeDelayHop2MsSum(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-hop2-ms-sum"}, "ms")) + , mTxsetTreeDelayHop2Count(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-hop2-count"}, "txset")) + , mTxsetTreeDelayLe50(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-le-50ms"}, "txset")) + , mTxsetTreeDelayLe100(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-le-100ms"}, "txset")) + , mTxsetTreeDelayLe150(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-le-150ms"}, "txset")) + , mTxsetTreeDelayLe200(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-le-200ms"}, "txset")) + , mTxsetTreeDelayLe300(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-le-300ms"}, "txset")) + , mTxsetTreeDelayGt300(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "delay-gt-300ms"}, "txset")) + , mTxsetTreeWithheld(app.getMetrics().NewMeter( + {"overlay", "txset-tree", "withheld"}, "txset")) + , mTxsetCodedSetsEncoded(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "sets-encoded"}, "txset")) + , mTxsetCodedPiecesSent(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-sent"}, "piece")) + , mTxsetCodedPiecesRelayed(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-relayed"}, "piece")) + , mTxsetCodedPiecesReceived(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-received"}, "piece")) + , mTxsetCodedPiecesLate(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-late"}, "piece")) + , mTxsetCodedPiecesRefused(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-refused"}, "piece")) + , mTxsetCodedPiecesInvalid(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-invalid"}, "piece")) + , mTxsetCodedPiecesDuplicate(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-duplicate"}, "piece")) + , mTxsetCodedPiecesSkipped(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "pieces-skipped"}, "piece")) + , mTxsetCodedSetsRebuilt(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "sets-rebuilt"}, "txset")) + , mTxsetCodedSetsRebuiltWithParity(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "sets-rebuilt-with-parity"}, "txset")) + , mTxsetCodedSetsFailed(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "sets-failed"}, "txset")) + , mTxsetCodedSetsInconsistent(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "sets-inconsistent"}, "txset")) + , mTxsetCodedRebuildUsSum(app.getMetrics().NewMeter( + {"overlay", "txset-coded", "rebuild-us-sum"}, "microsecond")) , mTxDirectSent(app.getMetrics().NewMeter( {"overlay", "flood", "tx-direct-sent"}, "transaction")) , mTxDirectSentBytes(app.getMetrics().NewMeter( diff --git a/src/overlay/OverlayMetrics.h b/src/overlay/OverlayMetrics.h index cfaa696eff..581f10006a 100644 --- a/src/overlay/OverlayMetrics.h +++ b/src/overlay/OverlayMetrics.h @@ -75,6 +75,52 @@ struct OverlayMetrics medida::Meter& mTxSetPushDropped; medida::Meter& mTxSetPushEvicted; + // Spanning tree for the leader's tx-set push (kept by the Rust overlay; + // see overlay/src/metrics.rs for what each counts). + medida::Meter& mTxsetPushSentBytes; + medida::Meter& mTxsetTreeRootSent; + medida::Meter& mTxsetTreeRelayed; + medida::Meter& mTxsetTreeRelayedBytes; + medida::Meter& mTxsetTreeReceived; + medida::Meter& mTxsetTreeDuplicate; + medida::Meter& mTxsetTreeRefused; + medida::Meter& mTxsetTreeUnreachable; + medida::Meter& mTxsetTreeChildBusy; + medida::Meter& mTxsetTreeRelayBusy; + medida::Meter& mTxsetTreeRelayLimited; + medida::Meter& mTxsetTreeFetchDeferred; + medida::Meter& mTxsetTreeFetchFallback; + medida::Meter& mTxsetTreeDelayMsSum; + medida::Meter& mTxsetTreeDelayCount; + medida::Meter& mTxsetTreeDelayHop1MsSum; + medida::Meter& mTxsetTreeDelayHop1Count; + medida::Meter& mTxsetTreeDelayHop2MsSum; + medida::Meter& mTxsetTreeDelayHop2Count; + medida::Meter& mTxsetTreeDelayLe50; + medida::Meter& mTxsetTreeDelayLe100; + medida::Meter& mTxsetTreeDelayLe150; + medida::Meter& mTxsetTreeDelayLe200; + medida::Meter& mTxsetTreeDelayLe300; + medida::Meter& mTxsetTreeDelayGt300; + + // Erasure-coded pieces of the leader's tx set (EXPERIMENTAL_TX_SET_TREE + // = "coded") + medida::Meter& mTxsetTreeWithheld; + medida::Meter& mTxsetCodedSetsEncoded; + medida::Meter& mTxsetCodedPiecesSent; + medida::Meter& mTxsetCodedPiecesRelayed; + medida::Meter& mTxsetCodedPiecesReceived; + medida::Meter& mTxsetCodedPiecesLate; + medida::Meter& mTxsetCodedPiecesRefused; + medida::Meter& mTxsetCodedPiecesInvalid; + medida::Meter& mTxsetCodedPiecesDuplicate; + medida::Meter& mTxsetCodedPiecesSkipped; + medida::Meter& mTxsetCodedSetsRebuilt; + medida::Meter& mTxsetCodedSetsRebuiltWithParity; + medida::Meter& mTxsetCodedSetsFailed; + medida::Meter& mTxsetCodedSetsInconsistent; + medida::Meter& mTxsetCodedRebuildUsSum; + // Transactions sent to the target leaders only (tx-flood-to-leaders) medida::Meter& mTxDirectSent; medida::Meter& mTxDirectSentBytes; diff --git a/src/overlay/RustOverlayManager.cpp b/src/overlay/RustOverlayManager.cpp index f01015ce32..ee9e9564d2 100644 --- a/src/overlay/RustOverlayManager.cpp +++ b/src/overlay/RustOverlayManager.cpp @@ -272,6 +272,19 @@ RustOverlayManager::setTxFloodTargets( return false; } +bool +RustOverlayManager::setTxSetTree( + uint8_t mode, uint8_t fanout, + std::vector const& slots, + std::vector const& nodes) +{ + if (mOverlayIPC && !mShuttingDown) + { + return mOverlayIPC->setTxSetTree(mode, fanout, slots, nodes); + } + return false; +} + Hash RustOverlayManager::overlayIdentityBinding(Hash const& networkID, std::vector const& peerId) @@ -463,6 +476,49 @@ RustOverlayManager::syncOverlayMetrics() markDelta(m.mTxSetPushDropped, "txset_push_dropped"); markDelta(m.mTxSetPushEvicted, "txset_push_evicted"); + // Spanning tree for the leader's tx-set push + markDelta(m.mTxsetPushSentBytes, "txset_push_sent_bytes"); + markDelta(m.mTxsetTreeRootSent, "txset_tree_root_sent"); + markDelta(m.mTxsetTreeRelayed, "txset_tree_relayed"); + markDelta(m.mTxsetTreeRelayedBytes, "txset_tree_relayed_bytes"); + markDelta(m.mTxsetTreeReceived, "txset_tree_received"); + markDelta(m.mTxsetTreeDuplicate, "txset_tree_duplicate"); + markDelta(m.mTxsetTreeRefused, "txset_tree_refused"); + markDelta(m.mTxsetTreeUnreachable, "txset_tree_unreachable"); + markDelta(m.mTxsetTreeChildBusy, "txset_tree_child_busy"); + markDelta(m.mTxsetTreeRelayBusy, "txset_tree_relay_busy"); + markDelta(m.mTxsetTreeRelayLimited, "txset_tree_relay_limited"); + markDelta(m.mTxsetTreeFetchDeferred, "txset_tree_fetch_deferred"); + markDelta(m.mTxsetTreeFetchFallback, "txset_tree_fetch_fallback"); + markDelta(m.mTxsetTreeDelayMsSum, "txset_tree_delay_ms_sum"); + markDelta(m.mTxsetTreeDelayCount, "txset_tree_delay_count"); + markDelta(m.mTxsetTreeDelayHop1MsSum, "txset_tree_delay_hop1_ms_sum"); + markDelta(m.mTxsetTreeDelayHop1Count, "txset_tree_delay_hop1_count"); + markDelta(m.mTxsetTreeDelayHop2MsSum, "txset_tree_delay_hop2_ms_sum"); + markDelta(m.mTxsetTreeDelayHop2Count, "txset_tree_delay_hop2_count"); + markDelta(m.mTxsetTreeDelayLe50, "txset_tree_delay_le_50"); + markDelta(m.mTxsetTreeDelayLe100, "txset_tree_delay_le_100"); + markDelta(m.mTxsetTreeDelayLe150, "txset_tree_delay_le_150"); + markDelta(m.mTxsetTreeDelayLe200, "txset_tree_delay_le_200"); + markDelta(m.mTxsetTreeDelayLe300, "txset_tree_delay_le_300"); + markDelta(m.mTxsetTreeDelayGt300, "txset_tree_delay_gt_300"); + markDelta(m.mTxsetTreeWithheld, "txset_tree_withheld"); + markDelta(m.mTxsetCodedSetsEncoded, "txset_coded_sets_encoded"); + markDelta(m.mTxsetCodedPiecesSent, "txset_coded_pieces_sent"); + markDelta(m.mTxsetCodedPiecesRelayed, "txset_coded_pieces_relayed"); + markDelta(m.mTxsetCodedPiecesReceived, "txset_coded_pieces_received"); + markDelta(m.mTxsetCodedPiecesLate, "txset_coded_pieces_late"); + markDelta(m.mTxsetCodedPiecesRefused, "txset_coded_pieces_refused"); + markDelta(m.mTxsetCodedPiecesInvalid, "txset_coded_pieces_invalid"); + markDelta(m.mTxsetCodedPiecesDuplicate, "txset_coded_pieces_duplicate"); + markDelta(m.mTxsetCodedPiecesSkipped, "txset_coded_pieces_skipped"); + markDelta(m.mTxsetCodedSetsRebuilt, "txset_coded_sets_rebuilt"); + markDelta(m.mTxsetCodedSetsRebuiltWithParity, + "txset_coded_sets_rebuilt_with_parity"); + markDelta(m.mTxsetCodedSetsFailed, "txset_coded_sets_failed"); + markDelta(m.mTxsetCodedSetsInconsistent, "txset_coded_sets_inconsistent"); + markDelta(m.mTxsetCodedRebuildUsSum, "txset_coded_rebuild_us_sum"); + // Transactions sent to the target leaders only markDelta(m.mTxDirectSent, "flood_tx_direct_sent"); markDelta(m.mTxDirectSentBytes, "flood_tx_direct_sent_bytes"); diff --git a/src/overlay/RustOverlayManager.h b/src/overlay/RustOverlayManager.h index 1af7439f88..0beb51f2f9 100644 --- a/src/overlay/RustOverlayManager.h +++ b/src/overlay/RustOverlayManager.h @@ -101,6 +101,14 @@ class RustOverlayManager // Returns whether the overlay was told. bool setTxFloodTargets(std::optional> const& targets); + // Describe the spanning trees of the next slots to the overlay (see + // HerderImpl::updateTxSetTree): with them a leader's pushTxSet goes to + // its children in the slot's tree, and every node relays what its + // parent sends it. Returns whether the overlay was told. + bool setTxSetTree(uint8_t mode, uint8_t fanout, + std::vector const& slots, + std::vector const& nodes); + // What a validator signs to say "the overlay with this libp2p peer id is // mine": SHA256(networkID || "stellar-overlay-peer-id" || peerId). The // tag keeps it apart from everything else the node key signs. diff --git a/src/simulation/LoadGenerator.cpp b/src/simulation/LoadGenerator.cpp index c763d764ff..49d64847f0 100644 --- a/src/simulation/LoadGenerator.cpp +++ b/src/simulation/LoadGenerator.cpp @@ -3,6 +3,7 @@ // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 #include "simulation/LoadGenerator.h" +#include "crypto/KeyUtils.h" #include "database/Database.h" #include "herder/Herder.h" #include "ledger/LedgerManager.h" @@ -71,6 +72,16 @@ uint32_t const LoadGenerator::TX_SUBMIT_MAX_TRIES = 10; // After successfully submitting desired load, wait a bit to let it get into the // ledger. uint32_t const LoadGenerator::TIMEOUT_NUM_LEDGERS = 20; +uint32_t const LoadGenerator::DEFAULT_COMPLETION_TIMEOUT_NUM_LEDGERS = + LoadGenerator::TIMEOUT_NUM_LEDGERS; +// The payments of SAC_PAYMENT travel to the round leaders only, without an +// acknowledgement. A node's last few can be included much later than the +// rest (25 to 70 s were measured on a 90-validator network at 0.8 s per +// ledger, where 1 to 9 of 90 generators hit it per run): a leader dropped +// them from its build, and the origin sends them again only by its resend +// rules. They are applied in the end, so the run is complete, only late; +// the default wait of 20 ledgers reported such a run as failed. +uint32_t const LoadGenerator::SAC_PAYMENT_COMPLETION_TIMEOUT_NUM_LEDGERS = 150; // After successfully submitting desired load, wait for this many ledgers // without checking for account consistency. @@ -106,6 +117,29 @@ isMixedPregenMode(LoadGenMode mode) mode == LoadGenMode::MIXED_PREGEN_SOROSWAP_SWAP; } +LedgerKey +LoadGenerator::getNativeSACInstanceKey(Hash const& networkID) +{ + SCAddress contract(SC_ADDRESS_TYPE_CONTRACT); + contract.contractId() = getAssetContractID(networkID, makeNativeAsset()); + return makeContractInstanceKey(contract); +} + +SCAddress +LoadGenerator::getNativeSACPaymentDestination(NodeID const& node, + uint32_t ledgerNum, + uint64_t counter) +{ + // The name holds the generating node's key, so every generator of a + // network pays to destinations of its own; the ledger and the counter + // make the destinations of one generator distinct. + SCAddress dest(SC_ADDRESS_TYPE_CONTRACT); + dest.contractId() = + sha256(fmt::format("loadgen_native_sac_dest_{}_{}_{}", + KeyUtils::toStrKey(node), ledgerNum, counter)); + return dest; +} + LoadGenerator::LoadGenerator(Application& app) : mTxGenerator(app) , mApp(app) @@ -173,6 +207,14 @@ LoadGenerator::getMode(std::string const& mode) { return LoadGenMode::MIXED_PREGEN_SOROSWAP_SWAP; } + else if (mode == "sac_setup") + { + return LoadGenMode::SAC_SETUP; + } + else if (mode == "sac_payment") + { + return LoadGenMode::SAC_PAYMENT; + } else { throw std::runtime_error( @@ -345,6 +387,8 @@ LoadGenerator::reset() mSyntheticSoroswapState.reset(); mSyntheticSACDestCounter = 0; mSyntheticSoroswapSwapCounter = 0; + mNativeSACInstance.reset(); + mNativeSACDestCounter = 0; mWaitTillCompleteForLedgers = 0; mFailed = false; mStarted = false; @@ -401,8 +445,48 @@ LoadGenerator::start(GeneratedLoadConfig& cfg) cfg.txRate = 1; } + if (cfg.modeUsesNativeSAC()) + { + // These modes exist to exercise apply. In overlay only mode nothing + // is applied, so the instance would never appear and the payments + // would never complete. + if (mApp.getRunInOverlayOnlyMode()) + { + reset(); + throw std::runtime_error( + "SAC_SETUP / SAC_PAYMENT need transactions to be applied, " + "they can not run in overlay only mode"); + } + + auto instanceKey = getNativeSACInstanceKey(mApp.getNetworkID()); + bool instanceExists = static_cast( + mApp.getLedgerManager().getLCLView()->load(instanceKey)); + if (cfg.mode == LoadGenMode::SAC_SETUP) + { + // A single transaction creates the instance. If it is already + // there (e.g. another node ran the setup), there is nothing to do. + cfg.nTxs = instanceExists ? 0 : 1; + cfg.skipLowFeeTxs = false; + cfg.spikeInterval = std::chrono::seconds(0); + cfg.spikeSize = 0; + } + else if (!instanceExists) + { + reset(); + throw std::runtime_error( + "Before running MODE::SAC_PAYMENT, please run MODE::SAC_SETUP " + "to create the native asset contract first."); + } + + TxGenerator::ContractInstance instance; + instance.readOnlyKeys.emplace_back(instanceKey); + instance.contractID = instanceKey.contractData().contract; + mNativeSACInstance = instance; + } + // Setup config for soroban modes - if (cfg.isSoroban() && cfg.mode != LoadGenMode::SOROBAN_UPLOAD) + if (cfg.isSoroban() && cfg.mode != LoadGenMode::SOROBAN_UPLOAD && + !cfg.modeUsesNativeSAC()) { auto& sorobanLoadCfg = cfg.getMutSorobanConfig(); sorobanLoadCfg.nWasms = 1; @@ -715,6 +799,12 @@ GeneratedLoadConfig::getStatus() const case LoadGenMode::MIXED_PREGEN_SOROSWAP_SWAP: modeStr = "mixed_pregen_soroswap_swap"; break; + case LoadGenMode::SAC_SETUP: + modeStr = "sac_setup"; + break; + case LoadGenMode::SAC_PAYMENT: + modeStr = "sac_payment"; + break; } ret["mode"] = modeStr; @@ -947,6 +1037,19 @@ LoadGenerator::generateLoad(GeneratedLoadConfig cfg) return readTransactionFromFile(cfg); }; break; + case LoadGenMode::SAC_SETUP: + generateTx = [&]() { + return mTxGenerator.createSACTransaction( + ledgerNum, sourceAccountId, makeNativeAsset(), + cfg.maxGeneratedFeeRate); + }; + break; + case LoadGenMode::SAC_PAYMENT: + generateTx = [&]() { + return createNativeSACPaymentTransaction(ledgerNum, + sourceAccountId, cfg); + }; + break; } try @@ -1231,6 +1334,23 @@ LoadGenerator::createSyntheticSorobanTransaction(uint32_t ledgerNum, } } +std::pair +LoadGenerator::createNativeSACPaymentTransaction(uint32_t ledgerNum, + uint64_t sourceAccountId, + GeneratedLoadConfig const& cfg) +{ + releaseAssert(mNativeSACInstance); + // A payment writes only its source account and the balance entry of a + // destination no other transaction uses, so payments from different + // accounts never conflict. + auto dest = getNativeSACPaymentDestination( + mApp.getConfig().NODE_SEED.getPublicKey(), ledgerNum, + mNativeSACDestCounter++); + return mTxGenerator.invokeSACPayment(ledgerNum, sourceAccountId, dest, + *mNativeSACInstance, /* amount */ 100, + cfg.maxGeneratedFeeRate); +} + std::pair LoadGenerator::createUploadWasmTransaction(GeneratedLoadConfig const& cfg, uint32_t ledgerNum, @@ -1307,6 +1427,19 @@ LoadGenerator::checkSorobanStateSynced(Application& app, std::vector result; auto ledgerView = mApp.getLedgerManager().getLCLView(); + if (cfg.modeUsesNativeSAC()) + { + // SAC_SETUP is done once the instance is on the ledger. The payments + // do not create persistent loadgen state; their source accounts are + // checked by checkAccountSynced. + auto instanceKey = getNativeSACInstanceKey(mApp.getNetworkID()); + if (!ledgerView->load(instanceKey)) + { + result.emplace_back(instanceKey); + } + return result; + } + for (auto const& lk : mContractInstanceKeys) { if (!ledgerView->load(lk)) @@ -1382,6 +1515,18 @@ LoadGenerator::checkMinimumSorobanSuccess(GeneratedLoadConfig const& cfg) return (nSuccessful * 100) / nTxns >= cfg.getMinSorobanPercentSuccess(); } +uint32_t +LoadGenerator::completionTimeoutLedgers(GeneratedLoadConfig const& cfg) +{ + if (cfg.completionTimeoutLedgers != 0) + { + return cfg.completionTimeoutLedgers; + } + return cfg.mode == LoadGenMode::SAC_PAYMENT + ? SAC_PAYMENT_COMPLETION_TIMEOUT_NUM_LEDGERS + : DEFAULT_COMPLETION_TIMEOUT_NUM_LEDGERS; +} + void LoadGenerator::waitTillComplete(GeneratedLoadConfig cfg) { @@ -1440,7 +1585,7 @@ LoadGenerator::waitTillComplete(GeneratedLoadConfig cfg) // If we have an inconsistency, reset the timer and wait for another ledger else if (!classicIsDone || !sorobanIsDone) { - if (++mWaitTillCompleteForLedgers >= TIMEOUT_NUM_LEDGERS) + if (++mWaitTillCompleteForLedgers >= completionTimeoutLedgers(cfg)) { emitFailure(!sorobanIsDone); return; @@ -1634,6 +1779,12 @@ LoadGenerator::execute(TransactionFrameBasePtr txf, LoadGenMode mode, txm.mNativePaymentBytes.Mark(msgBytes); } break; + case LoadGenMode::SAC_SETUP: + txm.mSorobanSetupInvokeTxs.Mark(); + break; + case LoadGenMode::SAC_PAYMENT: + txm.mSorobanInvokeTxs.Mark(); + break; } txm.mTxnAttempted.Mark(); @@ -1951,7 +2102,7 @@ GeneratedLoadConfig::isSoroban() const mode == LoadGenMode::SOROBAN_UPLOAD || mode == LoadGenMode::SOROBAN_UPGRADE_SETUP || mode == LoadGenMode::SOROBAN_CREATE_UPGRADE || - mode == LoadGenMode::MIXED_CLASSIC_SOROBAN; + mode == LoadGenMode::MIXED_CLASSIC_SOROBAN || modeUsesNativeSAC(); } bool @@ -1968,7 +2119,8 @@ GeneratedLoadConfig::isLoad() const mode == LoadGenMode::SOROBAN_INVOKE || mode == LoadGenMode::SOROBAN_CREATE_UPGRADE || mode == LoadGenMode::MIXED_CLASSIC_SOROBAN || - mode == LoadGenMode::PAY_PREGENERATED || modeMixesPregen(); + mode == LoadGenMode::PAY_PREGENERATED || modeMixesPregen() || + modeUsesNativeSAC(); } bool @@ -1984,6 +2136,12 @@ GeneratedLoadConfig::modeMixesPregen() const return isMixedPregenMode(mode); } +bool +GeneratedLoadConfig::modeUsesNativeSAC() const +{ + return mode == LoadGenMode::SAC_SETUP || mode == LoadGenMode::SAC_PAYMENT; +} + bool GeneratedLoadConfig::modeSetsUpInvoke() const { diff --git a/src/simulation/LoadGenerator.h b/src/simulation/LoadGenerator.h index 97ca72ecd4..e357327abd 100644 --- a/src/simulation/LoadGenerator.h +++ b/src/simulation/LoadGenerator.h @@ -54,7 +54,17 @@ enum class LoadGenMode // is skipped so nothing is actually executed. MIXED_PREGEN_SAC_PAYMENT, MIXED_PREGEN_OZ_TOKEN_TRANSFER, - MIXED_PREGEN_SOROSWAP_SWAP + MIXED_PREGEN_SOROSWAP_SWAP, + // Create the Stellar Asset Contract instance of the native asset with one + // transaction, for SAC_PAYMENT. Completes once the instance is on the + // ledger; does nothing if it is already there. + SAC_SETUP, + // SAC payments of the native asset that are really applied: the same + // transaction shape as MIXED_PREGEN_SAC_PAYMENT, but against the instance + // SAC_SETUP created, so it needs apply (not overlay only mode). Each + // payment goes to a new contract address, so no two transactions write the + // same entry. + SAC_PAYMENT }; struct GeneratedLoadConfig @@ -135,6 +145,10 @@ struct GeneratedLoadConfig // True iff mode generates SOROBAN_UPLOAD load bool modeUploads() const; + // True iff mode works on the native asset's SAC instance (SAC_SETUP and + // SAC_PAYMENT). + bool modeUsesNativeSAC() const; + bool isDone() const; bool areTxsRemaining() const; Json::Value getStatus() const; @@ -158,6 +172,10 @@ struct GeneratedLoadConfig // to `TxQueueLimiter` limiting the operation count per ledger). Otherwise, // the load generation will fail after a couple of retries. bool skipLowFeeTxs = false; + // How many ledgers a completed submission waits for its transactions to + // be applied before the run is reported as failed. 0: the default of the + // mode (see LoadGenerator::completionTimeoutLedgers). + uint32_t completionTimeoutLedgers = 0; // Path to the pre-generated transactions file for PAY_PREGENERATED mode std::filesystem::path preloadedTransactionsFile; @@ -175,10 +193,29 @@ struct GeneratedLoadConfig class LoadGenerator { public: + // How long a run waits, after its last submission, for its + // transactions to be applied, unless its config says otherwise: 20 + // ledgers, and for SAC_PAYMENT, whose last payments can be included + // late, 150. + static uint32_t const DEFAULT_COMPLETION_TIMEOUT_NUM_LEDGERS; + static uint32_t const SAC_PAYMENT_COMPLETION_TIMEOUT_NUM_LEDGERS; + // The ledgers a run of `cfg` waits. + static uint32_t completionTimeoutLedgers(GeneratedLoadConfig const& cfg); + LoadGenerator(Application& app); static LoadGenMode getMode(std::string const& mode); + // Ledger key of the native asset's SAC instance (SAC_SETUP creates it, + // SAC_PAYMENT pays through it). + static LedgerKey getNativeSACInstanceKey(Hash const& networkID); + + // Destination of the `counter`-th SAC_PAYMENT transaction that the node + // `node` generated for ledger `ledgerNum`. + static SCAddress getNativeSACPaymentDestination(NodeID const& node, + uint32_t ledgerNum, + uint64_t counter); + // Returns true if loadgen has submitted all required TXs bool isDone(GeneratedLoadConfig const& cfg) const; @@ -336,6 +373,11 @@ class LoadGenerator std::optional mSyntheticSoroswapState; // Counter for generating unique SAC payment destination addresses. uint32_t mSyntheticSACDestCounter{0}; + + // SAC_SETUP / SAC_PAYMENT: the native asset's SAC instance, and the + // number of payment destinations generated so far in this run. + std::optional mNativeSACInstance; + uint64_t mNativeSACDestCounter{0}; // Round-robin counter for Soroswap pair selection + direction. uint32_t mSyntheticSoroswapSwapCounter{0}; // Per-stream submission counters, tracked for every mode. For single- @@ -393,6 +435,13 @@ class LoadGenerator uint64_t sourceAccountId, GeneratedLoadConfig const& cfg); + // Build a SAC_PAYMENT transaction: a native asset payment through the + // real SAC instance, to a destination no other transaction uses. + std::pair + createNativeSACPaymentTransaction(uint32_t ledgerNum, + uint64_t sourceAccountId, + GeneratedLoadConfig const& cfg); + std::pair createUploadWasmTransaction(GeneratedLoadConfig const& cfg, uint32_t ledgerNum, uint64_t sourceAccountId); diff --git a/src/simulation/test/LoadGeneratorTests.cpp b/src/simulation/test/LoadGeneratorTests.cpp index 7e482fb35e..ac9d17d600 100644 --- a/src/simulation/test/LoadGeneratorTests.cpp +++ b/src/simulation/test/LoadGeneratorTests.cpp @@ -7,12 +7,14 @@ #include "crypto/SecretKey.h" #include "ledger/ImmutableLedgerView.h" #include "ledger/LedgerManager.h" +#include "ledger/NetworkConfig.h" #include "main/Config.h" #include "simulation/ApplyLoad.h" #include "simulation/LoadGenerator.h" #include "simulation/Topologies.h" #include "test/Catch2.h" #include "test/test.h" +#include "transactions/TransactionUtils.h" #include "transactions/test/SorobanTxTestUtils.h" #include "util/Math.h" #include "util/MetricsRegistry.h" @@ -378,6 +380,225 @@ TEST_CASE("synthetic SAC load recycles source accounts in overlay-only mode", REQUIRE(bytes.count() - bytesBefore == nTxs * refMsgBytes); } +TEST_CASE("native SAC setup and payments are applied", "[loadgen][soroban]") +{ + // SAC_SETUP creates the native asset contract with a real transaction and + // SAC_PAYMENT pays through it, with apply on (unlike the MIXED_PREGEN_* + // modes, whose synthetic contract only exists in overlay-only mode). The + // run has more transactions than source accounts, so it also needs + // accounts to return to the pool after their transactions are applied. + uint32_t const nAccounts = 300; + uint32_t const txRate = 15; + uint32_t const nTxs = nAccounts + 100; + + Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE); + Simulation::pointer simulation = Topologies::pair(networkID, [&](int i) { + auto cfg = getTestConfig(i); + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + cfg.ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING = true; + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = + Config::CURRENT_LEDGER_PROTOCOL_VERSION; + cfg.GENESIS_TEST_ACCOUNT_COUNT = nAccounts; + return cfg; + }); + + simulation->startAllNodes(); + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(3, 1); }, + 10 * simulation->getExpectedLedgerCloseTime(), false); + + auto nodes = simulation->getNodes(); + auto& app = *nodes[0]; + auto& loadGen = app.getLoadGenerator(); + + // Ledger limits high enough that every submitted transaction fits in the + // next ledger. The per-transaction limits stay as perfnet's + // block_time_check leaves them: it only raises a limit that is under what + // its SAC load needs (4 footprint entries, 4 read and 2 written entries), + // so every other limit is the network's initial one. In particular a + // transaction's events may take 200 bytes at most, and both the setup + // transaction and the payments must be applied within that. + upgradeSorobanNetworkConfig( + [&](SorobanNetworkConfig& cfg) { + auto mx = std::numeric_limits::max(); + cfg.mLedgerMaxTxCount = mx; + cfg.mLedgerMaxInstructions = mx; + cfg.mLedgerMaxTransactionsSizeBytes = mx; + cfg.mLedgerMaxDiskReadEntries = mx; + cfg.mLedgerMaxDiskReadBytes = mx; + cfg.mLedgerMaxWriteLedgerEntries = mx; + cfg.mLedgerMaxWriteBytes = mx; + cfg.mTxMaxDiskReadEntries = + std::max(cfg.mTxMaxDiskReadEntries, 4); + cfg.mTxMaxWriteLedgerEntries = + std::max(cfg.mTxMaxWriteLedgerEntries, 2); + cfg.mTxMaxFootprintEntries = + std::max(cfg.mTxMaxFootprintEntries, 4); + }, + simulation); + { + auto const& sorobanCfg = + app.getLedgerManager().getLastClosedSorobanNetworkConfig(); + REQUIRE(sorobanCfg.txMaxContractEventsSizeBytes() == + MinimumSorobanNetworkConfig::TX_MAX_CONTRACT_EVENTS_SIZE_BYTES); + REQUIRE(sorobanCfg.txMaxInstructions() == + MinimumSorobanNetworkConfig::TX_MAX_INSTRUCTIONS); + } + + auto& completed = + app.getMetrics().NewMeter({"loadgen", "run", "complete"}, "run"); + auto& failed = + app.getMetrics().NewMeter({"loadgen", "run", "failed"}, "run"); + auto& attempted = + app.getMetrics().NewMeter({"loadgen", "txn", "attempted"}, "txn"); + auto applySuccess = [](Application& node) { + return node.getMetrics() + .NewCounter({"ledger", "apply-soroban", "success"}) + .count(); + }; + auto applyFailure = [](Application& node) { + return node.getMetrics() + .NewCounter({"ledger", "apply-soroban", "failure"}) + .count(); + }; + auto runToCompletion = [&](GeneratedLoadConfig cfg) { + auto const completedBefore = completed.count(); + auto const failedBefore = failed.count(); + loadGen.generateLoad(cfg); + simulation->crankUntil( + [&]() { + return completed.count() == completedBefore + 1 || + failed.count() != failedBefore; + }, + 120 * simulation->getExpectedLedgerCloseTime(), false); + REQUIRE(failed.count() == failedBefore); + REQUIRE(completed.count() == completedBefore + 1); + }; + + auto instanceKey = LoadGenerator::getNativeSACInstanceKey(networkID); + auto setupCfg = GeneratedLoadConfig::txLoad(LoadGenMode::SAC_SETUP, + nAccounts, /* nTxs */ 1, + /* txRate */ 1); + auto paymentCfg = GeneratedLoadConfig::txLoad(LoadGenMode::SAC_PAYMENT, + nAccounts, nTxs, txRate); + // Any payment that fails at apply fails the run. + paymentCfg.setMinSorobanPercentSuccess(100); + + // A node's last payments can be included late (they travel to the round + // leaders only), so the payment mode waits longer for them than other + // modes wait for theirs, unless the request says how long. + REQUIRE(LoadGenerator::completionTimeoutLedgers(setupCfg) == + LoadGenerator::DEFAULT_COMPLETION_TIMEOUT_NUM_LEDGERS); + REQUIRE(LoadGenerator::completionTimeoutLedgers(paymentCfg) == + LoadGenerator::SAC_PAYMENT_COMPLETION_TIMEOUT_NUM_LEDGERS); + REQUIRE(LoadGenerator::SAC_PAYMENT_COMPLETION_TIMEOUT_NUM_LEDGERS > + LoadGenerator::DEFAULT_COMPLETION_TIMEOUT_NUM_LEDGERS); + { + auto withTimeout = paymentCfg; + withTimeout.completionTimeoutLedgers = 7; + REQUIRE(LoadGenerator::completionTimeoutLedgers(withTimeout) == 7); + withTimeout.mode = LoadGenMode::PAY; + REQUIRE(LoadGenerator::completionTimeoutLedgers(withTimeout) == 7); + } + + // The payments need the contract. + REQUIRE(!app.getLedgerManager().getLCLView()->load(instanceKey)); + REQUIRE_THROWS_WITH( + loadGen.generateLoad(paymentCfg), + "Before running MODE::SAC_PAYMENT, please run MODE::SAC_SETUP to " + "create the native asset contract first."); + + // Neither mode can complete when nothing is applied. + app.setRunInOverlayOnlyMode(true); + REQUIRE_THROWS_WITH(loadGen.generateLoad(setupCfg), + "SAC_SETUP / SAC_PAYMENT need transactions to be " + "applied, they can not run in overlay only mode"); + REQUIRE_THROWS_WITH(loadGen.generateLoad(paymentCfg), + "SAC_SETUP / SAC_PAYMENT need transactions to be " + "applied, they can not run in overlay only mode"); + app.setRunInOverlayOnlyMode(false); + + // Setup: one applied transaction creates the instance on every node. + auto successBefore = applySuccess(app); + auto attemptedBefore = attempted.count(); + runToCompletion(setupCfg); + REQUIRE(attempted.count() - attemptedBefore == 1); + for (auto const& node : nodes) + { + REQUIRE(static_cast( + node->getLedgerManager().getLCLView()->load(instanceKey))); + REQUIRE(applySuccess(*node) == successBefore + 1); + REQUIRE(applyFailure(*node) == 0); + } + + // Setup again: the instance is there, so nothing is submitted. + attemptedBefore = attempted.count(); + runToCompletion(setupCfg); + REQUIRE(attempted.count() == attemptedBefore); + REQUIRE(applySuccess(app) == successBefore + 1); + + // Payments: every one is applied successfully on every node. + successBefore = applySuccess(app); + attemptedBefore = attempted.count(); + auto const firstLedger = + app.getLedgerManager().getLastClosedLedgerNum() + 1; + runToCompletion(paymentCfg); + auto const lastLedger = app.getLedgerManager().getLastClosedLedgerNum(); + REQUIRE(attempted.count() - attemptedBefore == nTxs); + for (auto const& node : nodes) + { + REQUIRE(applySuccess(*node) == successBefore + nTxs); + REQUIRE(applyFailure(*node) == 0); + } + + // The payments moved funds: the destination of the first payment of the + // run holds the amount paid. Its name contains the ledger the payment was + // generated for, which is one of the ledgers of the run. + auto balanceOf = [&](Application& node, + SCAddress const& dest) -> std::optional { + SCVal destVal(SCV_ADDRESS); + destVal.address() = dest; + LedgerKey balanceKey(CONTRACT_DATA); + balanceKey.contractData().contract = + instanceKey.contractData().contract; + balanceKey.contractData().key = + txtest::makeVecSCVal({makeSymbolSCVal("Balance"), destVal}); + balanceKey.contractData().durability = + ContractDataDurability::PERSISTENT; + auto entry = node.getLedgerManager().getLCLView()->load(balanceKey); + if (!entry) + { + return std::nullopt; + } + for (auto const& field : *entry.current().data.contractData().val.map()) + { + if (field.key.type() == SCV_SYMBOL && field.key.sym() == "amount") + { + REQUIRE(field.val.i128().hi == 0); + return field.val.i128().lo; + } + } + return std::nullopt; + }; + for (auto const& node : nodes) + { + uint32_t found = 0; + for (auto ledger = firstLedger; ledger <= lastLedger; ++ledger) + { + auto balance = balanceOf( + *node, LoadGenerator::getNativeSACPaymentDestination( + app.getConfig().NODE_SEED.getPublicKey(), ledger, + /* counter */ 0)); + if (balance) + { + REQUIRE(*balance == 100); + ++found; + } + } + REQUIRE(found == 1); + } +} + TEST_CASE("externalized tx set hashes and sources match transaction frames", "[loadgen][herder][txset]") { diff --git a/src/test/test.cpp b/src/test/test.cpp index 9015980cf4..476b1378a3 100644 --- a/src/test/test.cpp +++ b/src/test/test.cpp @@ -847,6 +847,17 @@ getTestConfig(int instanceNumber, Config::TestDbMode mode) thisConfig.NETWORK_PASSPHRASE = "(V) (;,,;) (V)"; + // STELLAR_TEST_SHARDED_TOP_BUCKET=N runs every test that takes its + // config from here with level 0 of the live bucket list written as + // shards under the commutative hash (see ShardedTopBucket). Tests + // that expect the SHA-256 of a level-0 bucket's file fail with it; + // everything else is expected to pass. + if (auto const* shards = getenv("STELLAR_TEST_SHARDED_TOP_BUCKET")) + { + thisConfig.EXPERIMENTAL_SHARDED_TOP_BUCKET = + static_cast(std::stoul(shards)); + } + std::ostringstream dbname; switch (mode) { diff --git a/src/util/BatchExecutor.cpp b/src/util/BatchExecutor.cpp index d508947730..fc792e8136 100644 --- a/src/util/BatchExecutor.cpp +++ b/src/util/BatchExecutor.cpp @@ -118,11 +118,17 @@ pinnedCpuOrder() } } // namespace -BatchExecutor::BatchExecutor() +BatchExecutor::BatchExecutor(bool pinWorkers) { auto pinning = pinnedCpuOrder(); mPinCpuOrder = std::move(pinning.order); mPhysicalCoreCount = pinning.physicalCoreCount; + if (!pinWorkers) + { + // The core count still sizes the batches. + mPinCpuOrder.clear(); + return; + } if (mPinCpuOrder.empty()) { CLOG_WARNING( diff --git a/src/util/BatchExecutor.h b/src/util/BatchExecutor.h index 85a9754303..ad1cebcfa2 100644 --- a/src/util/BatchExecutor.h +++ b/src/util/BatchExecutor.h @@ -36,7 +36,10 @@ namespace stellar class BatchExecutor : private NonMovableOrCopyable { public: - BatchExecutor(); + // `pinWorkers` false: the workers are left to the scheduler. For a second + // executor of an app (see the class comment): its batches may overlap + // those of the first, and two pinned threads would then share a core. + explicit BatchExecutor(bool pinWorkers = true); // Destroys the executor and joins all worker threads. // No batch should be in progress - batch execution is not supposed to be // async. diff --git a/src/util/RandomEvictionCache.h b/src/util/RandomEvictionCache.h index b503297f74..d8ad6f9b57 100644 --- a/src/util/RandomEvictionCache.h +++ b/src/util/RandomEvictionCache.h @@ -224,6 +224,19 @@ class RandomEvictionCache : public NonMovableOrCopyable } } + // Returns a pointer to the value if the key exists and nullptr + // otherwise, like `maybeGet`, without recording the access: no counter + // is touched and the entry does not become any younger for eviction. + // That makes it a read: any number of threads may call it at the same + // time as long as none changes the cache (`put`, `maybeGet`, ...) + // meanwhile. The pointer is good until the cache is changed. + V const* + peek(K const& k) const + { + auto it = mValueMap.find(k); + return it == mValueMap.end() ? nullptr : &it->second.mValue; + } + // `get` offers basic exception safety guarantee. V& get(K const& k) diff --git a/src/util/test/CacheTests.cpp b/src/util/test/CacheTests.cpp index c051ea7d52..e3a330af49 100644 --- a/src/util/test/CacheTests.cpp +++ b/src/util/test/CacheTests.cpp @@ -101,6 +101,32 @@ TEST_CASE("RandomEvictionCache works as a cache", "[randomevictioncache]") } } +TEST_CASE("RandomEvictionCache peek reads without recording the access", + "[randomevictioncache]") +{ + RandomEvictionCache cache(4); + RandomEvictionCache const& constCache = cache; + cache.put(1, 10); + cache.put(2, 20); + auto const before = cache.getCounters(); + REQUIRE(constCache.peek(1)); + REQUIRE(*constCache.peek(1) == 10); + REQUIRE(*constCache.peek(2) == 20); + REQUIRE(constCache.peek(3) == nullptr); + // Neither hits nor misses are counted, and nothing was inserted. + auto const after = cache.getCounters(); + REQUIRE(after.mHits == before.mHits); + REQUIRE(after.mMisses == before.mMisses); + REQUIRE(after.mInserts == before.mInserts); + REQUIRE(cache.size() == 2); + // An updated value is what peek returns next. + cache.put(1, 11); + REQUIRE(*constCache.peek(1) == 11); + // maybeGet still counts. + REQUIRE(cache.maybeGet(1)); + REQUIRE(cache.getCounters().mHits == before.mHits + 1); +} + TEST_CASE("RandomEvictionCache does not thrash", "[randomevictioncachethrash][!hide]") {