From 0f9255d47801ee48823a13c29e5c281fb7d2ceca Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Tue, 10 Nov 2026 22:55:01 +0000 Subject: [PATCH 1/4] Start QUIC connections with an 8 MiB congestion window quinn paces one congestion window over 0.8 round trips and does not grow the window while the sender is application limited. A connection that mostly idles therefore sends a burst of S bytes in about 0.8 * S / window round trips: a 640 KB tx set took 1 to 1.3 round trips on top of the one-way delay on a warm connection, and 6 to 7 on a cold one (two nodes over a loopback with a 150 ms round trip). With an initial window of 8 MiB it arrives about 20 ms after the one-way delay. Add an initial_congestion_window option to the vendored libp2p-quic config (CUBIC's initial window; recorded in congestion-window.patch) and set it to 8 MiB for every peer connection. CUBIC still shrinks the window on loss. STELLAR_OVERLAY_QUIC_INITIAL_CWND overrides it in bytes (0 = quinn's default); the effective window is logged at startup. This is a prototype setting for validators on good links: it applies to every stream of a connection, and a fresh connection may send 8 MiB without probing the path. The ignored test txset_transfer_latency_experiment is the harness behind the numbers; it needs a link with real delay (netem). --- docs/rust-overlay/transport.md | 12 ++ overlay/src/libp2p_overlay.rs | 39 +++++ .../libp2p_overlay/quic_window_experiment.rs | 151 ++++++++++++++++++ overlay/vendor/congestion-window.patch | 49 ++++++ overlay/vendor/libp2p-quic/src/config.rs | 17 ++ 5 files changed, 268 insertions(+) create mode 100644 overlay/src/libp2p_overlay/quic_window_experiment.rs create mode 100644 overlay/vendor/congestion-window.patch diff --git a/docs/rust-overlay/transport.md b/docs/rust-overlay/transport.md index 95dbefc955..0268b33462 100644 --- a/docs/rust-overlay/transport.md +++ b/docs/rust-overlay/transport.md @@ -108,6 +108,18 @@ Streams open when a peer connects and reopen on demand after failure. Every opening applies the route's priority before writing application data. A fetch reuses the existing stream; it does not open a new stream per request. +Every connection starts with a congestion window of 8 MiB instead of Quinn's +default of about ten packets (`initial_congestion_window` in the vendored +`libp2p-quic` config, see +[`congestion-window.patch`](../../overlay/vendor/congestion-window.patch); +`STELLAR_OVERLAY_QUIC_INITIAL_CWND` overrides it in bytes, 0 = Quinn's +default). Quinn paces one window over 0.8 round trips and does not grow the +window while the sender is application limited, so with the default a tx set +of 0.6 MB took 1 to 1.3 round trips on top of the one-way delay on a warm +connection and 6 to 7 on a cold one; with 8 MiB it leaves in one flight (about +20 ms after the one-way delay at a 150 ms round trip). CUBIC still shrinks the +window on loss. This is a prototype setting for validators on good links. + Streams still share connection-level flow control, congestion control, CPU, and link bandwidth. This separation removes dispatcher and stream-lock serialization; it does not guarantee SCP latency under link saturation. diff --git a/overlay/src/libp2p_overlay.rs b/overlay/src/libp2p_overlay.rs index a46b6e56b9..3b3ac6149a 100644 --- a/overlay/src/libp2p_overlay.rs +++ b/overlay/src/libp2p_overlay.rs @@ -470,6 +470,37 @@ pub struct StellarOverlay { sends: JoinSet<()>, } +/// See `quic_initial_congestion_window`. `None` = quinn's default. +/// +/// 8 MiB lets a tx set (0.5-0.8 MB compressed at 3,000 TPS) leave in one +/// flight. quinn paces one window over 0.8 round trips and does not grow the +/// window while the sender is application limited, so with its default a +/// 640 KB frame took 1 to 1.3 round trips on top of the one-way delay even on +/// a warm connection (and 6 to 7 on a cold one); with 8 MiB it arrives about +/// 20 ms after the one-way delay at a 150 ms round trip. CUBIC still shrinks +/// the window on loss. A prototype setting for validators on good links. +const DEFAULT_QUIC_INITIAL_CONGESTION_WINDOW: Option = Some(8 * 1024 * 1024); + +/// Initial QUIC congestion window for every peer connection, in bytes. +/// `STELLAR_OVERLAY_QUIC_INITIAL_CWND` overrides it (0 = quinn's default of +/// about ten packets); the override exists for latency experiments. +fn quic_initial_congestion_window() -> Option { + let Ok(value) = std::env::var("STELLAR_OVERLAY_QUIC_INITIAL_CWND") else { + return DEFAULT_QUIC_INITIAL_CONGESTION_WINDOW; + }; + match value.parse::() { + Ok(0) => None, + Ok(bytes) => Some(bytes), + Err(_) => { + warn!( + "STELLAR_OVERLAY_QUIC_INITIAL_CWND={:?} is not a byte count, using the default", + value + ); + DEFAULT_QUIC_INITIAL_CONGESTION_WINDOW + } + } +} + /// Create the overlay and return handle + event receivers /// /// Returns: @@ -499,6 +530,11 @@ pub fn create_overlay( let mut quic_config = libp2p::quic::Config::new(&keypair); quic_config.keep_alive_interval = Duration::from_secs(15); quic_config.max_idle_timeout = 60_000; // 60 seconds in ms + quic_config.initial_congestion_window = quic_initial_congestion_window(); + match quic_config.initial_congestion_window { + Some(bytes) => info!("QUIC_CWND: initial congestion window {} bytes", bytes), + None => info!("QUIC_CWND: initial congestion window: quinn default"), + } let swarm = SwarmBuilder::with_existing_identity(keypair.clone()) .with_tokio() @@ -2405,6 +2441,9 @@ fn create_test_overlay( #[cfg(test)] mod dispatch_tests; +#[cfg(test)] +mod quic_window_experiment; + #[cfg(test)] mod tests { use super::*; diff --git a/overlay/src/libp2p_overlay/quic_window_experiment.rs b/overlay/src/libp2p_overlay/quic_window_experiment.rs new file mode 100644 index 0000000000..64833331fe --- /dev/null +++ b/overlay/src/libp2p_overlay/quic_window_experiment.rs @@ -0,0 +1,151 @@ +//! Latency experiment behind the initial QUIC congestion window (see +//! `DEFAULT_QUIC_INITIAL_CONGESTION_WINDOW`). Ignored by default: it needs a +//! link with real delay. + +use super::*; +use rand::{RngCore, SeedableRng}; + +/// A valid `GeneralizedTransactionSet` whose zstd encoding is about +/// `encoded_bytes` long: one classic phase of transactions with random +/// accounts, memos and signatures (incompressible, like real payments). +fn large_test_txset(seed: u64, encoded_bytes: usize) -> ([u8; 32], Arc) { + use stellar_xdr::{ + BytesM, DecoratedSignature, GeneralizedTransactionSet, Hash, Limits, Memo, MuxedAccount, + Operation, SequenceNumber, Signature, SignatureHint, Transaction, TransactionEnvelope, + TransactionPhase, TransactionV1Envelope, TxSetComponent, + TxSetComponentTxsMaybeDiscountedFee, Uint256, VecM, WriteXdr, + }; + let mut rng = rand::rngs::StdRng::seed_from_u64(seed); + // Each envelope carries about 132 random bytes. + let count = encoded_bytes / 132 + 1; + let mut txs = Vec::with_capacity(count); + for i in 0..count { + let mut account = [0u8; 32]; + let mut memo = [0u8; 32]; + let mut hint = [0u8; 4]; + let mut signature = [0u8; 64]; + rng.fill_bytes(&mut account); + rng.fill_bytes(&mut memo); + rng.fill_bytes(&mut hint); + rng.fill_bytes(&mut signature); + let tx = Transaction { + source_account: MuxedAccount::Ed25519(Uint256(account)), + fee: 100, + seq_num: SequenceNumber(i as i64), + memo: Memo::Hash(Hash(memo)), + operations: VecM::try_from(vec![Operation::default()]).unwrap(), + ..Transaction::default() + }; + txs.push(TransactionEnvelope::Tx(TransactionV1Envelope { + tx, + signatures: VecM::try_from(vec![DecoratedSignature { + hint: SignatureHint(hint), + signature: Signature(BytesM::try_from(signature.to_vec()).unwrap()), + }]) + .unwrap(), + })); + } + let mut tx_set = GeneralizedTransactionSet::default(); + let GeneralizedTransactionSet::V1(v1) = &mut tx_set; + v1.phases = VecM::try_from(vec![TransactionPhase::V0( + VecM::try_from(vec![TxSetComponent::TxsetCompTxsMaybeDiscountedFee( + TxSetComponentTxsMaybeDiscountedFee { + base_fee: Some(100), + txs: VecM::try_from(txs).unwrap(), + }, + )]) + .unwrap(), + )]) + .unwrap(); + let bytes = tx_set.to_xdr(Limits::none()).unwrap(); + let hash = crate::xdr::sha256_hash(&bytes); + (hash, Arc::new(TxSetData::from_local(bytes).unwrap())) +} + +fn env_or(name: &str, default: T) -> T { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Latency experiment, not a pass/fail test: how long does one tx-set frame +/// take from `send_txset` on one node to `TxSetReceived` on the other, on a +/// link with real delay? Run it in a network namespace whose loopback has a +/// netem delay (75 ms each way = a 150 ms round trip): +/// +/// ```text +/// unshare -rn sh -c 'ip link set lo up; \ +/// tc qdisc add dev lo root netem delay 75ms limit 100000; \ +/// --ignored --nocapture txset_transfer_latency_experiment' +/// ``` +/// +/// Knobs: `EXP_ENCODED_KB` (frame size), `EXP_ROUNDS`, `EXP_GAP_MS` (idle +/// time between transfers), and `STELLAR_OVERLAY_QUIC_INITIAL_CWND`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore] +async fn txset_transfer_latency_experiment() { + let encoded_kb: usize = env_or("EXP_ENCODED_KB", 600); + let rounds: u64 = env_or("EXP_ROUNDS", 8); + let gap_ms: u64 = env_or("EXP_GAP_MS", 2000); + + let keypair1 = Keypair::generate_ed25519(); + let keypair2 = Keypair::generate_ed25519(); + let peer2 = keypair2.public().to_peer_id(); + let (handle1, mut events1, _admissions1, overlay1) = + create_test_overlay(keypair1, Arc::new(OverlayMetrics::new())).unwrap(); + let (handle2, mut events2, _admissions2, overlay2) = + create_test_overlay(keypair2, Arc::new(OverlayMetrics::new())).unwrap(); + let listen_port = 21701; + tokio::spawn(async move { overlay1.run("127.0.0.1", listen_port).await }); + tokio::spawn(async move { overlay2.run("127.0.0.1", listen_port + 1).await }); + tokio::time::sleep(Duration::from_millis(200)).await; + let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{}/quic-v1", listen_port) + .parse() + .unwrap(); + handle2.dial(addr).await; + // Handshake, identify and stream negotiation take several round trips. + tokio::time::sleep(Duration::from_millis(3000)).await; + while events1.try_recv().is_ok() {} + while events2.try_recv().is_ok() {} + + // One-way latency of a small control message, for reference. + let scp = test_scp_envelope_xdr(200); + let start = Instant::now(); + handle1.broadcast_scp(scp.clone()).await; + loop { + match tokio::time::timeout(Duration::from_secs(10), events2.recv()).await { + Ok(Some(OverlayEvent::ScpReceived { envelope, .. })) if envelope == scp => break, + Ok(Some(_)) => {} + other => panic!("no SCP message: {:?}", other.is_ok()), + } + } + println!( + "EXP scp one-way: {:.1} ms (initial cwnd {:?})", + start.elapsed().as_secs_f64() * 1e3, + quic_initial_congestion_window() + ); + + for round in 0..rounds { + let (hash, data) = large_test_txset(round, encoded_kb * 1024); + let encoded = data.encoded().len(); + let start = Instant::now(); + handle1.send_txset(hash, data, peer2).await; + loop { + match tokio::time::timeout(Duration::from_secs(30), events2.recv()).await { + Ok(Some(OverlayEvent::TxSetReceived { hash: got, .. })) if got == hash => break, + Ok(Some(_)) => {} + other => panic!("no tx set: {:?}", other.is_ok()), + } + } + println!( + "EXP round {} encoded {} bytes: {:.1} ms", + round, + encoded, + start.elapsed().as_secs_f64() * 1e3 + ); + tokio::time::sleep(Duration::from_millis(gap_ms)).await; + } + handle1.shutdown().await; + handle2.shutdown().await; +} diff --git a/overlay/vendor/congestion-window.patch b/overlay/vendor/congestion-window.patch new file mode 100644 index 0000000000..be222e263e --- /dev/null +++ b/overlay/vendor/congestion-window.patch @@ -0,0 +1,49 @@ +diff --git a/libp2p-quic/src/config.rs b/libp2p-quic/src/config.rs +index 2456ed3e3..fb8effcc1 100644 +--- a/libp2p-quic/src/config.rs ++++ b/libp2p-quic/src/config.rs +@@ -69,6 +69,16 @@ pub struct Config { + + /// Parameters governing MTU discovery. See [`MtuDiscoveryConfig`] for details. + mtu_discovery_config: Option, ++ ++ /// Initial congestion window in bytes for every connection (CUBIC). ++ /// `None` keeps quinn's default (about ten packets). ++ /// ++ /// quinn paces one congestion window over 0.8 RTT and does not grow the ++ /// window while the sender is application limited, so a connection that ++ /// mostly idles sends a burst of S bytes in about 0.8 * S / window round ++ /// trips. A window several times the largest expected burst lets that ++ /// burst leave in one flight. Loss still shrinks the window as usual. ++ pub initial_congestion_window: Option, + } + + impl Config { +@@ -95,6 +105,7 @@ impl Config { + max_stream_data: 10_000_000, + keypair: keypair.clone(), + mtu_discovery_config: Some(Default::default()), ++ initial_congestion_window: None, + } + } + +@@ -135,6 +146,7 @@ impl From for QuinnConfig { + handshake_timeout: _, + keypair, + mtu_discovery_config, ++ initial_congestion_window, + } = config; + let mut transport = quinn::TransportConfig::default(); + // Disable uni-directional streams. +@@ -148,6 +160,11 @@ impl From for QuinnConfig { + transport.stream_receive_window(max_stream_data.into()); + transport.receive_window(max_connection_data.into()); + transport.mtu_discovery_config(mtu_discovery_config); ++ if let Some(window) = initial_congestion_window { ++ let mut cubic = quinn::congestion::CubicConfig::default(); ++ cubic.initial_window(window); ++ transport.congestion_controller_factory(Arc::new(cubic)); ++ } + let transport = Arc::new(transport); + + let mut server_config = quinn::ServerConfig::with_crypto(server_tls_config); diff --git a/overlay/vendor/libp2p-quic/src/config.rs b/overlay/vendor/libp2p-quic/src/config.rs index 2456ed3e36..fb8effcc16 100644 --- a/overlay/vendor/libp2p-quic/src/config.rs +++ b/overlay/vendor/libp2p-quic/src/config.rs @@ -69,6 +69,16 @@ pub struct Config { /// Parameters governing MTU discovery. See [`MtuDiscoveryConfig`] for details. mtu_discovery_config: Option, + + /// Initial congestion window in bytes for every connection (CUBIC). + /// `None` keeps quinn's default (about ten packets). + /// + /// quinn paces one congestion window over 0.8 RTT and does not grow the + /// window while the sender is application limited, so a connection that + /// mostly idles sends a burst of S bytes in about 0.8 * S / window round + /// trips. A window several times the largest expected burst lets that + /// burst leave in one flight. Loss still shrinks the window as usual. + pub initial_congestion_window: Option, } impl Config { @@ -95,6 +105,7 @@ impl Config { max_stream_data: 10_000_000, keypair: keypair.clone(), mtu_discovery_config: Some(Default::default()), + initial_congestion_window: None, } } @@ -135,6 +146,7 @@ impl From for QuinnConfig { handshake_timeout: _, keypair, mtu_discovery_config, + initial_congestion_window, } = config; let mut transport = quinn::TransportConfig::default(); // Disable uni-directional streams. @@ -148,6 +160,11 @@ impl From for QuinnConfig { transport.stream_receive_window(max_stream_data.into()); transport.receive_window(max_connection_data.into()); transport.mtu_discovery_config(mtu_discovery_config); + if let Some(window) = initial_congestion_window { + let mut cubic = quinn::congestion::CubicConfig::default(); + cubic.initial_window(window); + transport.congestion_controller_factory(Arc::new(cubic)); + } let transport = Arc::new(transport); let mut server_config = quinn::ServerConfig::with_crypto(server_tls_config); From 8f7768be7aae15d9551295239f3e5bb8c72fe1b8 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Tue, 10 Nov 2026 22:57:54 +0000 Subject: [PATCH 2/4] Push the leader's tx set to peers when it is built (prototype) Tx-set dissemination was a request model: a follower saw the leader's proposal, asked for the tx set, and received it about two round trips later. The round's leader now sends the set it is about to propose to every connected peer as soon as the set is built, so a follower normally holds it when the proposal arrives. The request path is unchanged and remains the fallback for a node that did not get the push. Core: a new IPC message PUSH_TX_SET [hash][slot]. HerderImpl sends it right after CACHE_TX_SET from inside the build when the node leads the current round, after the build for a first-round leader's capacity-limited prepared set, and again when a prepared set is taken at the trigger. Overlay, sender: the dispatcher writes the cached, already compressed set to the tx-set stream of every connected peer that has not been sent it, one push in flight per peer. The first request from a peer for a set that is being written to it, or was written less than 500 ms ago, is not answered: the bytes are already on the same ordered stream. Every other request is answered, a failed push answers a skipped request after all, and a closed connection clears the record for that peer. Overlay, receiver: a pushed set is an ordinary tx-set frame nobody asked for. It is decompressed, strict-parsed and filed under the hash the receiver computed; it carries no claimed hash. Core is not told: it gets a set only through RequestTxSet for a hash named by an SCP statement, and re-checks the content hash. So a pushed set that does not match the leader's statement is never used, and the proposed set is requested. Since unrequested frames are now a normal event they are bounded: a per-peer token bucket applied before decompression (answers to this node's requests are free), and limits on cached sets from peers that Core has not asked for (per peer, and a byte budget over all peers). No change to tx-set validation, voting or apply. Six meters overlay.txset-push.* report what happened; see docs/rust-overlay/txset-fetching.md. --- docs/metrics.md | 6 + docs/rust-overlay/ipc.md | 1 + docs/rust-overlay/txset-fetching.md | 86 +- overlay/src/flood/mod.rs | 10 + overlay/src/flood/txset.rs | 39 + overlay/src/flood/txset_push.rs | 832 ++++++++++++++++++ overlay/src/ipc/messages.rs | 9 + overlay/src/libp2p_overlay.rs | 206 ++++- .../src/libp2p_overlay/txset_push_tests.rs | 402 +++++++++ overlay/src/main.rs | 431 ++++++++- overlay/src/metrics.rs | 32 + overlay/tests/e2e_binary.rs | 311 +++++++ src/herder/HerderImpl.cpp | 46 +- src/herder/HerderImpl.h | 11 +- src/herder/test/HerderTests.cpp | 4 +- src/herder/test/TxSetPushTests.cpp | 306 +++++++ src/overlay/IPC.h | 6 + src/overlay/OverlayIPC.cpp | 21 + src/overlay/OverlayIPC.h | 13 + src/overlay/OverlayMetrics.cpp | 12 + src/overlay/OverlayMetrics.h | 8 + src/overlay/RustOverlayManager.cpp | 23 + src/overlay/RustOverlayManager.h | 20 + src/overlay/test/IPCTests.cpp | 22 + 24 files changed, 2843 insertions(+), 14 deletions(-) create mode 100644 overlay/src/flood/txset_push.rs create mode 100644 overlay/src/libp2p_overlay/txset_push_tests.rs create mode 100644 src/herder/test/TxSetPushTests.cpp diff --git a/docs/metrics.md b/docs/metrics.md index 098823e7bc..0f290a46f0 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -187,6 +187,12 @@ overlay.recv. | timer | received message overlay.send. | meter | sent message overlay.timeout.idle | meter | idle peer timeout overlay.timeout.straggler | meter | straggler peer timeout +overlay.txset-push.dropped | meter | tx-set frames dropped undecoded by the per-peer limit on frames the node did not ask for +overlay.txset-push.evicted | meter | tx sets from peers evicted by the limits on sets Core has not asked for +overlay.txset-push.hit | meter | Core tx-set requests answered at once from a set a peer had pushed +overlay.txset-push.received | meter | tx sets received from a peer without a request +overlay.txset-push.request-skipped | meter | peer requests not answered because the set was already pushed to that peer +overlay.txset-push.sent | meter | tx sets pushed to a peer by a round leader, without a request (one per peer) process.action.queue | counter | number of items waiting in internal action-queue process.action.overloaded | counter | 0-or-1 value indicating action-queue overloading process.file.handles | counter | number of open file handles diff --git a/docs/rust-overlay/ipc.md b/docs/rust-overlay/ipc.md index a96778308a..433ea47418 100644 --- a/docs/rust-overlay/ipc.md +++ b/docs/rust-overlay/ipc.md @@ -72,6 +72,7 @@ A mempool pull of 2x the tx-set capacity plus a pending set's transactions | 11 | `RequestTxSet` | `[hash:32][slot:u32 LE]` | Fetch TX set body by hash | | 12 | `CacheTxSet` | `[hash:32][slot:u32 LE][txset_xdr]` | Tell overlay to cache a locally-built TX set | | 13 | `RequestOverlayMetrics` | (empty) | Request metrics snapshot | +| 14 | `PushTxSet` | `[hash:32][slot:u32 LE]` | Send a cached TX set to every peer now (leader push) | ### Overlay → Core diff --git a/docs/rust-overlay/txset-fetching.md b/docs/rust-overlay/txset-fetching.md index 1e4b899022..b9294e399a 100644 --- a/docs/rust-overlay/txset-fetching.md +++ b/docs/rust-overlay/txset-fetching.md @@ -1,8 +1,10 @@ # TX set fetching -SCP envelopes reference transaction sets by hash. Rust prefetches their bodies -from peers and keeps them in the tx-set cache. Core receives a set only after -it establishes demand with `RequestTxSet`. +SCP envelopes reference transaction sets by hash. A round's leader pushes the +set it is about to propose to its peers; a node that does not hold a +referenced set requests its body from a peer. Either way Rust keeps the bodies +in the tx-set cache, and Core receives a set only after it establishes demand +with `RequestTxSet`. ## Wire formats and routes @@ -57,6 +59,78 @@ borrow those encoded bytes directly, so relays never recompress received sets. Encoding is complete before cache insertion; sending has no lazy initialization or compression decision. Core continues receiving uncompressed XDR over IPC. +## Leader push + +A round leader does not wait to be asked. For the set it is about to propose, +Core sends `PushTxSet [hash:32][slot:u32 LE]` right after the `CacheTxSet` +that carried the set: from inside the build when the node leads the current +round, after the build for a first-round leader's early (prepared) set that +was capacity limited, and again when a prepared set is taken at the trigger. +App looks the hash up in the cache (a miss is logged and ignored) and the +dispatcher writes the prepared encoding to the tx-set stream of every +connected peer that has not been sent this set. The proposal itself follows +on the control stream once Core has validated and persisted it. + +The pushed frame is an ordinary tx-set frame that nobody requested. It +carries no hash and no slot: the receiver decompresses, strict-parses and +hashes it like a response and caches it under the hash it computed. Core is +not told. When an SCP statement names that hash, Core's `RequestTxSet` is a +cache hit (`overlay.txset-push.hit`). A pushed set whose hash no statement +names is never delivered to Core. + +Push against request. If the proposal reaches a peer before the pushed set is +complete, the peer requests the set as usual. The leader does not answer that +request (`overlay.txset-push.request-skipped`) if it is the first one from +that peer for a set that is still being written to it, or was written less +than 500 ms ago, on the current connection: the bytes are already on the same +ordered, reliable stream, and the peer's reservation is cleared when they +land. Every other request is answered: a repeated one, one that arrives later +(a peer only asks when it does not hold the set, so it lost or dropped the +pushed copy), and one from a peer the set was not pushed to. A failed push +write answers a skipped request after all, and any connection to the peer +closing drops the record for that peer. + +A leader keeps at most one push in flight per peer (`TXSET_PUSH_BUSY` in the +log when a peer is left out). A peer that does not read its tx-set stream is +not sent further sets (it requests them), so it pins at most one set in the +leader's memory; closing a connection does not release that count, only the +end of the write does. + +A node that did not receive the push (not connected to the leader, connected +later, or busy with an earlier push) has no cached set and uses the request +path below. A push that the receiver itself dropped (rate limit) or evicted +right away (limits on unclaimed sets) is not recovered that fast: the +receiver's request reaches the leader within the 500 ms and is skipped, so the +set only comes with Core's retry to another peer after one second. The window +helps requests that arrive later (an eviction after some time, a first request +that went to a relay). The push is not relayed: a +node that is not connected to the leader requests the set from the peer that +relayed the proposal, which holds the pushed copy. + +Limits on what a peer can make a node decode and keep: + +- Rate: every tx-set frame takes a token from a per-peer bucket (burst 8, 4 + per second). A frame that answers a request of this node returns its token, + so only frames nobody asked for use the allowance. With the bucket empty a + frame still passes if this node sent that peer a request it has not used up + (one frame per request, at most 8 saved). Anything else is dropped before + decompression (`overlay.txset-push.dropped`). +- Space: cached sets that came from a peer and that Core has not asked for + are limited per peer, oldest dropped first: 4 that the peer pushed and, + separately, 16 that answered this node's own requests (a peer answers a new + connection's SCP state request with about 12 slots at once). Together they + are limited to 128 MiB (the peer holding the most bytes loses its oldest + set). Evictions count as `overlay.txset-push.evicted`. A set leaves this + accounting when Core requests or builds it, or when the cache drops it. A + set is accounted to the peer that delivered it first. +- Not bounded by these: a peer can make a node send it requests, because the + overlay prefetches any hash named in an SCP envelope before Core checks the + signature; each such envelope buys one frame over the rate. A peer that + reconnects starts with a full allowance. Many colluding peers can use the + shared byte budget to push out a large honest set, and a peer that delivers + the leader's set first and then pushes four more evicts it (the receiver + then requests the set). + ## Core requests and network prefetches `RequestTxSet` IPC carries `[hash:32][slot:u32 LE]`. App records the newest @@ -121,6 +195,12 @@ See [mempool](mempool.md). ## Remaining limitations +- The push is not relayed and carries the full compressed set; nodes that are + not connected to the leader, and a compact (hash-list) form, are future work. +- A peer whose first copy of a proposal came through a relay requests the set + from that relay, which answers from its cache while the leader's push may + still be in flight: one duplicate transfer. + - A cache miss at the selected peer has no explicit negative response; the requester only notices through Core's one-second retry. - Selecting one connected peer does not guarantee it has the requested body. diff --git a/overlay/src/flood/mod.rs b/overlay/src/flood/mod.rs index ecefbc36db..864c7c5f30 100644 --- a/overlay/src/flood/mod.rs +++ b/overlay/src/flood/mod.rs @@ -10,6 +10,7 @@ mod pending_requests; mod tx_buffer; mod txset; pub(crate) mod txset_encoding; +pub(crate) mod txset_push; pub use inv_batcher::InvBatcher; pub use inv_messages::{GetData, InvBatch, InvEntry, TxStreamMessage, INV_BATCH_MAX_SIZE}; @@ -19,3 +20,12 @@ pub use pending_requests::PendingRequests; pub use tx_buffer::TxBuffer; pub use txset::{CachedTxSet, Hash256, TxSetCache}; pub use txset_encoding::TxSetData; +pub use txset_push::{PushRateLimiter, PushRecord, UnclaimedSetBudget}; + +/// Limits of the tx-set push, for tests outside this crate's library. +pub mod txset_push_limits { + pub use super::txset_push::{ + MAX_UNCLAIMED_BYTES, MAX_UNCLAIMED_REQUESTED_SETS_PER_PEER, MAX_UNCLAIMED_SETS_PER_PEER, + PUSH_BURST, PUSH_REFILL_PER_SEC, PUSH_SKIP_WINDOW, + }; +} diff --git a/overlay/src/flood/txset.rs b/overlay/src/flood/txset.rs index 95beb3c44b..8c5583d7d3 100644 --- a/overlay/src/flood/txset.rs +++ b/overlay/src/flood/txset.rs @@ -58,6 +58,16 @@ impl TxSetCache { self.by_hash.get(hash) } + /// Remove one TX set. Returns true if it was cached. + pub fn remove(&mut self, hash: &Hash256) -> bool { + if self.by_hash.remove(hash).is_some() { + self.order.retain(|h| h != hash); + true + } else { + false + } + } + /// Remove TX sets for ledgers before the given sequence. pub fn evict_before(&mut self, ledger_seq: u32) { self.order.retain(|hash| { @@ -121,6 +131,35 @@ mod tests { assert!(cache.get(&[2u8; 32]).is_some()); // kept } + #[test] + fn test_cache_remove_keeps_order_in_sync() { + let mut cache = TxSetCache::new(2); + for (seed, seq) in [(1u8, 100), (2u8, 101)] { + cache.insert(CachedTxSet { + hash: [seed; 32], + xdr: Arc::new(TxSetData::from_local(vec![]).unwrap()), + ledger_seq: seq, + }); + } + assert!(cache.remove(&[1u8; 32])); + assert!(!cache.remove(&[1u8; 32])); + assert_eq!(cache.len(), 1); + + // The removed hash no longer takes part in capacity eviction: adding + // two more sets evicts [2] and keeps the cache at capacity. + for (seed, seq) in [(3u8, 102), (4u8, 103)] { + cache.insert(CachedTxSet { + hash: [seed; 32], + xdr: Arc::new(TxSetData::from_local(vec![]).unwrap()), + ledger_seq: seq, + }); + } + assert_eq!(cache.len(), 2); + assert!(cache.get(&[2u8; 32]).is_none()); + assert!(cache.get(&[3u8; 32]).is_some()); + assert!(cache.get(&[4u8; 32]).is_some()); + } + #[test] fn test_cache_capacity_eviction() { let mut cache = TxSetCache::new(2); // Small cache diff --git a/overlay/src/flood/txset_push.rs b/overlay/src/flood/txset_push.rs new file mode 100644 index 0000000000..e09567e637 --- /dev/null +++ b/overlay/src/flood/txset_push.rs @@ -0,0 +1,832 @@ +//! Bookkeeping for the leader's tx-set push. +//! +//! A round's leader sends its tx set to every connected peer as soon as the +//! set is built, without waiting for a request. The pushed frame is the same +//! frame a request is answered with, and the receiver verifies and files it by +//! its own content hash, so nothing here is trusted: these types only decide +//! what to send, what to skip and how much unrequested data to keep. +//! +//! * [`PushRecord`] (sender): which peers a set went to, so that it is pushed +//! once per peer and a request that crosses the push is not answered with a +//! second copy. +//! * [`PushRateLimiter`] (receiver): how many tx-set frames a peer may send +//! beyond the answers to this node's requests before they are dropped +//! undecoded. +//! * [`UnclaimedSetBudget`] (receiver): how many sets from peers stay cached +//! while Core has not asked for them. + +use super::Hash256; +use libp2p::PeerId; +use std::collections::{HashMap, VecDeque}; +use std::num::NonZeroUsize; +use std::time::{Duration, Instant}; + +/// Sets a node remembers having pushed. A node pushes one set per round it +/// leads, and a request can only cross a push for a few round trips. +const PUSH_RECORD_SETS: usize = 32; + +/// How long after a push was written a request from that peer is still taken +/// to have crossed it. The peer asks when the proposal arrives, which the +/// leader sends about 70 ms after the push, so its request reaches the leader +/// about 70 ms plus one round trip after the push was written. A later +/// request means the peer does not hold the set (it only asks on a cache +/// miss) and is answered. +pub const PUSH_SKIP_WINDOW: Duration = Duration::from_millis(500); + +/// Pushes that may be queued or in flight to one peer at a time. A peer that +/// stops reading its tx-set stream then holds at most this many sets in +/// memory; further sets are not pushed to it and it requests them instead. +pub const MAX_IN_FLIGHT_PUSHES_PER_PEER: usize = 1; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PushProgress { + /// Queued on, or being written to, the peer's tx-set stream. + InFlight, + /// Handed to the transport at this time. + Written(Instant), +} + +#[derive(Clone, Copy, Debug)] +struct PeerPush { + progress: PushProgress, + /// The peer's first request for the set was skipped. + asked: bool, +} + +/// Sender side: the sets this node pushed, per peer, on the peer's current +/// connection. Entries for a peer are dropped when a connection to it closes. +pub struct PushRecord { + by_hash: lru::LruCache>, + /// Pushes not yet handed to the transport, per peer. + in_flight: HashMap, +} + +impl Default for PushRecord { + fn default() -> Self { + Self::new() + } +} + +impl PushRecord { + pub fn new() -> Self { + Self { + by_hash: lru::LruCache::new(NonZeroUsize::new(PUSH_RECORD_SETS).unwrap()), + in_flight: HashMap::new(), + } + } + + /// Record a push of `hash` and return the peers among `connected` to send + /// it to: those that have not been sent it yet and have no other push + /// still in flight. A repeated push only reaches peers that connected + /// since. A peer that is skipped because it is busy is not recorded, so + /// its request for the set is answered. + pub fn start_push( + &mut self, + hash: Hash256, + connected: impl IntoIterator, + ) -> Vec { + self.start_push_counting_busy(hash, connected).0 + } + + /// As [`Self::start_push`]; also returns how many peers were left out + /// because an earlier push to them is still in flight. + pub fn start_push_counting_busy( + &mut self, + hash: Hash256, + connected: impl IntoIterator, + ) -> (Vec, usize) { + let peers = self.by_hash.get_or_insert_mut(hash, HashMap::new); + let mut targets = Vec::new(); + let mut busy = 0; + for peer in connected { + if peers.contains_key(&peer) { + continue; + } + let in_flight = self.in_flight.entry(peer).or_insert(0); + if *in_flight >= MAX_IN_FLIGHT_PUSHES_PER_PEER { + busy += 1; + continue; + } + *in_flight += 1; + peers.insert( + peer, + PeerPush { + progress: PushProgress::InFlight, + asked: false, + }, + ); + targets.push(peer); + } + (targets, busy) + } + + fn finish_in_flight(&mut self, peer: &PeerId) { + if let Some(count) = self.in_flight.get_mut(peer) { + *count = count.saturating_sub(1); + if *count == 0 { + self.in_flight.remove(peer); + } + } + } + + /// The push to `peer` was handed to the transport. + pub fn push_written(&mut self, hash: &Hash256, peer: &PeerId, now: Instant) { + self.finish_in_flight(peer); + if let Some(push) = self + .by_hash + .peek_mut(hash) + .and_then(|peers| peers.get_mut(peer)) + { + push.progress = PushProgress::Written(now); + } + } + + /// The write to `peer` failed, so the peer does not have the set from us. + /// Returns true if a request from that peer was skipped because of the + /// push; the caller then answers it after all. + pub fn push_failed(&mut self, hash: &Hash256, peer: &PeerId) -> bool { + self.finish_in_flight(peer); + self.by_hash + .peek_mut(hash) + .and_then(|peers| peers.remove(peer)) + .is_some_and(|push| push.asked) + } + + /// `peer` asked for `hash`. Returns true if the request should not be + /// answered: the set is still being written to that peer, or was written + /// less than [`PUSH_SKIP_WINDOW`] ago, on the same ordered, reliable + /// stream an answer would use, so a second copy could only arrive later. + /// + /// Only the first such request is skipped. A repeated request, or one + /// that arrives after the window, means the peer does not hold the pushed + /// copy (it was dropped by the peer's limits, or evicted) and is answered. + pub fn should_skip_request(&mut self, hash: &Hash256, peer: &PeerId, now: Instant) -> bool { + let Some(push) = self + .by_hash + .peek_mut(hash) + .and_then(|peers| peers.get_mut(peer)) + else { + return false; + }; + if push.asked { + return false; + } + let crossing = match push.progress { + PushProgress::InFlight => true, + PushProgress::Written(at) => now.saturating_duration_since(at) < PUSH_SKIP_WINDOW, + }; + if crossing { + push.asked = true; + } + crossing + } + + /// A connection to the peer closed: the pushed sets may have been on its + /// streams and not arrived, so the peer's requests are answered again. + /// + /// The in-flight count is left alone. Every push task ends in + /// `push_written` or `push_failed` (a write on a closed connection fails), + /// so the count stays exact; resetting it here would let a peer that does + /// not read its stream collect one stuck push per connection it closes. + pub fn forget_peer(&mut self, peer: &PeerId) { + for (_, peers) in self.by_hash.iter_mut() { + peers.remove(peer); + } + } + + #[cfg(test)] + pub fn was_pushed(&self, hash: &Hash256, peer: &PeerId) -> bool { + self.by_hash + .peek(hash) + .is_some_and(|peers| peers.contains_key(peer)) + } +} + +/// Unrequested tx-set frames a peer may send in a burst. +pub const PUSH_BURST: f64 = 8.0; +/// Sustained rate of unrequested tx-set frames per peer, per second. An honest +/// leader pushes one set per round it leads (two if it also prepared one). +pub const PUSH_REFILL_PER_SEC: f64 = 4.0; +/// Frames beyond the rate that a peer may still send because this node sent +/// it tx-set requests: one per request, at most this many saved up. +pub const MAX_REQUEST_CREDITS: u32 = 8; + +/// Peers the rate limiter tracks before it starts over. +const MAX_TRACKED_PEERS: usize = 4096; + +struct PeerAllowance { + tokens: f64, + last_refill: Instant, + /// Requests this node sent the peer that have not been used up by a + /// frame over the rate. + request_credits: u32, +} + +/// Receiver side: how many tx-set frames a peer may have decoded. +/// +/// Every frame takes a token from the peer's bucket; a frame that turns out +/// to answer a request of this node gives it back, so answers are free and +/// only frames nobody asked for use the allowance. With the bucket empty, a +/// frame is still let through if this node sent that peer a request it has +/// not used up (the frame may be the answer). Anything else is dropped before +/// it is decompressed. +/// +/// What this does not bound: a peer can make this node send it requests (the +/// overlay prefetches any hash named in an SCP envelope, before Core checks +/// the signature), so it can buy one extra frame per envelope. +pub struct PushRateLimiter { + peers: HashMap, + burst: f64, + refill_per_sec: f64, +} + +impl Default for PushRateLimiter { + fn default() -> Self { + Self::new(PUSH_BURST, PUSH_REFILL_PER_SEC) + } +} + +impl PushRateLimiter { + pub fn new(burst: f64, refill_per_sec: f64) -> Self { + Self { + peers: HashMap::new(), + burst, + refill_per_sec, + } + } + + fn allowance(&mut self, peer: PeerId, now: Instant) -> &mut PeerAllowance { + let burst = self.burst; + let refill = self.refill_per_sec; + // Entries go when a peer disconnects. A reader task that is still + // draining can recreate one afterwards; keep the map bounded anyway. + if self.peers.len() >= MAX_TRACKED_PEERS && !self.peers.contains_key(&peer) { + self.peers.clear(); + } + let allowance = self.peers.entry(peer).or_insert(PeerAllowance { + tokens: burst, + last_refill: now, + request_credits: 0, + }); + let elapsed = now + .saturating_duration_since(allowance.last_refill) + .as_secs_f64(); + allowance.tokens = (allowance.tokens + elapsed * refill).min(burst); + allowance.last_refill = now; + allowance + } + + /// A tx-set frame arrived from `peer`. False means it is over the limit + /// and must be dropped undecoded. + pub fn allow(&mut self, peer: PeerId, now: Instant) -> bool { + let allowance = self.allowance(peer, now); + if allowance.tokens >= 1.0 { + allowance.tokens -= 1.0; + true + } else if allowance.request_credits > 0 { + allowance.request_credits -= 1; + true + } else { + false + } + } + + /// This node sent `peer` a tx-set request. + pub fn request_sent(&mut self, peer: PeerId, now: Instant) { + let allowance = self.allowance(peer, now); + allowance.request_credits = (allowance.request_credits + 1).min(MAX_REQUEST_CREDITS); + } + + /// A frame that was let through answered a request of this node: it does + /// not count against the peer. + pub fn answered_request(&mut self, peer: PeerId, now: Instant) { + let burst = self.burst; + let allowance = self.allowance(peer, now); + allowance.tokens = (allowance.tokens + 1.0).min(burst); + } + + /// The peer's last connection closed. A peer that reconnects starts with + /// a full allowance again. + pub fn forget_peer(&mut self, peer: &PeerId) { + self.peers.remove(peer); + } + + #[cfg(test)] + pub fn request_credits(&self, peer: &PeerId) -> u32 { + self.peers.get(peer).map_or(0, |a| a.request_credits) + } +} + +/// Unclaimed sets kept per peer that the peer sent unrequested (pushes). +pub const MAX_UNCLAIMED_SETS_PER_PEER: usize = 4; +/// Unclaimed sets kept per peer that answered this node's own requests +/// (prefetches for hashes named in SCP envelopes). A peer answers a new +/// connection's SCP state request with up to about 12 slots of envelopes, and +/// the overlay may receive all of their sets before Core asks for the first. +pub const MAX_UNCLAIMED_REQUESTED_SETS_PER_PEER: usize = 16; +/// Bytes of unclaimed sets kept in total (XDR plus encoding). +pub const MAX_UNCLAIMED_BYTES: usize = 128 * 1024 * 1024; + +/// Receiver side: the cached sets that came from a peer and that Core has not +/// asked for ("unclaimed"): pushed sets, and sets the overlay prefetched +/// because an SCP envelope named them. A set leaves this budget when Core +/// requests it or builds it (it is then an ordinary cache entry) or when the +/// cache drops it. +/// +/// Limits: a fixed number of sets per peer, oldest dropped first, and one byte +/// budget over all peers. Over the byte budget the peer holding the most bytes +/// loses its oldest set, so a few peers cannot push out the sets of the +/// others. (Many colluding peers can: the budget then evens out the holders, +/// and a large honest set can go. Its receiver falls back to a request.) +pub struct UnclaimedSetBudget { + /// Per peer, oldest first: (hash, bytes, arrived unrequested). + per_peer: HashMap>, + /// Hash -> (peer it came from, whether it arrived unrequested). + owner: HashMap, + total_bytes: usize, + max_pushed_per_peer: usize, + max_requested_per_peer: usize, + max_total_bytes: usize, +} + +impl Default for UnclaimedSetBudget { + fn default() -> Self { + Self::new( + MAX_UNCLAIMED_SETS_PER_PEER, + MAX_UNCLAIMED_REQUESTED_SETS_PER_PEER, + MAX_UNCLAIMED_BYTES, + ) + } +} + +impl UnclaimedSetBudget { + pub fn new( + max_pushed_per_peer: usize, + max_requested_per_peer: usize, + max_total_bytes: usize, + ) -> Self { + Self { + per_peer: HashMap::new(), + owner: HashMap::new(), + total_bytes: 0, + max_pushed_per_peer, + max_requested_per_peer, + max_total_bytes, + } + } + + /// Track a set of `bytes` from `peer` that Core has not asked for. + /// `pushed` says whether it arrived unrequested; pushed and requested + /// sets have separate per-peer limits. Returns the hashes the caller must + /// drop from its cache to stay within the limits; that can include `hash` + /// itself. + pub fn insert( + &mut self, + peer: PeerId, + hash: Hash256, + bytes: usize, + pushed: bool, + ) -> Vec { + if self.owner.contains_key(&hash) { + return Vec::new(); + } + self.owner.insert(hash, (peer, pushed)); + self.per_peer + .entry(peer) + .or_default() + .push_back((hash, bytes, pushed)); + self.total_bytes += bytes; + + let limit = if pushed { + self.max_pushed_per_peer + } else { + self.max_requested_per_peer + }; + let mut evicted = Vec::new(); + while self.count_of_kind(&peer, pushed) > limit { + evicted.extend(self.evict_oldest_of(&peer, Some(pushed))); + } + while self.total_bytes > self.max_total_bytes { + let Some(largest) = self + .per_peer + .iter() + .max_by_key(|(_, sets)| sets.iter().map(|(_, bytes, _)| *bytes).sum::()) + .map(|(peer, _)| *peer) + else { + break; + }; + evicted.extend(self.evict_oldest_of(&largest, None)); + } + evicted + } + + fn count_of_kind(&self, peer: &PeerId, pushed: bool) -> usize { + self.per_peer.get(peer).map_or(0, |sets| { + sets.iter().filter(|(_, _, kind)| *kind == pushed).count() + }) + } + + /// Drop the peer's oldest set, of the given kind if one is named. + fn evict_oldest_of(&mut self, peer: &PeerId, kind: Option) -> Option { + let sets = self.per_peer.get_mut(peer)?; + let pos = sets + .iter() + .position(|(_, _, pushed)| kind.map_or(true, |kind| kind == *pushed))?; + let (hash, bytes, _) = sets.remove(pos)?; + if sets.is_empty() { + self.per_peer.remove(peer); + } + self.owner.remove(&hash); + self.total_bytes -= bytes; + Some(hash) + } + + /// Stop counting `hash`: Core asked for it or built it, or the cache + /// dropped it. Returns `Some(pushed)` if it was an unclaimed set, where + /// `pushed` says whether it had arrived unrequested. + pub fn claim(&mut self, hash: &Hash256) -> Option { + let (peer, pushed) = self.owner.remove(hash)?; + if let Some(sets) = self.per_peer.get_mut(&peer) { + if let Some(pos) = sets.iter().position(|(h, _, _)| h == hash) { + let (_, bytes, _) = sets.remove(pos).unwrap(); + self.total_bytes -= bytes; + } + if sets.is_empty() { + self.per_peer.remove(&peer); + } + } + Some(pushed) + } + + /// Forget every set for which `keep` is false (the cache evicted it). + pub fn retain(&mut self, keep: impl Fn(&Hash256) -> bool) { + let gone: Vec = self.owner.keys().filter(|h| !keep(h)).copied().collect(); + for hash in gone { + self.claim(&hash); + } + } + + pub fn contains(&self, hash: &Hash256) -> bool { + self.owner.contains_key(hash) + } + + pub fn len(&self) -> usize { + self.owner.len() + } + + pub fn is_empty(&self) -> bool { + self.owner.is_empty() + } + + pub fn total_bytes(&self) -> usize { + self.total_bytes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn peer() -> PeerId { + PeerId::random() + } + + fn written(record: &mut PushRecord, hash: Hash256, peers: &[PeerId], now: Instant) { + for peer in peers { + record.push_written(&hash, peer, now); + } + } + + #[test] + fn push_goes_to_each_peer_once() { + let mut record = PushRecord::new(); + let (a, b, c) = (peer(), peer(), peer()); + let hash = [1u8; 32]; + let now = Instant::now(); + + let mut first = record.start_push(hash, [a, b]); + first.sort(); + let mut expected = vec![a, b]; + expected.sort(); + assert_eq!(first, expected); + written(&mut record, hash, &[a, b], now); + + // A repeated push reaches only the peer that connected since. + assert_eq!(record.start_push(hash, [a, b, c]), vec![c]); + written(&mut record, hash, &[c], now); + assert!(record.start_push(hash, [a, b, c]).is_empty()); + + // Another set is independent. + assert_eq!(record.start_push([2u8; 32], [a]), vec![a]); + } + + #[test] + fn a_peer_with_a_push_in_flight_is_not_sent_another() { + let mut record = PushRecord::new(); + let (stuck, fine) = (peer(), peer()); + let now = Instant::now(); + let (first, second, third) = ([1u8; 32], [2u8; 32], [3u8; 32]); + + let mut targets = record.start_push(first, [stuck, fine]); + targets.sort(); + assert_eq!(targets.len(), 2); + // Only `fine` finishes its write; `stuck` does not read. + record.push_written(&first, &fine, now); + + // The next set goes to `fine` only, and `stuck` is not recorded for + // it: its request for that set is answered through the request path. + assert_eq!(record.start_push(second, [stuck, fine]), vec![fine]); + assert!(!record.was_pushed(&second, &stuck)); + assert!(!record.should_skip_request(&second, &stuck, now)); + record.push_written(&second, &fine, now); + + // Once the stuck write ends (here: fails), pushes resume. + record.push_failed(&first, &stuck); + let mut targets = record.start_push(third, [stuck, fine]); + targets.sort(); + assert_eq!(targets.len(), 2); + } + + #[test] + fn a_crossing_request_is_skipped_once() { + let mut record = PushRecord::new(); + let (a, b) = (peer(), peer()); + let hash = [3u8; 32]; + let now = Instant::now(); + record.start_push(hash, [a]); + + // Not pushed to b, and not this hash to a: answer. + assert!(!record.should_skip_request(&hash, &b, now)); + assert!(!record.should_skip_request(&[4u8; 32], &a, now)); + + // In flight to a: the first request is skipped, a repeat is answered. + assert!(record.should_skip_request(&hash, &a, now)); + assert!(!record.should_skip_request(&hash, &a, now)); + assert!(!record.should_skip_request(&hash, &a, now)); + } + + #[test] + fn a_request_long_after_the_push_is_answered() { + let mut record = PushRecord::new(); + let (early, late, in_flight) = (peer(), peer(), peer()); + let hash = [6u8; 32]; + let start = Instant::now(); + record.start_push(hash, [early, late, in_flight]); + written(&mut record, hash, &[early, late], start); + + // Inside the window the request crossed the push. + let inside = start + PUSH_SKIP_WINDOW - Duration::from_millis(1); + assert!(record.should_skip_request(&hash, &early, inside)); + + // After the window the peer evidently does not hold the pushed copy + // (it lost or dropped it): the request is answered, every time. + let after = start + PUSH_SKIP_WINDOW; + assert!(!record.should_skip_request(&hash, &late, after)); + assert!(!record.should_skip_request(&hash, &late, after)); + + // A push that is still being written is skipped whatever the time. + let much_later = start + Duration::from_secs(60); + assert!(record.should_skip_request(&hash, &in_flight, much_later)); + } + + #[test] + fn failed_push_and_disconnect_restore_the_request_path() { + let mut record = PushRecord::new(); + let (a, b) = (peer(), peer()); + let hash = [5u8; 32]; + let now = Instant::now(); + record.start_push(hash, [a, b]); + + // No request was skipped for a: nothing to answer after the failure. + assert!(!record.push_failed(&hash, &a)); + assert!(!record.was_pushed(&hash, &a)); + assert!(!record.should_skip_request(&hash, &a, now)); + // The set can be pushed to a again. + assert_eq!(record.start_push(hash, [a, b]), vec![a]); + + // b asked while the push was in flight, then the push failed: the + // caller has to answer the skipped request. + assert!(record.should_skip_request(&hash, &b, now)); + assert!(record.push_failed(&hash, &b)); + assert!(!record.push_failed(&hash, &b)); + + record.forget_peer(&a); + assert!(!record.was_pushed(&hash, &a)); + assert!(!record.should_skip_request(&hash, &a, now)); + } + + #[test] + fn a_closed_connection_does_not_release_a_push_still_in_flight() { + let mut record = PushRecord::new(); + let a = peer(); + let now = Instant::now(); + let (first, second, third) = ([1u8; 32], [2u8; 32], [3u8; 32]); + assert_eq!(record.start_push(first, [a]), vec![a]); + + // A connection to the peer closes (possibly a duplicate one) while + // the write is still stuck: the peer's requests are answered again, + // but it is not sent another set until that write has ended. + record.forget_peer(&a); + assert!(!record.should_skip_request(&first, &a, now)); + assert!(record.start_push(second, [a]).is_empty()); + assert!(record.start_push(second, [a]).is_empty()); + + // The stuck write ends (on a closed connection it fails). Its + // completion is counted once, even though the record was cleared. + assert!(!record.push_failed(&first, &a)); + assert!(record.in_flight.is_empty()); + assert_eq!(record.start_push(third, [a]), vec![a]); + record.push_written(&third, &a, now); + assert!(record.in_flight.is_empty()); + } + + #[test] + fn push_record_is_bounded() { + let mut record = PushRecord::new(); + let a = peer(); + let now = Instant::now(); + for i in 0..(PUSH_RECORD_SETS as u32 + 10) { + let mut hash = [0u8; 32]; + hash[..4].copy_from_slice(&i.to_be_bytes()); + assert_eq!(record.start_push(hash, [a]), vec![a]); + record.push_written(&hash, &a, now); + } + assert_eq!(record.by_hash.len(), PUSH_RECORD_SETS); + assert!(record.in_flight.is_empty()); + // The oldest sets fell out: requests for them are answered normally. + assert!(!record.should_skip_request(&[0u8; 32], &a, now)); + } + + #[test] + fn rate_limiter_allows_a_burst_then_the_sustained_rate() { + let mut limiter = PushRateLimiter::new(3.0, 2.0); + let (a, b) = (peer(), peer()); + let start = Instant::now(); + + assert!(limiter.allow(a, start)); + assert!(limiter.allow(a, start)); + assert!(limiter.allow(a, start)); + assert!(!limiter.allow(a, start)); + // Peers are independent. + assert!(limiter.allow(b, start)); + + // Half a second refills one token at 2 per second. + let later = start + Duration::from_millis(500); + assert!(limiter.allow(a, later)); + assert!(!limiter.allow(a, later)); + + // A long pause refills to the burst, not beyond it. + let much_later = later + Duration::from_secs(3600); + for _ in 0..3 { + assert!(limiter.allow(a, much_later)); + } + assert!(!limiter.allow(a, much_later)); + + limiter.forget_peer(&a); + assert!(limiter.allow(a, much_later)); + } + + #[test] + fn answers_to_requests_do_not_use_the_allowance() { + let mut limiter = PushRateLimiter::new(2.0, 1.0); + let a = peer(); + let now = Instant::now(); + + // Many answered requests in a row: each frame is let through and + // gives its token back. + for _ in 0..50 { + limiter.request_sent(a, now); + assert!(limiter.allow(a, now)); + limiter.answered_request(a, now); + } + // The allowance for unrequested frames is intact ... + assert!(limiter.allow(a, now)); + assert!(limiter.allow(a, now)); + // ... and once it is used up, the saved-up requests (capped) let + // that many more frames through, which may be late answers. + for _ in 0..MAX_REQUEST_CREDITS { + assert!(limiter.allow(a, now)); + } + assert!(!limiter.allow(a, now)); + + // One request buys exactly one frame over the rate. + limiter.request_sent(a, now); + assert!(limiter.allow(a, now)); + assert!(!limiter.allow(a, now)); + } + + #[test] + fn rate_limiter_tolerates_a_clock_that_does_not_advance() { + let mut limiter = PushRateLimiter::new(1.0, 1.0); + let a = peer(); + let now = Instant::now(); + assert!(limiter.allow(a, now)); + // An earlier instant must not underflow or add tokens. + assert!(!limiter.allow(a, now - Duration::from_millis(1))); + } + + #[test] + fn budget_keeps_a_fixed_number_of_sets_per_peer() { + let mut budget = UnclaimedSetBudget::new(2, 3, 1_000_000); + let (a, b) = (peer(), peer()); + + assert!(budget.insert(a, [1u8; 32], 10, true).is_empty()); + assert!(budget.insert(a, [2u8; 32], 10, true).is_empty()); + // The third set from a pushes out a's oldest, not b's. + assert!(budget.insert(b, [9u8; 32], 10, true).is_empty()); + assert_eq!(budget.insert(a, [3u8; 32], 10, true), vec![[1u8; 32]]); + + assert!(!budget.contains(&[1u8; 32])); + assert!(budget.contains(&[2u8; 32])); + assert!(budget.contains(&[3u8; 32])); + assert!(budget.contains(&[9u8; 32])); + assert_eq!(budget.total_bytes(), 30); + assert_eq!(budget.len(), 3); + } + + #[test] + fn pushed_and_requested_sets_have_separate_per_peer_limits() { + // 2 pushed and 3 requested sets per peer. + let mut budget = UnclaimedSetBudget::new(2, 3, 1_000_000); + let a = peer(); + // Answers to this node's own requests (a burst after a reconnect). + for i in 0..3u8 { + assert!(budget.insert(a, [i; 32], 10, false).is_empty()); + } + // Pushes from the same peer do not evict them, and the other way + // round: each kind only pushes out its own oldest. + assert!(budget.insert(a, [10u8; 32], 10, true).is_empty()); + assert!(budget.insert(a, [11u8; 32], 10, true).is_empty()); + assert_eq!(budget.insert(a, [12u8; 32], 10, true), vec![[10u8; 32]]); + assert_eq!(budget.insert(a, [3u8; 32], 10, false), vec![[0u8; 32]]); + assert_eq!(budget.len(), 5); + assert_eq!(budget.total_bytes(), 50); + } + + #[test] + fn byte_budget_evicts_from_the_largest_holder() { + let mut budget = UnclaimedSetBudget::new(4, 4, 100); + let (honest, greedy) = (peer(), peer()); + + assert!(budget.insert(honest, [1u8; 32], 10, true).is_empty()); + assert!(budget.insert(greedy, [2u8; 32], 40, true).is_empty()); + assert!(budget.insert(greedy, [3u8; 32], 40, true).is_empty()); + // 130 bytes > 100: the greedy peer loses its own oldest set. + assert_eq!(budget.insert(greedy, [4u8; 32], 40, true), vec![[2u8; 32]]); + assert!(budget.contains(&[1u8; 32])); + assert_eq!(budget.total_bytes(), 90); + + // A single set larger than the whole budget evicts itself (and, being + // the largest holder, only that peer's sets). + let evicted = budget.insert(greedy, [5u8; 32], 500, true); + assert!(evicted.contains(&[5u8; 32])); + assert!(budget.contains(&[1u8; 32])); + assert!(budget.total_bytes() <= 100); + } + + #[test] + fn many_equal_holders_can_push_out_a_larger_one() { + // The documented limit of the byte budget: it evens out the holders. + let mut budget = UnclaimedSetBudget::new(4, 4, 100); + let honest = peer(); + assert!(budget.insert(honest, [0xaa; 32], 30, true).is_empty()); + let mut evicted = Vec::new(); + for i in 0..10u8 { + evicted.extend(budget.insert(peer(), [i; 32], 10, true)); + } + assert!(evicted.contains(&[0xaa; 32])); + assert!(budget.total_bytes() <= 100); + } + + #[test] + fn claimed_and_dropped_sets_leave_the_budget() { + let mut budget = UnclaimedSetBudget::new(2, 2, 1_000); + let a = peer(); + budget.insert(a, [1u8; 32], 100, true); + budget.insert(a, [2u8; 32], 100, false); + + // `claim` reports whether the set had arrived unrequested. + assert_eq!(budget.claim(&[1u8; 32]), Some(true)); + assert_eq!(budget.claim(&[1u8; 32]), None); + assert_eq!(budget.total_bytes(), 100); + + // The claimed set no longer counts toward the per-peer limit. + assert!(budget.insert(a, [3u8; 32], 100, true).is_empty()); + assert_eq!(budget.claim(&[2u8; 32]), Some(false)); + + budget.retain(|hash| *hash != [3u8; 32]); + assert!(!budget.contains(&[3u8; 32])); + assert!(budget.is_empty()); + assert_eq!(budget.total_bytes(), 0); + } + + #[test] + fn the_same_set_is_counted_once() { + let mut budget = UnclaimedSetBudget::new(2, 2, 1_000); + let (a, b) = (peer(), peer()); + assert!(budget.insert(a, [1u8; 32], 100, true).is_empty()); + assert!(budget.insert(b, [1u8; 32], 100, true).is_empty()); + assert_eq!(budget.len(), 1); + assert_eq!(budget.total_bytes(), 100); + } +} diff --git a/overlay/src/ipc/messages.rs b/overlay/src/ipc/messages.rs index 9930f96138..d326a82e34 100644 --- a/overlay/src/ipc/messages.rs +++ b/overlay/src/ipc/messages.rs @@ -56,6 +56,12 @@ pub enum MessageType { /// Request overlay metrics snapshot (empty payload) RequestOverlayMetrics = 13, + /// Push a cached TX set to every connected peer now, without waiting for + /// their requests. Core sends it for the set it is about to propose as a + /// round leader, after the `CacheTxSet` that carried the set. + /// Payload: [hash:32][slotSeq:4] + PushTxSet = 14, + // ═══ Overlay → Core (Critical Path) ═══ /// Received SCP envelope from network ScpReceived = 100, @@ -93,6 +99,7 @@ impl TryFrom for MessageType { 11 => Ok(MessageType::RequestTxSet), 12 => Ok(MessageType::CacheTxSet), 13 => Ok(MessageType::RequestOverlayMetrics), + 14 => Ok(MessageType::PushTxSet), 100 => Ok(MessageType::ScpReceived), 101 => Ok(MessageType::TopTxsResponse), 102 => Ok(MessageType::PeerRequestsScpState), @@ -294,6 +301,7 @@ mod tests { MessageType::RequestTxSet, MessageType::CacheTxSet, MessageType::RequestOverlayMetrics, + MessageType::PushTxSet, MessageType::ScpReceived, MessageType::TopTxsResponse, MessageType::PeerRequestsScpState, @@ -364,6 +372,7 @@ mod tests { MessageType::try_from(13).unwrap(), MessageType::RequestOverlayMetrics ); + assert_eq!(MessageType::try_from(14).unwrap(), MessageType::PushTxSet); assert_eq!( MessageType::try_from(100).unwrap(), MessageType::ScpReceived diff --git a/overlay/src/libp2p_overlay.rs b/overlay/src/libp2p_overlay.rs index 3b3ac6149a..d936fd5139 100644 --- a/overlay/src/libp2p_overlay.rs +++ b/overlay/src/libp2p_overlay.rs @@ -12,8 +12,8 @@ //! QUIC provides independent loss recovery per stream. use crate::flood::{ - GetData, InvBatch, InvBatcher, InvEntry, InvTracker, PendingRequests, TxBuffer, TxSetData, - TxStreamMessage, + GetData, InvBatch, InvBatcher, InvEntry, InvTracker, PendingRequests, PushRateLimiter, + PushRecord, TxBuffer, TxSetData, TxStreamMessage, }; #[cfg(test)] use crate::integrated::CoreCommand; @@ -114,6 +114,12 @@ pub enum OverlayCommand { }, /// Record that a peer has a specific TX set (learned from SCP message) RecordTxSetSource { hash: [u8; 32], peer: PeerId }, + /// Push a TX set to every connected peer that has not been sent it + /// (this node leads a round and is about to propose the set) + PushTxSet { + hash: [u8; 32], + data: Arc, + }, /// Connect to a peer by address (bootstrap — PeerId unknown) Dial(Multiaddr), /// Connect to a known peer by PeerId (reconnect — deduplicates automatically) @@ -291,6 +297,21 @@ impl OverlayHandle { } } + /// Push a TX set to every connected peer without waiting for requests. + /// Each peer is sent a given set once; see `PushRecord`. + pub async fn push_txset(&self, hash: [u8; 32], data: Arc) { + if let Err(e) = self + .cmd_tx + .send(OverlayCommand::PushTxSet { hash, data }) + .await + { + warn!( + "Overlay command channel closed, failed to send PushTxSet: {}", + e + ); + } + } + /// Record that a peer has a specific TX set (call when receiving SCP with txSetHash) pub async fn record_txset_source(&self, hash: [u8; 32], peer: PeerId) { if let Err(e) = self @@ -402,6 +423,12 @@ struct SharedState { /// Pending TX set requests: hash -> (peer, request_time) to avoid duplicate fetches and track latency /// hash -> (peer asked, request time, slot the set is for) pending_txset_requests: RwLock>, + /// TX sets this node pushed as a round leader, per peer. Never held + /// across an await. + txset_pushes: std::sync::Mutex, + /// Per-peer allowance for TX-set frames nobody asked for. Never held + /// across an await. + txset_push_limiter: std::sync::Mutex, /// Event sender for non-TX events (SCP, TxSet - critical path, unbounded) event_tx: mpsc::UnboundedSender, /// Direct, bounded admission into the same mempool FIFO as Core removal. @@ -446,6 +473,8 @@ impl SharedState { std::num::NonZeroUsize::new(1000).unwrap(), )), pending_txset_requests: RwLock::new(HashMap::new()), + txset_pushes: std::sync::Mutex::new(PushRecord::new()), + txset_push_limiter: std::sync::Mutex::new(PushRateLimiter::default()), event_tx, mempool, tx_dropped_count: AtomicU64::new(0), @@ -662,6 +691,9 @@ impl StellarOverlay { sources.put(hash, peer); debug!("Recorded peer {} as source for TX set {:02x?}...", peer, &hash[..4]); } + OverlayCommand::PushTxSet { hash, data } => { + self.push_txset(hash, data).await; + } OverlayCommand::Dial(addr) => { info!("Dialing peer at {}", addr); self.state.metrics.connection_pending.fetch_add(1, Ordering::Relaxed); @@ -803,6 +835,14 @@ impl StellarOverlay { num_established, .. } => { + // Whatever was pushed to the peer may have been on this + // connection's streams and not arrived: answer its requests + // normally again. This holds for a duplicate connection too. + self.state + .txset_pushes + .lock() + .unwrap() + .forget_peer(&peer_id); // Only clean up when the LAST connection to this peer closes. // Duplicate connections closing shouldn't tear down working streams. if num_established == 0 { @@ -832,6 +872,11 @@ impl StellarOverlay { ); } } + self.state + .txset_push_limiter + .lock() + .unwrap() + .forget_peer(&peer_id); // Notify main loop to clean up any pending requests for this peer if let Err(e) = self.state.event_tx.send(OverlayEvent::PeerDisconnected { peer_id: peer_id.clone(), @@ -1099,6 +1144,12 @@ impl StellarOverlay { .write() .await .insert(hash, pending_request); + // The answer must get past the limit on TX-set frames from that peer. + self.state + .txset_push_limiter + .lock() + .unwrap() + .request_sent(peer, Instant::now()); let request = crate::xdr::frame_get_tx_set(hash); let state = Arc::clone(&self.state); @@ -1127,6 +1178,102 @@ impl StellarOverlay { }); } + /// Push a TX set to every connected peer that has not been sent it. The + /// frame is the one a request is answered with, sent unsolicited on the + /// TX-set stream; receivers verify and cache it by its content hash. + async fn push_txset(&mut self, hash: [u8; 32], data: Arc) { + // Include the StellarMessage discriminant in the existing wire limit. + if data.len() > MAX_MESSAGE_SIZE - 4 { + warn!( + "TXSET_PUSH_SKIP: TX set {:02x?}... is too large to send ({} bytes)", + &hash[..4], + data.len() + ); + return; + } + let connected: Vec = self + .state + .peer_streams + .read() + .await + .keys() + .cloned() + .collect(); + let (targets, busy) = self + .state + .txset_pushes + .lock() + .unwrap() + .start_push_counting_busy(hash, connected); + if busy > 0 { + // Those peers request the set when they see the proposal. + warn!( + "TXSET_PUSH_BUSY: Not pushing TX set {:02x?}... to {} peer(s): an earlier push to them is still in flight", + &hash[..4], + busy + ); + } + if targets.is_empty() { + debug!( + "TXSET_PUSH_SKIP: TX set {:02x?}... has no peer left to push to", + &hash[..4] + ); + return; + } + info!( + "TXSET_PUSH: Pushing TX set {:02x?}... ({} bytes, {} on the wire) to {} peers", + &hash[..4], + data.len(), + data.encoded().len(), + targets.len() + ); + for peer in targets { + let state = Arc::clone(&self.state); + let data = Arc::clone(&data); + self.sends.spawn(async move { + match send_to_peer_stream(&state, peer, StreamType::TxSet, data.encoded()).await { + Ok(_) => { + state.txset_pushes.lock().unwrap().push_written( + &hash, + &peer, + Instant::now(), + ); + let metrics = &state.metrics; + metrics.txset_push_sent.fetch_add(1, Ordering::Relaxed); + metrics.send_txset.fetch_add(1, Ordering::Relaxed); + metrics.message_write.fetch_add(1, Ordering::Relaxed); + metrics + .byte_write + .fetch_add(data.encoded().len() as u64, Ordering::Relaxed); + debug!( + "TXSET_PUSH_OK: Queued TX set {:02x?}... to {}", + &hash[..4], + peer + ); + } + Err(e) => { + state.metrics.error_write.fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_PUSH_FAIL: Failed to push TX set {:02x?}... to {}: {}", + &hash[..4], + peer, + e + ); + // The peer does not have the set from us. If its + // request was skipped because of this push, answer it + // through the normal path now. + let skipped = state.txset_pushes.lock().unwrap().push_failed(&hash, &peer); + if skipped { + let _ = state + .event_tx + .send(OverlayEvent::TxSetRequested { hash, from: peer }); + } + } + } + }); + } + } + /// Forget requests for slots below `min_slot` that were never answered. async fn prune_txset_requests(&mut self, min_slot: u32) { let mut pending = self.state.pending_txset_requests.write().await; @@ -2165,6 +2312,29 @@ fn handle_txset_request(state: &SharedState, peer_id: PeerId, hash: [u8; 32]) { &hash[..4], peer_id ); + // A set this node is pushing to that peer, or pushed a moment ago, is + // already on the peer's TX-set stream on this connection. The request + // crossed it; a second copy could only arrive later. The peer's pending + // request completes when the pushed copy lands. A repeated request, or + // one that comes later than a crossing can explain, is answered (see + // `PushRecord::should_skip_request`). + if state + .txset_pushes + .lock() + .unwrap() + .should_skip_request(&hash, &peer_id, Instant::now()) + { + state + .metrics + .txset_push_request_skipped + .fetch_add(1, Ordering::Relaxed); + info!( + "TXSET_REQ_SKIP: TxSet {:02x?}... was already pushed to {}, not sending it again", + &hash[..4], + peer_id + ); + return; + } if let Err(e) = state.event_tx.send(OverlayEvent::TxSetRequested { hash, from: peer_id, @@ -2193,6 +2363,13 @@ async fn deliver_txset(state: &SharedState, peer_id: PeerId, txset_data: Arc, + state: Arc, + metrics: Arc, + peer_id: PeerId, + port: u16, + _admissions: mpsc::UnboundedReceiver, +} + +async fn start_node(port: u16) -> Node { + let keypair = Keypair::generate_ed25519(); + let peer_id = keypair.public().to_peer_id(); + let metrics = Arc::new(OverlayMetrics::new()); + let (handle, events, admissions, overlay) = + create_test_overlay(keypair, Arc::clone(&metrics)).unwrap(); + let state = Arc::clone(&overlay.state); + tokio::spawn(async move { overlay.run("127.0.0.1", port).await }); + tokio::time::sleep(Duration::from_millis(100)).await; + Node { + handle, + events, + state, + metrics, + peer_id, + port, + _admissions: admissions, + } +} + +/// Dial `listener` from `dialer` and wait until both sides have streams. +async fn connect(dialer: &Node, listener: &Node) { + let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{}/quic-v1", listener.port) + .parse() + .unwrap(); + dialer.handle.dial(addr).await; + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let a = dialer + .state + .peer_streams + .read() + .await + .contains_key(&listener.peer_id); + let b = listener + .state + .peer_streams + .read() + .await + .contains_key(&dialer.peer_id); + if a && b { + break; + } + assert!(Instant::now() < deadline, "peers did not connect"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + // Let the SCP state exchange of a new connection settle. + tokio::time::sleep(Duration::from_millis(200)).await; +} + +fn small_txset(seed: u8) -> ([u8; 32], Arc) { + let (hash, xdr) = test_txset_xdr(seed); + (hash, Arc::new(TxSetData::from_local(xdr).unwrap())) +} + +/// Wait for a `TxSetReceived` of `hash`; returns the slot it carries (None = +/// nobody on this node asked for it) and the sender. +async fn expect_txset(node: &mut Node, hash: [u8; 32]) -> (Option, PeerId) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + match tokio::time::timeout_at(deadline, node.events.recv()).await { + Ok(Some(OverlayEvent::TxSetReceived { + hash: got, + slot, + from, + data, + })) if got == hash => { + assert_eq!(crate::xdr::sha256_hash(data.as_slice()), hash); + return (slot, from); + } + Ok(Some(_)) => {} + _ => panic!("no TxSetReceived for {:02x?}", &hash[..4]), + } + } +} + +/// Wait for a `TxSetRequested` of `hash` and return the requester. +async fn expect_txset_request(node: &mut Node, hash: [u8; 32]) -> PeerId { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + match tokio::time::timeout_at(deadline, node.events.recv()).await { + Ok(Some(OverlayEvent::TxSetRequested { hash: got, from })) if got == hash => { + return from + } + Ok(Some(_)) => {} + _ => panic!("no TxSetRequested for {:02x?}", &hash[..4]), + } + } +} + +/// Collect the tx-set events (received hashes, requested hashes) that arrive +/// within `window`. +async fn txset_events_within(node: &mut Node, window: Duration) -> (Vec<[u8; 32]>, Vec<[u8; 32]>) { + let deadline = tokio::time::Instant::now() + window; + let (mut received, mut requested) = (Vec::new(), Vec::new()); + loop { + match tokio::time::timeout_at(deadline, node.events.recv()).await { + Ok(Some(OverlayEvent::TxSetReceived { hash, .. })) => received.push(hash), + Ok(Some(OverlayEvent::TxSetRequested { hash, .. })) => requested.push(hash), + Ok(Some(_)) => {} + _ => return (received, requested), + } + } +} + +fn load(counter: &AtomicU64) -> u64 { + counter.load(Ordering::Relaxed) +} + +/// The leader's push reaches every connected peer without any request, and +/// each peer is sent a given set once. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn push_reaches_every_connected_peer_once() { + let leader = start_node(21711).await; + let mut follower_a = start_node(21712).await; + let mut follower_b = start_node(21713).await; + connect(&follower_a, &leader).await; + connect(&follower_b, &leader).await; + + let (hash, data) = small_txset(0x51); + leader.handle.push_txset(hash, Arc::clone(&data)).await; + + for follower in [&mut follower_a, &mut follower_b] { + let (slot, from) = expect_txset(follower, hash).await; + assert_eq!(slot, None, "a pushed set arrives unrequested"); + assert_eq!(from, leader.peer_id); + assert_eq!(load(&follower.metrics.fetch_txset_count), 0); + } + assert_eq!(load(&leader.metrics.txset_push_sent), 2); + + // Pushing the same set again sends nothing. + leader.handle.push_txset(hash, data).await; + for follower in [&mut follower_a, &mut follower_b] { + let (received, _) = txset_events_within(follower, Duration::from_millis(300)).await; + assert!(received.is_empty(), "set was sent twice"); + } + assert_eq!(load(&leader.metrics.txset_push_sent), 2); +} + +/// A follower whose request crosses the push is not sent a second copy. A +/// repeated request (the follower lost the pushed copy) is answered. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn request_for_a_pushed_set_is_skipped_once_then_answered() { + let mut leader = start_node(21721).await; + let mut follower = start_node(21722).await; + connect(&follower, &leader).await; + + let (hash, data) = small_txset(0x52); + leader.handle.push_txset(hash, data).await; + expect_txset(&mut follower, hash).await; + + // First request: the leader already sent the set on this connection. + follower.handle.fetch_txset(hash, 7).await; + let (_, requested) = txset_events_within(&mut leader, Duration::from_millis(400)).await; + assert!(requested.is_empty(), "request should have been skipped"); + assert_eq!(load(&leader.metrics.txset_push_request_skipped), 1); + + // Core's retry: asked again, the leader answers through the request path. + follower.handle.refetch_txset(hash, 7).await; + assert_eq!( + expect_txset_request(&mut leader, hash).await, + follower.peer_id + ); + assert_eq!(load(&leader.metrics.txset_push_request_skipped), 1); +} + +/// A request that arrives later than a crossing can explain means the peer +/// does not hold the pushed copy (it dropped or evicted it): the leader +/// answers it, the first time. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn request_long_after_the_push_is_answered() { + let mut leader = start_node(21771).await; + let mut follower = start_node(21772).await; + connect(&follower, &leader).await; + + let (hash, data) = small_txset(0x57); + leader.handle.push_txset(hash, data).await; + expect_txset(&mut follower, hash).await; + + tokio::time::sleep(crate::flood::txset_push::PUSH_SKIP_WINDOW + Duration::from_millis(100)) + .await; + follower.handle.fetch_txset(hash, 7).await; + assert_eq!( + expect_txset_request(&mut leader, hash).await, + follower.peer_id + ); + assert_eq!(load(&leader.metrics.txset_push_request_skipped), 0); +} + +/// Fallback: a peer that was not connected when the set was pushed gets it +/// through the request path, and a later push only goes to that peer. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn peer_that_missed_the_push_is_served_on_request() { + let mut leader = start_node(21731).await; + let mut early = start_node(21732).await; + connect(&early, &leader).await; + + let (hash, data) = small_txset(0x53); + leader.handle.push_txset(hash, Arc::clone(&data)).await; + expect_txset(&mut early, hash).await; + + let mut late = start_node(21733).await; + connect(&late, &leader).await; + + // The late peer has nothing and asks: the request is not skipped. + late.handle.fetch_txset(hash, 9).await; + let from = expect_txset_request(&mut leader, hash).await; + assert_eq!(from, late.peer_id); + assert_eq!(load(&leader.metrics.txset_push_request_skipped), 0); + leader + .handle + .send_txset(hash, Arc::clone(&data), from) + .await; + let (slot, _) = expect_txset(&mut late, hash).await; + assert_eq!(slot, Some(9), "delivered as the answer to the request"); + assert_eq!(load(&late.metrics.fetch_txset_count), 1); + + // Another set, pushed after both connected, reaches both; re-pushing the + // first set reaches only the late peer. + leader.handle.push_txset(hash, data).await; + let (slot, _) = expect_txset(&mut late, hash).await; + assert_eq!(slot, None); + let (received, _) = txset_events_within(&mut early, Duration::from_millis(300)).await; + assert!(received.is_empty()); + assert_eq!(load(&leader.metrics.txset_push_sent), 2); +} + +/// A connection closing drops the record of what was pushed to that peer, +/// so its requests are answered again. +/// The close event is injected: two nodes in one process keep each other's +/// streams alive, so a real close only shows up after the idle timeout. +#[tokio::test] +async fn disconnect_forgets_what_was_pushed_to_the_peer() { + use libp2p::core::{transport::PortUse, ConnectedPoint, Endpoint}; + use libp2p::swarm::ConnectionId; + + let (_handle, _events, _admissions, mut overlay) = + create_test_overlay(Keypair::generate_ed25519(), Arc::new(OverlayMetrics::new())).unwrap(); + let gone = PeerId::random(); + let stays = PeerId::random(); + let hash = [0x54; 32]; + overlay + .state + .txset_pushes + .lock() + .unwrap() + .start_push(hash, [gone, stays]); + // Use up the unrequested-frame allowance of the peer that will leave. + let now = Instant::now(); + while overlay + .state + .txset_push_limiter + .lock() + .unwrap() + .allow(gone, now) + {} + + let closed = |peer_id, num_established| SwarmEvent::ConnectionClosed { + peer_id, + connection_id: ConnectionId::new_unchecked(1), + endpoint: ConnectedPoint::Dialer { + address: "/ip4/127.0.0.1/udp/1/quic-v1".parse().unwrap(), + role_override: Endpoint::Dialer, + port_use: PortUse::Reuse, + }, + num_established, + cause: None, + }; + + // Any connection to the peer closing clears the record of what was + // pushed to it (the set may have been on that connection's streams); the + // other peer is not affected. The rate-limit state stays while another + // connection to the peer is still up. + overlay.handle_swarm_event(closed(gone, 1)).await; + { + let mut pushes = overlay.state.txset_pushes.lock().unwrap(); + assert!(!pushes.was_pushed(&hash, &gone)); + assert!(pushes.was_pushed(&hash, &stays)); + assert!(!pushes.should_skip_request(&hash, &gone, now)); + } + assert!(!overlay + .state + .txset_push_limiter + .lock() + .unwrap() + .allow(gone, now)); + + // The last connection closing also resets the peer's allowance. + overlay.handle_swarm_event(closed(gone, 0)).await; + assert!(overlay + .state + .txset_push_limiter + .lock() + .unwrap() + .allow(gone, now)); +} + +/// A peer that sends more unrequested sets than the limit allows has the +/// excess dropped undecoded. A set this node asked that peer for still +/// arrives when the peer is over its limit, and does not use the allowance. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unrequested_frames_over_the_rate_limit_are_dropped() { + let mut pusher = start_node(21751).await; + let mut receiver = start_node(21752).await; + connect(&receiver, &pusher).await; + // An allowance of exactly 3 frames that never refills, so the counts + // below do not depend on timing. + *receiver.state.txset_push_limiter.lock().unwrap() = PushRateLimiter::new(3.0, 0.0); + let credits = |node: &Node, peer: &PeerId| { + node.state + .txset_push_limiter + .lock() + .unwrap() + .request_credits(peer) + }; + + // Frames written straight to the tx-set stream, as a peer that ignores + // the sender-side rules would. + for seed in 0..5u8 { + let (_, data) = small_txset(seed); + send_to_peer_stream( + &pusher.state, + receiver.peer_id, + StreamType::TxSet, + data.encoded(), + ) + .await + .unwrap(); + } + let (received, _) = txset_events_within(&mut receiver, Duration::from_millis(1000)).await; + assert_eq!(received.len(), 3, "the allowance is honoured"); + assert_eq!(load(&receiver.metrics.txset_push_dropped), 2); + + // The peer is over its limit. This node asks it for a set: the request + // buys the answer's way in ... + let (wanted, wanted_data) = small_txset(0xf0); + receiver.handle.fetch_txset(wanted, 11).await; + let from = expect_txset_request(&mut pusher, wanted).await; + assert_eq!(credits(&receiver, &pusher.peer_id), 1); + pusher.handle.send_txset(wanted, wanted_data, from).await; + let (slot, _) = expect_txset(&mut receiver, wanted).await; + assert_eq!(slot, Some(11)); + assert_eq!(credits(&receiver, &pusher.peer_id), 0); + assert_eq!(load(&receiver.metrics.txset_push_dropped), 2); + + // ... and, having answered a request, gave its token back: exactly one + // more unrequested frame passes, the next one is dropped. + for seed in 200..202u8 { + let (_, data) = small_txset(seed); + send_to_peer_stream( + &pusher.state, + receiver.peer_id, + StreamType::TxSet, + data.encoded(), + ) + .await + .unwrap(); + } + let (received, _) = txset_events_within(&mut receiver, Duration::from_millis(1000)).await; + assert_eq!(received.len(), 1); + assert_eq!(load(&receiver.metrics.txset_push_dropped), 3); +} + +/// A frame that is not a valid compressed tx set is dropped, and the stream +/// keeps working: the next (valid) push is delivered. Nothing is filed under +/// a hash the content does not have. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn malformed_push_is_dropped_and_the_stream_survives() { + let pusher = start_node(21761).await; + let mut receiver = start_node(21762).await; + connect(&receiver, &pusher).await; + + let (hash, data) = small_txset(0x56); + // Garbage, a truncated frame, and a well-formed zstd frame whose content + // is not a GeneralizedTransactionSet. + let truncated = &data.encoded()[..data.encoded().len() - 1]; + let not_a_txset = TxSetData::from_local(vec![0xab; 64]).unwrap(); + for frame in [&b"not a tx set"[..], truncated, not_a_txset.encoded()] { + send_to_peer_stream(&pusher.state, receiver.peer_id, StreamType::TxSet, frame) + .await + .unwrap(); + } + pusher.handle.push_txset(hash, data).await; + + let (received, _) = txset_events_within(&mut receiver, Duration::from_millis(1000)).await; + assert_eq!(received, vec![hash], "only the valid set is delivered"); +} diff --git a/overlay/src/main.rs b/overlay/src/main.rs index 5b2dfa3f6a..5e07ff57f0 100644 --- a/overlay/src/main.rs +++ b/overlay/src/main.rs @@ -20,7 +20,7 @@ use tracing::{debug, error, info, warn}; use libp2p::identity::Keypair as Libp2pKeypair; use libp2p::{Multiaddr, PeerId}; use stellar_overlay::config::Config; -use stellar_overlay::flood::{CachedTxSet, Hash256, TxSetCache, TxSetData}; +use stellar_overlay::flood::{CachedTxSet, Hash256, TxSetCache, TxSetData, UnclaimedSetBudget}; use stellar_overlay::integrated::{Overlay, OverlayHandle}; use stellar_overlay::ipc::{CoreIpc, Message, MessageType}; use stellar_overlay::libp2p_overlay::{ @@ -397,6 +397,10 @@ struct App { /// Missing sets requested by Core, with the newest slot needing each set. /// Prefetches stay in the cache until Core establishes this demand. pending_core_tx_sets: HashMap, + /// Cached sets that came from a peer (pushed, or prefetched because an + /// SCP envelope named them) and that Core has not requested yet. Bounded + /// per peer and in total. + unclaimed_tx_sets: UnclaimedSetBudget, /// Current ledger sequence current_ledger_seq: u32, /// libp2p overlay handle (QUIC-based SCP + TX) @@ -484,6 +488,7 @@ impl App { overlay_handle, tx_set_cache: TxSetCache::new(100), pending_core_tx_sets: HashMap::new(), + unclaimed_tx_sets: UnclaimedSetBudget::default(), current_ledger_seq: 0, libp2p_handle, libp2p_events: libp2p_event_rx, @@ -730,6 +735,31 @@ impl App { from ); + // A set nobody here asked for was pushed by its sender (a + // round leader does that for the set it is about to propose). + // It is cached like any other, but Core only receives it if a + // statement Core processes names its hash (RequestTxSet). + let already_cached = self.tx_set_cache.get(&hash).is_some(); + if slot.is_none() { + self.metrics + .txset_push_received + .fetch_add(1, Ordering::Relaxed); + if already_cached { + // Already held (requested earlier, built locally, or + // pushed twice): keep the existing entry. + self.send_requested_tx_set(&hash); + return; + } + } + // Whether pushed or prefetched, a new set that Core is not + // waiting for counts against the limits for unclaimed sets + // until Core asks for it. A set this node already held (for + // example one it built) is never made evictable by a copy + // arriving from a peer. + let unclaimed_bytes = (!already_cached + && !self.pending_core_tx_sets.contains_key(&hash)) + .then(|| data.len() + data.encoded().len()); + // IMPORTANT: Cache the TxSet FIRST, before pushing to Core // This ensures the TxSet is available when SCP processing resumes // Stamp with the slot the set was requested for so eviction is @@ -740,6 +770,21 @@ impl App { hash, data, ); + if let Some(bytes) = unclaimed_bytes { + let pushed = slot.is_none(); + for evicted in self.unclaimed_tx_sets.insert(from, hash, bytes, pushed) { + self.tx_set_cache.remove(&evicted); + self.metrics + .txset_push_evicted + .fetch_add(1, Ordering::Relaxed); + warn!( + "TXSET_PUSH_EVICT: Dropped TxSet {:02x?}... from the cache (limit on sets from peers that Core has not asked for)", + &evicted[..4] + ); + } + // The cache's own capacity limit may have dropped a set. + self.sync_unclaimed_tx_sets(); + } self.send_requested_tx_set(&hash); } @@ -914,6 +959,13 @@ impl App { } } + /// Stop counting pushed sets that the cache no longer holds. + fn sync_unclaimed_tx_sets(&mut self) { + let cache = &self.tx_set_cache; + self.unclaimed_tx_sets + .retain(|hash| cache.get(hash).is_some()); + } + /// Core rejects sets it has not requested. Keep proactive network fetches /// cached, and deliver only after RequestTxSet. Both event orderings run /// on this App task, so a request racing a prefetch cannot lose its reply. @@ -1024,14 +1076,23 @@ impl App { .and_modify(|needed_slot| *needed_slot = (*needed_slot).max(slot)) .or_insert(slot); + // Core asks only for a hash named by an SCP statement it is + // processing: from here on a pushed set is an ordinary cache + // entry, no longer subject to the limits for unclaimed sets. + let pushed = self.unclaimed_tx_sets.claim(&hash) == Some(true); + // First check local cache if let Some(cached) = self.tx_set_cache.get(&hash) { info!( - "TXSET_FROM_CACHE: Sending TX set {:02x?}... ({} bytes) from local cache", + "TXSET_FROM_CACHE: Sending TX set {:02x?}... ({} bytes) from local cache{}", &hash[..4], - cached.xdr.len() + cached.xdr.len(), + if pushed { " (pushed by a peer)" } else { "" } ); + if pushed { + self.metrics.txset_push_hit.fetch_add(1, Ordering::Relaxed); + } self.send_requested_tx_set(&hash); } else { // Not in local cache - request from peers. Spawn so the main @@ -1113,10 +1174,47 @@ impl App { } }; cache_tx_set(&mut self.tx_set_cache, slot, hash, data); + // Core built this set itself: whatever a peer sent under the + // same hash is no longer an unclaimed set that the limits may + // evict (this node must be able to serve what it proposes). + self.unclaimed_tx_sets.claim(&hash); + // The cache's own capacity limit may have dropped a set. + self.sync_unclaimed_tx_sets(); self.send_requested_tx_set(&hash); } + MessageType::PushTxSet => { + // Core leads a round and is about to propose this set: send + // it to every peer now instead of waiting for their requests. + // Payload: [hash:32][slotSeq:4]. The set itself came with the + // CacheTxSet before this message on the same ordered channel. + if msg.payload.len() < 36 { + warn!("PushTxSet payload too short"); + return true; + } + let mut hash = [0u8; 32]; + hash.copy_from_slice(&msg.payload[0..32]); + let slot = u32::from_le_bytes(msg.payload[32..36].try_into().unwrap()); + + let Some(cached) = self.tx_set_cache.get(&hash) else { + // E.g. the set failed to encode. Peers request it. + warn!( + "TXSET_PUSH_SKIP: TX set {:02x?}... for slot {} is not cached, not pushing", + &hash[..4], + slot + ); + return true; + }; + let data = Arc::clone(&cached.xdr); + let handle = self.libp2p_handle.clone(); + // Spawn so the main loop never awaits on the bounded libp2p + // command channel. + tokio::spawn(async move { + handle.push_txset(hash, data).await; + }); + } + MessageType::SubmitTx => { // Parse payload: [fee:i64][numOps:u32][txEnvelope...] if msg.payload.len() < 12 { @@ -1186,6 +1284,7 @@ impl App { .evict_before(ledger_seq.saturating_sub(12)); self.pending_core_tx_sets .retain(|_, slot| *slot >= ledger_seq.saturating_sub(12)); + self.sync_unclaimed_tx_sets(); // Requests that were never answered expire with the same // retained-slot window. let handle = self.libp2p_handle.clone(); @@ -2009,6 +2108,7 @@ mod tests { overlay_handle, tx_set_cache: TxSetCache::new(100), pending_core_tx_sets: HashMap::new(), + unclaimed_tx_sets: UnclaimedSetBudget::default(), current_ledger_seq: 0, libp2p_handle, libp2p_events, @@ -2248,6 +2348,331 @@ mod tests { } } + fn pushed_txset(hash: Hash256, data: &[u8], from: PeerId) -> LibP2pOverlayEvent { + LibP2pOverlayEvent::TxSetReceived { + hash, + data: Arc::new(TxSetData::from_local(data.to_vec()).unwrap()), + from, + // Nobody on this node asked for it: a push. + slot: None, + } + } + + fn push_counters(app: &App) -> (u64, u64, u64) { + ( + app.metrics.txset_push_received.load(Ordering::Relaxed), + app.metrics.txset_push_hit.load(Ordering::Relaxed), + app.metrics.txset_push_evicted.load(Ordering::Relaxed), + ) + } + + /// A pushed set is held in the overlay and reaches Core only when Core + /// asks for that hash; the request is then answered at once. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn pushed_txset_reaches_core_only_on_request() { + let (mut app, mut core) = test_app(); + let (hash, data) = test_txset_xdr(41); + let event = pushed_txset(hash, &data, PeerId::random()); + app.handle_libp2p_event(event).await; + + assert!(app.tx_set_cache.get(&hash).is_some()); + assert!(app.unclaimed_tx_sets.contains(&hash)); + assert_no_core_txset(&mut core); + assert_eq!(push_counters(&app), (1, 0, 0)); + + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&hash, 100), + )) + .await; + assert_core_txset(&mut core, &hash, &data); + // Claimed by Core: an ordinary cache entry from here on. + assert!(!app.unclaimed_tx_sets.contains(&hash)); + assert!(!app.pending_core_tx_sets.contains_key(&hash)); + assert_eq!(push_counters(&app), (1, 1, 0)); + + // A second request is served too, but is not another push hit. + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&hash, 100), + )) + .await; + assert_core_txset(&mut core, &hash, &data); + assert_eq!(push_counters(&app), (1, 1, 0)); + } + + /// A pushed set whose content is not the set Core asks for is never + /// delivered: Core's request for the proposed hash stays pending (and + /// goes to the network), whatever was pushed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn pushed_txset_with_another_hash_does_not_answer_a_request() { + let (mut app, mut core) = test_app(); + let (pushed_hash, pushed_data) = test_txset_xdr(42); + let (proposed_hash, proposed_data) = test_txset_xdr(43); + let leader = PeerId::random(); + let event = pushed_txset(pushed_hash, &pushed_data, leader); + app.handle_libp2p_event(event).await; + + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&proposed_hash, 100), + )) + .await; + assert_no_core_txset(&mut core); + assert_eq!(app.pending_core_tx_sets.get(&proposed_hash), Some(&100)); + // The pushed set is still unclaimed and was not counted as a hit. + assert!(app.unclaimed_tx_sets.contains(&pushed_hash)); + assert_eq!(push_counters(&app), (1, 0, 0)); + + // An event that claims the proposed hash but carries other content + // cannot exist: the reader files a set under the hash it computed. + // The proposed set arriving (by request) completes the fetch. + app.handle_libp2p_event(LibP2pOverlayEvent::TxSetReceived { + hash: proposed_hash, + data: Arc::new(TxSetData::from_local(proposed_data.clone()).unwrap()), + from: leader, + slot: Some(100), + }) + .await; + assert_core_txset(&mut core, &proposed_hash, &proposed_data); + assert_no_core_txset(&mut core); + } + + /// A push that lands after Core asked for the hash satisfies the request. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn push_arriving_after_the_core_request_completes_it() { + let (mut app, mut core) = test_app(); + let (hash, data) = test_txset_xdr(44); + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&hash, 100), + )) + .await; + assert_no_core_txset(&mut core); + + let event = pushed_txset(hash, &data, PeerId::random()); + app.handle_libp2p_event(event).await; + assert_core_txset(&mut core, &hash, &data); + // Core was already waiting for it, so it never counted as unclaimed. + assert!(!app.unclaimed_tx_sets.contains(&hash)); + assert!(app.unclaimed_tx_sets.is_empty()); + + // A duplicate push changes nothing and delivers nothing. + let event = pushed_txset(hash, &data, PeerId::random()); + app.handle_libp2p_event(event).await; + assert_no_core_txset(&mut core); + assert_eq!(app.tx_set_cache.len(), 1); + } + + /// Sets a peer pushes and Core never asks for are bounded per peer; sets + /// Core asked for are not affected by the limit. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unclaimed_pushed_txsets_are_bounded_per_peer() { + use stellar_overlay::flood::txset_push_limits::MAX_UNCLAIMED_SETS_PER_PEER as LIMIT; + let (mut app, mut core) = test_app(); + let pusher = PeerId::random(); + let other = PeerId::random(); + + // One set from the pusher that Core uses. + let (used_hash, used_data) = test_txset_xdr(50); + let event = pushed_txset(used_hash, &used_data, pusher); + app.handle_libp2p_event(event).await; + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&used_hash, 100), + )) + .await; + assert_core_txset(&mut core, &used_hash, &used_data); + + // One unclaimed set from another peer. + let (other_hash, other_data) = test_txset_xdr(51); + let event = pushed_txset(other_hash, &other_data, other); + app.handle_libp2p_event(event).await; + + // The pusher sends two more sets than it may keep unclaimed. + let hashes: Vec = (0..LIMIT + 2) + .map(|i| { + let (hash, _) = test_txset_xdr(60 + i as u8); + hash + }) + .collect(); + for i in 0..LIMIT + 2 { + let (hash, data) = test_txset_xdr(60 + i as u8); + let event = pushed_txset(hash, &data, pusher); + app.handle_libp2p_event(event).await; + } + + // Its two oldest unclaimed sets are gone from the cache ... + assert!(app.tx_set_cache.get(&hashes[0]).is_none()); + assert!(app.tx_set_cache.get(&hashes[1]).is_none()); + for hash in &hashes[2..] { + assert!(app.tx_set_cache.get(hash).is_some()); + assert!(app.unclaimed_tx_sets.contains(hash)); + } + // ... while the set Core used and the other peer's set stay. + assert!(app.tx_set_cache.get(&used_hash).is_some()); + assert!(app.tx_set_cache.get(&other_hash).is_some()); + assert_eq!(app.unclaimed_tx_sets.len(), LIMIT + 1); + let (received, hits, evicted) = push_counters(&app); + assert_eq!((received, hits, evicted), (LIMIT as u64 + 4, 1, 2)); + + // An evicted set that turns out to be needed is fetched like any + // other missing set: the request stays pending instead of failing. + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&hashes[0], 100), + )) + .await; + assert_no_core_txset(&mut core); + assert!(app.pending_core_tx_sets.contains_key(&hashes[0])); + + // Slot eviction of the cache also releases the budget. + app.handle_core_message(Message::new( + MessageType::LedgerClosed, + ledger_closed_payload(200), + )) + .await; + assert_eq!(app.tx_set_cache.len(), 0); + assert!(app.unclaimed_tx_sets.is_empty()); + assert_eq!(app.unclaimed_tx_sets.total_bytes(), 0); + } + + /// A set this node built itself is never evicted by the limits for + /// unclaimed sets, even if a peer pushed the same set first (two nodes + /// build identical empty sets on an idle network). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn locally_built_txset_is_not_evicted_as_unclaimed() { + use stellar_overlay::flood::txset_push_limits::MAX_UNCLAIMED_SETS_PER_PEER as LIMIT; + let (mut app, _core) = test_app(); + let pusher = PeerId::random(); + let (hash, data) = test_txset_xdr(70); + + // The peer's push arrives first, then Core builds the same set. + app.handle_libp2p_event(pushed_txset(hash, &data, pusher)) + .await; + assert!(app.unclaimed_tx_sets.contains(&hash)); + let mut payload = request_tx_set_payload(&hash, 100); + payload.extend_from_slice(&data); + app.handle_core_message(Message::new(MessageType::CacheTxSet, payload)) + .await; + assert!(!app.unclaimed_tx_sets.contains(&hash)); + + // Another copy from a peer does not make it evictable again. + app.handle_libp2p_event(LibP2pOverlayEvent::TxSetReceived { + hash, + data: Arc::new(TxSetData::from_local(data.clone()).unwrap()), + from: pusher, + slot: Some(100), + }) + .await; + assert!(!app.unclaimed_tx_sets.contains(&hash)); + + // The same peer now pushes more sets than it may keep unclaimed. + for i in 0..LIMIT + 2 { + let (other, other_data) = test_txset_xdr(80 + i as u8); + app.handle_libp2p_event(pushed_txset(other, &other_data, pusher)) + .await; + } + assert!(app.tx_set_cache.get(&hash).is_some()); + } + + /// Sets the overlay prefetched (an SCP envelope named them, a peer + /// answered) but Core has not asked for are bounded too, with their own, + /// larger per-peer limit: a peer answers a new connection's SCP state + /// request with many slots at once. They are not counted as push hits + /// when Core does ask, and pushes from the same peer do not evict them. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn prefetched_txsets_core_did_not_ask_for_are_bounded() { + use stellar_overlay::flood::txset_push_limits::{ + MAX_UNCLAIMED_REQUESTED_SETS_PER_PEER as REQUESTED_LIMIT, + MAX_UNCLAIMED_SETS_PER_PEER as PUSHED_LIMIT, + }; + let (mut app, mut core) = test_app(); + let peer = PeerId::random(); + let sets: Vec<(Hash256, Vec)> = (0..REQUESTED_LIMIT + 1) + .map(|i| test_txset_xdr(90 + i as u8)) + .collect(); + // A full window of answers arrives before Core asks for any: all kept. + for (hash, data) in &sets[..REQUESTED_LIMIT] { + app.handle_libp2p_event(LibP2pOverlayEvent::TxSetReceived { + hash: *hash, + data: Arc::new(TxSetData::from_local(data.clone()).unwrap()), + from: peer, + slot: Some(100), + }) + .await; + } + assert_eq!(app.unclaimed_tx_sets.len(), REQUESTED_LIMIT); + for (hash, _) in &sets[..REQUESTED_LIMIT] { + assert!(app.tx_set_cache.get(hash).is_some()); + } + // Pushes from the same peer are counted separately. + for i in 0..PUSHED_LIMIT { + let (hash, data) = test_txset_xdr(150 + i as u8); + app.handle_libp2p_event(pushed_txset(hash, &data, peer)) + .await; + } + assert!(app.tx_set_cache.get(&sets[0].0).is_some()); + assert_eq!(push_counters(&app), (PUSHED_LIMIT as u64, 0, 0)); + + // One more answer than the limit drops the oldest prefetched set. + let (extra_hash, extra_data) = &sets[REQUESTED_LIMIT]; + app.handle_libp2p_event(LibP2pOverlayEvent::TxSetReceived { + hash: *extra_hash, + data: Arc::new(TxSetData::from_local(extra_data.clone()).unwrap()), + from: peer, + slot: Some(100), + }) + .await; + assert!(app.tx_set_cache.get(&sets[0].0).is_none()); + assert_eq!(app.unclaimed_tx_sets.len(), REQUESTED_LIMIT + PUSHED_LIMIT); + assert_no_core_txset(&mut core); + + // Core asks for a prefetched set: served, and not a push hit. + let (hash, data) = &sets[1]; + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(hash, 100), + )) + .await; + assert_core_txset(&mut core, hash, data); + assert!(!app.unclaimed_tx_sets.contains(hash)); + assert_eq!(push_counters(&app), (PUSHED_LIMIT as u64, 0, 1)); + + // Core asks for the evicted one: it is fetched again, not lost. + app.handle_core_message(Message::new( + MessageType::RequestTxSet, + request_tx_set_payload(&sets[0].0, 100), + )) + .await; + assert_no_core_txset(&mut core); + assert!(app.pending_core_tx_sets.contains_key(&sets[0].0)); + } + + /// PushTxSet for a set the overlay does not hold, or with a short + /// payload, is ignored: no crash, no shutdown, nothing sent to Core. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn push_txset_ipc_for_unknown_or_malformed_input_is_ignored() { + let (mut app, mut core) = test_app(); + let (hash, _) = test_txset_xdr(45); + assert!( + app.handle_core_message(Message::new( + MessageType::PushTxSet, + request_tx_set_payload(&hash, 100), + )) + .await + ); + assert!( + app.handle_core_message(Message::new(MessageType::PushTxSet, vec![1, 2, 3])) + .await + ); + assert!( + app.handle_core_message(Message::new(MessageType::PushTxSet, vec![])) + .await + ); + assert_no_core_txset(&mut core); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn pending_core_txset_delivers_once_despite_duplicate_arrivals() { let (mut app, mut core) = test_app(); diff --git a/overlay/src/metrics.rs b/overlay/src/metrics.rs index 9371beae3d..4cc5e94c5a 100644 --- a/overlay/src/metrics.rs +++ b/overlay/src/metrics.rs @@ -97,6 +97,20 @@ pub struct OverlayMetrics { /// overlay.send.txset — TX set messages sent pub send_txset: AtomicU64, + // Leader tx-set push + /// overlay.txset-push.sent — tx sets pushed to a peer as round leader (one per peer) + pub txset_push_sent: AtomicU64, + /// overlay.txset-push.received — tx sets received without a request + pub txset_push_received: AtomicU64, + /// overlay.txset-push.hit — Core requests answered at once from a pushed set + pub txset_push_hit: AtomicU64, + /// overlay.txset-push.request-skipped — peer requests not answered because the set was already pushed to that peer + pub txset_push_request_skipped: AtomicU64, + /// overlay.txset-push.dropped — unrequested tx-set frames dropped by the per-peer rate limit + pub txset_push_dropped: AtomicU64, + /// overlay.txset-push.evicted — unrequested tx sets evicted by the unclaimed-set limits + pub txset_push_evicted: AtomicU64, + // Receive timers (per message type, tracked as sum_us + count) /// overlay.recv.scp-message — time processing SCP messages pub recv_scp_sum_us: AtomicU64, @@ -152,6 +166,12 @@ impl Default for OverlayMetrics { send_scp_message: AtomicU64::new(0), send_transaction: AtomicU64::new(0), send_txset: AtomicU64::new(0), + txset_push_sent: AtomicU64::new(0), + txset_push_received: AtomicU64::new(0), + txset_push_hit: AtomicU64::new(0), + txset_push_request_skipped: AtomicU64::new(0), + txset_push_dropped: AtomicU64::new(0), + txset_push_evicted: AtomicU64::new(0), recv_scp_sum_us: AtomicU64::new(0), recv_scp_count: AtomicU64::new(0), fetch_txset_sum_us: AtomicU64::new(0), @@ -213,6 +233,12 @@ impl OverlayMetrics { send_scp_message: self.send_scp_message.load(ORD), send_transaction: self.send_transaction.load(ORD), send_txset: self.send_txset.load(ORD), + txset_push_sent: self.txset_push_sent.load(ORD), + txset_push_received: self.txset_push_received.load(ORD), + txset_push_hit: self.txset_push_hit.load(ORD), + txset_push_request_skipped: self.txset_push_request_skipped.load(ORD), + txset_push_dropped: self.txset_push_dropped.load(ORD), + txset_push_evicted: self.txset_push_evicted.load(ORD), recv_scp_sum_us: self.recv_scp_sum_us.load(ORD), recv_scp_count: self.recv_scp_count.load(ORD), fetch_txset_sum_us: self.fetch_txset_sum_us.load(ORD), @@ -282,6 +308,12 @@ pub struct MetricsSnapshot { pub send_scp_message: u64, pub send_transaction: u64, pub send_txset: u64, + pub txset_push_sent: u64, + pub txset_push_received: u64, + pub txset_push_hit: u64, + pub txset_push_request_skipped: u64, + pub txset_push_dropped: u64, + pub txset_push_evicted: u64, pub recv_scp_sum_us: u64, pub recv_scp_count: u64, pub fetch_txset_sum_us: u64, diff --git a/overlay/tests/e2e_binary.rs b/overlay/tests/e2e_binary.rs index 3a7d9bbf14..e7148e9ea6 100644 --- a/overlay/tests/e2e_binary.rs +++ b/overlay/tests/e2e_binary.rs @@ -18,6 +18,12 @@ mod ipc { pub const BROADCAST_SCP: u32 = 1; pub const SET_PEER_CONFIG: u32 = 8; pub const SHUTDOWN: u32 = 7; + pub const REQUEST_TX_SET: u32 = 11; + pub const CACHE_TX_SET: u32 = 12; + pub const REQUEST_OVERLAY_METRICS: u32 = 13; + pub const PUSH_TX_SET: u32 = 14; + pub const TX_SET_AVAILABLE: u32 = 103; + pub const OVERLAY_METRICS_RESPONSE: u32 = 105; pub const SCP_RECEIVED: u32 = 100; pub const PEER_REQUESTS_SCP_STATE: u32 = 102; @@ -150,6 +156,116 @@ fn valid_scp_envelope_xdr(slot_index: u64) -> Vec { envelope.to_xdr(Limits::none()).unwrap() } +/// A minimal valid GeneralizedTransactionSet and its content hash. +fn txset_xdr(seed: u8) -> ([u8; 32], Vec) { + use sha2::{Digest, Sha256}; + use stellar_xdr::{GeneralizedTransactionSet, Hash}; + let mut tx_set = GeneralizedTransactionSet::default(); + let GeneralizedTransactionSet::V1(v1) = &mut tx_set; + v1.previous_ledger_hash = Hash([seed; 32]); + let bytes = tx_set.to_xdr(Limits::none()).unwrap(); + let hash: [u8; 32] = Sha256::digest(&bytes).into(); + (hash, bytes) +} + +/// `[hash:32][slot:4 LE]`, the payload of REQUEST_TX_SET and PUSH_TX_SET and +/// the prefix of CACHE_TX_SET. +fn txset_key(hash: &[u8; 32], slot: u32) -> Vec { + let mut payload = hash.to_vec(); + payload.extend_from_slice(&slot.to_le_bytes()); + payload +} + +/// Read IPC messages until one of `msg_type` arrives (skipping others, such +/// as SCP state requests of a new connection). None on timeout. +fn recv_until(stream: &mut UnixStream, msg_type: u32, timeout: Duration) -> Option> { + let deadline = Instant::now() + timeout; + loop { + let remaining = deadline.checked_duration_since(Instant::now())?; + stream + .set_read_timeout(Some(remaining.max(Duration::from_millis(1)))) + .unwrap(); + match ipc::recv_message(stream) { + Ok((got, payload)) if got == msg_type => return Some(payload), + Ok(_) => {} + Err(_) => return None, + } + } +} + +/// One counter of the overlay's metrics snapshot. +fn overlay_metric(stream: &mut UnixStream, name: &str) -> u64 { + ipc::send_message(stream, ipc::REQUEST_OVERLAY_METRICS, &[]).unwrap(); + let payload = recv_until( + stream, + ipc::OVERLAY_METRICS_RESPONSE, + Duration::from_secs(5), + ) + .expect("metrics response"); + let json: serde_json::Value = serde_json::from_slice(&payload).unwrap(); + json[name] + .as_u64() + .unwrap_or_else(|| panic!("no metric {name}")) +} + +/// Two overlay processes, B connected to A, each with its Core-side socket. +struct TwoOverlays { + child_a: Child, + child_b: Child, + core_a: UnixStream, + core_b: UnixStream, + socket_a: String, + socket_b: String, +} + +impl TwoOverlays { + fn start(name: &str, port_a: u16, port_b: u16) -> Self { + let socket_a = format!("/tmp/e2e-{}-a-{}.sock", name, std::process::id()); + let socket_b = format!("/tmp/e2e-{}-b-{}.sock", name, std::process::id()); + let _ = std::fs::remove_file(&socket_a); + let _ = std::fs::remove_file(&socket_b); + let child_a = spawn_overlay(&socket_a, port_a); + let child_b = spawn_overlay(&socket_b, port_b); + let core_a = wait_for_socket(&socket_a, 5000).expect("Socket A should be ready"); + let mut core_b = wait_for_socket(&socket_b, 5000).expect("Socket B should be ready"); + let peer_config = format!( + r#"{{"known_peers":["127.0.0.1:{}"],"preferred_peers":[],"listen_port":{}}}"#, + port_a, port_b + ); + ipc::send_message(&mut core_b, ipc::SET_PEER_CONFIG, peer_config.as_bytes()) + .expect("Should send peer config"); + // Connection and stream setup. + thread::sleep(Duration::from_millis(1000)); + Self { + child_a, + child_b, + core_a, + core_b, + socket_a, + socket_b, + } + } + + fn stop(self) { + let TwoOverlays { + mut child_a, + mut child_b, + mut core_a, + mut core_b, + socket_a, + socket_b, + } = self; + ipc::send_message(&mut core_a, ipc::SHUTDOWN, &[]).ok(); + ipc::send_message(&mut core_b, ipc::SHUTDOWN, &[]).ok(); + drop(core_a); + drop(core_b); + wait_for_child_exit(&mut child_a, "overlay A"); + wait_for_child_exit(&mut child_b, "overlay B"); + let _ = std::fs::remove_file(&socket_a); + let _ = std::fs::remove_file(&socket_b); + } +} + /// Wait for socket to be ready and return the connected stream fn wait_for_socket(path: &str, timeout_ms: u64) -> Option { let start = std::time::Instant::now(); @@ -329,4 +445,199 @@ mod tests { } } } + + /// The leader's Core hands its set to its overlay and asks for the push + /// (CACHE_TX_SET, PUSH_TX_SET). The follower's overlay then holds the + /// set before its Core knows the hash: Core's later request is answered + /// from the pushed copy, without any network request. + #[test] + fn test_leader_push_delivers_txset_before_the_request() { + let mut net = TwoOverlays::start("push", 11720, 11721); + let (hash, xdr) = txset_xdr(0x61); + + let mut cache = txset_key(&hash, 5); + cache.extend_from_slice(&xdr); + ipc::send_message(&mut net.core_a, ipc::CACHE_TX_SET, &cache).unwrap(); + ipc::send_message(&mut net.core_a, ipc::PUSH_TX_SET, &txset_key(&hash, 5)).unwrap(); + + // The pushed set reaches B's overlay but not B's Core. + let deadline = Instant::now() + Duration::from_secs(10); + while overlay_metric(&mut net.core_b, "txset_push_received") == 0 { + assert!(Instant::now() < deadline, "push did not arrive"); + thread::sleep(Duration::from_millis(50)); + } + assert!( + recv_until( + &mut net.core_b, + ipc::TX_SET_AVAILABLE, + Duration::from_millis(300) + ) + .is_none(), + "Core must not receive a set it did not ask for" + ); + + // B's Core sees the proposal and asks for the hash. + ipc::send_message(&mut net.core_b, ipc::REQUEST_TX_SET, &txset_key(&hash, 5)).unwrap(); + let payload = recv_until( + &mut net.core_b, + ipc::TX_SET_AVAILABLE, + Duration::from_secs(5), + ) + .expect("requested set"); + assert_eq!(&payload[..32], &hash); + assert_eq!(&payload[32..], &xdr[..]); + + assert_eq!(overlay_metric(&mut net.core_a, "txset_push_sent"), 1); + assert_eq!(overlay_metric(&mut net.core_b, "txset_push_received"), 1); + assert_eq!(overlay_metric(&mut net.core_b, "txset_push_hit"), 1); + // No request went over the network. + assert_eq!(overlay_metric(&mut net.core_b, "fetch_txset_count"), 0); + net.stop(); + } + + /// Fallback: the leader's overlay holds the set but the push never + /// happened (lost). The follower's request fetches it from the leader. + #[test] + fn test_txset_request_path_works_without_a_push() { + let mut net = TwoOverlays::start("nopush", 11730, 11731); + let (hash, xdr) = txset_xdr(0x62); + + let mut cache = txset_key(&hash, 5); + cache.extend_from_slice(&xdr); + ipc::send_message(&mut net.core_a, ipc::CACHE_TX_SET, &cache).unwrap(); + thread::sleep(Duration::from_millis(300)); + assert_eq!(overlay_metric(&mut net.core_b, "txset_push_received"), 0); + + ipc::send_message(&mut net.core_b, ipc::REQUEST_TX_SET, &txset_key(&hash, 5)).unwrap(); + let payload = recv_until( + &mut net.core_b, + ipc::TX_SET_AVAILABLE, + Duration::from_secs(5), + ) + .expect("requested set"); + assert_eq!(&payload[..32], &hash); + assert_eq!(&payload[32..], &xdr[..]); + + assert_eq!(overlay_metric(&mut net.core_a, "txset_push_sent"), 0); + assert_eq!(overlay_metric(&mut net.core_b, "txset_push_hit"), 0); + assert_eq!(overlay_metric(&mut net.core_b, "fetch_txset_count"), 1); + net.stop(); + } + + /// The leader pushes set A but its proposal names B. The follower's + /// request for B is not answered with A: it is fetched from the leader, + /// which still answers requests for sets it did not push. + #[test] + fn test_pushed_txset_does_not_answer_a_request_for_another_hash() { + let mut net = TwoOverlays::start("mismatch", 11740, 11741); + let (pushed_hash, pushed_xdr) = txset_xdr(0x63); + let (proposed_hash, proposed_xdr) = txset_xdr(0x64); + + for (hash, xdr) in [(&pushed_hash, &pushed_xdr), (&proposed_hash, &proposed_xdr)] { + let mut cache = txset_key(hash, 5); + cache.extend_from_slice(xdr); + ipc::send_message(&mut net.core_a, ipc::CACHE_TX_SET, &cache).unwrap(); + } + ipc::send_message( + &mut net.core_a, + ipc::PUSH_TX_SET, + &txset_key(&pushed_hash, 5), + ) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while overlay_metric(&mut net.core_b, "txset_push_received") == 0 { + assert!(Instant::now() < deadline, "push did not arrive"); + thread::sleep(Duration::from_millis(50)); + } + + ipc::send_message( + &mut net.core_b, + ipc::REQUEST_TX_SET, + &txset_key(&proposed_hash, 5), + ) + .unwrap(); + let payload = recv_until( + &mut net.core_b, + ipc::TX_SET_AVAILABLE, + Duration::from_secs(5), + ) + .expect("requested set"); + assert_eq!(&payload[..32], &proposed_hash); + assert_eq!(&payload[32..], &proposed_xdr[..]); + // Only the requested set was delivered. + assert!(recv_until( + &mut net.core_b, + ipc::TX_SET_AVAILABLE, + Duration::from_millis(300) + ) + .is_none()); + + assert_eq!(overlay_metric(&mut net.core_b, "txset_push_hit"), 0); + assert_eq!(overlay_metric(&mut net.core_b, "fetch_txset_count"), 1); + assert_eq!( + overlay_metric(&mut net.core_a, "txset_push_request_skipped"), + 0 + ); + net.stop(); + } + + /// A push that the receiver drops (here: by its per-peer rate limit and + /// by its limit on sets Core has not asked for) is recovered through the + /// request path: when the follower's Core later asks for such a set, the + /// leader answers although it had pushed it. Every set arrives. + #[test] + fn test_dropped_push_is_recovered_by_a_request() { + let mut net = TwoOverlays::start("dropped", 11750, 11751); + // More unrequested sets in a burst than the follower accepts. + let sets: Vec<([u8; 32], Vec)> = (0..14).map(|i| txset_xdr(0x70 + i)).collect(); + for (hash, xdr) in &sets { + let mut cache = txset_key(hash, 5); + cache.extend_from_slice(xdr); + ipc::send_message(&mut net.core_a, ipc::CACHE_TX_SET, &cache).unwrap(); + ipc::send_message(&mut net.core_a, ipc::PUSH_TX_SET, &txset_key(hash, 5)).unwrap(); + thread::sleep(Duration::from_millis(15)); + } + // Later than any request could have crossed a push. + thread::sleep(Duration::from_millis(800)); + let pushed = overlay_metric(&mut net.core_a, "txset_push_sent"); + let received = overlay_metric(&mut net.core_b, "txset_push_received"); + let dropped = overlay_metric(&mut net.core_b, "txset_push_dropped"); + assert!(dropped >= 1, "the burst should exceed the follower's limit"); + assert_eq!(received + dropped, pushed); + + for (hash, xdr) in &sets { + ipc::send_message(&mut net.core_b, ipc::REQUEST_TX_SET, &txset_key(hash, 5)).unwrap(); + let payload = recv_until( + &mut net.core_b, + ipc::TX_SET_AVAILABLE, + Duration::from_secs(5), + ) + .expect("requested set"); + assert_eq!(&payload[..32], hash); + assert_eq!(&payload[32..], &xdr[..]); + } + // Of the sets that arrived, the follower kept only as many as one + // peer may have unclaimed; the older ones were evicted. + let evicted = overlay_metric(&mut net.core_b, "txset_push_evicted"); + assert_eq!( + received - evicted, + stellar_overlay::flood::txset_push_limits::MAX_UNCLAIMED_SETS_PER_PEER as u64 + ); + // Every set the follower did not hold (dropped by the rate limit, + // evicted, or not pushed because an earlier push was still in + // flight) came through a request, which the leader answered. + assert_eq!( + overlay_metric(&mut net.core_b, "fetch_txset_count"), + sets.len() as u64 - (received - evicted) + ); + assert_eq!( + overlay_metric(&mut net.core_b, "txset_push_hit"), + received - evicted + ); + assert_eq!( + overlay_metric(&mut net.core_a, "txset_push_request_skipped"), + 0 + ); + net.stop(); + } } diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp index 1a66412bb7..f0896bd20b 100644 --- a/src/herder/HerderImpl.cpp +++ b/src/herder/HerderImpl.cpp @@ -1874,16 +1874,36 @@ HerderImpl::prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime) { return; } - mPreparedTxSet = buildTxSet(ledgerSeq, closeTime); + mPreparedTxSet = buildTxSet(ledgerSeq, closeTime, /*pushToPeers=*/false); CLOG_INFO(Herder, "Prepared TX set for ledger {} before trigger: {} transactions, " "close time {}", ledgerSeq, mPreparedTxSet->txSet->sizeTxTotal(), closeTime.toString()); + // A capacity-limited prepared set is proposed unchanged at the trigger + // (makeProposalValue), so peers can have it before the proposal exists. + // Any other prepared set is rebuilt at the trigger and is not pushed. + if (mPreparedTxSet->capacityLimited && mPreparedTxSet->applicableTxSet) + { + pushTxSetToPeers(mPreparedTxSet->txSet, ledgerSeq); + } +} + +void +HerderImpl::pushTxSetToPeers(TxSetXDRFrameConstPtr const& txSet, + uint32_t ledgerSeq) +{ + // Legacy (pre-protocol 20) sets cannot be carried by the Rust overlay. + if (!txSet || !txSet->isGeneralizedTxSet()) + { + return; + } + mApp.getOverlayManager().pushTxSet(txSet->getContentsHash(), ledgerSeq); } HerderImpl::PreparedTxSet -HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime) +HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, + bool pushToPeers) { auto const txSetBuildStart = std::chrono::steady_clock::now(); auto const lcl = mLedgerManager.getLastClosedLedgerHeader(); @@ -2014,6 +2034,15 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime) // the set's XDR first. overlayMgr.cacheTxSet(txSet->getContentsHash(), txSet->getGeneralizedXDR(), ledgerSeq); + // Leader's push: this set is about to be proposed, so send + // it to the peers now, while the builder's remaining + // validation and the proposal's persistence still run. If + // that validation fails nothing is proposed, and peers drop + // the unused set. + if (pushToPeers) + { + pushTxSetToPeers(txSet, ledgerSeq); + } } }, targets); @@ -2480,12 +2509,23 @@ HerderImpl::makeProposalValue(uint32_t ledgerSeqToTrigger) ctValidityOffset(prepared->closeTime) != std::chrono::milliseconds::zero()) { - prepared = buildTxSet(ledgerSeqToTrigger, nextCloseTime); + // Push the set from inside the build if this node proposes it as + // the current round's leader. (Every validator also builds here for + // legacy tx sets, which are neither cached nor pushed.) + uint32_t const round = getSCP().getLeaderRound(ledgerSeqToTrigger); + bool const activeLeader = + round != 0 && + mHerderSCPDriver.isLocalNodeRoundLeader(ledgerSeqToTrigger, round); + prepared = buildTxSet(ledgerSeqToTrigger, nextCloseTime, + /*pushToPeers=*/activeLeader); } else { CLOG_INFO(Herder, "Using prepared TX set for ledger {}", ledgerSeqToTrigger); + // Pushed when it was prepared; this reaches peers that connected + // since (the overlay sends a set to each peer once). + pushTxSetToPeers(prepared->txSet, ledgerSeqToTrigger); } nextCloseTime = prepared->closeTime; // The offset is computed in *apply* time rather than consensus time: diff --git a/src/herder/HerderImpl.h b/src/herder/HerderImpl.h index f46ad8bf7d..e2d6b2ceaf 100644 --- a/src/herder/HerderImpl.h +++ b/src/herder/HerderImpl.h @@ -269,7 +269,16 @@ class HerderImpl : public Herder bool capacityLimited; }; std::optional mPreparedTxSet; - PreparedTxSet buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime); + // `pushToPeers`: this is the set the node is about to propose as a round + // leader, so the overlay sends it to every peer as soon as its XDR exists + // instead of waiting for their requests (see pushTxSetToPeers). + PreparedTxSet buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, + bool pushToPeers); + // Leader's tx-set push: ask the overlay to send a set it already holds + // (cacheTxSet) to every connected peer. Peers use a pushed set only if + // an SCP statement names its hash; one that missed it requests it. + void pushTxSetToPeers(TxSetXDRFrameConstPtr const& txSet, + uint32_t ledgerSeq); void prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime); void discardPreparedTxSet(); ValueWrapperPtr makeProposalValue(uint32_t ledgerSeq); diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index f151cbe81c..6259ee2f46 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -5388,7 +5388,7 @@ class LeaderBuildTestAccess static TxSetXDRFrameConstPtr build(HerderImpl& herder, uint32_t ledgerSeq, ConsensusTime closeTime) { - return herder.buildTxSet(ledgerSeq, closeTime).txSet; + return herder.buildTxSet(ledgerSeq, closeTime, false).txSet; } // The same, with the hashes of the invalid candidates it found. @@ -5396,7 +5396,7 @@ class LeaderBuildTestAccess buildWithInvalid(HerderImpl& herder, uint32_t ledgerSeq, ConsensusTime closeTime) { - auto prepared = herder.buildTxSet(ledgerSeq, closeTime); + auto prepared = herder.buildTxSet(ledgerSeq, closeTime, false); return {prepared.txSet, prepared.invalidTxHashes}; } diff --git a/src/herder/test/TxSetPushTests.cpp b/src/herder/test/TxSetPushTests.cpp new file mode 100644 index 0000000000..cc0ff3a839 --- /dev/null +++ b/src/herder/test/TxSetPushTests.cpp @@ -0,0 +1,306 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// Network-level tests of the leader's tx-set push: the leader of a round +// sends the tx set it is about to propose to every peer as soon as the set +// is built, instead of waiting for each follower to request it. Followers +// keep a pushed set in their overlay and hand it to Core only when an SCP +// statement names its hash; a follower that did not get it (or got another +// set) requests the proposed one as before. + +#include "crypto/SHA.h" +#include "herder/Herder.h" +#include "herder/HerderImpl.h" +#include "herder/TxSetFrame.h" +#include "ledger/LedgerManager.h" +#include "main/Application.h" +#include "overlay/RustOverlayManager.h" +#include "simulation/Simulation.h" +#include "simulation/Topologies.h" +#include "test/Catch2.h" +#include "test/test.h" +#include "util/Logging.h" +#include "util/MetricsRegistry.h" +#include "util/ProtocolVersion.h" +#include +#include + +using namespace stellar; + +namespace +{ +Config +acceleratedConfig(int configNum) +{ + auto cfg = getTestConfig(configNum); + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + return cfg; +} + +// Network totals of the tx-set dissemination counters. +struct TxSetCounts +{ + // Leader side: sets written to a peer without a request, and peer + // requests not answered because the set was already pushed to that peer. + int64_t mPushSent{0}; + int64_t mRequestSkipped{0}; + // Follower side: sets that arrived unrequested, Core requests answered + // at once from such a set, sets that had to be awaited from the network, + // and unrequested sets dropped by the limits. + int64_t mPushReceived{0}; + int64_t mPushHit{0}; + int64_t mNetworkFetches{0}; + int64_t mPushDropped{0}; + // Core: sets delivered by the overlay, and requests repeated after the + // one-second retry interval. + int64_t mCoreFetches{0}; + int64_t mCoreRetries{0}; + + std::string + toString() const + { + return fmt::format( + "push sent {} received {} hit {} dropped {}; request skipped {}; " + "network fetches {}; core fetches {} retries {}", + mPushSent, mPushReceived, mPushHit, mPushDropped, mRequestSkipped, + mNetworkFetches, mCoreFetches, mCoreRetries); + } +}; + +TxSetCounts +countTxSetDissemination(std::vector const& nodes) +{ + TxSetCounts res; + for (auto const& app : nodes) + { + app->getOverlayManager().syncOverlayMetrics(); + auto& metrics = app->getMetrics(); + auto push = [&](char const* name) { + return static_cast( + metrics.NewMeter({"overlay", "txset-push", name}, "txset") + .count()); + }; + res.mPushSent += push("sent"); + res.mRequestSkipped += push("request-skipped"); + res.mPushReceived += push("received"); + res.mPushHit += push("hit"); + res.mPushDropped += push("dropped") + push("evicted"); + res.mNetworkFetches += static_cast( + metrics.NewTimer({"overlay", "fetch", "txset"}).count()); + res.mCoreFetches += static_cast( + metrics.NewTimer({"scp", "fetch", "txset"}).count()); + res.mCoreRetries += static_cast( + metrics.NewCounter({"scp", "fetch", "txset-retry"}).count()); + } + return res; +} + +// Cranks until `app` closed `lastLedger` and returns the tx-set hash of +// every ledger it closed on the way, keyed by ledger sequence, with the hash +// of the ledger before it. +std::map> +crankAndRecordTxSets(Simulation::pointer const& simulation, + Application::pointer const& app, uint32_t lastLedger) +{ + std::map> res; + auto& lm = app->getLedgerManager(); + auto const deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(300); + uint32_t seen = lm.getLastClosedLedgerNum(); + while (seen < lastLedger) + { + REQUIRE(std::chrono::steady_clock::now() < deadline); + simulation->crankForAtMost(std::chrono::milliseconds(20), false); + auto const& lcl = lm.getLastClosedLedgerHeader(); + if (lcl.header.ledgerSeq > seen) + { + REQUIRE(lcl.header.ledgerSeq == seen + 1); + seen = lcl.header.ledgerSeq; + res[seen] = {lcl.header.scpValue.txSetHash, + lcl.header.previousLedgerHash}; + } + } + return res; +} + +// A 4-node network with a 3-of-4 quorum, every node connected to every +// other one. +Simulation::pointer +makeNetwork(std::function const& makeConfig) +{ + return Topologies::core(4, 0.75, sha256(getTestConfig().NETWORK_PASSPHRASE), + makeConfig); +} +} + +TEST_CASE("leader pushes its tx set to followers", + "[herder][txset-push][simulation]") +{ + auto simulation = makeNetwork(acceleratedConfig); + simulation->startAllNodes(); + auto nodes = simulation->getNodes(); + + uint32_t const lastLedger = 14; + auto closed = crankAndRecordTxSets(simulation, nodes[0], lastLedger); + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(lastLedger, 3); }, + std::chrono::seconds(60), false); + REQUIRE(closed.size() >= 10); + + auto counts = countTxSetDissemination(nodes); + INFO(counts.toString()); + + // Every closed ledger had a leader that pushed its set to its 3 peers + // (a few ledgers may still be in flight when the counters are read). + auto const ledgers = static_cast(closed.size()); + REQUIRE(counts.mPushSent >= 3 * (ledgers - 2)); + // A follower either already held the pushed set when Core asked for it + // (a hit), or its request crossed the push and was completed by it. On + // a loopback network the push normally wins. + REQUIRE(counts.mPushHit > 0); + REQUIRE(counts.mPushHit + counts.mNetworkFetches >= 3 * (ledgers - 2)); + // Every follower got every set from its overlay ... + REQUIRE(counts.mCoreFetches >= 3 * (ledgers - 2)); + // ... and none had to fall back to the one-second retry: a request that + // the leader skipped (because it had pushed the set to that peer) was + // always completed by the pushed copy. + REQUIRE(counts.mCoreRetries == 0); + REQUIRE(counts.mPushDropped == 0); +} + +TEST_CASE("consensus uses the request path when the tx set push is lost", + "[herder][txset-push][simulation]") +{ + auto simulation = makeNetwork(acceleratedConfig); + auto nodes = simulation->getNodes(); + // No node's push leaves Core, as if every push were lost at its source. + // (A push that is sent and then lost at the receiver is covered by the + // overlay's own tests: test_dropped_push_is_recovered_by_a_request.) + for (auto const& app : nodes) + { + app->getOverlayManager().setDropTxSetPushForTesting(true); + } + simulation->startAllNodes(); + + uint32_t const lastLedger = 12; + auto closed = crankAndRecordTxSets(simulation, nodes[0], lastLedger); + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(lastLedger, 3); }, + std::chrono::seconds(60), false); + REQUIRE(closed.size() >= 8); + + auto counts = countTxSetDissemination(nodes); + INFO(counts.toString()); + auto const ledgers = static_cast(closed.size()); + + // Nothing was pushed, so nothing arrived unrequested and no request was + // skipped: every follower asked the network for every set, as on the + // base, and got it without waiting for the retry. + REQUIRE(counts.mPushSent == 0); + REQUIRE(counts.mPushReceived == 0); + REQUIRE(counts.mPushHit == 0); + REQUIRE(counts.mRequestSkipped == 0); + REQUIRE(counts.mNetworkFetches >= 3 * (ledgers - 2)); + REQUIRE(counts.mCoreFetches >= 3 * (ledgers - 2)); + REQUIRE(counts.mCoreRetries == 0); +} + +TEST_CASE("some nodes lose the tx set push and still follow", + "[herder][txset-push][simulation]") +{ + auto simulation = makeNetwork(acceleratedConfig); + auto nodes = simulation->getNodes(); + // Nodes 0 and 1 push when they lead; the pushes of nodes 2 and 3 are + // lost. The network sees both paths on alternating ledgers. + nodes[2]->getOverlayManager().setDropTxSetPushForTesting(true); + nodes[3]->getOverlayManager().setDropTxSetPushForTesting(true); + simulation->startAllNodes(); + + uint32_t const lastLedger = 20; + auto closed = crankAndRecordTxSets(simulation, nodes[0], lastLedger); + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(lastLedger, 3); }, + std::chrono::seconds(60), false); + + auto counts = countTxSetDissemination(nodes); + INFO(counts.toString()); + REQUIRE(closed.size() >= 16); + REQUIRE(counts.mCoreRetries == 0); + REQUIRE(counts.mCoreFetches >= + 3 * (static_cast(closed.size()) - 2)); +} + +// Every leader pushes the set it built (A) but proposes another one (B): a +// push whose content does not match the hash in the leader's SCP statement. +// Followers must not use A. They request B, validate it and externalize it. +// +// B is a tx set without transactions that is not encoded like +// `TxSetXDRFrame::makeEmpty` (its parallel Soroban phase carries a base +// fee). It is valid before CANONICAL_EMPTY_SOROBAN_PHASE_PROTOCOL_VERSION, +// so the network runs on the protocol before that gate. A, built from an +// empty mempool, is the canonical empty set. +TEST_CASE("pushed tx set that does not match the proposal is not used", + "[herder][txset-push][simulation]") +{ + auto const protocolVersion = + static_cast(CANONICAL_EMPTY_SOROBAN_PHASE_PROTOCOL_VERSION) - + 1; + auto makeConfig = [&](int i) { + auto cfg = acceleratedConfig(i); + cfg.LEDGER_PROTOCOL_VERSION = protocolVersion; + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = protocolVersion; + return cfg; + }; + auto makeProposedTxSet = [](Hash const& prevHash) { + GeneralizedTransactionSet xdr(1); + xdr.v1TxSet().previousLedgerHash = prevHash; + xdr.v1TxSet().phases.emplace_back(0); + xdr.v1TxSet() + .phases.emplace_back(1) + .parallelTxsComponent() + .baseFee.activate() = 12345; + return TxSetXDRFrame::makeFromWire(xdr); + }; + + auto simulation = makeNetwork(makeConfig); + auto nodes = simulation->getNodes(); + for (auto const& app : nodes) + { + static_cast(app->getHerder()).mProposedTxSetOverride = + [&](LedgerHeaderHistoryEntry const& lcl) { + return makeProposedTxSet(lcl.hash); + }; + } + simulation->startAllNodes(); + + uint32_t const lastLedger = 12; + auto closed = crankAndRecordTxSets(simulation, nodes[1], lastLedger); + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(lastLedger, 3); }, + std::chrono::seconds(60), false); + REQUIRE(closed.size() >= 8); + + // Every ledger externalized the proposed set B, never the pushed set A. + for (auto const& [seq, hashes] : closed) + { + auto const& [txSetHash, prevHash] = hashes; + CAPTURE(seq); + REQUIRE(txSetHash == makeProposedTxSet(prevHash)->getContentsHash()); + } + + auto counts = countTxSetDissemination(nodes); + INFO(counts.toString()); + auto const ledgers = static_cast(closed.size()); + // The built sets were pushed and arrived ... + REQUIRE(counts.mPushSent >= 3 * (ledgers - 2)); + REQUIRE(counts.mPushReceived > 0); + // ... but no Core request was ever answered from one: the proposed + // hash is not the hash of anything that was pushed. Every follower + // fetched the proposed set through the request path instead, and the + // leader answered (it never pushed that set, so it skipped no request). + REQUIRE(counts.mPushHit == 0); + REQUIRE(counts.mRequestSkipped == 0); + REQUIRE(counts.mNetworkFetches >= 3 * (ledgers - 2)); + REQUIRE(counts.mCoreRetries == 0); +} diff --git a/src/overlay/IPC.h b/src/overlay/IPC.h index cfe81b3007..841968daa1 100644 --- a/src/overlay/IPC.h +++ b/src/overlay/IPC.h @@ -76,6 +76,12 @@ enum class IPCMessageType : uint32_t /// Response: OVERLAY_METRICS_RESPONSE with JSON payload REQUEST_OVERLAY_METRICS = 13, + /// Push a cached TX set to every connected peer now, without waiting + /// for their requests. Sent for the set this node is about to propose + /// as a round leader, after the CACHE_TX_SET that carried the set. + /// Payload: [hash:32][slotSeq:4] + PUSH_TX_SET = 14, + // ═══ Overlay → Core (Critical Path) ═══ /// Received SCP envelope from network diff --git a/src/overlay/OverlayIPC.cpp b/src/overlay/OverlayIPC.cpp index 284dd3db4c..af81eb9254 100644 --- a/src/overlay/OverlayIPC.cpp +++ b/src/overlay/OverlayIPC.cpp @@ -756,6 +756,27 @@ OverlayIPC::cacheTxSet(Hash const& hash, GeneralizedTransactionSet const& txSet, mChannel->send(msg); } +void +OverlayIPC::pushTxSet(Hash const& hash, uint32_t slotIndex) +{ + if (!mChannel || !mChannel->isConnected()) + { + return; + } + + IPCMessage msg; + msg.type = IPCMessageType::PUSH_TX_SET; + // Payload: [hash:32][slotSeq:4] + msg.payload.resize(36); + std::memcpy(msg.payload.data(), hash.data(), 32); + std::memcpy(msg.payload.data() + 32, &slotIndex, 4); + + CLOG_DEBUG(Overlay, "Pushing TX set {} for slot {} to peers", + hexAbbrev(hash), slotIndex); + std::lock_guard lock(mSendMutex); + mChannel->send(msg); +} + void OverlayIPC::setPeerConfig(std::vector const& knownPeers, std::vector const& preferredPeers, diff --git a/src/overlay/OverlayIPC.h b/src/overlay/OverlayIPC.h index 4fd231ba23..e5eba401cc 100644 --- a/src/overlay/OverlayIPC.h +++ b/src/overlay/OverlayIPC.h @@ -194,6 +194,19 @@ class OverlayIPC void cacheTxSet(Hash const& hash, GeneralizedTransactionSet const& txSet, uint32_t slotIndex); + /** + * Push a TX set to every connected peer (leader's tx-set push). + * + * The set must have been handed to Rust with cacheTxSet first. Rust + * sends it to each connected peer once, without waiting for requests; + * peers verify and keep it by its content hash and use it only if an + * SCP statement names that hash. + * + * @param hash The TX set hash + * @param slotIndex The slot the set is proposed for (for logging) + */ + void pushTxSet(Hash const& hash, uint32_t slotIndex); + /// Set callback for received SCP envelopes void setOnSCPReceived(SCPReceivedCallback cb); diff --git a/src/overlay/OverlayMetrics.cpp b/src/overlay/OverlayMetrics.cpp index 0af11d70d7..fbf16589c6 100644 --- a/src/overlay/OverlayMetrics.cpp +++ b/src/overlay/OverlayMetrics.cpp @@ -57,6 +57,18 @@ OverlayMetrics::OverlayMetrics(Application& app) , mAuthenticatedPeersSize(app.getMetrics().NewCounter( {"overlay", "connection", "authenticated"})) , mFetchTxSetTimer(app.getMetrics().NewTimer({"overlay", "fetch", "txset"})) + , mTxSetPushSent( + app.getMetrics().NewMeter({"overlay", "txset-push", "sent"}, "txset")) + , mTxSetPushReceived(app.getMetrics().NewMeter( + {"overlay", "txset-push", "received"}, "txset")) + , mTxSetPushHit( + app.getMetrics().NewMeter({"overlay", "txset-push", "hit"}, "txset")) + , mTxSetPushRequestSkipped(app.getMetrics().NewMeter( + {"overlay", "txset-push", "request-skipped"}, "txset")) + , mTxSetPushDropped(app.getMetrics().NewMeter( + {"overlay", "txset-push", "dropped"}, "txset")) + , mTxSetPushEvicted(app.getMetrics().NewMeter( + {"overlay", "txset-push", "evicted"}, "txset")) { } } diff --git a/src/overlay/OverlayMetrics.h b/src/overlay/OverlayMetrics.h index d59a038e38..3fbb7475db 100644 --- a/src/overlay/OverlayMetrics.h +++ b/src/overlay/OverlayMetrics.h @@ -66,5 +66,13 @@ struct OverlayMetrics // ── TxSet fetch latency ── medida::Timer& mFetchTxSetTimer; + + // ── Leader tx-set push (synced from the Rust overlay) ── + medida::Meter& mTxSetPushSent; + medida::Meter& mTxSetPushReceived; + medida::Meter& mTxSetPushHit; + medida::Meter& mTxSetPushRequestSkipped; + medida::Meter& mTxSetPushDropped; + medida::Meter& mTxSetPushEvicted; }; } diff --git a/src/overlay/RustOverlayManager.cpp b/src/overlay/RustOverlayManager.cpp index 9326e4e08e..780b1610f9 100644 --- a/src/overlay/RustOverlayManager.cpp +++ b/src/overlay/RustOverlayManager.cpp @@ -228,6 +228,21 @@ RustOverlayManager::cacheTxSet(Hash const& txSetHash, } } +void +RustOverlayManager::pushTxSet(Hash const& txSetHash, uint32_t slotIndex) +{ +#ifdef BUILD_TESTS + if (mDropTxSetPushForTesting) + { + return; + } +#endif + if (mOverlayIPC && !mShuttingDown) + { + mOverlayIPC->pushTxSet(txSetHash, slotIndex); + } +} + std::vector RustOverlayManager::getTopTransactions(size_t count) { @@ -378,6 +393,14 @@ RustOverlayManager::syncOverlayMetrics() markDelta(m.mSendTransactionMeter, "send_transaction"); markDelta(m.mSendTxSetMeter, "send_txset"); + // Leader tx-set push + markDelta(m.mTxSetPushSent, "txset_push_sent"); + markDelta(m.mTxSetPushReceived, "txset_push_received"); + markDelta(m.mTxSetPushHit, "txset_push_hit"); + markDelta(m.mTxSetPushRequestSkipped, "txset_push_request_skipped"); + markDelta(m.mTxSetPushDropped, "txset_push_dropped"); + markDelta(m.mTxSetPushEvicted, "txset_push_evicted"); + // Connection lifecycle — these aren't registered as medida meters on // the C++ side yet, so they'll just be tracked by the existing counters. // The inbound/outbound attempt/establish/drop are already covered diff --git a/src/overlay/RustOverlayManager.h b/src/overlay/RustOverlayManager.h index 31506106e8..7dde7b7243 100644 --- a/src/overlay/RustOverlayManager.h +++ b/src/overlay/RustOverlayManager.h @@ -74,6 +74,23 @@ class RustOverlayManager void cacheTxSet(Hash const& txSetHash, GeneralizedTransactionSet const& txSet, uint32_t slotIndex); + // Send a TX set that was handed to the overlay with cacheTxSet to every + // connected peer now, without waiting for their requests. Called by a + // round leader for the set it is about to propose. Peers keep a pushed + // set by its content hash and use it only if an SCP statement names it; + // a peer that did not get it requests it as before. + void pushTxSet(Hash const& txSetHash, uint32_t slotIndex); + +#ifdef BUILD_TESTS + // Make pushTxSet a no-op, as if every push were lost: peers then get + // the leader's set through the request path only. + void + setDropTxSetPushForTesting(bool drop) + { + mDropTxSetPushForTesting = drop; + } +#endif + // Get top transactions from Rust overlay's mempool for TX set building. // Blocks until the overlay responds, shuts down, or disconnects. std::vector getTopTransactions(size_t count); @@ -100,6 +117,9 @@ class RustOverlayManager std::unique_ptr mOverlayIPC; std::atomic mShuttingDown{false}; std::vector mExtraKnownPeers; +#ifdef BUILD_TESTS + bool mDropTxSetPushForTesting{false}; +#endif // Config KNOWN_PEERS plus any peers added via addKnownPeerForTesting. std::vector effectiveKnownPeers() const; diff --git a/src/overlay/test/IPCTests.cpp b/src/overlay/test/IPCTests.cpp index fde1d04a8c..7024021496 100644 --- a/src/overlay/test/IPCTests.cpp +++ b/src/overlay/test/IPCTests.cpp @@ -100,6 +100,28 @@ TEST_CASE("IPC message types", "[overlay][ipc]") close(sockets[1]); } + SECTION("CoreToOverlay::PushTxSet") + { + int sockets[2]; + REQUIRE(socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) == 0); + + // [hash:32][slotSeq:4]; the value must match the Rust overlay's + // MessageType::PushTxSet. + REQUIRE(static_cast(IPCMessageType::PUSH_TX_SET) == 14); + IPCMessage msg; + msg.type = IPCMessageType::PUSH_TX_SET; + msg.payload.assign(36, 7); + + REQUIRE(ipc::sendMessage(sockets[0], msg)); + auto recvMsg = ipc::receiveMessage(sockets[1]); + REQUIRE(recvMsg.has_value()); + REQUIRE(recvMsg->type == IPCMessageType::PUSH_TX_SET); + REQUIRE(recvMsg->payload == msg.payload); + + close(sockets[0]); + close(sockets[1]); + } + SECTION("OverlayToCore::SCPReceived") { int sockets[2]; From d0fe62652119447fac61a1cc356e1d6ff80409a1 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Tue, 10 Nov 2026 23:54:56 +0000 Subject: [PATCH 3/4] Document the remaining gaps of the tx-set push Corrections to docs/rust-overlay/txset-fetching.md from review: what happens to a push that is in flight when a connection closes, that the one-push-in-flight bound covers pushes only, which peer gets a token back, and that the 100-entry cache is not protected by the limits on unclaimed sets. Also mark the congestion-window patch record as whitespace-exact, like the other vendor patch. No code change. --- docs/rust-overlay/txset-fetching.md | 22 +++++++++++++++------- overlay/vendor/.gitattributes | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/rust-overlay/txset-fetching.md b/docs/rust-overlay/txset-fetching.md index b9294e399a..a9f3fd306f 100644 --- a/docs/rust-overlay/txset-fetching.md +++ b/docs/rust-overlay/txset-fetching.md @@ -88,13 +88,17 @@ land. Every other request is answered: a repeated one, one that arrives later (a peer only asks when it does not hold the set, so it lost or dropped the pushed copy), and one from a peer the set was not pushed to. A failed push write answers a skipped request after all, and any connection to the peer -closing drops the record for that peer. +closing drops the record for that peer. (If a connection closes while a push +is still being written, that push is no longer recorded: a request that +crosses it gets a second copy, and if the write then fails, a request that +was skipped before the close is not answered and falls to Core's retry.) A leader keeps at most one push in flight per peer (`TXSET_PUSH_BUSY` in the log when a peer is left out). A peer that does not read its tx-set stream is -not sent further sets (it requests them), so it pins at most one set in the -leader's memory; closing a connection does not release that count, only the -end of the write does. +not sent further sets (it requests them), so it pins at most one pushed set +in the leader's memory; closing a connection does not release that count, +only the end of the write does. Answers to that peer's requests queue behind +the stuck write under the send permits that bound all tx-set responses. A node that did not receive the push (not connected to the leader, connected later, or busy with an earlier push) has no cached set and uses the request @@ -110,8 +114,9 @@ relayed the proposal, which holds the pushed copy. Limits on what a peer can make a node decode and keep: - Rate: every tx-set frame takes a token from a per-peer bucket (burst 8, 4 - per second). A frame that answers a request of this node returns its token, - so only frames nobody asked for use the allowance. With the bucket empty a + per second). A frame that delivers a set this node has a request pending + for returns its token (to the peer that delivered it, whichever peer was + asked), so only frames nobody is waiting for use the allowance. With the bucket empty a frame still passes if this node sent that peer a request it has not used up (one frame per request, at most 8 saved). Anything else is dropped before decompression (`overlay.txset-push.dropped`). @@ -129,7 +134,10 @@ Limits on what a peer can make a node decode and keep: reconnects starts with a full allowance. Many colluding peers can use the shared byte budget to push out a large honest set, and a peer that delivers the leader's set first and then pushes four more evicts it (the receiver - then requests the set). + then requests the set). The cache itself still holds 100 sets of any kind, + first in, first out: with up to 20 unclaimed sets per peer, five peers can + fill it and cycle out sets the node built or already uses (on the base one + peer could, through prefetches). ## Core requests and network prefetches diff --git a/overlay/vendor/.gitattributes b/overlay/vendor/.gitattributes index 8b01954fbc..7874c6519d 100644 --- a/overlay/vendor/.gitattributes +++ b/overlay/vendor/.gitattributes @@ -3,3 +3,4 @@ libp2p-core/CHANGELOG.md -whitespace libp2p-core/src/generated/* -whitespace multistream-select/CHANGELOG.md -whitespace stream-priority.patch -whitespace +congestion-window.patch -whitespace From 2f9baef2435fc7e9d153c16718e260519348ff83 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Wed, 11 Nov 2026 02:10:35 +0000 Subject: [PATCH 4/4] Describe the tx-set push experiment and its perfnet results in the README The branch README now covers the latest experiment (the leader pushes its tx set, measured on 8f7768be7) with the previous experiment's values (the leader schedule alone, daa965f4c) as "before": 1,000 ms target at 3,000 TPS, block time windows, stage timings, the push counters, the duplicate inclusions that overlay-only mode allows and why they are accepted, and the test setup. README only. --- README.md | 312 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 193 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index f6d812cfd7..9a5a5b1a51 100644 --- a/README.md +++ b/README.md @@ -1,131 +1,188 @@ -# stellar-core: leader schedule without SCP nomination (experimental branch) +# stellar-core: leader schedule and leader tx-set push (experimental branch) **This branch is an experiment. It is not production code and must not run on a real network.** It has no feature flag, no protocol gate, and it is not compatible with nodes that nominate. -Results below were measured on commit `daa965f4c` on 2026-11-10 and are one run per setting. +The latest experiment (the leader pushes its tx set) was measured on commit `8f7768be7` on +2026-11-11. "Before" is the previous experiment on this branch (the leader schedule alone), commit +`daa965f4c`, measured on 2026-11-10 on the same fixture. One run per setting on each side. ## What this branch is It is the overlay-v2 stack up to [#5459](https://github.com/stellar/stellar-core/pull/5459) (Rust overlay over QUIC, millisecond close times, compressed tx-set responses, leader-only tx-set -construction) plus one change: **the SCP nomination phase is deleted and replaced by a leader -schedule that every node computes locally.** Draft PR: -[#5495](https://github.com/stellar/stellar-core/pull/5495). +construction) plus two experiments: + +1. **No nomination** ([#5495](https://github.com/stellar/stellar-core/pull/5495), the "before" + below). The SCP nomination phase is deleted and replaced by a leader schedule that every node + computes locally. + - **Schedule.** For slot N, the nodes of the transitive quorum are sorted by + `SHA256(hash of ledger header N-1 || node id)`. The leader of round r is entry `(r-1) mod n`. + - **Rounds.** A round is a ballot counter. The round's leader builds its value and goes straight + to the ballot protocol with `(r, value)`. The other nodes vote for the leader's ballot once it + passes proposal validation. A timeout (the ballot timeout settings) hands the slot to the next + leader. +2. **The leader pushes its tx set** ([#5497](https://github.com/stellar/stellar-core/pull/5497), + the latest experiment). Before, a follower saw the leader's proposal, asked for the tx set and + received it about two round trips later. Now: + - **Push.** The round's leader sends the set it is about to propose to every connected peer as + soon as the set is built, without waiting for a request. The proposal follows about 70 ms + later. + - **Use.** A follower's overlay verifies a pushed set and keeps it under the hash of its own + content. Core only gets it when an SCP statement names that hash, so a pushed set that does not + match the proposal is never used. + - **Fallback.** The request path is unchanged, for a node that did not get the push. + - **One flight.** Every QUIC connection starts with an 8 MiB congestion window, so that a set + leaves in one flight instead of being paced over a round trip. + - Details and limits: [docs/rust-overlay/txset-fetching.md](docs/rust-overlay/txset-fetching.md). + +Unchanged by both: the ballot protocol's accept, confirm and commit rules, value validation, apply, +ledger state and meta. -- **Schedule.** For slot N, the nodes of the transitive quorum are sorted by - `SHA256(hash of ledger header N-1 || node id)`. The leader of round r is entry `(r-1) mod n`. -- **Rounds.** A round is a ballot counter. The round's leader builds its value and goes straight to - the ballot protocol with `(r, value)`. The other nodes vote for the leader's ballot once it passes - proposal validation. A timeout (the ballot timeout settings) hands the slot to the next leader. -- **Unchanged.** The ballot protocol's accept, confirm and commit rules, value validation, apply, - ledger state and meta. +## Performance results -The question it answers: how much of a ledger's time is nomination, and what limits block time once -nomination is gone? +Target block time 1,000 ms, 3,000 TPS, 30 validators in 4 regions. -## Performance results +| | Before (`daa965f4c`) | With the tx-set push (`8f7768be7`) | +|---|---|---| +| Block time windows | **missed** the first window; the load stops there | **on target in all four windows**, on all 30 nodes | +| p75, average over 30 nodes | 1,386 ms (limit 1,200) | 1,091 / 1,096 / 1,109 / 1,115 ms | +| p99, average over 30 nodes | 1,599 ms (limit 2,000) | 1,227 / 1,243 / 1,253 / 1,290 ms | +| Time per ledger | 1.33 s | 1.05-1.07 s | +| Tx-set fetch as seen by a follower's Core | 217 ms | 16 ms | -| Target block time | Load | Verdict | Block time measured (average over 30 nodes) | -|---|---|---|---| -| **2,000 ms** | 3,000 TPS | **PASS**, all four windows | p75 2,066-2,109 ms, p99 2,286-2,324 ms | -| **1,000 ms** | 3,000 TPS | **FAIL**, first window | p75 1,386 ms (limit 1,200), p99 1,599 ms | +In short: before, the stages of one ledger added up to more than a second and the network backed +up. With the push the tx set is already at the follower when the proposal arrives, a ledger's +stages fit into about 0.9 s, and the network keeps up with the load at a 1 s target. -In short: at 2 s the network keeps up and has more slack per ledger than with nomination. At 1 s it -settles at about 1.33 s per ledger. Nothing breaks; the stages of one ledger simply add up to more -than a second. +**One caveat on correctness checks.** Every node's ledgers held 60 transactions more than the +2,880,000 offered: about 60 transactions were included in two ledgers. This is an artifact of the +test mode, in which ledgers are not applied (see [The inclusion check](#the-inclusion-check)). The +run was judged with the exact check and reported FAIL for that check alone; the check has since +been relaxed to allow such duplicates +([performance-testing#4](https://github.com/stellar-experimental/performance-testing/pull/4)). +All other checks passed. -### 2,000 ms target: PASS +### 1,000 ms target, with the tx-set push -Load ran its full 900 s. Every node passed every window. +Load ran its full 900 s. Every node was on target in every window. | Window | p75, average (range over nodes) | p99, average (range over nodes) | |---|---|---| -| 1 | 2,109 ms (2,086-2,128) | 2,324 ms (2,282-2,359) | -| 2 | 2,066 ms (2,058-2,076) | 2,286 ms (2,239-2,328) | -| 3 | 2,082 ms (2,067-2,091) | 2,296 ms (2,251-2,333) | -| 4 | 2,094 ms (2,080-2,105) | 2,303 ms (2,262-2,344) | - -Checks, all passed: -- **Inclusion:** every node's ledgers hold exactly the 2,880,000 offered transactions. -- **No fork**, all 30 nodes in sync, still closing ledgers after the load. +| 1 | 1,091 ms (1,085-1,096) | 1,227 ms (1,205-1,273) | +| 2 | 1,096 ms (1,090-1,105) | 1,243 ms (1,227-1,272) | +| 3 | 1,109 ms (1,104-1,116) | 1,253 ms (1,231-1,360) | +| 4 (2 minutes) | 1,115 ms (1,109-1,125) | 1,290 ms (1,245-1,396) | + +Limits: p75 below 1,200 ms, p99 at most 2,000 ms. + +Checks: +- **No fork**, all 30 nodes in sync, still closing ledgers after the load: passed. - **Error counters zero** (invalid signatures, invariant failures, publish failures, invalid values, - lost sync), no node restarts. + lost sync), no node restarts: passed. - **No wasted rounds:** every ledger was decided by the first leader. 0 leader timeouts, 0 ballot timeouts, 0 empty transaction sets. +- **Inclusion:** 2,880,060 transactions in every node's ledgers against 2,880,000 offered. See + below. -Against the same fixture on #5459's head `ebdd73635` (with nomination), run the same day on the same -instance type: +### Before and after, stage by stage -| 5-minute means, median node | #5459 `ebdd73635` | This branch `daa965f4c` | -|---|---:|---:| -| Verdict | PASS | PASS | -| p75 per window | 2,112 / 2,099 / 2,101 / 2,099 ms | 2,109 / 2,066 / 2,082 / 2,094 ms | -| p99 per window | 2,334 / 2,391 / 2,324 / 2,354 ms | 2,324 / 2,286 / 2,296 / 2,303 ms | -| Ledgers per 5 minutes | 147-148 | 149-150 | -| Transactions per ledger | 6,092 | 6,003 | -| Trigger to first ballot | 1.21 s | 0.88 s | -| First ballot to externalize | 0.29 s | 0.43 s | -| **Trigger to externalize** | **1.50 s** | **1.31 s** | -| Leader tx-set build | 0.28 s | 0.27 s | -| Tx-set fetch (follower) | 0.24 s | 0.26 s | -| Tx-set validation (follower) | 0.20 s | 0.19 s | -| Ledger close | 0.164 s | 0.162 s | - -- Block time is the same on both, as expected: when the network keeps up, the 2 s trigger sets the - cadence. -- The time from trigger to externalize is 0.19 s shorter. The part before the first ballot that is - not build, fetch or validation fell from about 0.48 s (nomination rounds) to about 0.15 s. -- Some of that came back in the ballot phase. Nodes now start their ballot at different times, as - each finishes validating the leader's set, so the early ones wait for a quorum of the others. - -### 1,000 ms target: FAIL - -The first 5-minute window missed on all 30 nodes, and the test stops the load at the first miss. - -| Window 1 | Value | -|---|---| -| p75, average (range over nodes) | 1,386 ms (1,375-1,399); the limit is below 1,200 ms | -| p99, average (range over nodes) | 1,599 ms (1,582-1,628); the limit is 2,000 ms | -| Ledgers per 5 minutes | 225 (1.33 s each) | -| Transactions per ledger | 3,750, the cap, on every ledger | -| Leader timeouts, ballot timeouts, empty tx sets | 0, 0, 0 | -| Error counters, in sync, no fork, still closing | all passed | - -The network was over capacity at this block time: 3,000 TPS x 1.33 s is 4,000 transactions offered -per ledger against a cap of 3,750, so the mempool backed up and every ledger started the instant -the previous one closed. Inclusion is not checked after an early stop. No baseline was run at this -setting. - -### Where a ledger's time goes +5-minute means at the end of the first window, median over the 30 nodes. -| Stage (5-minute means, median node) | 2,000 ms target (6,000 txs) | 1,000 ms target (3,750 txs) | +| | Before (`daa965f4c`) | With the tx-set push (`8f7768be7`) | |---|---:|---:| -| Leader builds the tx set | 0.27 s | 0.25 s | -| Trigger to first ballot (includes the build, the fetch and validation) | 0.88 s | 0.70 s | -| First ballot to externalize | 0.43 s | 0.41 s | -| Ledger close (150 ms / 187 ms of it is simulated apply) | 0.16 s | 0.20 s | -| **Sum** | **1.47 s** | **1.30 s** | - -Halving the transaction set took only 0.17 s off the cycle. What remains is mostly cross-region -round trips, not CPU: - -1. **Tx-set dissemination takes about 2.5 round trips.** A follower receives the proposal, requests - the set from the proposer, and the response takes two flights. Seen from us-west-2, the fetch - took 27 ms from the same region, 150 ms from us-east-1, 301 ms from eu-central-1 and 362 ms from - ap-southeast-1. Nothing is pushed, and transactions are sent in full although followers already - hold nearly all of them. -2. **Followers validate before they vote, and the ballot then needs four one-way trips.** -3. **About 70 ms pass between the leader finishing its set and the proposal reaching a peer in the - same region.** SCP state, including the tx set, is written to the database before each - broadcast. That this write accounts for most of the 70 ms is inferred, not timed. -4. **The leader's build is on the critical path.** At 1 s nothing can be built ahead of the trigger. +| Ledgers per 5 minutes | 225 | 285 | +| Transactions per ledger | 3,750, the cap, on every ledger | 3,155 (what was offered; below the cap) | +| Leader tx-set build | 0.25 s | 0.15 s | +| Tx-set fetch (follower's Core, request to delivery) | 0.217 s | **0.016 s** | +| Tx sets a follower had to wait for from the network | every one | none in this window | +| Tx-set validation (follower) | 0.12 s | 0.11 s | +| Trigger to first ballot (includes the build, the fetch and validation) | 0.70 s | **0.42 s** | +| First ballot to externalize | 0.41 s | 0.33 s | +| Ledger close (simulated apply, 50 us per operation) | 0.20 s | 0.16 s | +| **Sum: trigger to close** | **1.30 s** | **0.91 s** | +| Leader timeouts, ballot timeouts, empty tx sets | 0, 0, 0 | 0, 0, 0 | + +What the push did, over the whole load (totals of the 30 nodes): + +| | | +|---|---:| +| Tx sets pushed (one per peer) | 28,507 | +| Deliveries that were already at the follower when its Core asked | 28,181 | +| Requests that crossed a push (the leader did not send a second copy) | 64 | +| Pushes dropped or evicted by a follower's limits, write errors | 0 | +| Requests Core had to repeat after its 1 s retry interval | 0 | + +- **The tx-set round trips are gone.** On one follower whose logs were saved (us-west-2), all 817 + sets it received during the load arrived by push before its Core asked, a median 38 ms ahead of + the request (minimum 18 ms). Measured from four nodes in four regions, a follower's Core held the + set a median 126 ms after the leader finished building it (76-169 ms by region pair); before, that + took about 480 ms on a 147 ms link. +- **The two runs are in different regimes.** Before, a ledger could not close in less than 1.33 s, + so 3,000 TPS filled every set to the cap and the mempool backed up. Now ledgers hold what was + offered. That also makes stages the push does not touch cheaper: the leader pulls 3,200 + candidates instead of 7,700, and sets are smaller. +- **Two changes were measured together:** the push and the 8 MiB congestion window. One run cannot + say how much each contributes. +- **Block time is about 1.05 s although the stages sum to 0.91 s**, and the windows drifted up by + about 25 ms over the 16 minutes. Neither was investigated. + +### The inclusion check + +The test compares the transactions in every node's ledgers with the number offered. All 30 nodes +held exactly 60 more (the same on every node, so no fork). The generators had submitted exactly +2,880,000 and every node had pulled each transaction once, so about 60 transactions (0.002%) were +included in two ledgers. + +Why this happens in the test and not on a network that applies ledgers: + +1. A leader that is not backed up proposes everything in its mempool, including a transaction + that was submitted a moment ago and is still being flooded to the far nodes (up to about 0.5 s). +2. With the push, a follower closes the ledger about 0.5 s after the leader built its set, so now + and then a far node closes the ledger before its own copy of that transaction has arrived. +3. The overlay's mempool removes a ledger's transactions when the ledger closes and keeps no memory + of them, on purpose. The late copy is admitted as a new transaction. +4. On a real network Core rejects that copy when the node next builds a set, because its sequence + number was already used. The test runs in "overlay-only" mode: ledgers are not applied, sequence + numbers never advance, and that check is switched off. So the node includes the transaction a + second time. + +That this path produced the 60 is inferred from the counters; the logs carry no transaction hashes. +Before, a follower closed the ledger about a second after the leader's build, well after flooding +had finished, so the two never met: the completed "before" run at 2,000 ms held exactly 2,880,000. + +It is accepted as an artifact of overlay-only mode. The check now allows up to 1 in 10,000 +transactions more than offered and reports how many there were; fewer than offered, more than +that, and nodes that disagree still fail. + +### Not rerun: the 2,000 ms target + +The 2,000 ms target was measured for the "before" commit only: PASS in all four windows (p75 +2,066-2,109 ms, p99 2,286-2,324 ms), with exactly 2,880,000 transactions on every node, and 0.19 s +less from trigger to externalize than with nomination (1.31 s against 1.50 s on #5459). The details +are in [#5495](https://github.com/stellar/stellar-core/pull/5495). It was not repeated with the +push. + +### Where a ledger's time goes now + +With the tx-set round trips gone, what is left of the 0.91 s from trigger to close at 1,000 ms: + +1. **The ballot: 0.33 s.** Followers validate before they vote, and the ballot then needs four + one-way trips. +2. **The leader's build: 0.15 s**, on the critical path. With the trigger mostly on time, the + first-round leader also builds once before the trigger and discards that set unless it was + capacity limited (562 builds for 285 ledgers). +3. **Validation: 0.11 s.** +4. **About 70 ms between the leader finishing its set and its proposal leaving.** SCP state, + including the tx set, is written to the database before each broadcast. That this write accounts + for most of the 70 ms is inferred, not timed. The push already leaves during this time. +5. **Ledger close: 0.16 s**, almost all of it the simulated apply. ## How it was measured -Both runs use the `block_time_check` mission of perfnet, SDF's tooling for performance networks on -AWS. Each run launches a fresh network, brings it to the target settings, runs the load, judges -it, and terminates the network. +Both runs (before and after) use the `block_time_check` mission of perfnet, SDF's tooling for +performance networks on AWS. Each run launches a fresh network, brings it to the target settings, +runs the load, judges it, and terminates the network. ### Network topology @@ -144,8 +201,9 @@ it, and terminates the network. | | | |---|---| | Instance type | c6id.2xlarge: 8 vCPU (Intel Ice Lake, with SHA extensions), 16 GiB RAM, local NVMe | -| Build | Commit `daa965f4c`, the perftests package `30.0.1-3674.daa965f4ce.noble~perf.leader.schedule.no.nomination~perftests`. All 30 nodes reported that commit in their build label | -| Overlay | The Rust overlay of this stack: QUIC, one connection per peer | +| Build, with the push | Commit `8f7768be7`, the perftests package `30.0.1-3675.8f7768be7a.noble~perf.leader.txset.push~perftests`. All 30 nodes reported that commit in their build label | +| Build, before | Commit `daa965f4c`, the perftests package `30.0.1-3674.daa965f4ce.noble~perf.leader.schedule.no.nomination~perftests` | +| Overlay | The Rust overlay of this stack: QUIC, one connection per peer. With the push, every connection starts with an 8 MiB congestion window (quinn's default before) | | Ledger protocol | 30, upgraded from genesis during preparation | | Database | SQLite | @@ -167,7 +225,7 @@ TESTING_IGNORE_LEDGER_TIME_UPGRADE_BOUNDS=true TESTING_MAX_CLASSIC_BYTE_ALLOWANCE=1048576 TESTING_MAX_SOROBAN_BYTE_ALLOWANCE=9437184 LOADGEN_PREGENERATED_TRANSACTIONS_FILE="/var/lib/stellar-core/loadgen-txs.xdr" -OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING=[25] # [50] in the 1,000 ms run +OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING=[50] # [25] in the 2,000 ms run OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING=[100] ``` @@ -176,16 +234,16 @@ off, so a node that crashes or falls behind stays that way and fails the run. ### Network settings (applied by upgrade before the load) -| Setting | 2,000 ms run | 1,000 ms run | +| Setting | 1,000 ms runs (before and after) | 2,000 ms run (before only) | |---|---:|---:| -| Target ledger close time | 2,000 ms | 1,000 ms | +| Target ledger close time | 1,000 ms | 2,000 ms | | SCP timeouts (ballot and nomination, initial and increment) | 2,000 ms | 2,000 ms | -| Maximum transactions per ledger (125% of the offered load per block) | 7,500 | 3,750 | +| Maximum transactions per ledger (125% of the offered load per block) | 3,750 | 7,500 | | Maximum tx-set size in bytes | 10 MiB (1 MiB classic + 9 MiB Soroban) | same | | Soroban limits | raised to fit the load | raised to fit the load | The leader timer of this branch uses the ballot timeout, so a round's leader had 2 s (4 s with its -one extension) in both runs. +one extension) in every run. ### Load @@ -195,14 +253,14 @@ one extension) in both runs. | Rate | 3,000 TPS offered: every validator generates 100 TPS | | Accounts | 1,000,000 genesis accounts, 33,333 per generator | | Duration | 60 s warm-up, then 900 s measured | -| Apply | Skipped ("overlay-only" mode) and replaced by a sleep per operation, sized as a 150 ms budget per ledger at the offered load: 25 us per operation at 2,000 ms, 50 us at 1,000 ms | +| Apply | Skipped ("overlay-only" mode) and replaced by a sleep per operation, sized as a 150 ms budget per ledger at the offered load: 50 us per operation at 1,000 ms, 25 us at 2,000 ms | -Commands (the two runs differ only in the block time): +Command (before and after differ only in the build): ``` block_time_check.py \ - --stellar-core-version 30.0.1-3674.daa965f4ce.noble~perf.leader.schedule.no.nomination~perftests \ - --block-time-ms 2000 \ # 1000 for the second run + --stellar-core-version 30.0.1-3675.8f7768be7a.noble~perf.leader.txset.push~perftests \ + --block-time-ms 1000 \ --soroban-tps 3000 --soroban-type sac --classic-tps 0 \ --simulate-apply-budget-ms 150 --load-duration-s 900 \ --instance-type c6id.2xlarge --save-logs always @@ -217,21 +275,37 @@ Defaults that matter: 10 organizations, validators as generators, `--scp-timeout 5-minute windows after the warm-up. - **PASS** needs, on every node in every window: p75 within [0.8 T, 1.2 T), p99 at most 2 T, and at least one close per 2 T on average, where T is the target block time. It also needs every node's - ledgers to hold exactly the offered transactions, zero error counters, all nodes in sync, no - fork, and the network still closing ledgers afterwards. -- The load stops at the first window that misses, so a FAIL can rest on one window. + ledgers to hold the offered transactions, zero error counters, all nodes in sync, no fork, and + the network still closing ledgers afterwards. +- "Hold the offered transactions" was an exact comparison when these runs were judged. It now + allows up to 1 in 10,000 more, for the duplicates overlay-only mode lets through (see + [The inclusion check](#the-inclusion-check)). +- The load stops at the first window that misses, so a FAIL can rest on one window (as in the + "before" run). ### Limits of these numbers -- One run per setting, and one baseline run (at 2,000 ms only). +- One run on each side. The "before" numbers at 1,000 ms come from a single 5-minute window, and + from a commit that differs from this branch's base in formatting only. +- The push and the 8 MiB congestion window were not measured separately. - Apply is simulated, so these are consensus and overlay numbers, not end-to-end ledger close. -- `scp.timing.trigger-to-ballot` records 0 when a follower votes before its own trigger fires, so it - is not strictly comparable with the nomination build. Triggers were on time in the 2,000 ms run. -- The per-stage timeline and the fetch times by region come from one node's logs at 2,000 ms and - four nodes' logs at 1,000 ms. +- `scp.timing.trigger-to-ballot` records 0 when a follower votes before its own trigger fires. +- The timings between a leader and its followers come from live logs of four nodes (one per + region) and the saved logs of one node. ## Known limitations of the prototype +Tx-set push: +- The push is not relayed. A node that is not connected to the leader requests the set from the + peer that relayed the proposal. +- A follower that drops a pushed set on arrival (its own limits for unrequested sets) only gets + it through Core's 1 s retry. Honest leaders do not trigger those limits. +- The limits on unrequested sets can be stepped around by a peer that sends SCP envelopes naming + unknown tx sets, and the 8 MiB congestion window applies to every stream of a connection. Both + are prototype settings. +- Transactions can be included twice in overlay-only mode (above). + +Leader schedule: - The leader schedule's node set is whatever transitive quorum each node currently knows. It assumes a symmetric, static quorum. - Upgrades: a value needs a quorum of nodes that armed the same upgrades; there is no stripping or @@ -245,4 +319,4 @@ Defaults that matter: 10 organizations, validators as generators, `--scp-timeout ## Building and the rest of the documentation Unchanged from stellar-core: see [INSTALL.md](./INSTALL.md), [CONTRIBUTING.md](./CONTRIBUTING.md) -and [`./docs`](./docs). The full test suite passes on `daa965f4c`, also with `--all-versions`. +and [`./docs`](./docs). The full test suite passes on `8f7768be7`, also with `--all-versions`.