diff --git a/overlay/src/flood/mod.rs b/overlay/src/flood/mod.rs index 9e796310e..59a6adcaa 100644 --- a/overlay/src/flood/mod.rs +++ b/overlay/src/flood/mod.rs @@ -11,7 +11,7 @@ mod tx_buffer; mod txset; pub use inv_batcher::InvBatcher; -pub use inv_messages::{GetData, InvBatch, InvEntry, TxStreamMessage}; +pub use inv_messages::{GetData, InvBatch, InvEntry, TxStreamMessage, INV_BATCH_MAX_SIZE}; pub use inv_tracker::InvTracker; pub use mempool::Mempool; pub use pending_requests::PendingRequests; diff --git a/overlay/src/libp2p_overlay.rs b/overlay/src/libp2p_overlay.rs index 3ec67a470..0e11b7864 100644 --- a/overlay/src/libp2p_overlay.rs +++ b/overlay/src/libp2p_overlay.rs @@ -999,18 +999,7 @@ impl StellarOverlay { .flood_advertised .fetch_add(peers.len() as u64, Ordering::Relaxed); - let inv_entry = InvEntry { hash, fee_per_op }; - - // Add to batcher for each peer, send batch immediately when full - for peer in &peers { - let batch_to_send = { - let mut batcher = self.state.inv_batcher.write().await; - batcher.add(*peer, inv_entry.clone()) - }; - if let Some(batch) = batch_to_send { - send_inv_batch(&self.state, *peer, batch).await; - } - } + queue_inv(&self.state, &peers, InvEntry { hash, fee_per_op }).await; } /// Fetch TX set from a peer - preferring the peer who sent us the SCP message referencing it @@ -1367,13 +1356,30 @@ async fn send_to_peer_stream( send_to_peer_stream_parts(state, peer_id, stream_type, &[data]).await } -/// All parts form one frame, including across partial writes and retries. -/// The per-route lock stays held until that entire frame has been written. +/// Send one frame made of `parts`, without assembling them into one buffer. async fn send_to_peer_stream_parts( state: &SharedState, peer_id: PeerId, stream_type: StreamType, parts: &[&[u8]], +) -> io::Result<()> { + send_peer_frames(state, peer_id, stream_type, framed_io::Frames::One(parts)).await +} + +/// Write `frames` to the peer's stream for `stream_type`, opening it if +/// needed. The per-route lock stays held until every frame has been written, +/// including across partial writes, so frames never interleave mid-frame. +/// +/// A failed write drops the stream and retries the whole of `frames` on a +/// reopened one (3 attempts). For a `Frames::Batch` this can deliver complete +/// frames that reached the peer before the failure a second time; TX +/// receivers drop such duplicates via `tx_seen`. The peer's reader on the old +/// stream sees it end, possibly mid-frame, and exits. +async fn send_peer_frames( + state: &SharedState, + peer_id: PeerId, + stream_type: StreamType, + frames: framed_io::Frames<'_>, ) -> io::Result<()> { // Retry up to 2 times (3 attempts total) for reliability const MAX_RETRIES: usize = 2; @@ -1453,15 +1459,15 @@ async fn send_to_peer_stream_parts( // Never send a request to an old SCP reader: it would silently // discard it. Keep requests isolated from legacy response writes. drop(stream_guard); - return Box::pin(send_to_peer_stream_parts( + return Box::pin(send_peer_frames( state, peer_id, StreamType::LegacyTxSetRequest, - parts, + frames, )) .await; } - match framed_io::write_frame_parts(&mut stream.stream, parts).await { + match frames.write(&mut stream.stream).await { Ok(()) => return Ok(()), Err(e) => { // Clear the broken stream @@ -1506,6 +1512,26 @@ async fn flush_inv_batch_to_peer(state: &Arc, peer: PeerId) { } } +/// Queue `entry` for every peer in `peers` under one INV batcher lock, rather +/// than one lock round trip per peer, then send the batches it completed. +/// `send_inv_batch` only spawns the write, so they still go out at once. +async fn queue_inv(state: &Arc, peers: &[PeerId], entry: InvEntry) { + let full_batches: Vec<(PeerId, InvBatch)> = { + let mut batcher = state.inv_batcher.write().await; + peers + .iter() + .filter_map(|peer| { + batcher + .add(*peer, entry.clone()) + .map(|batch| (*peer, batch)) + }) + .collect() + }; + for (peer, batch) in full_batches { + send_inv_batch(state, peer, batch).await; + } +} + /// Send an INV batch to a peer async fn send_inv_batch(state: &Arc, peer: PeerId, batch: InvBatch) { let batch_size = batch.entries.len() as u64; @@ -1548,7 +1574,7 @@ async fn send_inv_batch(state: &Arc, peer: PeerId, batch: InvBatch) } /// Read length-prefixed frame from stream -async fn read_framed(stream: &mut Stream) -> io::Result> { +async fn read_framed(stream: &mut R) -> io::Result> { let mut len_buf = [0u8; 4]; stream.read_exact(&mut len_buf).await?; let len = u32::from_be_bytes(len_buf) as usize; @@ -1713,14 +1739,24 @@ async fn handle_inbound_scp_streams( } } +/// Inbound TX streams carry many small frames, and GETDATA responses arrive +/// as batches of them. Reading through a buffer copies whatever QUIC has +/// received in one call, instead of two reads (each taking the connection +/// lock) per frame. It never waits for more than is available, and larger +/// frames bypass it once it is empty. +fn buffered_tx_stream(stream: R) -> futures::io::BufReader { + futures::io::BufReader::with_capacity(TX_RESPONSE_BATCH_BYTES, stream) +} + /// Handle inbound TX streams from peers async fn handle_inbound_tx_streams(mut incoming: IncomingStreams, state: Arc) { - while let Some((peer_id, mut stream)) = incoming.next().await { + while let Some((peer_id, stream)) = incoming.next().await { info!("TX_STREAM: Accepted inbound TX stream from {}", peer_id); state.metrics.inbound_live.fetch_add(1, Ordering::Relaxed); let state = state.clone(); tokio::spawn(async move { + let mut stream = buffered_tx_stream(stream); loop { match read_framed(&mut stream).await { Ok(data) => { @@ -1730,7 +1766,7 @@ async fn handle_inbound_tx_streams(mut incoming: IncomingStreams, state: Arc { state.metrics.error_read.fetch_add(1, Ordering::Relaxed); @@ -1745,18 +1781,13 @@ async fn handle_inbound_tx_streams(mut incoming: IncomingStreams, state: Arc, - peer_id: &PeerId, - data: &[u8], - stream: &mut Stream, -) { +async fn handle_tx_stream_message(state: &Arc, peer_id: &PeerId, data: &[u8]) { match TxStreamMessage::decode(data) { Ok(TxStreamMessage::InvBatch(batch)) => { handle_inv_batch(state, peer_id, batch).await; } Ok(TxStreamMessage::GetData(getdata)) => { - handle_getdata(state, peer_id, getdata, stream).await; + handle_getdata(state, peer_id, getdata); } Ok(TxStreamMessage::Tx(tx)) => { handle_tx_response(state, peer_id, tx).await; @@ -1778,29 +1809,29 @@ async fn handle_inv_batch(state: &Arc, peer_id: &PeerId, batch: Inv peer_id ); - let mut to_request: Vec<[u8; 32]> = Vec::new(); - - for entry in batch.entries { - // Check if we already have this TX - { - let seen = state.tx_seen.read().await; - if seen.contains(&entry.hash) { - // Already have it, just record this peer as a source (for relay tracking) - continue; - } - } - - // Record this peer as a source for round-robin GETDATA - let is_first = { - let mut tracker = state.inv_tracker.write().await; - tracker.record_source(entry.hash, *peer_id) - }; - - // If this is the first INV for this TX, we should request it - if is_first { - to_request.push(entry.hash); - } - } + // Each structure is locked once per batch rather than once per entry, and + // the two locks are never held together. Entries keep their batch order. + // Skip TXs we already have; they are not recorded as sources. + let unseen: Vec<[u8; 32]> = { + let seen = state.tx_seen.read().await; + batch + .entries + .iter() + .map(|entry| entry.hash) + .filter(|hash| !seen.contains(hash)) + .collect() + }; + // Record this peer as a source for round-robin GETDATA retries, and + // request only TXs it is the first to advertise. + let to_request: Vec<[u8; 32]> = if unseen.is_empty() { + Vec::new() + } else { + let mut tracker = state.inv_tracker.write().await; + unseen + .into_iter() + .filter(|hash| tracker.record_source(*hash, *peer_id)) + .collect() + }; // Send GETDATA for TXs we don't have if !to_request.is_empty() { @@ -1855,67 +1886,156 @@ async fn handle_inv_batch(state: &Arc, peer_id: &PeerId, batch: Inv } /// Handle GETDATA message - respond with requested TXs -async fn handle_getdata( - state: &Arc, - peer_id: &PeerId, - getdata: GetData, - _stream: &mut Stream, -) { +fn handle_getdata(state: &Arc, peer_id: &PeerId, getdata: GetData) { debug!( "TX_GETDATA_RECV: Peer {} requesting {} TXs", peer_id, getdata.hashes.len() ); - for hash in getdata.hashes { - // Look up TX in our buffer - let tx = { - let mut buffer = state.tx_buffer.write().await; - buffer.get(&hash) - }; + let state = Arc::clone(state); + let peer = *peer_id; + tokio::spawn(async move { serve_getdata(&state, peer, getdata.hashes).await }); +} - if let Some(tx) = tx { - state - .metrics - .flood_fulfilled - .fetch_add(1, Ordering::Relaxed); - // Send TX response. Buffered bytes were validated on entry, so - // framing them (concat) yields valid wire XDR by construction. - let encoded = tx.to_flood_frame(); +/// GETDATA responses are written as batches of complete TX frames of at most +/// this many bytes. Inbound TX streams are read through a buffer of the same +/// size, so one read can take in a whole batch. +const TX_RESPONSE_BATCH_BYTES: usize = 64 * 1024; + +/// A transaction's length-prefixed `StellarMessage::Transaction` frame size. +fn tx_frame_len(tx: &ValidatedTx) -> usize { + 4 + tx + .flood_frame_parts() + .iter() + .map(|part| part.len()) + .sum::() +} - let state_clone = Arc::clone(state); - let peer_clone = *peer_id; - tokio::spawn(async move { - if let Err(e) = - send_to_peer_stream(&state_clone, peer_clone, StreamType::Tx, &encoded).await - { - state_clone - .metrics - .error_write - .fetch_add(1, Ordering::Relaxed); - warn!("Failed to send TX to {}: {}", peer_clone, e); - } else { - state_clone +/// Serve one demand in order, as one job that retains only one bounded batch +/// while a peer is backpressured. Every TX still has its own wire frame. +/// Batching avoids one task, stream lock and short QUIC write per transaction. +/// +/// Each batch is looked up under one TX buffer lock, which is released before +/// its frames are copied (straight from the shared TX bytes) and written. +/// Lookups are lazy: batch k+1 is looked up after batch k is written, so a +/// peer backpressured for longer than the buffer's retention gets misses for +/// the rest rather than late responses; it re-demands after its timeout. +/// +/// The job stops at the first write that fails after `send_peer_frames`' +/// retries, usually because the peer disconnected. If the connection is in +/// fact still up (a reset stream that also failed to reopen), the requester +/// re-demands the rest after its response timeout. +async fn serve_getdata(state: &SharedState, peer: PeerId, hashes: Vec<[u8; 32]>) { + let mut hashes = hashes.into_iter(); + // A hit that did not fit the previous batch; it starts the next one. + let mut carried: Option> = None; + let mut txs: Vec> = Vec::new(); + let mut batch = Vec::new(); + loop { + txs.clear(); + let mut bytes = 0; + if let Some(tx) = carried.take() { + bytes += tx_frame_len(&tx); + txs.push(tx); + } + if bytes < TX_RESPONSE_BATCH_BYTES { + let mut buffer = state.tx_buffer.write().await; + for hash in hashes.by_ref() { + let Some(tx) = buffer.get(&hash) else { + state .metrics - .message_write + .flood_unfulfilled_unknown .fetch_add(1, Ordering::Relaxed); - state_clone - .metrics - .byte_write - .fetch_add(encoded.len() as u64, Ordering::Relaxed); - debug!("TX_SEND: Sent TX {:02x?}... to {}", &hash[..4], peer_clone); + trace!( + "TX_GETDATA_MISS: Don't have TX {:02x?}... for {}", + &hash[..4], + peer + ); + continue; + }; + let len = tx_frame_len(&tx); + if !txs.is_empty() && bytes + len > TX_RESPONSE_BATCH_BYTES { + carried = Some(tx); + break; } - }); + bytes += len; + txs.push(tx); + if bytes >= TX_RESPONSE_BATCH_BYTES { + break; + } + } + } + let Some(first) = txs.first() else { + return; + }; + state + .metrics + .flood_fulfilled + .fetch_add(txs.len() as u64, Ordering::Relaxed); + let written = if bytes > TX_RESPONSE_BATCH_BYTES { + // Only a lone transaction can exceed the budget. Write it as the + // existing segmented frame, straight from the shared TX bytes. + let parts = first.flood_frame_parts(); + let result = send_to_peer_stream_parts(state, peer, StreamType::Tx, &parts).await; + record_tx_response_write(state, peer, result, 1, bytes - 4) } else { + batch.clear(); + batch.reserve_exact(bytes); + for tx in &txs { + let parts = tx.flood_frame_parts(); + // Within the batch budget, so the length fits the u32 prefix. + let payload = parts.iter().map(|part| part.len()).sum::(); + batch.extend_from_slice(&(payload as u32).to_be_bytes()); + for part in parts { + batch.extend_from_slice(part); + } + } + let frames = framed_io::Frames::Batch(&batch); + let result = send_peer_frames(state, peer, StreamType::Tx, frames).await; + record_tx_response_write(state, peer, result, txs.len(), bytes - 4 * txs.len()) + }; + if !written { + let remaining = hashes.len() + usize::from(carried.is_some()); + if remaining > 0 { + debug!( + "TX_GETDATA_ABANDON: Not serving {} more requested hashes to {} after a failed write", + remaining, peer + ); + } + return; + } + } +} + +/// Returns whether the write succeeded. +fn record_tx_response_write( + state: &SharedState, + peer: PeerId, + result: io::Result<()>, + messages: usize, + bytes: usize, +) -> bool { + match result { + Ok(()) => { state .metrics - .flood_unfulfilled_unknown - .fetch_add(1, Ordering::Relaxed); - trace!( - "TX_GETDATA_MISS: Don't have TX {:02x?}... for {}", - &hash[..4], - peer_id - ); + .message_write + .fetch_add(messages as u64, Ordering::Relaxed); + state + .metrics + .byte_write + .fetch_add(bytes as u64, Ordering::Relaxed); + debug!("TX_SEND: Sent {} TXs to {}", messages, peer); + true + } + Err(e) => { + state + .metrics + .error_write + .fetch_add(messages as u64, Ordering::Relaxed); + warn!("Failed to send {} TXs to {}: {}", messages, peer, e); + false } } } @@ -2018,18 +2138,7 @@ async fn handle_tx_response(state: &Arc, peer_id: &PeerId, tx: Arc< peers_to_announce.len() ); - let inv_entry = InvEntry { hash, fee_per_op }; - - // Add to batcher for each peer, send batch immediately when full - for peer in &peers_to_announce { - let batch_to_send = { - let mut batcher = state.inv_batcher.write().await; - batcher.add(*peer, inv_entry.clone()) - }; - if let Some(batch) = batch_to_send { - send_inv_batch(state, *peer, batch).await; - } - } + queue_inv(state, &peers_to_announce, InvEntry { hash, fee_per_op }).await; } // Record recv-transaction timing diff --git a/overlay/src/libp2p_overlay/dispatch_tests.rs b/overlay/src/libp2p_overlay/dispatch_tests.rs index fde3941fc..1cb00aa20 100644 --- a/overlay/src/libp2p_overlay/dispatch_tests.rs +++ b/overlay/src/libp2p_overlay/dispatch_tests.rs @@ -169,6 +169,502 @@ impl Drop for TestNode { } } +#[tokio::test] +async fn getdata_batches_preserve_frames_and_leave_control_responsive() { + use stellar_xdr::{ + HostFunction, InvokeHostFunctionOp, Limits, Operation, OperationBody, ReadXdr, + TransactionEnvelope, WriteXdr, + }; + + let mut sender = TestNode::start().await; + let mut receiver = TestNode::start().await; + sender.connect(&receiver).await; + + let mut transactions = Vec::new(); + for sequence in 0..160 { + let bytes = crate::xdr::tests::valid_transaction_xdr(1000, sequence, 16); + transactions.push(ValidatedTx::from_core_trusted(bytes, 1000, 16).unwrap()); + } + // A wire-valid upload larger than the batching budget exercises the + // standalone write between two batches. Ledger validation is unrelated + // to this transport test. + let mut envelope = TransactionEnvelope::from_xdr( + crate::xdr::tests::valid_transaction_xdr(1000, 200, 1), + Limits::none(), + ) + .unwrap(); + let TransactionEnvelope::Tx(v1) = &mut envelope else { + unreachable!() + }; + v1.tx.operations = vec![Operation { + source_account: None, + body: OperationBody::InvokeHostFunction(InvokeHostFunctionOp { + host_function: HostFunction::UploadContractWasm(vec![0; 70_000].try_into().unwrap()), + auth: Default::default(), + }), + }] + .try_into() + .unwrap(); + let large = + ValidatedTx::from_core_trusted(envelope.to_xdr(Limits::none()).unwrap(), 1000, 1).unwrap(); + transactions.insert(80, large); + let mut demand = GetData::new(); + demand.push([0xaa; 32]); + for tx in &transactions { + sender.state.tx_buffer.write().await.insert(Arc::clone(tx)); + demand.push(*tx.hash()); + } + demand.push([0xbb; 32]); + + let streams = sender.streams_to(receiver.peer).await; + let blocked = streams.tx.lock().await; + for (encoded, _) in demand.encode_chunked().unwrap() { + send_to_peer_stream(&receiver.state, sender.peer, StreamType::Tx, &encoded) + .await + .unwrap(); + } + // A second demand on the same incoming stream is a barrier: its misses + // prove the reader dispatched both requests while the first TX writer is + // still blocked. The original per-TX task path has already looked up all + // transactions by this point; a bounded response job retains only its next + // batch. Use unknown hashes so the barrier needs no response stream. + let mut barrier = GetData::new(); + for byte in 0xc0..0xe0 { + barrier.push([byte; 32]); + } + for (encoded, _) in barrier.encode_chunked().unwrap() { + send_to_peer_stream(&receiver.state, sender.peer, StreamType::Tx, &encoded) + .await + .unwrap(); + } + tokio::time::timeout(Duration::from_secs(5), async { + while sender + .state + .metrics + .flood_unfulfilled_unknown + .load(Ordering::Relaxed) + < 32 + || sender.state.metrics.flood_fulfilled.load(Ordering::Relaxed) == 0 + { + tokio::task::yield_now().await; + } + }) + .await + .expect("GETDATA reader stopped progressing behind a blocked TX writer"); + assert!( + sender.state.metrics.flood_fulfilled.load(Ordering::Relaxed) < transactions.len() as u64, + "blocked response retained every requested transaction" + ); + // Lookups hold the TX buffer lock per batch, never across a write. + assert!( + sender.state.tx_buffer.try_write().is_ok(), + "a blocked GETDATA write holds the TX buffer lock" + ); + let scp = test_scp_envelope_xdr(987); + sender.handle.broadcast_scp(scp.clone()).await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Some(OverlayEvent::ScpReceived { envelope, .. }) = receiver.events.recv().await { + if envelope == scp { + break; + } + } + } + }) + .await + .expect("a blocked GETDATA response held up the control stream"); + assert!(receiver._admissions.try_recv().is_err()); + drop(blocked); + + tokio::time::timeout(Duration::from_secs(5), async { + for expected in &transactions { + let Some(CoreCommand::SubmitTx { tx, .. }) = receiver._admissions.recv().await else { + panic!("expected transaction admission"); + }; + assert_eq!(tx.bytes(), expected.bytes()); + assert_eq!(tx.hash(), expected.hash()); + } + }) + .await + .expect("batched, oversized, and trailing transactions must all arrive"); + assert_eq!( + sender.state.metrics.flood_fulfilled.load(Ordering::Relaxed), + transactions.len() as u64 + ); + assert_eq!( + sender + .state + .metrics + .flood_unfulfilled_unknown + .load(Ordering::Relaxed), + 34 + ); + sender.stop().await; + receiver.stop().await; +} + +/// A wire-valid transaction carrying a `wasm_len`-byte contract upload, for +/// controlling frame sizes. Ledger validity is irrelevant to the transport. +fn upload_tx(sequence: i64, wasm_len: usize) -> Arc { + use stellar_xdr::{ + HostFunction, InvokeHostFunctionOp, Limits, Operation, OperationBody, ReadXdr, + TransactionEnvelope, WriteXdr, + }; + let mut envelope = TransactionEnvelope::from_xdr( + crate::xdr::tests::valid_transaction_xdr(1000, sequence, 1), + Limits::none(), + ) + .unwrap(); + let TransactionEnvelope::Tx(v1) = &mut envelope else { + unreachable!() + }; + v1.tx.operations = vec![Operation { + source_account: None, + body: OperationBody::InvokeHostFunction(InvokeHostFunctionOp { + host_function: HostFunction::UploadContractWasm(vec![0; wasm_len].try_into().unwrap()), + auth: Default::default(), + }), + }] + .try_into() + .unwrap(); + ValidatedTx::from_core_trusted(envelope.to_xdr(Limits::none()).unwrap(), 1000, 1).unwrap() +} + +#[tokio::test] +async fn getdata_batch_edges_preserve_order_and_bytes() { + let mut sender = TestNode::start().await; + let mut receiver = TestNode::start().await; + sender.connect(&receiver).await; + + let small = |sequence| { + let bytes = crate::xdr::tests::valid_transaction_xdr(1000, sequence, 16); + ValidatedTx::from_core_trusted(bytes, 1000, 16).unwrap() + }; + // XDR sizes are multiples of 4, so this lands exactly on the budget. + let exact = |sequence| { + let probe = tx_frame_len(&upload_tx(sequence, 60_000)); + let tx = upload_tx(sequence, 60_000 + TX_RESPONSE_BATCH_BYTES - probe); + assert_eq!(tx_frame_len(&tx), TX_RESPONSE_BATCH_BYTES); + tx + }; + // An oversized TX as the first hit, then another straight after it; a + // batch closed by an exactly budget-sized frame, which then goes out as a + // batch of its own; batches spanning several writes; an oversized TX + // carried over from a full batch at the end. Misses are interleaved. + let mut transactions = vec![upload_tx(1, 70_000), upload_tx(2, 70_000)]; + transactions.extend((10..13).map(small)); + transactions.push(exact(20)); + transactions.push(exact(21)); + transactions.extend((30..130).map(small)); + transactions.push(upload_tx(3, 100_000)); + let mut demand = vec![[0xaa; 32]]; + for (index, tx) in transactions.iter().enumerate() { + sender.state.tx_buffer.write().await.insert(Arc::clone(tx)); + demand.push(*tx.hash()); + if index % 40 == 3 { + demand.push([index as u8; 32]); + } + } + let misses = demand.len() - transactions.len(); + + serve_getdata(&sender.state, receiver.peer, demand).await; + + tokio::time::timeout(Duration::from_secs(5), async { + for expected in &transactions { + let Some(CoreCommand::SubmitTx { tx, .. }) = receiver._admissions.recv().await else { + panic!("expected transaction admission"); + }; + assert_eq!(tx.bytes(), expected.bytes()); + } + }) + .await + .expect("every served transaction must arrive, in demand order"); + let metrics = &sender.state.metrics; + let served = transactions.len() as u64; + assert_eq!(metrics.flood_fulfilled.load(Ordering::Relaxed), served); + assert_eq!(metrics.message_write.load(Ordering::Relaxed), served); + assert_eq!(metrics.error_write.load(Ordering::Relaxed), 0); + assert_eq!( + metrics.flood_unfulfilled_unknown.load(Ordering::Relaxed), + misses as u64 + ); + sender.stop().await; + receiver.stop().await; +} + +#[tokio::test] +async fn failed_getdata_batch_is_rewritten_whole_on_a_reopened_stream() { + use futures::AsyncWriteExt; + + let mut sender = TestNode::start().await; + let mut receiver = TestNode::start().await; + sender.connect(&receiver).await; + let transactions: Vec<_> = (0..200) + .map(|sequence| { + let bytes = crate::xdr::tests::valid_transaction_xdr(1000, sequence, 16); + ValidatedTx::from_core_trusted(bytes, 1000, 16).unwrap() + }) + .collect(); + for tx in &transactions { + sender.state.tx_buffer.write().await.insert(Arc::clone(tx)); + } + // Finish the cached TX stream: the first batch write then fails, and + // send_peer_frames must reopen the stream and write that batch again. + { + let streams = sender.streams_to(receiver.peer).await; + let mut cached = streams.tx.lock().await; + cached.as_mut().unwrap().stream.close().await.unwrap(); + } + + let demand = transactions.iter().map(|tx| *tx.hash()).collect(); + serve_getdata(&sender.state, receiver.peer, demand).await; + + tokio::time::timeout(Duration::from_secs(5), async { + for expected in &transactions { + let Some(CoreCommand::SubmitTx { tx, .. }) = receiver._admissions.recv().await else { + panic!("expected transaction admission"); + }; + assert_eq!(tx.bytes(), expected.bytes()); + } + }) + .await + .expect("the batch that hit the closed stream must be delivered on a new one"); + let metrics = &sender.state.metrics; + assert_eq!(metrics.error_write.load(Ordering::Relaxed), 0); + assert_eq!( + metrics.message_write.load(Ordering::Relaxed), + transactions.len() as u64 + ); + assert_eq!( + receiver + .state + .metrics + .flood_duplicate_recv + .load(Ordering::Relaxed), + 0, + "a frame was delivered twice" + ); + sender.stop().await; + receiver.stop().await; +} + +#[tokio::test] +async fn getdata_job_stops_after_a_failed_write() { + let (_handle, _events, _admissions, overlay) = + create_test_overlay(Keypair::generate_ed25519(), Arc::new(OverlayMetrics::new())).unwrap(); + let state = Arc::clone(&overlay.state); + let mut hashes = Vec::new(); + for sequence in 0..200 { + let bytes = crate::xdr::tests::valid_transaction_xdr(1000, sequence, 16); + let tx = ValidatedTx::from_core_trusted(bytes, 1000, 16).unwrap(); + hashes.push(*tx.hash()); + state.tx_buffer.write().await.insert(tx); + } + // A peer without outbound streams, as after a disconnect: every write + // fails at once. The demand spans several batches. + serve_getdata(&state, PeerId::random(), hashes.clone()).await; + + let failed = state.metrics.error_write.load(Ordering::Relaxed); + let attempted = state.metrics.flood_fulfilled.load(Ordering::Relaxed); + assert!(failed > 0, "the first batch must have been attempted"); + assert_eq!(attempted, failed, "only the first batch was attempted"); + assert!( + attempted < hashes.len() as u64, + "the job kept serving a peer it could not write to" + ); + assert_eq!(state.metrics.message_write.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn received_tx_is_announced_to_every_peer_except_known_sources() { + use crate::flood::{InvEntry, INV_BATCH_MAX_SIZE}; + + let (_handle, _events, _admissions, overlay) = + create_test_overlay(Keypair::generate_ed25519(), Arc::new(OverlayMetrics::new())).unwrap(); + let state = Arc::clone(&overlay.state); + let source = PeerId::random(); + let advertiser = PeerId::random(); + let others: Vec = (0..5).map(|_| PeerId::random()).collect(); + { + let mut streams = state.peer_streams.write().await; + for peer in others.iter().chain([&source, &advertiser]) { + streams.insert(*peer, Arc::new(PeerOutboundStreams::new())); + } + } + // One peer's batch is one entry short of full, so this TX completes it. + let full = others[0]; + { + let mut batcher = state.inv_batcher.write().await; + for index in 0..INV_BATCH_MAX_SIZE - 1 { + let mut hash = [0xa5; 32]; + hash[..8].copy_from_slice(&(index as u64).to_be_bytes()); + assert!(batcher + .add( + full, + InvEntry { + hash, + fee_per_op: 1 + } + ) + .is_none()); + } + } + let tx = test_tx(1); + state + .inv_tracker + .write() + .await + .record_source(*tx.hash(), advertiser); + + handle_tx_response(&state, &source, Arc::clone(&tx)).await; + + let mut batcher = state.inv_batcher.write().await; + // The completed batch was handed to a sender, not left queued. + assert!(batcher.flush(&full).is_none()); + for peer in &others[1..] { + let batch = batcher + .flush(peer) + .expect("INV queued for every other peer"); + assert_eq!(batch.entries.len(), 1); + assert_eq!(batch.entries[0].hash, *tx.hash()); + } + assert!(batcher.flush(&source).is_none(), "INV echoed to the sender"); + assert!( + batcher.flush(&advertiser).is_none(), + "INV sent to a peer that already advertised the TX" + ); +} + +#[tokio::test] +async fn inv_batches_completed_by_relay_or_local_broadcast_reach_the_peer() { + use crate::flood::{InvEntry, INV_BATCH_MAX_SIZE}; + + let mut sender = TestNode::start().await; + let mut receiver = TestNode::start().await; + sender.connect(&receiver).await; + for round in 0..2u8 { + // Fill the receiver's pending batch to one short of full. The + // housekeeping flush needs 100 ms, so only the completing TX can + // send it; if that send were lost, these hashes would never arrive. + { + let mut batcher = sender.state.inv_batcher.write().await; + for index in 0..INV_BATCH_MAX_SIZE - 1 { + let mut hash = [round; 32]; + hash[..8].copy_from_slice(&(index as u64).to_be_bytes()); + assert!(batcher + .add( + receiver.peer, + InvEntry { + hash, + fee_per_op: 1 + } + ) + .is_none()); + } + } + let tx = test_tx(i64::from(round) + 1); + if round == 0 { + handle_tx_response(&sender.state, &PeerId::random(), tx).await; + } else { + sender.handle.broadcast_tx(tx).await; + } + let expected = (u64::from(round) + 1) * INV_BATCH_MAX_SIZE as u64; + tokio::time::timeout(Duration::from_secs(5), async { + while receiver + .state + .metrics + .flood_demanded + .load(Ordering::Relaxed) + < expected + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("round {round}: the completed INV batch never arrived")); + } + sender.stop().await; + receiver.stop().await; +} + +#[tokio::test] +async fn inv_batch_demands_only_first_unseen_advertisements_in_order() { + use crate::flood::{InvBatch, InvEntry}; + + let advertiser = TestNode::start().await; + let mut receiver = TestNode::start().await; + advertiser.connect(&receiver).await; + let txs: Vec<_> = (1..=6).map(test_tx).collect(); + for tx in &txs { + advertiser + .state + .tx_buffer + .write() + .await + .insert(Arc::clone(tx)); + } + let hash = |index: usize| *txs[index].hash(); + // txs[1] was already received; txs[2] was first advertised by another + // peer, which owns its demand; txs[5] is never advertised. + receiver.state.tx_seen.write().await.put(hash(1), ()); + let other = PeerId::random(); + assert!(receiver + .state + .inv_tracker + .write() + .await + .record_source(hash(2), other)); + + // Unsorted, with a repeat: demands must follow first-advertisement order. + let order = [4, 1, 0, 2, 4, 3]; + let batch = InvBatch { + entries: order + .iter() + .map(|&index| InvEntry { + hash: hash(index), + fee_per_op: 0, + }) + .collect(), + }; + handle_inv_batch(&receiver.state, &advertiser.peer, batch).await; + + // Counted when the demand is built, before any response can arrive. A + // demand for the seen TX would be counted here even though the receiver + // would then drop the response as a duplicate. + let demanded = [4, 0, 3]; + assert_eq!( + receiver + .state + .metrics + .flood_demanded + .load(Ordering::Relaxed), + demanded.len() as u64 + ); + { + let tracker = receiver.state.inv_tracker.read().await; + let sources = |index: usize| tracker.peek_sources(&hash(index)).cloned(); + // A seen TX is not tracked; a repeat does not add the peer twice; + // a later advertiser is recorded behind the first. + assert_eq!(sources(1), None); + assert_eq!(sources(4), Some(vec![advertiser.peer])); + assert_eq!(sources(2), Some(vec![other, advertiser.peer])); + assert_eq!(sources(5), None); + } + // The advertiser serves the one GETDATA in its hash order. + tokio::time::timeout(Duration::from_secs(5), async { + for index in demanded { + let Some(CoreCommand::SubmitTx { tx, .. }) = receiver._admissions.recv().await else { + panic!("expected transaction admission"); + }; + assert_eq!(tx.hash(), &hash(index), "demand order"); + } + }) + .await + .expect("demanded transactions must arrive"); + assert!(receiver._admissions.try_recv().is_err()); + receiver.stop().await; +} + #[tokio::test] async fn scp_state_sends_preserve_slot_order_without_blocking_other_peers() { let mut sender = TestNode::start().await; diff --git a/overlay/src/libp2p_overlay/framed_io.rs b/overlay/src/libp2p_overlay/framed_io.rs index 1332b066b..b5a9e8e84 100644 --- a/overlay/src/libp2p_overlay/framed_io.rs +++ b/overlay/src/libp2p_overlay/framed_io.rs @@ -2,6 +2,26 @@ use futures::{AsyncWrite, AsyncWriteExt}; use std::io; +/// Either one segmented frame or several complete length-prefixed frames. +/// A batch preserves the wire boundaries of its individual messages. +#[derive(Clone, Copy)] +pub(super) enum Frames<'a> { + One(&'a [&'a [u8]]), + Batch(&'a [u8]), +} + +impl Frames<'_> { + pub(super) async fn write(self, writer: &mut W) -> io::Result<()> { + match self { + Self::One(parts) => write_frame_parts(writer, parts).await, + Self::Batch(bytes) => { + writer.write_all(bytes).await?; + writer.flush().await + } + } + } +} + /// Write a single length-prefixed frame without assembling a bulk copy. /// Callers sharing a stream must hold its lock over this entire future. pub(super) async fn write_frame_parts( @@ -26,6 +46,7 @@ pub(super) async fn write_frame_parts( #[cfg(test)] mod tests { use super::*; + use futures::AsyncRead; use std::pin::Pin; use std::task::{Context, Poll}; @@ -80,4 +101,114 @@ mod tests { expected_frames.extend_from_slice(b"next frame"); assert_eq!(writer.bytes, expected_frames); } + + /// Delivers at most `max` bytes per read and yields between reads, like + /// a QUIC stream handing over a batch as packets arrive. + struct ShortReader { + bytes: Vec, + pos: usize, + max: usize, + yield_next: bool, + } + + impl AsyncRead for ShortReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + if self.yield_next { + self.yield_next = false; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + self.yield_next = true; + let pos = self.pos; + let n = buf.len().min(self.max).min(self.bytes.len() - pos); + buf[..n].copy_from_slice(&self.bytes[pos..pos + n]); + self.pos += n; + Poll::Ready(Ok(n)) + } + } + + #[tokio::test] + async fn batched_frames_read_back_through_a_buffered_reader() { + // Frames from 1 byte to more than twice the read buffer: the large + // ones exercise the reader's bypass once its buffer is drained. + let frames: Vec> = (0..40u8).map(|i| vec![i; 1 + 61 * i as usize]).collect(); + let mut batch = Vec::new(); + for frame in &frames[..30] { + batch.extend_from_slice(&(frame.len() as u32).to_be_bytes()); + batch.extend_from_slice(frame); + } + let mut writer = ShortWriter::default(); + Frames::Batch(&batch).write(&mut writer).await.unwrap(); + for frame in &frames[30..] { + Frames::One(&[frame]).write(&mut writer).await.unwrap(); + } + + let reader = ShortReader { + bytes: writer.bytes, + pos: 0, + max: 700, + yield_next: false, + }; + let mut reader = futures::io::BufReader::with_capacity(1024, reader); + for frame in &frames { + assert_eq!( + &super::super::read_framed(&mut reader).await.unwrap(), + frame + ); + } + let end = super::super::read_framed(&mut reader).await.unwrap_err(); + assert_eq!(end.kind(), io::ErrorKind::UnexpectedEof); + } + + /// Delivers its bytes, then never has more and never wakes the reader. + struct StallingReader { + bytes: Vec, + pos: usize, + } + + impl AsyncRead for StallingReader { + fn poll_read( + mut self: Pin<&mut Self>, + _: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let pos = self.pos; + if pos == self.bytes.len() { + return Poll::Pending; + } + let n = buf.len().min(self.bytes.len() - pos); + buf[..n].copy_from_slice(&self.bytes[pos..pos + n]); + self.pos += n; + Poll::Ready(Ok(n)) + } + } + + #[tokio::test] + async fn buffered_tx_stream_takes_in_available_frames_without_waiting_for_more() { + let frames = [&b"first"[..], &b"second frame"[..]]; + let mut bytes = Vec::new(); + for frame in frames { + bytes.extend_from_slice(&(frame.len() as u32).to_be_bytes()); + bytes.extend_from_slice(frame); + } + let available = bytes.len(); + let mut reader = super::super::buffered_tx_stream(StallingReader { bytes, pos: 0 }); + + for (index, frame) in frames.into_iter().enumerate() { + let read = futures::poll!(Box::pin(super::super::read_framed(&mut reader))); + assert!( + matches!(&read, Poll::Ready(Ok(data)) if data == frame), + "frame {index} was available but not returned: {read:?}" + ); + // One inner read took in both frames. + assert_eq!(reader.get_ref().pos, available); + } + // Only a read past the available data waits. + let next = futures::poll!(Box::pin(super::super::read_framed(&mut reader))); + assert!(next.is_pending()); + } } diff --git a/overlay/src/wire.rs b/overlay/src/wire.rs index 6680c7fa4..b5332e2ef 100644 --- a/overlay/src/wire.rs +++ b/overlay/src/wire.rs @@ -109,10 +109,19 @@ impl ValidatedTx { self.fee / i64::from(self.num_ops.max(1)) } - /// The `StellarMessage::Transaction(..)` wire framing for flooding this tx. + /// The `StellarMessage::Transaction(..)` wire framing for flooding this tx, + /// as one owned buffer. The reference encoding; the flooding path itself + /// writes [`Self::flood_frame_parts`] without this copy. pub fn to_flood_frame(&self) -> Vec { xdr::frame_transaction(&self.bytes) } + + /// The same framing as [`Self::to_flood_frame`], as borrowed parts + /// (discriminant, envelope) whose concatenation is the message. Writers + /// copy or send these in place, without allocating a framed copy. + pub fn flood_frame_parts(&self) -> [&[u8]; 2] { + [&xdr::TRANSACTION_DISCRIMINANT, &self.bytes] + } } impl fmt::Debug for ValidatedTx { @@ -194,5 +203,7 @@ mod tests { .to_xdr(Limits::none()) .unwrap(); assert_eq!(tx.to_flood_frame(), expected); + assert_eq!(tx.flood_frame_parts().concat(), expected); + assert!(std::ptr::eq(tx.flood_frame_parts()[1], tx.bytes())); } } diff --git a/overlay/src/xdr.rs b/overlay/src/xdr.rs index 8f0ded0bb..e65607157 100644 --- a/overlay/src/xdr.rs +++ b/overlay/src/xdr.rs @@ -74,6 +74,11 @@ pub(crate) fn frame_transaction(envelope_bytes: &[u8]) -> Vec { frame(MessageType::Transaction, envelope_bytes) } +/// The discriminant `frame_transaction` prefixes, for writers that place the +/// envelope bytes directly instead of first building a framed copy. +pub(crate) const TRANSACTION_DISCRIMINANT: [u8; 4] = + (MessageType::Transaction as i32).to_be_bytes(); + /// `StellarMessage::ScpMessage(..)` framing over canonical envelope bytes. pub(crate) fn frame_scp(envelope_bytes: &[u8]) -> Vec { frame(MessageType::ScpMessage, envelope_bytes)