From cd6e51116fca27f700d39610d28eb5b2cddd04a6 Mon Sep 17 00:00:00 2001 From: ethan Date: Sun, 13 Sep 2026 01:24:07 +0800 Subject: [PATCH 01/17] fix: add bip68 mask check for timelock --- crates/bitvm-gc/src/timelocks.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/bitvm-gc/src/timelocks.rs b/crates/bitvm-gc/src/timelocks.rs index 72b930ac..917a404f 100644 --- a/crates/bitvm-gc/src/timelocks.rs +++ b/crates/bitvm-gc/src/timelocks.rs @@ -155,6 +155,12 @@ pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Re ("operator_commit", config.operator_commit), ("connector_f", config.connector_f), ] { + const BIP68_BLOCKS_MASK: u32 = 0x0000_ffff; + if value & !BIP68_BLOCKS_MASK != 0 { + bail!( + "timelock_config.{name} must use a BIP68 height-based sequence value, got {value:#010x}" + ); + } if value < budget.reaction_blocks { bail!( "timelock_config.{name} must be at least {} reaction blocks, got {value}", From a1acaa100dbbe5f3df6c0b63335e5ca09890278e Mon Sep 17 00:00:00 2001 From: ethan Date: Sun, 13 Sep 2026 02:52:53 +0800 Subject: [PATCH 02/17] mark send-challenge as rpc-debug-endpoints --- .github/workflows/ci.yml | 4 ++-- node/Cargo.toml | 1 + node/src/rpc_service/bitvm.rs | 1 + node/src/rpc_service/handler/bitvm_handler.rs | 5 ++++- node/src/rpc_service/mod.rs | 9 ++++----- node/src/rpc_service/routes.rs | 1 + 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65ed43d2..5dfdb879 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,7 @@ jobs: run: sudo apt update && sudo apt install protobuf-compiler - run: | source ~/.zkm-toolchain/env - cargo clippy --all-targets -- -D warnings + cargo clippy --all-targets --features rpc-debug-endpoints -- -D warnings test: name: Cargo Test needs: tla-plus @@ -173,4 +173,4 @@ jobs: run: | set -e source ~/.zkm-toolchain/env - cargo test -r --all --all-targets + cargo test -r --all --all-targets --features rpc-debug-endpoints diff --git a/node/Cargo.toml b/node/Cargo.toml index e1359f36..9a072d34 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -14,6 +14,7 @@ path = "src/bin/send_pegin_request.rs" [[bin]] name = "challenge" path = "src/bin/send_challenge.rs" +required-features = ["rpc-debug-endpoints"] [[bin]] name = "verifier-challenge" diff --git a/node/src/rpc_service/bitvm.rs b/node/src/rpc_service/bitvm.rs index b73057bf..26dde07c 100644 --- a/node/src/rpc_service/bitvm.rs +++ b/node/src/rpc_service/bitvm.rs @@ -47,6 +47,7 @@ pub struct InstanceSettingResponse { } #[derive(Debug, Deserialize, Serialize)] +#[cfg(feature = "rpc-debug-endpoints")] pub struct SendChallengeResponse { pub challenge_txid: String, } diff --git a/node/src/rpc_service/handler/bitvm_handler.rs b/node/src/rpc_service/handler/bitvm_handler.rs index 64da9d0c..1095775d 100644 --- a/node/src/rpc_service/handler/bitvm_handler.rs +++ b/node/src/rpc_service/handler/bitvm_handler.rs @@ -10,9 +10,11 @@ use crate::rpc_service::response::{ ApiErrorExt, ApiResult, ErrorResponse, error_response, ok_response, }; use crate::rpc_service::validation::InputValidator; +#[cfg(feature = "rpc-debug-endpoints")] +use crate::utils::send_challenge_tx; use crate::utils::{ gen_instance_parameters_local, get_bridge_out_global_stats, load_validated_graph_definition, - obsolete_graph, send_challenge_tx, + obsolete_graph, }; use alloy::primitives::U256; use axum::Json; @@ -1211,6 +1213,7 @@ pub async fn get_unsigned_pegin_txn( /// /// - `200 OK`: Challenge transaction broadcasted successfully, returns txid /// - `500 Internal Server Error`: Graph not found or broadcast failed +#[cfg(feature = "rpc-debug-endpoints")] #[axum::debug_handler] pub async fn send_challenge( Path(graph_id): Path, diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index a16185ef..b4b0c353 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -17,12 +17,12 @@ use crate::rpc_service::handler::{ get_chain_proof_desc, get_graph, get_graph_neighbor_ids, get_graph_tx, get_graph_txn, get_graphs, get_instance, get_instances, get_instances_overview, get_node, get_nodes, get_nodes_overview, get_operator_proof_desc, get_ready_to_kickoff_graph, get_swap, get_swaps, - get_unsigned_pegin_txn, instance_settings, pegout, send_challenge, + get_unsigned_pegin_txn, instance_settings, pegout, }; #[cfg(feature = "rpc-debug-endpoints")] use crate::rpc_service::handler::{ get_debug_message_details, get_debug_status, get_graph_debug_messages, - get_instance_debug_messages, send_verifier_challenge, + get_instance_debug_messages, send_challenge, send_verifier_challenge, }; use anyhow::Context; use axum::body::Body; @@ -172,12 +172,11 @@ pub(crate) fn build_business_router(app_state: Arc) -> Router { .route(routes::v1::DEBUG_INSTANCE_MESSAGES, get(get_instance_debug_messages)) .route(routes::v1::DEBUG_MESSAGE_DETAILS, get(get_debug_message_details)); - let signed_routes = Router::new() - .route(routes::v1::GRAPHS_SEND_CHALLENGE, post(send_challenge)) - .route(routes::v1::PEGOUT, post(pegout)); + let signed_routes = Router::new().route(routes::v1::PEGOUT, post(pegout)); #[cfg(feature = "rpc-debug-endpoints")] let signed_routes = signed_routes + .route(routes::v1::GRAPHS_SEND_CHALLENGE, post(send_challenge)) .route(routes::v1::GRAPHS_SEND_VERIFIER_CHALLENGE, post(send_verifier_challenge)); let signed_routes = signed_routes diff --git a/node/src/rpc_service/routes.rs b/node/src/rpc_service/routes.rs index 61e8fe18..93f150cb 100644 --- a/node/src/rpc_service/routes.rs +++ b/node/src/rpc_service/routes.rs @@ -20,6 +20,7 @@ pub(crate) mod v1 { pub const GRAPHS_TXN_BY_ID: &str = "/v1/graphs/{:id}/txn"; pub const GRAPHS_NEIGHBOR_IDS: &str = "/v1/graphs/{:id}/neighbor-ids"; pub const GRAPHS_TX_BY_ID: &str = "/v1/graphs/{:id}/tx"; + #[cfg(feature = "rpc-debug-endpoints")] pub const GRAPHS_SEND_CHALLENGE: &str = "/v1/graphs/{:id}/send-challenge"; #[cfg(feature = "rpc-debug-endpoints")] pub const GRAPHS_SEND_VERIFIER_CHALLENGE: &str = "/v1/graphs/{:id}/send-verifier-challenge"; From f3a2012592b0001ea2509ad6df35c1da3338c36f Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 14 Sep 2026 09:13:27 +0800 Subject: [PATCH 03/17] prevent instance upserts from overwriting progressed state --- crates/store/src/localdb.rs | 98 +++++++++---------- node/src/bin/mock_rpc.rs | 2 +- node/src/rpc_service/mod.rs | 2 +- node/src/scheduled_tasks/event_watch_task.rs | 15 ++- .../instance_maintenance_tasks.rs | 19 +++- 5 files changed, 78 insertions(+), 58 deletions(-) diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index aa8e2f9b..bd31ae4d 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -1108,65 +1108,61 @@ impl<'a> StorageProcessor<'a> { Ok(counts) } - /// Insert or update an instance - /// - /// Performs an INSERT OR REPLACE operation on the instance table. - /// If an instance with the same instance_id exists, it will be updated. - /// If no instance exists, a new one will be created. + /// Insert an instance only when its ID is not already present. /// /// Parameters: - /// - instance: The complete instance data to insert or update + /// - instance: The complete instance data to insert /// /// Returns: - /// - Ok(true) if the operation affected at least one row - /// - Ok(false) if no rows were affected + /// - Ok(true) if the instance was inserted + /// - Ok(false) if an instance with the same ID already exists /// - Err if the operation failed - pub async fn upsert_instance(&mut self, instance: &Instance) -> anyhow::Result { + pub async fn insert_instance_if_absent(&mut self, instance: &Instance) -> anyhow::Result { let committees_answers_json = serde_json::to_string(&instance.committees_answers)?; - let res = sqlx::query!( - "INSERT OR - REPLACE INTO instance (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, status, goat_tx_hash, goat_tx_height, + let res = sqlx::query( + "INSERT INTO instance (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, status, goat_tx_hash, goat_tx_height, user_xonly_pubkey, user_change_addr, user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers, pegin_data_tx_hash, btc_height, parameters, status_updated_at, post_pegin_txhash, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - instance.instance_id, - instance.network, - instance.from_addr, - instance.to_addr, - instance.amount, - instance.fees, - instance.input_utxos, - instance.status, - instance.goat_tx_hash, - instance.goat_tx_height, - instance.user_xonly_pubkey, - instance.user_change_addr, - instance.user_refund_addr, - instance.btc_txid, - instance.pegin_confirm_txid, - instance.pegin_cancel_txid, - committees_answers_json, - instance.pegin_data_tx_hash, - instance.btc_height, - instance.parameters, - instance.status_updated_at, - instance.post_pegin_txhash, - instance.created_at, - instance.updated_at + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(instance_id) DO NOTHING", ) - .execute(self.conn()) - .await?; + .bind(instance.instance_id) + .bind(&instance.network) + .bind(&instance.from_addr) + .bind(&instance.to_addr) + .bind(instance.amount) + .bind(instance.fees) + .bind(&instance.input_utxos) + .bind(&instance.status) + .bind(&instance.goat_tx_hash) + .bind(instance.goat_tx_height) + .bind(instance.user_xonly_pubkey) + .bind(&instance.user_change_addr) + .bind(&instance.user_refund_addr) + .bind(&instance.btc_txid) + .bind(&instance.pegin_confirm_txid) + .bind(&instance.pegin_cancel_txid) + .bind(committees_answers_json) + .bind(&instance.pegin_data_tx_hash) + .bind(instance.btc_height) + .bind(&instance.parameters) + .bind(instance.status_updated_at) + .bind(&instance.post_pegin_txhash) + .bind(instance.created_at) + .bind(instance.updated_at) + .execute(self.conn()) + .await?; Ok(res.rows_affected() > 0) } /// Create a bridge-in instance from a pegin request, or refresh one that /// has not moved past the pegin-request stage yet. /// - /// `PeginRequest` is a re-deliverable P2P message, so `upsert_instance` is - /// unsafe here: its `INSERT OR REPLACE` lets a replayed or forged request - /// roll a live instance back to its initial row and clear everything the - /// later stages wrote. The write is therefore a compare-and-swap over the - /// current status, and it only touches the columns a pegin request owns: + /// `PeginRequest` is a re-deliverable P2P message, so a full-row replacement + /// would let a replayed or forged request roll a live instance back to its + /// initial row and clear everything the later stages wrote. The write is + /// therefore a compare-and-swap over the current status, and it only touches + /// the columns a pegin request owns: /// committee answers, instance parameters and the BTC-side fields are never /// overwritten. The caller is responsible for passing canonical request /// metadata - `goat_tx_hash`/`goat_tx_height` are refreshed from it, so that @@ -4516,7 +4512,7 @@ mod tests { let mut initing = pegin_instance(instance_id, "UserIniting"); initing.to_addr = "0xuser".to_string(); initing.from_addr = "bcrt1quser".to_string(); - assert!(s.upsert_instance(&initing).await.unwrap()); + assert!(s.insert_instance_if_absent(&initing).await.unwrap()); let mut request = pegin_instance(instance_id, "UserInited"); request.to_addr = "0xuser".to_string(); @@ -4553,7 +4549,7 @@ mod tests { inited.goat_tx_height = 500; inited.committees_answers = IndexMap::from([("0xcommittee".to_string(), vec![1u8, 2, 3])]); inited.parameters = Some("{}".to_string()); - assert!(s.upsert_instance(&inited).await.unwrap()); + assert!(s.insert_instance_if_absent(&inited).await.unwrap()); // A re-delivered request refreshes the row in place; everything the // instance accrued after the request must survive it. @@ -4578,7 +4574,7 @@ mod tests { minted.goat_tx_height = 500; minted.parameters = Some("{}".to_string()); minted.post_pegin_txhash = Some("0xmint".to_string()); - assert!(s.upsert_instance(&minted).await.unwrap()); + assert!(s.insert_instance_if_absent(&minted).await.unwrap()); // Once the instance moves on, a re-delivered request may not pull it back. let replay = pegin_instance(instance_id, "UserInited"); @@ -4593,20 +4589,20 @@ mod tests { } #[tokio::test] - async fn test_upsert_pegin_request_instance_rejects_bridge_out_collision() { + async fn test_upsert_pegin_request_instance_rejects_progressed_instance() { let db = setup_db().await; let mut s = db.acquire().await.unwrap(); let instance_id = Uuid::new_v4(); - let bridge_out = pegin_instance(instance_id, "Initialize"); - assert!(s.upsert_instance(&bridge_out).await.unwrap()); + let progressed = pegin_instance(instance_id, "CommitteesAnswered"); + assert!(s.insert_instance_if_absent(&progressed).await.unwrap()); let request = pegin_instance(instance_id, "UserInited"); assert!( !s.upsert_pegin_request_instance(&request, &pegin_request_statuses()).await.unwrap() ); let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); - assert_eq!(stored.status, "Initialize"); + assert_eq!(stored.status, "CommitteesAnswered"); } #[tokio::test] diff --git a/node/src/bin/mock_rpc.rs b/node/src/bin/mock_rpc.rs index 6f011f30..cff02fca 100644 --- a/node/src/bin/mock_rpc.rs +++ b/node/src/bin/mock_rpc.rs @@ -336,7 +336,7 @@ async fn seed_mock_data( tx.upsert_node(&node).await?; } for instance in [bridge_in_success, bridge_in_pending] { - tx.upsert_instance(&instance).await?; + tx.insert_instance_if_absent(&instance).await?; } tx.insert_swap_escrow_if_absent(&swap_escrow).await?; for graph in [ready_graph, challenge_graph] { diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index b4b0c353..77b38968 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -786,7 +786,7 @@ mod tests { ) -> anyhow::Result<()> { let mut tx = local_db.start_transaction().await?; for instance in instances { - tx.upsert_instance(instance).await?; + tx.insert_instance_if_absent(instance).await?; } for graph in graphs { seed_graph_runtime(&mut tx, graph).await?; diff --git a/node/src/scheduled_tasks/event_watch_task.rs b/node/src/scheduled_tasks/event_watch_task.rs index 2fd39bc9..60cb1ac3 100644 --- a/node/src/scheduled_tasks/event_watch_task.rs +++ b/node/src/scheduled_tasks/event_watch_task.rs @@ -727,12 +727,13 @@ async fn handle_bridge_in_events<'a>( bridge_in_events: Vec, ) -> anyhow::Result<()> { for event in bridge_in_events { + let post_pegin_txhash = event.transaction_hash.clone(); if let Ok(instance_id) = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)) && !storage_processor .update_instance( &InstanceUpdate::new_with_instance_id(instance_id) .with_status(InstanceBridgeInStatus::RelayerL2Minted.to_string()) - .with_post_pegin(event.transaction_hash), + .with_post_pegin(post_pegin_txhash.clone()), ) .await? && let Some(tx_record) = storage_processor @@ -757,7 +758,17 @@ async fn handle_bridge_in_events<'a>( { info!("Instance {instance_id} is created and set status to RelayerL2Minted"); instance.status = InstanceBridgeInStatus::RelayerL2Minted.to_string(); - storage_processor.upsert_instance(&instance).await?; + instance.post_pegin_txhash = Some(post_pegin_txhash.clone()); + if !storage_processor.insert_instance_if_absent(&instance).await? { + info!("Instance {instance_id} was created concurrently; applying mint event"); + storage_processor + .update_instance( + &InstanceUpdate::new_with_instance_id(instance_id) + .with_status(InstanceBridgeInStatus::RelayerL2Minted.to_string()) + .with_post_pegin(post_pegin_txhash.clone()), + ) + .await?; + } } if tx_record.processing_status == GoatTxProcessingStatus::Pending.to_string() { diff --git a/node/src/scheduled_tasks/instance_maintenance_tasks.rs b/node/src/scheduled_tasks/instance_maintenance_tasks.rs index 0126560a..122eff63 100644 --- a/node/src/scheduled_tasks/instance_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/instance_maintenance_tasks.rs @@ -186,8 +186,20 @@ pub async fn instance_answers_monitor( let mut tx = local_db.start_transaction().await?; if let Some(event) = event { if is_outside_response_window { - if let Some(instance) = discarded_instance { - tx.upsert_instance(&instance).await?; + if let Some(instance) = discarded_instance + && !tx.insert_instance_if_absent(&instance).await? + && !tx + .update_instance( + &InstanceUpdate::new_with_instance_id(instance.instance_id) + .with_status(InstanceBridgeInStatus::UserDiscarded.to_string()) + .with_only_if_status_in(vec![ + InstanceBridgeInStatus::UserIniting.to_string(), + InstanceBridgeInStatus::UserInited.to_string(), + ]), + ) + .await? + { + info!("skip stale UserDiscarded update for instance {}", instance.instance_id); } } else { upsert_message( @@ -497,7 +509,8 @@ pub async fn instance_btc_tx_monitor( update_instance( &mut storage_processor, &InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(InstanceBridgeInStatus::UserDiscarded.to_string()), + .with_status(InstanceBridgeInStatus::UserDiscarded.to_string()) + .with_only_if_status_in(vec![instance.status.clone()]), ) .await?; } From 3be67caf7c22b58c887ffcfae0a8bf62c54ace99 Mon Sep 17 00:00:00 2001 From: ethan Date: Tue, 15 Sep 2026 20:14:51 +0800 Subject: [PATCH 04/17] fix: harden message claims and retry handling --- .github/workflows/ci.yml | 11 +- ...20260914000000_add_queue_poison_guards.sql | 18 + crates/store/src/localdb.rs | 897 ++++++++++++-- crates/store/src/schema.rs | 21 +- deployment/regtest/bitvm-noded/stop_nodes.sh | 2 +- deployment/testnet4/bitvm-noded/stop_nodes.sh | 2 +- node/src/action.rs | 1031 +++++++++++++---- node/src/main.rs | 12 +- node/src/metrics_service.rs | 40 + node/src/middleware/swarm.rs | 19 +- node/src/p2p_msg_handler.rs | 18 + node/src/rpc_service/mod.rs | 5 +- node/src/scheduled_tasks/event_watch_task.rs | 22 +- .../graph_maintenance_tasks.rs | 2 + .../instance_maintenance_tasks.rs | 96 +- node/src/utils.rs | 68 +- node/tla/MessageStateRace.cfg | 4 +- node/tla/MessageStateRace.tla | 122 +- node/tla/MessageStateRaceFixed.cfg | 2 +- 19 files changed, 1881 insertions(+), 511 deletions(-) create mode 100644 crates/store/migrations/20260914000000_add_queue_poison_guards.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5dfdb879..4d22c2f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,13 +49,10 @@ jobs: mkdir -p ~/.local/share/tlaplus curl -sL -o ~/.local/share/tlaplus/tla2tools.jar \ https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar - # This audit pass proves bugs exist in CURRENT code and proves correct - # fix designs for them - the fixes are NOT yet applied to the Rust code - # (see node/README.md's "Known gap" sections). These configs model the - # verified fix designs (or a baseline that was never buggy) and must - # always pass. See root README.md's "Formal verification (TLA+)" - # section for what each spec covers. - - name: Run baseline + proposed-fix specs (must pass) + # These configs model the implemented fixes (or a baseline that was + # never buggy) and must always pass. The separate configs below retain + # the pre-fix counterexamples as historical regression documentation. + - name: Run baseline + fixed specs (must pass) working-directory: node/tla run: | set -e diff --git a/crates/store/migrations/20260914000000_add_queue_poison_guards.sql b/crates/store/migrations/20260914000000_add_queue_poison_guards.sql new file mode 100644 index 00000000..964e1f85 --- /dev/null +++ b/crates/store/migrations/20260914000000_add_queue_poison_guards.sql @@ -0,0 +1,18 @@ +-- Poison-message guards for the two durable work queues. +-- +-- Both queues previously had no way to tell "the handler returned Err and asked +-- for a retry" apart from "the attempt never finished because the process died". +-- Only the latter indicates a message that reproducibly takes the node down, so +-- it needs its own counter and a much smaller ceiling: a transient RPC or SQLite +-- outage must not push legitimate messages toward quarantine. +-- +-- `abandon_count` is incremented when an expired `Processing` lease is reclaimed, +-- so the next worker durably records that the previous dispatch never finished. + +ALTER TABLE p2p_inbox ADD COLUMN abandon_count BIGINT NOT NULL DEFAULT 0; + +ALTER TABLE message ADD COLUMN attempt_count BIGINT NOT NULL DEFAULT 0; +ALTER TABLE message ADD COLUMN abandon_count BIGINT NOT NULL DEFAULT 0; +ALTER TABLE message ADD COLUMN last_error TEXT; + +CREATE INDEX IF NOT EXISTS idx_message_claimable ON message (state, lock_time_until, created_at); diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index bd31ae4d..8a70b889 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -2,11 +2,11 @@ use crate::utils::{QueryBuilder, QueryParam, create_place_holders}; use crate::{ BridgeOutGlobalStats, EventWatchMetricsSnapshot, GoatTxRecord, Graph, GraphBtcTxVoutMonitor, GraphRawData, GraphStatus, GraphStatusSource, GraphStatusTransitionOutcome, Instance, - LongRunningTaskProof, Message, MessageDebugOverview, MessageDebugReason, MetricsStateCount, - Node, NodeAlertMetricsSnapshot, NodesOverview, OperatorProof, P2pInboxMessage, - P2pOutboxMessage, PeginGraphProcessData, PeginInstanceProcessData, PendingGraphInit, - SequencerSetHashChange, SequencerSetScanState, SerializableTxid, SwapEscrow, SwapEscrowStatus, - WatchContract, WatchtowerProof, + LongRunningTaskProof, Message, MessageDebugOverview, MessageDebugReason, MessageState, + MetricsStateCount, Node, NodeAlertMetricsSnapshot, NodesOverview, OperatorProof, + P2pInboxMessage, P2pOutboxMessage, PeginGraphProcessData, PeginInstanceProcessData, + PendingGraphInit, SequencerSetHashChange, SequencerSetScanState, SerializableTxid, SwapEscrow, + SwapEscrowStatus, WatchContract, WatchtowerProof, }; use indexmap::IndexMap; @@ -23,6 +23,11 @@ fn get_current_timestamp_secs() -> i64 { SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 } +/// Columns every `message` SELECT must fetch. +const MESSAGE_COLUMNS: &str = "message_id, business_id, from_peer, actor, msg_type, content, \ + message_version, state, weight, lock_time_until, attempt_count, abandon_count, last_error, \ + created_at"; + fn message_from_row(row: &SqliteRow) -> Result { Ok(Message { message_id: row.try_get("message_id")?, @@ -35,6 +40,9 @@ fn message_from_row(row: &SqliteRow) -> Result { message_version: row.try_get("message_version")?, weight: row.try_get("weight")?, lock_time_until: row.try_get("lock_time_until")?, + attempt_count: row.try_get("attempt_count")?, + abandon_count: row.try_get("abandon_count")?, + last_error: row.try_get("last_error")?, created_at: row.try_get("created_at")?, }) } @@ -50,6 +58,7 @@ fn p2p_inbox_message_from_row(row: &SqliteRow) -> Result StorageProcessor<'a> { } } - /// Returns grouped instance, graph, and message state counts for Node metrics. + /// Returns grouped instance, graph, and queue state counts for Node metrics. pub async fn node_metrics_state_counts(&mut self) -> anyhow::Result> { let counts = sqlx::query_as::<_, MetricsStateCount>( r#" @@ -998,6 +1007,15 @@ impl<'a> StorageProcessor<'a> { NULL AS last_success_at FROM message GROUP BY state + UNION ALL + SELECT + 'p2p_inbox' AS category, + state, + COUNT(*) AS count, + MIN(created_at) AS oldest_created_at, + NULL AS last_success_at + FROM p2p_inbox + GROUP BY state ORDER BY category, state "#, ) @@ -2438,19 +2456,19 @@ impl<'a> StorageProcessor<'a> { Ok((total, alive)) } - pub async fn update_messages_state( + /// Complete the exact local queue claim handed to a worker. + pub async fn complete_local_message( &mut self, message_id: &str, message_version: i64, - state: String, ) -> anyhow::Result { let current_time = get_current_timestamp_secs(); let res = sqlx::query( "UPDATE message \ - SET state = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state != 'Cancelled'", + SET state = 'Processed', content = X'', lock_time_until = 0, \ + abandon_count = 0, last_error = NULL, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state = 'Processing'", ) - .bind(state) .bind(current_time) .bind(message_id) .bind(message_version) @@ -2460,63 +2478,42 @@ impl<'a> StorageProcessor<'a> { Ok(res.rows_affected() > 0) } - pub async fn update_messages_state_by_business_id( + /// Cancel queued local work for a business object, including work which has + /// already been claimed. Claim completion/defer writes require `Processing`, + /// so an in-flight worker cannot overwrite the terminal cancellation. + pub async fn cancel_messages_by_business_id( &mut self, business_id: &Uuid, msg_type: Option, - old_state: String, - new_state: String, - ) -> anyhow::Result { + ) -> anyhow::Result { let current_time = get_current_timestamp_secs(); let res = match msg_type { Some(msg_type) => { sqlx::query( "UPDATE message \ - SET state = ?, updated_at = ? \ - WHERE business_id = ? AND msg_type = ? AND state = ? AND state != 'Cancelled'", + SET state = 'Cancelled', lock_time_until = 0, updated_at = ? \ + WHERE business_id = ? AND msg_type = ? \ + AND state IN ('Pending', 'Processing')", ) - .bind(new_state) .bind(current_time) .bind(business_id) .bind(msg_type) - .bind(old_state) .execute(self.conn()) .await? } None => { sqlx::query( "UPDATE message \ - SET state = ?, updated_at = ? \ - WHERE business_id = ? AND state = ? AND state != 'Cancelled'", + SET state = 'Cancelled', lock_time_until = 0, updated_at = ? \ + WHERE business_id = ? AND state IN ('Pending', 'Processing')", ) - .bind(new_state) .bind(current_time) .bind(business_id) - .bind(old_state) .execute(self.conn()) .await? } }; - Ok(res.rows_affected() > 0) - } - - pub async fn update_messages_lock_time_until( - &mut self, - message_id: &str, - message_version: i64, - lock_time_until: i64, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "Update message Set lock_time_until = ?, updated_at = ? WHERE message_id = ? AND message_version = ?", - lock_time_until, - current_time, - message_id, - message_version - - ).execute(self.conn()).await?; - - Ok(res.rows_affected() > 0) + Ok(res.rows_affected()) } pub async fn set_messages_expired(&mut self, expired: i64) -> anyhow::Result<()> { @@ -2544,21 +2541,9 @@ impl<'a> StorageProcessor<'a> { business_id: &Uuid, msg_type: &str, ) -> anyhow::Result> { - let row = sqlx::query( - "SELECT message_id, - business_id, - from_peer, - actor, - msg_type, - content, - message_version, - state, - weight, - lock_time_until, - created_at - FROM message - WHERE business_id = ? AND msg_type = ?", - ) + let row = sqlx::query(&format!( + "SELECT {MESSAGE_COLUMNS} FROM message WHERE business_id = ? AND msg_type = ?" + )) .bind(business_id) .bind(msg_type) .fetch_optional(self.conn()) @@ -2569,68 +2554,194 @@ impl<'a> StorageProcessor<'a> { &mut self, message_id: &str, ) -> anyhow::Result> { - let row = sqlx::query( - "SELECT message_id, - business_id, - from_peer, - actor, - msg_type, - content, - message_version, - state, - weight, - lock_time_until, - created_at - FROM message - WHERE message_id = ?", + let row = + sqlx::query(&format!("SELECT {MESSAGE_COLUMNS} FROM message WHERE message_id = ?")) + .bind(message_id) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| message_from_row(&row)).transpose()?) + } + + /// Retire local messages whose claims keep failing to report an outcome. + /// + /// A surviving `message` row makes the producer treat the work as already + /// created, so retiring on repeated handler errors would strand it until the + /// reaper; repeated abandons mean the node went down mid-dispatch on this + /// payload, which retrying cannot fix. + pub async fn quarantine_local_messages( + &mut self, + now: i64, + max_abandons: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Quarantined', \ + abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ + lock_time_until = 0, \ + last_error = ?, updated_at = ? \ + WHERE state IN ('Pending', 'Processing') \ + AND lock_time_until <= ? \ + AND abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END >= ?", ) - .bind(message_id) - .fetch_optional(self.conn()) + .bind(format!("quarantined: exceeded max abandoned claims ({max_abandons})")) + .bind(now) + .bind(now) + .bind(max_abandons) + .execute(self.conn()) .await?; - Ok(row.map(|row| message_from_row(&row)).transpose()?) + Ok(result.rows_affected()) } - pub async fn filter_messages( + /// Claim a batch of local messages for dispatch. + /// + /// This replaces the previous select-only pop, which wrote nothing before + /// handing work to the dispatcher. Without a durable claim, a handler that + /// panicked left the row `Pending` with its lock untouched, so the very next + /// tick re-read it and panicked again — a crash loop with no backoff at all. + /// Charging the claim up front means the record survives an abort or a kill, + /// not just an unwinding panic. + pub async fn claim_local_messages( &mut self, - state: String, - weight: i64, - lock_time_until: i64, + now: i64, + lease_until: i64, expired: i64, limit: i64, - offset: i64, + max_abandons: i64, ) -> anyhow::Result> { - let rows = sqlx::query( - "SELECT message_id, - business_id, - from_peer, - actor, - msg_type, - content, - message_version, - state, - weight, - lock_time_until, - created_at - FROM message - WHERE state = ? - AND weight >= ? - AND lock_time_until <= ? - AND updated_at >= ? - ORDER BY created_at ASC - LIMIT ? OFFSET ?", - ) - .bind(state) - .bind(weight) - .bind(lock_time_until) + let rows = sqlx::query(&format!( + "SELECT {MESSAGE_COLUMNS} FROM message \ + WHERE state IN ('Pending', 'Processing') \ + AND lock_time_until <= ? \ + AND updated_at >= ? \ + AND abandon_count < ? \ + ORDER BY created_at ASC \ + LIMIT ?" + )) + .bind(now) .bind(expired) + .bind(max_abandons) .bind(limit) - .bind(offset) .fetch_all(self.conn()) .await?; - rows.into_iter() - .map(|row| message_from_row(&row)) - .collect::, _>>() - .map_err(Into::into) + + let mut claimed = Vec::with_capacity(rows.len()); + for row in rows { + let mut message = message_from_row(&row)?; + let was_abandoned = message.state == MessageState::Processing.to_string(); + // `message_version` is deliberately left alone: callers guard their + // completion writes with the version they were handed, and bumping + // it here would make every one of those writes miss. + let result = sqlx::query( + "UPDATE message \ + SET state = 'Processing', \ + abandon_count = abandon_count + ?, \ + lock_time_until = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? \ + AND state IN ('Pending', 'Processing') \ + AND lock_time_until <= ?", + ) + .bind(i64::from(was_abandoned)) + .bind(lease_until) + .bind(now) + .bind(&message.message_id) + .bind(message.message_version) + .bind(now) + .execute(self.conn()) + .await?; + if result.rows_affected() > 0 { + message.state = MessageState::Processing.to_string(); + message.abandon_count += i64::from(was_abandoned); + message.lock_time_until = lease_until; + claimed.push(message); + } + } + Ok(claimed) + } + + /// Record a failed dispatch attempt and reschedule it with backoff. + /// + /// `attempt_count` is observability only. Only an unfinished claim increments + /// `abandon_count` and contributes to quarantine. + pub async fn defer_local_message( + &mut self, + message_id: &str, + message_version: i64, + lock_time_until: i64, + error: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Pending', attempt_count = attempt_count + 1, \ + abandon_count = 0, lock_time_until = ?, last_error = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state = 'Processing'", + ) + .bind(lock_time_until) + .bind(error.chars().take(1024).collect::()) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .bind(message_version) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + /// Record a handler panic before shutting down the process. Unlike a normal + /// defer, this increments the consecutive unfinished-attempt counter. + pub async fn abandon_local_message( + &mut self, + message_id: &str, + message_version: i64, + error: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Pending', abandon_count = abandon_count + 1, \ + lock_time_until = 0, last_error = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state = 'Processing'", + ) + .bind(error.chars().take(1024).collect::()) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .bind(message_version) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + /// Terminally drop a local message whose payload the node can never process. + pub async fn fail_local_message( + &mut self, + message_id: &str, + message_version: i64, + error: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Failed', content = X'', lock_time_until = 0, \ + last_error = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state = 'Processing'", + ) + .bind(error.chars().take(1024).collect::()) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .bind(message_version) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + /// Release local claims during a graceful process shutdown. The database is + /// process-local, so every `Processing` row belongs to this node process. + pub async fn release_processing_local_messages(&mut self) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Pending', lock_time_until = 0, updated_at = ? \ + WHERE state = 'Processing'", + ) + .bind(get_current_timestamp_secs()) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) } pub async fn get_message_queue_stats( @@ -2642,7 +2753,7 @@ impl<'a> StorageProcessor<'a> { r#"SELECT COALESCE(SUM(CASE WHEN state = 'Pending' AND lock_time_until <= ? THEN 1 ELSE 0 END), 0) AS pending_ready, COALESCE(SUM(CASE WHEN state = 'Pending' AND lock_time_until > ? THEN 1 ELSE 0 END), 0) AS pending_locked, - COALESCE(SUM(CASE WHEN state = 'Failed' THEN 1 ELSE 0 END), 0) AS failed, + COALESCE(SUM(CASE WHEN state IN ('Failed', 'Quarantined') THEN 1 ELSE 0 END), 0) AS failed, MIN(CASE WHEN state = 'Pending' THEN created_at END) AS oldest_pending_at FROM message WHERE actor = ?"#, @@ -2774,6 +2885,13 @@ impl<'a> StorageProcessor<'a> { Ok(()) } + /// Insert a local message, or refresh an existing one. + /// + /// Terminal states are excluded so a retired row stays retired. + /// [`Self::fail_local_message`] and [`Self::quarantine_local_messages`] + /// both mean the payload must not run again automatically; + /// without the exclusion a periodic producer re-upserting the same + /// deterministic message_id would resurrect it on every tick. pub async fn upsert_message(&mut self, msg: Message) -> anyhow::Result { let current_time = get_current_timestamp_secs(); let res = sqlx::query( @@ -2788,8 +2906,9 @@ impl<'a> StorageProcessor<'a> { message_version = message.message_version + 1, lock_time_until = excluded.lock_time_until, weight = excluded.weight, + last_error = NULL, updated_at = excluded.updated_at - WHERE message.state != 'Cancelled'"#, + WHERE message.state NOT IN ('Processing', 'Cancelled', 'Failed', 'Quarantined')"#, ) .bind(msg.message_id) .bind(msg.business_id) @@ -2837,13 +2956,45 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } - /// Claim ready work with a lease. The state predicate on the update keeps - /// this safe when more than one worker observes the same pending rows. + /// Move inbox rows that repeatedly abandoned a claim out of the work set. + /// + /// `attempt_count` remains a pure diagnostic counter. Only `abandon_count` + /// is a poison-message signal: the claim + /// charge is committed before dispatch, so it is recorded even when the + /// process is aborted or killed rather than unwinding. + /// + /// Content is retained until the terminal-row TTL so an operator can inspect + /// and manually requeue the message. + pub async fn quarantine_p2p_inbox_messages( + &mut self, + now: i64, + max_abandons: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE p2p_inbox \ + SET state = 'Quarantined', \ + abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ + lease_until = 0, next_retry_at = 0, \ + last_error = ?, updated_at = ? \ + WHERE state IN ('Pending', 'Processing') \ + AND lease_until <= ? \ + AND abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END >= ?", + ) + .bind(format!("quarantined: exceeded max abandoned claims ({max_abandons})")) + .bind(now) + .bind(now) + .bind(max_abandons) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + pub async fn claim_p2p_inbox_messages( &mut self, now: i64, lease_until: i64, limit: i64, + max_abandons: i64, excluded_message_ids: &[String], ) -> anyhow::Result> { let excluded_predicate = if excluded_message_ids.is_empty() { @@ -2853,14 +3004,15 @@ impl<'a> StorageProcessor<'a> { }; let query = format!( "SELECT message_id, business_id, actor, from_peer, msg_type, content, content_size, \ - state, attempt_count, next_retry_at, lease_until, lease_token, last_error, created_at, updated_at \ + state, attempt_count, abandon_count, next_retry_at, lease_until, lease_token, last_error, created_at, updated_at \ FROM p2p_inbox \ WHERE ((state = 'Pending' AND next_retry_at <= ?) \ - OR (state = 'Processing' AND lease_until <= ?)){excluded_predicate} \ + OR (state = 'Processing' AND lease_until <= ?)) \ + AND abandon_count < ?{excluded_predicate} \ ORDER BY created_at ASC \ LIMIT ?" ); - let mut query = sqlx::query(&query).bind(now).bind(now); + let mut query = sqlx::query(&query).bind(now).bind(now).bind(max_abandons); for message_id in excluded_message_ids { query = query.bind(message_id); } @@ -2869,14 +3021,23 @@ impl<'a> StorageProcessor<'a> { let mut claimed = Vec::with_capacity(rows.len()); for row in rows { let mut message = p2p_inbox_message_from_row(&row)?; + // Re-claiming a row that is still `Processing` means the previous + // attempt never reported an outcome: the worker panicked, the + // process died, or it hung past the lease. That is charged + // separately from an ordinary handler error so that a transient + // outage cannot push healthy messages toward quarantine. + let was_abandoned = message.state == "Processing"; let lease_token = Uuid::new_v4().to_string(); let result = sqlx::query( "UPDATE p2p_inbox \ - SET state = 'Processing', attempt_count = attempt_count + 1, lease_until = ?, lease_token = ?, updated_at = ? \ + SET state = 'Processing', attempt_count = attempt_count + 1, \ + abandon_count = abandon_count + ?, \ + lease_until = ?, lease_token = ?, updated_at = ? \ WHERE message_id = ? \ AND ((state = 'Pending' AND next_retry_at <= ?) \ OR (state = 'Processing' AND lease_until <= ?))", ) + .bind(i64::from(was_abandoned)) .bind(lease_until) .bind(&lease_token) .bind(now) @@ -2888,6 +3049,7 @@ impl<'a> StorageProcessor<'a> { if result.rows_affected() > 0 { message.state = "Processing".to_owned(); message.attempt_count += 1; + message.abandon_count += i64::from(was_abandoned); message.lease_until = lease_until; message.lease_token = lease_token; message.updated_at = now; @@ -2925,7 +3087,8 @@ impl<'a> StorageProcessor<'a> { ) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_inbox \ - SET state = 'Pending', lease_until = 0, next_retry_at = ?, last_error = ?, updated_at = ? \ + SET state = 'Pending', abandon_count = 0, lease_until = 0, \ + next_retry_at = ?, last_error = ?, updated_at = ? \ WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", ) .bind(next_retry_at) @@ -2938,6 +3101,28 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Record a panic from the current lease before terminating the process. + pub async fn abandon_p2p_inbox_message( + &mut self, + message_id: &str, + lease_token: &str, + error: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE p2p_inbox \ + SET state = 'Pending', abandon_count = abandon_count + 1, lease_until = 0, \ + lease_token = '', next_retry_at = 0, last_error = ?, updated_at = ? \ + WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", + ) + .bind(error.chars().take(1024).collect::()) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .bind(lease_token) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + /// Return claimed work to the queue without charging it as a processing /// attempt. This is used when capacity is unavailable before dispatch. pub async fn defer_p2p_inbox_message( @@ -2971,7 +3156,7 @@ impl<'a> StorageProcessor<'a> { ) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_inbox \ - SET state = 'Failed', lease_until = 0, next_retry_at = 0, \ + SET state = 'Failed', content = X'', lease_until = 0, next_retry_at = 0, \ last_error = ?, updated_at = ? \ WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", ) @@ -2984,6 +3169,25 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Drop terminal inbox rows once they are older than `expired_before`. + /// + /// Processed/failed payloads are already released. Quarantined payloads are + /// retained only for this bounded inspection/requeue window. Anything still + /// claimable is left alone. + pub async fn purge_terminal_p2p_inbox_messages( + &mut self, + expired_before: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "DELETE FROM p2p_inbox \ + WHERE state IN ('Processed', 'Failed', 'Quarantined') AND updated_at < ?", + ) + .bind(expired_before) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + pub async fn renew_p2p_inbox_lease( &mut self, message_id: &str, @@ -3003,12 +3207,27 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Release inbox claims during a graceful process shutdown without charging + /// them as abandoned executions. + pub async fn release_processing_p2p_inbox_messages(&mut self) -> anyhow::Result { + let result = sqlx::query( + "UPDATE p2p_inbox \ + SET state = 'Pending', lease_until = 0, lease_token = '', \ + next_retry_at = 0, updated_at = ? \ + WHERE state = 'Processing'", + ) + .bind(get_current_timestamp_secs()) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + pub async fn requeue_p2p_inbox_message(&mut self, message_id: &str) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_inbox \ - SET state = 'Pending', attempt_count = 0, next_retry_at = 0, lease_until = 0, \ + SET state = 'Pending', abandon_count = 0, next_retry_at = 0, lease_until = 0, \ lease_token = '', last_error = NULL, updated_at = ? \ - WHERE message_id = ? AND state = 'Failed' AND length(content) > 0", + WHERE message_id = ? AND state = 'Quarantined' AND length(content) > 0", ) .bind(get_current_timestamp_secs()) .bind(message_id) @@ -4672,6 +4891,444 @@ mod tests { ); } + /// A retired row must stay retired: a periodic producer re-upserting the same + /// deterministic message_id would otherwise resurrect it on every tick. + #[tokio::test] + async fn upsert_does_not_resurrect_a_retired_message() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + let msg = Message { + message_id: "retired-1".to_string(), + business_id, + actor: "Operator".to_string(), + from_peer: "self".to_string(), + msg_type: "AssertReady".to_string(), + content: vec![1, 2], + state: MessageState::Pending.to_string(), + ..Default::default() + }; + assert!(s.upsert_message(msg.clone()).await.unwrap()); + + let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert!( + s.fail_local_message("retired-1", claimed[0].message_version, "handler panicked") + .await + .unwrap() + ); + + // The producer tries again with the same deterministic id. + assert!(!s.upsert_message(msg).await.unwrap(), "a retired row must not be revived"); + let row = s.find_messages_by_id("retired-1").await.unwrap().unwrap(); + assert_eq!(row.state, "Failed"); + assert!(row.content.is_empty(), "the retired payload must stay released"); + } + + #[tokio::test] + async fn upsert_does_not_replace_an_active_local_claim() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + let mut message = Message { + message_id: "active-1".to_owned(), + business_id, + actor: "Operator".to_owned(), + from_peer: "self".to_owned(), + msg_type: "AssertReady".to_owned(), + content: vec![1, 2], + state: MessageState::Pending.to_string(), + ..Default::default() + }; + assert!(s.upsert_message(message.clone()).await.unwrap()); + let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); + assert_eq!(claimed.len(), 1); + + message.content = vec![9, 9]; + assert!(!s.upsert_message(message).await.unwrap()); + let stored = s.find_messages_by_id("active-1").await.unwrap().unwrap(); + assert_eq!(stored.state, "Processing"); + assert_eq!(stored.content, vec![1, 2]); + assert_eq!(stored.message_version, claimed[0].message_version); + } + + #[tokio::test] + async fn producer_replay_does_not_forgive_an_abandoned_local_claim() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let message = Message { + message_id: "abandoned-replay-1".to_owned(), + business_id: Uuid::new_v4(), + actor: "Operator".to_owned(), + from_peer: "self".to_owned(), + msg_type: "AssertReady".to_owned(), + content: vec![1, 2], + state: MessageState::Pending.to_string(), + ..Default::default() + }; + assert!(s.upsert_message(message.clone()).await.unwrap()); + sqlx::query("UPDATE message SET abandon_count = 2 WHERE message_id = ?") + .bind(&message.message_id) + .execute(s.conn()) + .await + .unwrap(); + + assert!(s.upsert_message(message.clone()).await.unwrap()); + let stored = s.find_messages_by_id(&message.message_id).await.unwrap().unwrap(); + assert_eq!(stored.abandon_count, 2); + } + + #[tokio::test] + async fn self_defer_cannot_be_overwritten_by_stale_completion() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES ('self-defer-1', ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", + ) + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); + assert!( + s.defer_local_message("self-defer-1", claimed[0].message_version, 150, "not ready") + .await + .unwrap() + ); + assert!( + !s.complete_local_message("self-defer-1", claimed[0].message_version).await.unwrap() + ); + let stored = s.find_messages_by_id("self-defer-1").await.unwrap().unwrap(); + assert_eq!(stored.state, "Pending"); + assert_eq!(stored.attempt_count, 1); + } + + #[tokio::test] + async fn cancellation_reaches_pending_and_processing_local_messages() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + for (message_id, msg_type, state, lock_time_until) in [ + ("cancel-processing", "AssertReady", "Processing", 500), + ("cancel-pending", "PostReady", "Pending", 400), + ("keep-processed", "KickoffReady", "Processed", 0), + ] { + sqlx::query( + "INSERT INTO message \ + (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES (?, ?, 'Operator', ?, X'01', ?, ?, 10, 10)", + ) + .bind(message_id) + .bind(business_id) + .bind(msg_type) + .bind(state) + .bind(lock_time_until) + .execute(s.conn()) + .await + .unwrap(); + } + + assert_eq!( + s.cancel_messages_by_business_id(&business_id, Some("AssertReady".to_owned())) + .await + .unwrap(), + 1 + ); + let processing = s.find_messages_by_id("cancel-processing").await.unwrap().unwrap(); + assert_eq!(processing.state, "Cancelled"); + assert_eq!(processing.lock_time_until, 0); + assert_eq!(s.cancel_messages_by_business_id(&business_id, None).await.unwrap(), 1); + assert_eq!( + s.find_messages_by_id("cancel-pending").await.unwrap().unwrap().state, + "Cancelled" + ); + assert_eq!( + s.find_messages_by_id("keep-processed").await.unwrap().unwrap().state, + "Processed" + ); + } + + /// Every `message` SELECT must fetch the full column set `message_from_row` + /// reads. Adding a column and updating only some of the hand-written SELECT + /// lists fails at runtime, not at compile time, so pin all of them here. + #[tokio::test] + async fn every_message_query_hydrates_the_full_row() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES (?, ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", + ) + .bind("hydrate-1") + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + + let by_id = s.find_messages_by_id("hydrate-1").await.unwrap().expect("row by id"); + assert_eq!(by_id.attempt_count, 0); + assert_eq!(by_id.abandon_count, 0); + assert!(by_id.last_error.is_none()); + + let by_business = s + .find_message_by_business_id(&business_id, "AssertReady") + .await + .unwrap() + .expect("row by business id"); + assert_eq!(by_business.message_id, "hydrate-1"); + + let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].message_id, "hydrate-1"); + } + + /// A message is only charged an abandon when its previous claim never + /// reported an outcome. An ordinary deferred retry must not count, otherwise + /// a transient outage would drive healthy work into quarantine. + #[tokio::test] + async fn test_inbox_abandon_is_charged_only_for_unfinished_claims() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let message = P2pInboxMessage { + message_id: "inbox-abandon-1".to_string(), + actor: "Operator".to_string(), + from_peer: "peer".to_string(), + msg_type: "CreateGraph".to_string(), + content: vec![1, 2, 3], + content_size: 3, + ..Default::default() + }; + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + + // First claim of a Pending row: a retry attempt, not an abandon. + let claimed = s.claim_p2p_inbox_messages(100, 200, 10, 3, &[]).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].attempt_count, 1); + assert_eq!(claimed[0].abandon_count, 0); + + // The handler returned a retryable error and the row went back to Pending. + assert!( + s.retry_p2p_inbox_message( + &message.message_id, + &claimed[0].lease_token, + 150, + "storage busy" + ) + .await + .unwrap() + ); + let claimed = s.claim_p2p_inbox_messages(160, 260, 10, 3, &[]).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].attempt_count, 2, "a retry is recorded"); + assert_eq!(claimed[0].abandon_count, 0, "a retry must not charge the abandon budget"); + + // Now simulate a worker that died mid-dispatch: the row is still + // Processing and its lease has expired. + let claimed = s.claim_p2p_inbox_messages(400, 500, 10, 3, &[]).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].attempt_count, 3); + assert_eq!(claimed[0].abandon_count, 1, "an unfinished claim charges the abandon budget"); + + assert!( + s.retry_p2p_inbox_message( + &message.message_id, + &claimed[0].lease_token, + 450, + "dependency pending", + ) + .await + .unwrap() + ); + let claimed = s.claim_p2p_inbox_messages(460, 560, 10, 3, &[]).await.unwrap(); + assert_eq!(claimed[0].abandon_count, 0, "a reported outcome resets consecutive abandons"); + } + + /// A payload that keeps taking the node down is quarantined rather than + /// dispatched again. Its content remains available for manual requeue until + /// terminal-row cleanup removes it. + #[tokio::test] + async fn test_inbox_quarantines_repeatedly_abandoned_message() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let message = P2pInboxMessage { + message_id: "inbox-poison-1".to_string(), + actor: "Operator".to_string(), + from_peer: "peer".to_string(), + msg_type: "GraphFinalize".to_string(), + content: vec![9; 64], + content_size: 64, + ..Default::default() + }; + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + + // Claims that never report an outcome, each one lease apart. On the + // sweep after the third expired claim, that final abandon is recorded + // as part of the quarantine transition. + let mut now = 100; + for attempt in 1..=3 { + let claimed = s.claim_p2p_inbox_messages(now, now + 10, 10, 3, &[]).await.unwrap(); + assert_eq!(claimed.len(), 1, "claim {attempt} should still be served"); + assert_eq!( + claimed[0].abandon_count, + attempt - 1, + "claim {attempt} charges one abandon per unfinished predecessor" + ); + now += 100; + } + + let quarantined = s.quarantine_p2p_inbox_messages(now, 3).await.unwrap(); + assert_eq!(quarantined, 1); + + let claimed = s.claim_p2p_inbox_messages(now, now + 10, 10, 3, &[]).await.unwrap(); + assert!(claimed.is_empty(), "a quarantined message must not be claimed again"); + + let row = + sqlx::query("SELECT state, content, last_error FROM p2p_inbox WHERE message_id = ?") + .bind(&message.message_id) + .fetch_one(s.conn()) + .await + .unwrap(); + assert_eq!(row.get::("state"), "Quarantined"); + assert_eq!(row.get::, _>("content"), message.content); + assert!(row.get::, _>("last_error").is_some()); + + assert!(s.requeue_p2p_inbox_message(&message.message_id).await.unwrap()); + let requeued = s.claim_p2p_inbox_messages(now, now + 10, 10, 3, &[]).await.unwrap(); + assert_eq!(requeued.len(), 1); + assert_eq!(requeued[0].abandon_count, 0); + } + + /// The local queue used to hand out work without writing anything, so a + /// handler that panicked left the row immediately claimable again. A claim + /// must hold the message for the length of its lease. + #[tokio::test] + async fn test_local_claim_holds_lease_and_charges_abandon() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES (?, ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", + ) + .bind("local-claim-1") + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + + let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].abandon_count, 0); + + // Still inside the lease: the message must not be handed out again. + let claimed_again = s.claim_local_messages(150, 250, 0, 10, 3).await.unwrap(); + assert!(claimed_again.is_empty(), "a leased message must not be re-claimed"); + + // Lease expired with no outcome reported: that is an abandon. + let reclaimed = s.claim_local_messages(300, 400, 0, 10, 3).await.unwrap(); + assert_eq!(reclaimed.len(), 1); + assert_eq!(reclaimed[0].abandon_count, 1); + } + + #[tokio::test] + async fn graceful_shutdown_releases_claims_without_charging_abandon() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES ('shutdown-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 10, 10)", + ) + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + let inbox = P2pInboxMessage { + message_id: "shutdown-inbox".to_owned(), + actor: "Operator".to_owned(), + from_peer: "peer".to_owned(), + msg_type: "CreateGraph".to_owned(), + content: vec![1], + content_size: 1, + ..Default::default() + }; + assert!(s.insert_p2p_inbox_message(&inbox).await.unwrap()); + assert_eq!(s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap().len(), 1); + assert_eq!(s.claim_p2p_inbox_messages(100, 200, 10, 3, &[]).await.unwrap().len(), 1); + + assert_eq!(s.release_processing_local_messages().await.unwrap(), 1); + assert_eq!(s.release_processing_p2p_inbox_messages().await.unwrap(), 1); + + let local = s.find_messages_by_id("shutdown-local").await.unwrap().unwrap(); + assert_eq!(local.state, "Pending"); + assert_eq!(local.abandon_count, 0); + let inbox = sqlx::query( + "SELECT state, abandon_count, lease_token FROM p2p_inbox WHERE message_id = ?", + ) + .bind("shutdown-inbox") + .fetch_one(s.conn()) + .await + .unwrap(); + assert_eq!(inbox.get::("state"), "Pending"); + assert_eq!(inbox.get::("abandon_count"), 0); + assert!(inbox.get::("lease_token").is_empty()); + } + + /// Deferring a local message records the retry, but only abandoned claims + /// contribute to quarantine. + #[tokio::test] + async fn test_local_defer_charges_attempts_but_only_abandons_quarantine() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES (?, ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", + ) + .bind("local-defer-1") + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + + let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); + assert_eq!(claimed.len(), 1); + assert!( + s.defer_local_message(&claimed[0].message_id, claimed[0].message_version, 150, "boom") + .await + .unwrap() + ); + + let reclaimed = s.claim_local_messages(160, 260, 0, 10, 3).await.unwrap(); + assert_eq!(reclaimed.len(), 1); + assert_eq!(reclaimed[0].attempt_count, 1, "defer records the attempt"); + assert_eq!(reclaimed[0].abandon_count, 0, "defer must not charge the abandon budget"); + + // A deferred message is NOT quarantined however many times it errors: + // the local queue has no error budget, because a surviving row makes the + // producer treat the work as already created. + assert_eq!(s.quarantine_local_messages(300, 3).await.unwrap(), 0); + + // Abandoned claims are what retires it. The quarantine sweep counts the + // final expired Processing lease. + let mut now = 400; + for _ in 0..3 { + s.claim_local_messages(now, now + 10, 0, 10, 3).await.unwrap(); + now += 100; + } + assert_eq!(s.quarantine_local_messages(now, 3).await.unwrap(), 1); + let row = sqlx::query("SELECT state, content FROM message WHERE message_id = ?") + .bind("local-defer-1") + .fetch_one(s.conn()) + .await + .unwrap(); + assert_eq!(row.get::("state"), "Quarantined"); + assert_eq!(row.get::, _>("content"), vec![1, 2]); + } + #[tokio::test] async fn test_message_debug_reasons_are_deduplicated() { let db = setup_db().await; diff --git a/crates/store/src/schema.rs b/crates/store/src/schema.rs index 21319df2..40f9f431 100644 --- a/crates/store/src/schema.rs +++ b/crates/store/src/schema.rs @@ -544,8 +544,15 @@ pub struct GraphBtcTxVoutMonitor { #[derive(Clone, Debug, Display, EnumString)] pub enum MessageState { Pending, + /// Claimed by a worker and currently being dispatched. A row left in this + /// state past its `lock_time_until` means the attempt never finished, which + /// is charged as an abandon rather than a retry. + Processing, Processed, Failed, + /// Repeated claims expired without reporting an outcome. Kept separate + /// from deterministic handler failures so operators can inspect/requeue it. + Quarantined, Expired, Cancelled, } @@ -562,6 +569,14 @@ pub struct Message { pub message_version: i64, pub weight: i64, pub lock_time_until: i64, + /// Dispatch attempts that ended in a handler `Err` and were rescheduled. + /// Observability only; this counter never retires a message. + pub attempt_count: i64, + /// Claims whose previous attempt never reported an outcome, i.e. the worker + /// panicked or the process died mid-dispatch. Incremented when an expired + /// `Processing` lease is reclaimed. + pub abandon_count: i64, + pub last_error: Option, pub created_at: i64, } @@ -569,8 +584,8 @@ pub struct Message { /// /// Unlike `Message`, which is used for locally generated compensation work, /// this row retains the original sender and is consumed before dispatching the -/// external message. Processed content is cleared, while failed content is -/// retained for manual requeue and later TTL cleanup. +/// external message. Processed/failed content is cleared; quarantined content is +/// retained temporarily so an operator can inspect or manually requeue it. #[derive(Clone, FromRow, Debug, Serialize, Deserialize, Default)] pub struct P2pInboxMessage { pub message_id: String, @@ -582,6 +597,8 @@ pub struct P2pInboxMessage { pub content_size: i64, pub state: String, pub attempt_count: i64, + /// Claims whose attempt never reported an outcome. See `Message::abandon_count`. + pub abandon_count: i64, pub next_retry_at: i64, pub lease_until: i64, pub lease_token: String, diff --git a/deployment/regtest/bitvm-noded/stop_nodes.sh b/deployment/regtest/bitvm-noded/stop_nodes.sh index 726eadb4..48bc8e01 100644 --- a/deployment/regtest/bitvm-noded/stop_nodes.sh +++ b/deployment/regtest/bitvm-noded/stop_nodes.sh @@ -1 +1 @@ -killall -9 bitvm-noded \ No newline at end of file +killall -TERM bitvm-noded diff --git a/deployment/testnet4/bitvm-noded/stop_nodes.sh b/deployment/testnet4/bitvm-noded/stop_nodes.sh index 726eadb4..48bc8e01 100644 --- a/deployment/testnet4/bitvm-noded/stop_nodes.sh +++ b/deployment/testnet4/bitvm-noded/stop_nodes.sh @@ -1 +1 @@ -killall -9 bitvm-noded \ No newline at end of file +killall -TERM bitvm-noded diff --git a/node/src/action.rs b/node/src/action.rs index 7628f734..79c772dc 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -3,7 +3,7 @@ #![allow(clippy::collapsible_else_if)] use crate::env::{ - get_local_node_info, get_p2p_graph_setup_retry_interval_secs, + MESSAGE_EXPIRE_TIME, get_local_node_info, get_p2p_graph_setup_retry_interval_secs, get_p2p_graph_setup_retry_window_secs, get_p2p_inbox_batch_size, get_p2p_outbox_batch_size, }; use crate::handle::{ @@ -27,6 +27,7 @@ use client::{ btc_chain::{BTCClient, BtcRpcTimeoutError}, goat_chain::GOATClient, }; +use futures::FutureExt; use libp2p::gossipsub::MessageId; use libp2p::{PeerId, Swarm, gossipsub}; use musig2::{PartialSignature, PubNonce}; @@ -56,6 +57,91 @@ const P2P_INBOX_LEASE_SECS: i64 = 5 * 60; const P2P_INBOX_LEASE_RENEW_INTERVAL_SECS: u64 = 60; const P2P_INBOX_ENQUEUE_ATTEMPTS: usize = 3; +/// Budget for claims that never reported an outcome. Small on purpose: reaching +/// this means the node went down mid-dispatch more than once on the same +/// payload, which is the signature of a message that reproducibly kills it. +const QUEUE_MAX_ABANDONS: i64 = 3; +/// How long a claimed local message stays claimed before another tick may take +/// it over. Heavy work is routed through the durable P2P inbox instead, so local +/// handlers are expected to be short. +const LOCAL_MESSAGE_LEASE_SECS: i64 = 10 * 60; +/// Backoff applied to a local message whose handler returned a non-transient error. +const LOCAL_MESSAGE_RETRY_DELAY_SECS: i64 = 600; +const LOCAL_MESSAGE_BATCH_SIZE: i64 = 50; +/// A dispatch future erased behind a box to keep the enclosing task's state +/// machine reasonably small. +type BoxedDispatch<'a> = std::pin::Pin> + 'a>>; + +enum DispatchExecution { + Completed(T), + Shutdown, + Panicked(String), +} + +struct LocalMessageClaim { + message_id: String, + message_version: i64, +} + +tokio::task_local! { + static ACTIVE_LOCAL_MESSAGE_CLAIM: LocalMessageClaim; +} + +fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_owned() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "non-string panic payload".to_owned() + } +} + +/// Catch only at the worker-supervisor boundary. A panic is returned separately +/// so the caller can record an abandon and stop the node; it is never converted +/// into an ordinary handler error or followed by more business work. +async fn supervise_dispatch(future: F, shutdown: &CancellationToken) -> DispatchExecution +where + F: std::future::Future, +{ + match std::panic::AssertUnwindSafe(async { + tokio::select! { + biased; + _ = shutdown.cancelled() => None, + result = future => Some(result), + } + }) + .catch_unwind() + .await + { + Ok(Some(result)) => DispatchExecution::Completed(result), + Ok(None) => DispatchExecution::Shutdown, + Err(payload) => DispatchExecution::Panicked(panic_payload_message(payload.as_ref())), + } +} + +/// Log a per-message bookkeeping failure without aborting the rest of the batch. +/// +/// Propagating here used to abandon every message still claimed in the batch. +/// Those rows stay `Processing` until their lease lapses and are then charged an +/// abandon they never earned — so one transient storage blip could push a whole +/// batch of healthy messages toward quarantine. +fn log_queue_bookkeeping_failure( + queue: &'static str, + message_id: &str, + operation: &str, + error: &anyhow::Error, +) { + tracing::error!( + event = queue, + outcome = "bookkeeping_failed", + message_id, + operation, + error = %error, + "failed to persist a message outcome; leaving it claimed for its lease to lapse" + ); +} + /// Delivery semantics for externally received P2P messages. /// /// Protocol-state messages remain durable. Ephemeral messages carry @@ -79,6 +165,17 @@ struct HeavyTaskPermit { lease_token: String, } +/// Ensure a panicking background task cannot leave its detached lease renewer +/// running forever. Once renewal stops, the durable row becomes claimable and +/// the unfinished execution is counted as an abandon. +struct LeaseRenewalGuard(CancellationToken); + +impl Drop for LeaseRenewalGuard { + fn drop(&mut self) { + self.0.cancel(); + } +} + impl Drop for HeavyTaskPermit { fn drop(&mut self) { if let Ok(mut active) = ACTIVE_HEAVY_TASK.lock() @@ -1063,6 +1160,90 @@ fn log_stale_p2p_inbox_lease(message_id: &str, lease_token: &str, operation: &st ); } +async fn fail_p2p_inbox_without_aborting_batch( + local_db: &LocalDB, + message_id: &str, + lease_token: &str, + error: &str, +) { + let result = async { + let mut storage = local_db.acquire().await?; + storage.fail_p2p_inbox_message(message_id, lease_token, error).await + } + .await; + match result { + Ok(true) => {} + Ok(false) => log_stale_p2p_inbox_lease(message_id, lease_token, "fail"), + Err(error) => log_queue_bookkeeping_failure("p2p_inbox", message_id, "fail", &error), + } +} + +async fn defer_p2p_inbox_without_aborting_batch( + local_db: &LocalDB, + message_id: &str, + lease_token: &str, + next_retry_at: i64, + reason: &str, +) { + let result = async { + let mut storage = local_db.acquire().await?; + storage.defer_p2p_inbox_message(message_id, lease_token, next_retry_at, reason).await + } + .await; + match result { + Ok(true) => {} + Ok(false) => log_stale_p2p_inbox_lease(message_id, lease_token, "defer"), + Err(error) => log_queue_bookkeeping_failure("p2p_inbox", message_id, "defer", &error), + } +} + +async fn abandon_p2p_inbox_after_panic( + local_db: &LocalDB, + message_id: &str, + lease_token: &str, + detail: &str, +) { + let error = format!("handler panicked: {detail}"); + let result = async { + let mut storage = local_db.acquire().await?; + storage.abandon_p2p_inbox_message(message_id, lease_token, &error).await + } + .await; + match result { + Ok(true) => {} + Ok(false) => log_stale_p2p_inbox_lease(message_id, lease_token, "abandon"), + Err(error) => log_queue_bookkeeping_failure("p2p_inbox", message_id, "abandon", &error), + } +} + +async fn abandon_local_message_after_panic( + local_db: &LocalDB, + message_id: &str, + message_version: i64, + detail: &str, +) { + let error = format!("handler panicked: {detail}"); + let result = async { + let mut storage = local_db.acquire().await?; + storage.abandon_local_message(message_id, message_version, &error).await + } + .await; + match result { + Ok(true) => {} + Ok(false) => tracing::warn!( + event = "local_message_queue", + outcome = "stale_claim", + message_id, + message_version, + operation = "abandon", + "ignored local message update from a stale claim" + ), + Err(error) => { + log_queue_bookkeeping_failure("local_message_queue", message_id, "abandon", &error) + } + } +} + async fn renew_p2p_inbox_lease_until_cancelled( local_db: LocalDB, message_id: String, @@ -1117,36 +1298,57 @@ async fn handle_p2p_inbox_messages( soldering_builder: &Option>, actor: Actor, metrics_state: &MetricsState, + shutdown: &CancellationToken, ) -> Result<()> { let now = current_time_secs(); let active_heavy_task_ids = active_heavy_task_message_ids(); let mut storage = local_db.start_immediate_transaction().await?; + // Quarantine rows whose dispatch repeatedly failed to report any outcome. + // Returned retryable errors do not consume this budget. + let quarantined = storage.quarantine_p2p_inbox_messages(now, QUEUE_MAX_ABANDONS).await?; + // Bound terminal metadata and the temporary payload retained for manual + // inspection of quarantined rows. + let purged = storage.purge_terminal_p2p_inbox_messages(now - MESSAGE_EXPIRE_TIME).await?; let messages = storage .claim_p2p_inbox_messages( now, now + P2P_INBOX_LEASE_SECS, get_p2p_inbox_batch_size(), + QUEUE_MAX_ABANDONS, &active_heavy_task_ids, ) .await?; storage.commit().await?; + if quarantined > 0 { + tracing::warn!( + event = "p2p_inbox", + outcome = "quarantined", + quarantined, + max_abandons = QUEUE_MAX_ABANDONS, + "quarantined inbox messages that repeatedly abandoned their lease" + ); + } + if purged > 0 { + tracing::info!( + event = "p2p_inbox", + outcome = "purged", + purged, + "removed terminal inbox rows past their retention window" + ); + } + for message in messages { let from_peer_id = match PeerId::from_str(&message.from_peer) { Ok(peer_id) => peer_id, Err(error) => { - let updated = local_db - .acquire() - .await? - .fail_p2p_inbox_message( - &message.message_id, - &message.lease_token, - &format!("invalid stored source peer: {error}"), - ) - .await?; - if !updated { - log_stale_p2p_inbox_lease(&message.message_id, &message.lease_token, "fail"); - } + fail_p2p_inbox_without_aborting_batch( + local_db, + &message.message_id, + &message.lease_token, + &format!("invalid stored source peer: {error}"), + ) + .await; continue; } }; @@ -1155,38 +1357,24 @@ async fn handle_p2p_inbox_messages( let decoded = match GOATMessage::deserialize_message(&message.content).await { Ok(message) => message, Err(error) => { - let updated = local_db - .acquire() - .await? - .fail_p2p_inbox_message( - &message.message_id, - &message.lease_token, - &error.to_string(), - ) - .await?; - if !updated { - log_stale_p2p_inbox_lease( - &message.message_id, - &message.lease_token, - "fail", - ); - } - continue; - } - }; - let Some(task) = heavy_task_from_content(decoded.content(), &actor) else { - let updated = local_db - .acquire() - .await? - .fail_p2p_inbox_message( + fail_p2p_inbox_without_aborting_batch( + local_db, &message.message_id, &message.lease_token, - &format!("inbox message type does not match {} content", message.msg_type), + &error.to_string(), ) - .await?; - if !updated { - log_stale_p2p_inbox_lease(&message.message_id, &message.lease_token, "fail"); + .await; + continue; } + }; + let Some(task) = heavy_task_from_content(decoded.content(), &actor) else { + fail_p2p_inbox_without_aborting_batch( + local_db, + &message.message_id, + &message.lease_token, + &format!("inbox message type does not match {} content", message.msg_type), + ) + .await; continue; }; Some(task) @@ -1208,19 +1396,14 @@ async fn handle_p2p_inbox_messages( let graph_id = heavy_task.graph_id(); let Some(permit) = try_acquire_heavy_task_permit(&message_id, &lease_token) else { let retry_after_secs = 5; - let updated = local_db - .acquire() - .await? - .defer_p2p_inbox_message( - &message_id, - &lease_token, - current_time_secs() + retry_after_secs, - RetryableDispatchReason::ResourceLocked.code(), - ) - .await?; - if !updated { - log_stale_p2p_inbox_lease(&message.message_id, &message.lease_token, "defer"); - } + defer_p2p_inbox_without_aborting_batch( + &local_db, + &message_id, + &lease_token, + current_time_secs() + retry_after_secs, + RetryableDispatchReason::ResourceLocked.code(), + ) + .await; tracing::debug!( event = "p2p_inbox", outcome = "deferred", @@ -1232,9 +1415,11 @@ async fn handle_p2p_inbox_messages( ); continue; }; + let shutdown = shutdown.clone(); tokio::spawn(async move { let _permit = permit; let lease_cancellation = CancellationToken::new(); + let _lease_guard = LeaseRenewalGuard(lease_cancellation.clone()); let lease_renewal = tokio::spawn(renew_p2p_inbox_lease_until_cancelled( local_db.clone(), message_id.clone(), @@ -1249,7 +1434,8 @@ async fn handle_p2p_inbox_messages( metrics_state: metrics_state.clone(), from_peer_id, }; - let result = run_heavy_task(&context, heavy_task).await; + let execution = + supervise_dispatch(run_heavy_task(&context, heavy_task), &shutdown).await; lease_cancellation.cancel(); let lease_is_current = match lease_renewal.await { Ok(lease_is_current) => lease_is_current, @@ -1258,25 +1444,62 @@ async fn handle_p2p_inbox_messages( false } }; - if !lease_is_current { - return; - } - metrics_state.record_message_dispatch( - task_type, - if result.is_ok() { "success" } else { "failed" }, - ); - if let Err(error) = finish_p2p_inbox_attempt( - &local_db, - &metrics_state, - &message_id, - &lease_token, - task_type, - attempt_count, - result, - ) - .await - { - tracing::error!(error = %error, message_id, "failed to persist heavy task result"); + match execution { + DispatchExecution::Completed(result) => { + if !lease_is_current { + return; + } + metrics_state.record_message_dispatch( + task_type, + if result.is_ok() { "success" } else { "failed" }, + ); + if let Err(error) = finish_p2p_inbox_attempt( + &local_db, + &metrics_state, + &message_id, + &lease_token, + task_type, + attempt_count, + result, + ) + .await + { + tracing::error!(error = %error, message_id, "failed to persist heavy task result"); + } + } + DispatchExecution::Shutdown => { + if lease_is_current { + defer_p2p_inbox_without_aborting_batch( + &local_db, + &message_id, + &lease_token, + current_time_secs(), + "graceful_shutdown", + ) + .await; + } + } + DispatchExecution::Panicked(detail) => { + metrics_state.record_message_dispatch(task_type, "failed"); + tracing::error!( + event = "heavy_task_panic", + outcome = "node_shutdown", + message_id, + graph_id = %graph_id, + message_type = task_type, + task_kind, + detail, + "heavy task panicked; recorded an abandon and stopping the node" + ); + abandon_p2p_inbox_after_panic( + &local_db, + &message_id, + &lease_token, + &detail, + ) + .await; + shutdown.cancel(); + } } }); tracing::info!( @@ -1293,23 +1516,20 @@ async fn handle_p2p_inbox_messages( let raw_message_id = match hex::decode(&message.message_id) { Ok(message_id) => MessageId(message_id), Err(error) => { - let updated = local_db - .acquire() - .await? - .fail_p2p_inbox_message( - &message.message_id, - &message.lease_token, - &format!("invalid stored message id: {error}"), - ) - .await?; - if !updated { - log_stale_p2p_inbox_lease(&message.message_id, &message.lease_token, "fail"); - } + fail_p2p_inbox_without_aborting_batch( + local_db, + &message.message_id, + &message.lease_token, + &format!("invalid stored message id: {error}"), + ) + .await; continue; } }; - let result = recv_and_dispatch( + // Keep the deeply nested dispatch future out of the enclosing task's + // inline state machine. + let dispatch: BoxedDispatch<'_> = Box::pin(recv_and_dispatch( swarm, local_db, btc_client, @@ -1321,78 +1541,56 @@ async fn handle_p2p_inbox_messages( raw_message_id, &message.content, metrics_state, - ) - .await; - - let mut storage = local_db.acquire().await?; - match result { - Ok(()) => { - if !storage - .complete_p2p_inbox_message(&message.message_id, &message.lease_token) - .await? - { - log_stale_p2p_inbox_lease( - &message.message_id, - &message.lease_token, - "complete", - ); - } + )); + let result = match supervise_dispatch(dispatch, shutdown).await { + DispatchExecution::Completed(result) => result, + DispatchExecution::Shutdown => { + defer_p2p_inbox_without_aborting_batch( + local_db, + &message.message_id, + &message.lease_token, + current_time_secs(), + "graceful_shutdown", + ) + .await; + return Ok(()); } - Err(error) => { - let Some((reason, requested_retry_after_secs)) = - p2p_retryable_dispatch_error(&error) - else { - if !storage - .fail_p2p_inbox_message( - &message.message_id, - &message.lease_token, - &error.to_string(), - ) - .await? - { - log_stale_p2p_inbox_lease( - &message.message_id, - &message.lease_token, - "fail", - ); - } - tracing::warn!( - event = "p2p_inbox", - outcome = "failed", - message_id = %message.message_id, - message_type = %message.msg_type, - attempt_count = message.attempt_count, - error = %error, - "cached P2P message failed permanently" - ); - continue; - }; - let retry_after_secs = requested_retry_after_secs - .unwrap_or_else(|| p2p_retry_delay_secs(message.attempt_count)); - if !storage - .retry_p2p_inbox_message( - &message.message_id, - &message.lease_token, - current_time_secs() + retry_after_secs, - &error.to_string(), - ) - .await? - { - log_stale_p2p_inbox_lease(&message.message_id, &message.lease_token, "retry"); - } - metrics_state.record_message_retry(); - tracing::warn!( - event = "p2p_inbox", - outcome = "deferred", - reason = reason.code(), + DispatchExecution::Panicked(detail) => { + tracing::error!( + event = "p2p_dispatch_panic", + outcome = "node_shutdown", message_id = %message.message_id, message_type = %message.msg_type, - attempt_count = message.attempt_count, - retry_after_secs, - error = %error, - "deferred cached P2P message for retry" + detail, + "P2P message handler panicked; recorded an abandon and stopping the node" ); + abandon_p2p_inbox_after_panic( + local_db, + &message.message_id, + &message.lease_token, + &detail, + ) + .await; + shutdown.cancel(); + bail!("P2P message handler panicked: {detail}"); } + }; + metrics_state.record_message_dispatch( + &message.msg_type, + if result.is_ok() { "success" } else { "failed" }, + ); + if let Err(error) = finish_p2p_inbox_attempt( + local_db, + metrics_state, + &message.message_id, + &message.lease_token, + &message.msg_type, + message.attempt_count, + result, + ) + .await + { + log_queue_bookkeeping_failure("p2p_inbox", &message.message_id, "finish", &error); } } Ok(()) @@ -1423,6 +1621,15 @@ async fn finish_p2p_inbox_attempt( { log_stale_p2p_inbox_lease(message_id, lease_token, "fail"); } + tracing::warn!( + event = "p2p_inbox", + outcome = "failed", + message_id, + message_type, + attempt_count, + error = %error, + "cached P2P message failed permanently" + ); return Ok(()); }; let retry_after_secs = @@ -1554,6 +1761,7 @@ pub async fn handle_self_p2p_msg( id: MessageId, message: &[u8], metrics_state: &MetricsState, + shutdown: &CancellationToken, ) -> Result<()> { if id != GOATMessage::default_message_id() { tracing::warn!( @@ -1575,42 +1783,105 @@ pub async fn handle_self_p2p_msg( "received local queue trigger" ); - let messages = - pop_batch_local_unhandle_msg(local_db, actor.clone(), current_time_secs(), 0, 50).await?; + let (messages, quarantined) = claim_batch_local_msg( + local_db, + LOCAL_MESSAGE_LEASE_SECS, + QUEUE_MAX_ABANDONS, + LOCAL_MESSAGE_BATCH_SIZE, + ) + .await?; + if quarantined > 0 { + tracing::warn!( + event = "local_message_queue", + outcome = "quarantined", + quarantined, + max_abandons = QUEUE_MAX_ABANDONS, + "retired local messages that kept failing to report an outcome" + ); + } tracing::info!( event = "local_message_queue", outcome = "batch_loaded", role = %actor, batch_size = messages.len(), - "loaded pending local messages" + "claimed pending local messages" ); for message in messages { let queue_wait_secs = current_time_secs().saturating_sub(message.created_at); let started_at = Instant::now(); - match recv_and_dispatch( - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor.clone(), - from_peer_id, - id.clone(), - &message.content, - metrics_state, - ) - .await - { + let claim = LocalMessageClaim { + message_id: message.message_id.clone(), + message_version: message.message_version, + }; + let dispatch: BoxedDispatch<'_> = Box::pin(ACTIVE_LOCAL_MESSAGE_CLAIM.scope( + claim, + recv_and_dispatch( + swarm, + local_db, + btc_client, + goat_client, + http_client, + soldering_builder, + actor.clone(), + from_peer_id, + id.clone(), + &message.content, + metrics_state, + ), + )); + let result = match supervise_dispatch(dispatch, shutdown).await { + DispatchExecution::Completed(result) => result, + DispatchExecution::Shutdown => return Ok(()), + DispatchExecution::Panicked(detail) => { + tracing::error!( + event = "local_message_dispatch_panic", + outcome = "node_shutdown", + queued_message_id = %message.message_id, + business_id = %message.business_id, + message_type = %message.msg_type, + detail, + "local message handler panicked; recorded an abandon and stopping the node" + ); + abandon_local_message_after_panic( + local_db, + &message.message_id, + message.message_version, + &detail, + ) + .await; + shutdown.cancel(); + bail!("local message handler panicked: {detail}"); + } + }; + match result { Ok(_) => { - let mut storage_processor = local_db.acquire().await?; - let state_updated = storage_processor - .update_messages_state( - &message.message_id, - message.message_version, - MessageState::Processed.to_string(), - ) - .await?; + let mut storage_processor = match local_db.acquire().await { + Ok(storage_processor) => storage_processor, + Err(error) => { + log_queue_bookkeeping_failure( + "local_message_queue", + &message.message_id, + "acquire", + &error, + ); + continue; + } + }; + let state_updated = match storage_processor + .complete_local_message(&message.message_id, message.message_version) + .await + { + Ok(state_updated) => state_updated, + Err(error) => { + log_queue_bookkeeping_failure( + "local_message_queue", + &message.message_id, + "complete", + &error, + ); + continue; + } + }; if state_updated { tracing::info!( event = "local_message_queue", @@ -1624,42 +1895,63 @@ pub async fn handle_self_p2p_msg( "processed local message" ); } else { - tracing::warn!( - event = "local_message_queue", - outcome = "state_update_conflict", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "local message handler completed but its processed state was not persisted" - ); + let current_state = storage_processor + .find_messages_by_id(&message.message_id) + .await + .ok() + .flatten() + .map(|message| message.state); + if current_state.as_deref() == Some("Pending") { + tracing::debug!( + event = "local_message_queue", + outcome = "self_deferred", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "local message handler rescheduled its own queue entry" + ); + } else { + tracing::warn!( + event = "local_message_queue", + outcome = "state_update_conflict", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + current_state = ?current_state, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "local message handler completed but its processed state was not persisted" + ); + } } } Err(err) => { - let lock_time: i64 = if is_retryable_sqlite_error(&err) - && is_pegin_message_type(&message.msg_type) - { - TRANSIENT_PEGIN_RETRY_DELAY_SECS as i64 - } else { - 600 + let is_transient = is_retryable_sqlite_error(&err); + let requested_retry_delay = + p2p_retryable_dispatch_error(&err).and_then(|(_, delay)| delay); + let lock_time: i64 = requested_retry_delay.unwrap_or_else(|| { + if is_transient && is_pegin_message_type(&message.msg_type) { + TRANSIENT_PEGIN_RETRY_DELAY_SECS as i64 + } else { + LOCAL_MESSAGE_RETRY_DELAY_SECS + } + }); + let mut storage_processor = match local_db.acquire().await { + Ok(storage_processor) => storage_processor, + Err(error) => { + log_queue_bookkeeping_failure( + "local_message_queue", + &message.message_id, + "acquire", + &error, + ); + continue; + } }; - metrics_state.record_message_retry(); - tracing::warn!( - event = "local_message_queue", - outcome = "deferred", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - retry_after_secs = lock_time, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - error = %err, - "failed to process local message; deferred for retry" - ); - let mut storage_processor = local_db.acquire().await?; if let Err(reason_error) = storage_processor .upsert_message_debug_reason( &message.message_id, @@ -1676,18 +1968,61 @@ pub async fn handle_self_p2p_msg( "failed to persist local message debug reason" ); } - storage_processor - .update_messages_lock_time_until( + let deferred = storage_processor + .defer_local_message( &message.message_id, message.message_version, current_time_secs() + lock_time, + &err.to_string(), ) - .await?; + .await; + match deferred { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + event = "local_message_queue", + outcome = "stale_claim", + queued_message_id = %message.message_id, + message_version = message.message_version, + operation = "defer", + "ignored local message update from a stale claim" + ); + continue; + } + Err(error) => { + log_queue_bookkeeping_failure( + "local_message_queue", + &message.message_id, + "defer", + &error, + ); + continue; + } + } + metrics_state.record_message_retry(); + tracing::warn!( + event = "local_message_queue", + outcome = "deferred", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + retry_after_secs = lock_time, + attempt_count = message.attempt_count + 1, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %err, + "failed to process local message; deferred for retry" + ); } } } - handle_p2p_outbox_messages(swarm, local_db).await?; - handle_p2p_inbox_messages( + // The three queues share a tick but must not share a failure: propagating + // here would let one stalled queue starve the other two every tick. + if let Err(error) = handle_p2p_outbox_messages(swarm, local_db).await { + tracing::error!(error = %error, "failed to drain the durable P2P outbox"); + } + if let Err(error) = handle_p2p_inbox_messages( swarm, local_db, btc_client, @@ -1696,8 +2031,15 @@ pub async fn handle_self_p2p_msg( soldering_builder, actor, metrics_state, + shutdown, ) - .await?; + .await + { + tracing::error!(error = %error, "failed to drain the durable P2P inbox"); + if shutdown.is_cancelled() { + return Err(error); + } + } Ok(()) } @@ -1936,36 +2278,83 @@ pub async fn push_local_unhandled_messages_with_reason( reason: MessageDeferReason, reason_detail: &str, ) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; + let mut storage_processor = local_db.start_immediate_transaction().await?; let actor = message.actor.clone(); let content: GOATMessageContent = message.content().clone(); - upsert_message( - &mut storage_processor, - true, - business_id, - None, - SELF_SENDER.to_string(), - actor, - content, - 0, - delay_secs as i64, - ) - .await?; - let persist_result = match storage_processor - .find_message_by_business_id(&business_id, message.content.event_type()) - .await + let message_type = message.content.event_type(); + let target_message_id = generate_message_id(business_id, message_type.to_owned(), None); + let active_claim = ACTIVE_LOCAL_MESSAGE_CLAIM + .try_with(|claim| (claim.message_id.clone(), claim.message_version)) + .ok(); + let claimed_message = if let Some((message_id, _)) = active_claim.as_ref() { + storage_processor.find_messages_by_id(message_id).await? + } else { + None + }; + let owns_requeued_message = claimed_message.as_ref().is_some_and(|existing| { + active_claim.as_ref().is_some_and(|(message_id, message_version)| { + existing.message_id == message_id.as_str() + && existing.message_version == *message_version + && existing.business_id == business_id + && existing.msg_type == message_type + }) + }); + let self_deferred = if let Some(existing) = claimed_message.as_ref() + && existing.state == MessageState::Processing.to_string() + && owns_requeued_message { - Ok(Some(queued_message)) => { + storage_processor + .defer_local_message( + &existing.message_id, + existing.message_version, + current_time_secs() + delay_secs as i64, + reason_detail, + ) + .await? + } else { + false + }; + if !self_deferred { + let upserted = upsert_message( + &mut storage_processor, + true, + business_id, + None, + SELF_SENDER.to_string(), + actor, + content, + 0, + delay_secs as i64, + ) + .await?; + if !upserted { + let current = storage_processor.find_messages_by_id(&target_message_id).await?; + if current + .as_ref() + .is_some_and(|message| message.state == MessageState::Processing.to_string()) + { + return Err(retryable_dispatch_error( + RetryableDispatchReason::ResourceLocked, + Some(delay_secs.max(1) as i64), + format!( + "local message {business_id}:{message_type} is owned by another active claim" + ), + )); + } + } + } + let queued_message_id = if self_deferred { + claimed_message.as_ref().map(|message| message.message_id.as_str()) + } else { + Some(target_message_id.as_str()) + }; + let persist_result = match queued_message_id { + Some(message_id) => { storage_processor - .upsert_message_debug_reason( - &queued_message.message_id, - reason.code(), - reason_detail, - ) + .upsert_message_debug_reason(message_id, reason.code(), reason_detail) .await } - Ok(None) => Ok(()), - Err(error) => Err(error), + None => Ok(()), }; if let Err(error) = persist_result { tracing::warn!( @@ -1975,6 +2364,7 @@ pub async fn push_local_unhandled_messages_with_reason( "failed to persist local message defer reason" ); } + storage_processor.commit().await?; if delay_secs > 0 && let Some(metrics_state) = crate::metrics_service::node_metrics_state() { @@ -2093,4 +2483,161 @@ mod tests { assert!(!object.contains_key("setup_package")); assert!(!object.contains_key("verifier_pubkey")); } + + #[tokio::test] + async fn genuine_transient_errors_are_still_retryable() { + let error = anyhow!("database is locked"); + assert!( + p2p_retryable_dispatch_error(&error).is_some(), + "a real SQLite-busy error must remain retryable" + ); + } + + #[tokio::test] + async fn dispatch_supervisor_distinguishes_shutdown_and_panic() { + let shutdown = CancellationToken::new(); + shutdown.cancel(); + assert!(matches!( + supervise_dispatch(std::future::pending::<()>(), &shutdown).await, + DispatchExecution::Shutdown + )); + + let running = CancellationToken::new(); + let result = supervise_dispatch( + async { + panic!("poison message"); + }, + &running, + ) + .await; + assert!(matches!( + result, + DispatchExecution::Panicked(detail) if detail == "poison message" + )); + } + + #[tokio::test] + async fn non_owner_requeue_reports_processing_conflict() { + let local_db = store::create_local_db("sqlite::memory:").await; + let business_id = Uuid::new_v4(); + let message = GOATMessage::new(Actor::Operator, GOATMessageContent::Tick); + push_local_unhandled_messages_with_reason( + &local_db, + business_id, + &message, + 0, + MessageDeferReason::HandlerError, + "initial", + ) + .await + .unwrap(); + + let claimed = { + let mut storage = local_db.acquire().await.unwrap(); + storage + .claim_local_messages( + current_time_secs() + 1, + current_time_secs() + 300, + 0, + 1, + QUEUE_MAX_ABANDONS, + ) + .await + .unwrap() + }; + assert_eq!(claimed.len(), 1); + + let error = push_local_unhandled_messages_with_reason( + &local_db, + business_id, + &message, + 30, + MessageDeferReason::HandlerError, + "retry", + ) + .await + .unwrap_err(); + let retryable = error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .expect("Processing conflict must be retryable"); + assert_eq!(retryable.reason, RetryableDispatchReason::ResourceLocked); + assert_eq!(retryable.retry_after_secs, Some(30)); + + let mut storage = local_db.acquire().await.unwrap(); + let stored = storage.find_messages_by_id(&claimed[0].message_id).await.unwrap().unwrap(); + assert_eq!(stored.state, MessageState::Processing.to_string()); + assert_eq!(stored.message_version, claimed[0].message_version); + } + + #[tokio::test] + async fn owner_requeue_uses_exact_subtyped_message_id() { + let local_db = store::create_local_db("sqlite::memory:").await; + let business_id = Uuid::new_v4(); + let message = GOATMessage::new(Actor::Operator, GOATMessageContent::Tick); + let subtyped_message_id = generate_message_id( + business_id, + message.content.event_type().to_owned(), + Some("7".to_owned()), + ); + { + let mut storage = local_db.acquire().await.unwrap(); + assert!( + upsert_message( + &mut storage, + false, + business_id, + Some("7".to_owned()), + SELF_SENDER.to_owned(), + message.actor.clone(), + message.content.clone(), + 0, + 0, + ) + .await + .unwrap() + ); + } + let claimed = { + let mut storage = local_db.acquire().await.unwrap(); + storage + .claim_local_messages( + current_time_secs() + 1, + current_time_secs() + 300, + 0, + 1, + QUEUE_MAX_ABANDONS, + ) + .await + .unwrap() + .pop() + .unwrap() + }; + assert_eq!(claimed.message_id, subtyped_message_id); + + ACTIVE_LOCAL_MESSAGE_CLAIM + .scope( + LocalMessageClaim { + message_id: claimed.message_id.clone(), + message_version: claimed.message_version, + }, + push_local_unhandled_messages_with_reason( + &local_db, + business_id, + &message, + 30, + MessageDeferReason::HandlerError, + "retry subtyped message", + ), + ) + .await + .unwrap(); + + let mut storage = local_db.acquire().await.unwrap(); + let stored = storage.find_messages_by_id(&subtyped_message_id).await.unwrap().unwrap(); + assert_eq!(stored.state, MessageState::Pending.to_string()); + let base_message_id = + generate_message_id(business_id, message.content.event_type().to_owned(), None); + assert!(storage.find_messages_by_id(&base_message_id).await.unwrap().is_none()); + } } diff --git a/node/src/main.rs b/node/src/main.rs index b19d273b..0528c4d7 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -210,6 +210,7 @@ async fn main() -> Result<(), Box> { soldering_builder: matches!(actor, Actor::Verifier | Actor::Operator) .then(|| Arc::new(BabeBundleBuilder::new())), metrics_state: metrics_state.clone(), + shutdown_token: cancellation_token.clone(), }; tracing::info!( @@ -513,7 +514,7 @@ async fn main() -> Result<(), Box> { "all node background tasks have been started" ); - tokio::select! { + let fatal_error = tokio::select! { (result, index, remaining_handles) = future::select_all(task_handles) => { let task_name = task_names[index]; // Log the specific failure @@ -581,9 +582,9 @@ async fn main() -> Result<(), Box> { // Handle panic propagation if let Err(join_error) = result && join_error.is_panic() { - std::panic::resume_unwind(join_error.into_panic()); - + std::panic::resume_unwind(join_error.into_panic()); } + Some(anyhow::anyhow!("core task {task_name} stopped: {failure_reason}")) } _ = shutdown_signal() => { tracing::info!( @@ -603,7 +604,12 @@ async fn main() -> Result<(), Box> { outcome = "completed", "node graceful shutdown completed" ); + None } + }; + + if let Some(error) = fatal_error { + return Err(error.into()); } Ok(()) diff --git a/node/src/metrics_service.rs b/node/src/metrics_service.rs index d4ca387c..319c3eaa 100644 --- a/node/src/metrics_service.rs +++ b/node/src/metrics_service.rs @@ -131,6 +131,8 @@ pub struct MetricsState { graphs: Family, messages: Family, oldest_pending_message_age_seconds: Gauge, + p2p_inbox_messages: Family, + oldest_pending_p2p_inbox_age_seconds: Gauge, ready: Gauge, db_busy_retries_total: Counter, db_errors_total: Counter, @@ -191,6 +193,8 @@ impl MetricsState { let graphs = Family::default(); let messages = Family::default(); let oldest_pending_message_age_seconds = Gauge::default(); + let p2p_inbox_messages = Family::default(); + let oldest_pending_p2p_inbox_age_seconds = Gauge::default(); let ready = Gauge::default(); let db_busy_retries_total = Counter::default(); let db_errors_total = Counter::default(); @@ -284,6 +288,16 @@ impl MetricsState { "Age in seconds of the oldest pending message", oldest_pending_message_age_seconds.clone(), ); + registry.register( + "bitvm_node_p2p_inbox_messages", + "Number of durable P2P inbox messages by state", + p2p_inbox_messages.clone(), + ); + registry.register( + "bitvm_node_oldest_pending_p2p_inbox_age_seconds", + "Age in seconds of the oldest pending durable P2P inbox message", + oldest_pending_p2p_inbox_age_seconds.clone(), + ); registry.register( "bitvm_node_ready", "Whether the node is ready to process work", @@ -459,6 +473,8 @@ impl MetricsState { graphs, messages, oldest_pending_message_age_seconds, + p2p_inbox_messages, + oldest_pending_p2p_inbox_age_seconds, ready, db_busy_retries_total, db_errors_total, @@ -685,6 +701,8 @@ impl MetricsState { self.graphs.clear(); self.messages.clear(); self.oldest_pending_message_age_seconds.set(0); + self.p2p_inbox_messages.clear(); + self.oldest_pending_p2p_inbox_age_seconds.set(0); let now = current_time_secs(); for count in counts { @@ -718,6 +736,19 @@ impl MetricsState { ); } } + "p2p_inbox" => { + let status = known_status::(&count.state); + self.p2p_inbox_messages + .get_or_create(&StatusLabels { status: status.clone() }) + .inc_by(count.count); + if status == "Pending" { + self.oldest_pending_p2p_inbox_age_seconds.set( + count + .oldest_created_at + .map_or(0, |created_at| now.saturating_sub(created_at).max(0)), + ); + } + } _ => {} } } @@ -875,6 +906,13 @@ mod tests { oldest_created_at: None, last_success_at: None, }, + store::MetricsStateCount { + category: "p2p_inbox".to_string(), + state: "Pending".to_string(), + count: 4, + oldest_created_at: Some(current_time_secs() - 30), + last_success_at: None, + }, ]); let output = encoded(&state); @@ -885,6 +923,8 @@ mod tests { assert!(!output.contains("bitvm_node_graphs{status=\"OperatorPresigned\"}")); assert!(!output.contains("unexpected-id-like-value")); assert!(!output.contains("another-unexpected-value")); + assert!(output.contains("bitvm_node_p2p_inbox_messages{status=\"Pending\"} 4")); + assert!(output.contains("bitvm_node_oldest_pending_p2p_inbox_age_seconds 30")); } #[test] diff --git a/node/src/middleware/swarm.rs b/node/src/middleware/swarm.rs index 1a01c3ef..bf7b5e50 100644 --- a/node/src/middleware/swarm.rs +++ b/node/src/middleware/swarm.rs @@ -90,6 +90,10 @@ pub trait P2pMessageHandler { actor: Actor, topic: &str, ) -> anyhow::Result<()>; + + async fn graceful_shutdown(&self) -> anyhow::Result<()> { + Ok(()) + } } #[derive(Clone, Debug)] @@ -212,13 +216,20 @@ impl BitvmNetworkManager { select! { _ = cancellation_token.cancelled() => { info!("Swarm received shutdown signal"); + msg_handler.graceful_shutdown().await?; return Ok("swarm_shutdown".to_string()); } _ticker = interval.tick() => { match msg_handler.handle_tick_message(&mut self.swarm, self.peer_id, actor.clone(), TickMessageType::RegularlyAction).await { Ok(_) => {} - Err(e) => { tracing::error!("Fail to handle tick message {e:?}") } + Err(e) => { + tracing::error!("Fail to handle tick message {e:?}"); + if cancellation_token.is_cancelled() { + msg_handler.graceful_shutdown().await?; + return Err(e); + } + } } self.refresh_required_topics_health(&actor); @@ -255,7 +266,11 @@ impl BitvmNetworkManager { data_prefix, data_starts_with_goatbin, "Fail to handle p2p message" - ) + ); + if cancellation_token.is_cancelled() { + msg_handler.graceful_shutdown().await?; + return Err(e); + } } } } diff --git a/node/src/p2p_msg_handler.rs b/node/src/p2p_msg_handler.rs index 303d935c..d8ccdd86 100644 --- a/node/src/p2p_msg_handler.rs +++ b/node/src/p2p_msg_handler.rs @@ -13,6 +13,7 @@ use libp2p::PeerId; use libp2p::gossipsub::MessageId; use std::sync::Arc; use store::localdb::LocalDB; +use tokio_util::sync::CancellationToken; pub struct BitvmNodeProcessor { pub local_db: LocalDB, @@ -21,6 +22,7 @@ pub struct BitvmNodeProcessor { pub http_client: HttpAsyncClient, pub soldering_builder: Option>, pub metrics_state: MetricsState, + pub shutdown_token: CancellationToken, } impl P2pMessageHandler for BitvmNodeProcessor { async fn recv_and_dispatch( @@ -84,6 +86,7 @@ impl P2pMessageHandler for BitvmNodeProcessor { GOATMessage::default_message_id(), &tick_data, &self.metrics_state, + &self.shutdown_token, ) .await } @@ -107,6 +110,21 @@ impl P2pMessageHandler for BitvmNodeProcessor { } Ok(()) } + + async fn graceful_shutdown(&self) -> anyhow::Result<()> { + let mut storage = self.local_db.start_immediate_transaction().await?; + let local_released = storage.release_processing_local_messages().await?; + let inbox_released = storage.release_processing_p2p_inbox_messages().await?; + storage.commit().await?; + tracing::info!( + event = "message_queue_shutdown", + outcome = "claims_released", + local_released, + inbox_released, + "released active queue claims without charging abandon counters" + ); + Ok(()) + } } #[cfg(test)] diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index 77b38968..94fdfc6c 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -535,8 +535,7 @@ mod tests { let keypair = set_test_auth_key(); let cancellation_token = CancellationToken::new(); let (addr, server) = spawn_business_listener(cancellation_token.clone()).await?; - let graph_id = Uuid::new_v4(); - let request_target = format!("/v1/graphs/{graph_id}/send-challenge"); + let request_target = routes::v1::PEGOUT.to_owned(); let url = format!("http://{addr}{request_target}"); let (timestamp, nonce, signature) = sign_request_auth(&keypair, &Method::POST, &request_target, &[]); @@ -549,7 +548,7 @@ mod tests { .header(AUTH_SIGNATURE_HEADER, &signature) .send() .await?; - assert_eq!(first.status().as_u16(), 500); + assert_ne!(first.status().as_u16(), 409); let replay = client .post(&url) diff --git a/node/src/scheduled_tasks/event_watch_task.rs b/node/src/scheduled_tasks/event_watch_task.rs index 60cb1ac3..5066d17a 100644 --- a/node/src/scheduled_tasks/event_watch_task.rs +++ b/node/src/scheduled_tasks/event_watch_task.rs @@ -45,8 +45,8 @@ use store::localdb::{ }; use store::{ GoatTxProcessingStatus, GoatTxRecord, GoatTxType, GraphStatus, GraphStatusSource, - GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, MessageState, SwapEscrow, - SwapEscrowStatus, WatchContract, WatchContractStatus, normalize_escrow_hash, + GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, SwapEscrow, SwapEscrowStatus, + WatchContract, WatchContractStatus, normalize_escrow_hash, }; use tokio::time::sleep; use tokio_util::sync::CancellationToken; @@ -568,14 +568,7 @@ async fn handle_withdraw_paths_events<'a>( if is_new_event { add_node_reward(storage_processor, &goat_addr.unwrap(), reward_add).await?; } - storage_processor - .update_messages_state_by_business_id( - &graph_id, - None, - MessageState::Pending.to_string(), - MessageState::Cancelled.to_string(), - ) - .await?; + storage_processor.cancel_messages_by_business_id(&graph_id, None).await?; } Ok(()) } @@ -655,14 +648,7 @@ async fn handle_withdraw_disproved_events<'a>( ) .await?; } - storage_processor - .update_messages_state_by_business_id( - &graph_id, - None, - MessageState::Pending.to_string(), - MessageState::Cancelled.to_string(), - ) - .await?; + storage_processor.cancel_messages_by_business_id(&graph_id, None).await?; } Ok(()) } diff --git a/node/src/scheduled_tasks/graph_maintenance_tasks.rs b/node/src/scheduled_tasks/graph_maintenance_tasks.rs index 1cd3b5bc..ed3a9600 100644 --- a/node/src/scheduled_tasks/graph_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/graph_maintenance_tasks.rs @@ -303,6 +303,7 @@ async fn enqueue_kickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::Resu 0, ) .await + .map(|_| ()) } async fn enqueue_prekickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::Result<()> { @@ -322,6 +323,7 @@ async fn enqueue_prekickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::R 0, ) .await + .map(|_| ()) } fn is_kickoff_pending_status(status: &str) -> bool { diff --git a/node/src/scheduled_tasks/instance_maintenance_tasks.rs b/node/src/scheduled_tasks/instance_maintenance_tasks.rs index 122eff63..4ee932c7 100644 --- a/node/src/scheduled_tasks/instance_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/instance_maintenance_tasks.rs @@ -1,7 +1,7 @@ use crate::action::{ ConfirmInstance, GOATMessage, GOATMessageContent, MessageDeferReason, PeginConfirmNonce, PeginConfirmNonceConsensus, PeginConfirmPartialSig, PeginRequest, PostReady, - push_local_unhandled_messages_with_reason, + RetryableDispatchError, RetryableDispatchReason, push_local_unhandled_messages_with_reason, }; use crate::env::{ COMMITTEE_INSTANCE_KEYS_DIR, get_bitvm_key, get_committee_instance_key_delete_timelock_blocks, @@ -67,6 +67,34 @@ fn advance_instance_page_state(task_key: &'static str, watermark: i64, last: (i6 state.insert(task_key, InstancePageState { watermark, cursor: Some(last) }); } +fn finish_recovery_enqueue( + result: anyhow::Result<()>, + instance_id: Uuid, + action: &'static str, +) -> anyhow::Result { + match result { + Ok(()) => Ok(true), + Err(error) + if error.chain().any(|cause| { + cause.downcast_ref::().is_some_and(|retryable| { + retryable.reason == RetryableDispatchReason::ResourceLocked + }) + }) => + { + warn!( + event = "pegin_confirm_recovery", + outcome = "resource_locked", + instance_id = %instance_id, + action, + error = %error, + "skip recovery enqueue while the local message is actively claimed" + ); + Ok(false) + } + Err(error) => Err(error), + } +} + async fn find_one_instance_page( local_db: &LocalDB, task_key: &'static str, @@ -593,15 +621,21 @@ pub async fn pegin_confirm_recovery_monitor( endorse_sig, }), ); - push_local_unhandled_messages_with_reason( - local_db, + if !finish_recovery_enqueue( + push_local_unhandled_messages_with_reason( + local_db, + instance_id, + &message, + 0, + MessageDeferReason::RecoveryRepublish, + "re-publishing persisted pegin-confirm partial signature", + ) + .await, instance_id, - &message, - 0, - MessageDeferReason::RecoveryRepublish, - "re-publishing persisted pegin-confirm partial signature", - ) - .await?; + "republish_partial_signature", + )? { + continue; + } tracing::info!( event = "pegin_confirm_recovery", action = "republish_partial_signature", @@ -648,15 +682,21 @@ pub async fn pegin_confirm_recovery_monitor( nonce_sig, }), ); - push_local_unhandled_messages_with_reason( - local_db, + if !finish_recovery_enqueue( + push_local_unhandled_messages_with_reason( + local_db, + instance_id, + &message, + 0, + MessageDeferReason::RecoveryRepublish, + "re-publishing persisted pegin-confirm nonce", + ) + .await, instance_id, - &message, - 0, - MessageDeferReason::RecoveryRepublish, - "re-publishing persisted pegin-confirm nonce", - ) - .await?; + "republish_nonce", + )? { + continue; + } tracing::info!( event = "pegin_confirm_recovery", action = "republish_nonce", @@ -678,15 +718,21 @@ pub async fn pegin_confirm_recovery_monitor( signature, }), ); - push_local_unhandled_messages_with_reason( - local_db, + if !finish_recovery_enqueue( + push_local_unhandled_messages_with_reason( + local_db, + instance_id, + &message, + 0, + MessageDeferReason::RecoveryRepublish, + "re-publishing persisted PeginConfirm nonce consensus", + ) + .await, instance_id, - &message, - 0, - MessageDeferReason::RecoveryRepublish, - "re-publishing persisted PeginConfirm nonce consensus", - ) - .await?; + "republish_nonce_consensus", + )? { + continue; + } tracing::info!( event = "pegin_confirm_recovery", action = "republish_nonce_consensus", diff --git a/node/src/utils.rs b/node/src/utils.rs index bb7f8546..b3493e3b 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -3751,7 +3751,11 @@ pub async fn outpoint_spent_txin( } } -fn generate_message_id(business_id: Uuid, msg_type: String, sub_type: Option) -> String { +pub(crate) fn generate_message_id( + business_id: Uuid, + msg_type: String, + sub_type: Option, +) -> String { match sub_type { Some(sub_type) => { format!("{business_id}_{msg_type}_{sub_type}") @@ -3771,7 +3775,7 @@ pub async fn upsert_message( message_content: GOATMessageContent, weight: i64, lock_time: i64, -) -> Result<()> { +) -> Result { let message = GOATMessage::new(actor.clone(), message_content.clone()); let msg_type = get_goat_message_content_type(&message_content); let message_id = generate_message_id(business_id, msg_type.to_string().clone(), sub_type); @@ -3783,7 +3787,7 @@ pub async fn upsert_message( notify_to_cancel_proof_task(storage_processor, business_id, cancel_msg_type).await?; } - storage_processor + return storage_processor .upsert_message(Message { message_id, business_id, @@ -3795,14 +3799,17 @@ pub async fn upsert_message( lock_time_until: current_time_secs() + lock_time, state: MessageState::Pending.to_string(), message_version: 0, + attempt_count: 0, + abandon_count: 0, + last_error: None, created_at: 0, }) - .await?; + .await; } else { info!("{message_id} is already created for create action"); } - Ok(()) + Ok(false) } pub async fn notify_to_cancel_proof_task( @@ -3831,7 +3838,7 @@ pub async fn notify_to_cancel_proof_task( storage_processor.find_message_by_business_id(&business_id, &msg_type.to_string()).await? && let Some(graph) = storage_processor.find_graph(&business_id).await? { - if MessageState::Pending.to_string() != message.state { + if !matches!(message.state.as_str(), "Pending" | "Processing") { warn!( "message {business_id}, msg_type: {msg_type} no need to cancel.as state is {}", message.state @@ -3873,12 +3880,7 @@ pub async fn notify_to_cancel_proof_task( if notify_result { // cancel unfinished p2p message; when notify success! storage_processor - .update_messages_state_by_business_id( - &business_id, - Some(msg_type.to_string()), - MessageState::Pending.to_string(), - MessageState::Cancelled.to_string(), - ) + .cancel_messages_by_business_id(&business_id, Some(msg_type.to_string())) .await?; } } else { @@ -4230,29 +4232,34 @@ pub fn reflect_goat_address(addr_op: Option) -> (bool, Option) { (false, None) } -pub async fn pop_batch_local_unhandle_msg( +/// Claim a batch of local messages for dispatch, retiring exhausted ones first. +/// +/// Returns `(claimed, quarantined)`. Unlike the select-only pop this replaces, +/// every returned message carries a durable claim, so an attempt that never +/// reports an outcome is visible to the next tick instead of replaying forever. +pub async fn claim_batch_local_msg( local_db: &LocalDB, - _actor: Actor, - lock_time_until: i64, - offset: i64, + lease_secs: i64, + max_abandons: i64, limit: i64, -) -> Result> { +) -> Result<(Vec, u64)> { let mut tx = local_db.start_transaction().await?; let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; - tx.set_messages_expired(current_time - MESSAGE_EXPIRE_TIME).await?; - tx.delete_old_messages(current_time - MESSAGE_EXPIRE_TIME).await?; + let expired_before = current_time - MESSAGE_EXPIRE_TIME; + tx.set_messages_expired(expired_before).await?; + tx.delete_old_messages(expired_before).await?; + let quarantined = tx.quarantine_local_messages(current_time, max_abandons).await?; let messages = tx - .filter_messages( - MessageState::Pending.to_string(), - 0, - lock_time_until, - current_time - MESSAGE_EXPIRE_TIME, + .claim_local_messages( + current_time, + current_time + lease_secs, + expired_before, limit, - offset, + max_abandons, ) .await?; tx.commit().await?; - Ok(messages) + Ok((messages, quarantined)) } pub async fn operator_scan_ready_proof( @@ -5852,14 +5859,7 @@ pub(crate) async fn obsolete_graph( // `None` means no message-type filter: cancel every durable pending message // for this terminal graph so stale retries cannot consume queue capacity or // trigger a later graph action. - storage_processor - .update_messages_state_by_business_id( - &graph_id, - None, - MessageState::Pending.to_string(), - MessageState::Cancelled.to_string(), - ) - .await?; + storage_processor.cancel_messages_by_business_id(&graph_id, None).await?; Ok(true) } diff --git a/node/tla/MessageStateRace.cfg b/node/tla/MessageStateRace.cfg index 13a129bf..17e5d6ae 100644 --- a/node/tla/MessageStateRace.cfg +++ b/node/tla/MessageStateRace.cfg @@ -1,5 +1,5 @@ -\* Models the CURRENT, actual code (upsert_message resurrects unconditionally) -\* - expected to FAIL. This is a live bug, not a historical artifact. +\* Historical pre-fix behavior: an unconditional upsert can resurrect a +\* Cancelled row. This config must continue to produce a counterexample. SPECIFICATION FairSpec CHECK_DEADLOCK FALSE INVARIANT TypeOK diff --git a/node/tla/MessageStateRace.tla b/node/tla/MessageStateRace.tla index 9ab76eab..b8bf3dfd 100644 --- a/node/tla/MessageStateRace.tla +++ b/node/tla/MessageStateRace.tla @@ -1,78 +1,100 @@ ---- MODULE MessageStateRace ---- (***************************************************************************) -(* Formal model of a race found while auditing every remaining stateful *) -(* enum after GraphStatus/InstanceBridgeInStatus (see audit/TLAPlus-*.md). *) +(* Model the local-message cancellation race, including a worker which has *) +(* already claimed a Pending row. The current Rust implementation uses *) +(* state/version CAS operations for claim completion and owner defer, and *) +(* cancel_messages_by_business_id reaches both Pending and Processing. *) (* *) -(* `MessageState` (crates/store/src/schema.rs:431-437: Pending, Processed, *) -(* Failed, Expired, Cancelled) tracks P2P message delivery/processing *) -(* status. Unlike the GraphStatus/InstanceBridgeOutStatus findings, the *) -(* "cancel" writer here IS correctly guarded - `update_messages_state_by_ *) -(* business_id` (crates/store/src/localdb.rs) does a real CAS: `UPDATE ... *) -(* WHERE business_id=? AND state='Pending'`, called from *) -(* node/src/scheduled_tasks/event_watch_task.rs's handle_withdraw_paths_/ *) -(* disproved_events when a graph reaches a closed on-chain status *) -(* (OperatorTake1/OperatorTake2/Disprove) - bulk-cancelling any still- *) -(* Pending message for that graph as moot. *) -(* *) -(* The bug is on the OTHER side: `upsert_message` (node/src/utils.rs, *) -(* called by push_local_unhandled_messages - the generic "defer/retry *) -(* this p2p message" primitive used ~30 times across node/src/handle.rs) *) -(* with `is_update=true` unconditionally sets state back to Pending via *) -(* `INSERT ... ON CONFLICT(message_id) DO UPDATE SET state=excluded.state` *) -(* - no WHERE clause is possible on an upsert, so a message the system *) -(* just administratively marked Cancelled (because its graph is already *) -(* finalized) can be silently resurrected to Pending and re-dispatched *) -(* the next time a handler in the swarm-message task calls a retry/defer *) -(* on it, unrelated to the cancellation. *) +(* FairSpec retains the pre-fix unconditional upsert as a historical bug *) +(* reproduction. FairSpecFixed models the guarded upsert now implemented *) +(* by crates/store/src/localdb.rs. *) (***************************************************************************) -Statuses == {"Pending", "Cancelled"} -\* Cancelled is an administrative "this message is moot, stop touching it" -\* marker tied to its graph reaching a closed status - it must stay final. +Statuses == {"Pending", "Processing", "Processed", "Cancelled"} TerminalStatuses == {"Cancelled"} -VARIABLE status -vars == <> +VARIABLES status, workerActive +vars == <> -TypeOK == status \in Statuses +TypeOK == + /\ status \in Statuses + /\ workerActive \in BOOLEAN -Init == status = "Pending" +Init == + /\ status = "Pending" + /\ workerActive = FALSE --------------------------------------------------------------------------- -\* event_watch_task.rs's handle_withdraw_paths_events / handle_withdraw_ -\* disproved_events, via update_messages_state_by_business_id - a genuine -\* CAS, correctly guarded in the real code. -BulkCancelOnGraphClose == +----------------------------------------------------------------------------- +\* claim_local_messages: only an available Pending row can be claimed. +Claim == /\ status = "Pending" + /\ ~workerActive + /\ status' = "Processing" + /\ workerActive' = TRUE + +\* Terminal graph/instance handling cancels queued and already-claimed work. +BulkCancelOnGraphClose == + /\ status \in {"Pending", "Processing"} /\ status' = "Cancelled" + /\ UNCHANGED workerActive + +\* A live owner may complete or defer only the Processing row it claimed. +WorkerComplete == + /\ workerActive + /\ status = "Processing" + /\ status' = "Processed" + /\ workerActive' = FALSE + +OwnerDefer == + /\ workerActive + /\ status = "Processing" + /\ status' = "Pending" + /\ workerActive' = FALSE + +\* After cancellation, the old worker's guarded write affects zero rows. +StaleWorkerReturns == + /\ workerActive + /\ status # "Processing" + /\ UNCHANGED status + /\ workerActive' = FALSE + +\* Historical behavior: a periodic producer could resurrect any state. +UnconditionalUpsert == + /\ status' = "Pending" + /\ UNCHANGED workerActive -\* push_local_unhandled_messages -> utils::upsert_message(is_update=true) -\* -> store upsert_message's `ON CONFLICT DO UPDATE SET state=excluded.state` -\* - confirmed NO guard of any kind. Fires from ~30 call sites in -\* node/src/handle.rs whenever a message handler needs to defer/retry, -\* with no awareness of whether the message was since cancelled. -ResurrectPendingUnconditional == status' = "Pending" +\* Current behavior: Processing and terminal rows reject fallback upserts. +GuardedUpsert == + /\ status \notin {"Processing", "Cancelled"} + /\ status' = "Pending" + /\ UNCHANGED workerActive Next == + \/ Claim \/ BulkCancelOnGraphClose - \/ ResurrectPendingUnconditional + \/ WorkerComplete + \/ OwnerDefer + \/ StaleWorkerReturns + \/ UnconditionalUpsert Spec == Init /\ [][Next]_vars FairSpec == Spec /\ WF_vars(Next) -\* Proposed fix design (not applied to code): guard the resurrect-to-Pending -\* write the same way - only apply it if the message isn't already in a -\* terminal status, folded into the UPDATE/upsert's WHERE clause. NextFixed == + \/ Claim \/ BulkCancelOnGraphClose - \/ (status \notin TerminalStatuses /\ ResurrectPendingUnconditional) + \/ WorkerComplete + \/ OwnerDefer + \/ StaleWorkerReturns + \/ GuardedUpsert SpecFixed == Init /\ [][NextFixed]_vars FairSpecFixed == SpecFixed /\ WF_vars(NextFixed) --------------------------------------------------------------------------- -\* Safety property: once a message is administratively Cancelled, it must -\* never be resurrected and re-dispatched. -TerminalStatusesAreAbsorbing == [][(status \in TerminalStatuses => status' = status)]_status +----------------------------------------------------------------------------- +\* Once administratively cancelled, neither an old worker nor a producer may +\* make the message dispatchable again. +TerminalStatusesAreAbsorbing == + [][(status \in TerminalStatuses => status' = status)]_status ==== diff --git a/node/tla/MessageStateRaceFixed.cfg b/node/tla/MessageStateRaceFixed.cfg index ffdbda46..3661193d 100644 --- a/node/tla/MessageStateRaceFixed.cfg +++ b/node/tla/MessageStateRaceFixed.cfg @@ -1,4 +1,4 @@ -\* Proposed fix design (verified, NOT applied to code) - expected to pass. +\* Current behavior: claims, cancellation and worker writes are guarded. SPECIFICATION FairSpecFixed CHECK_DEADLOCK FALSE INVARIANT TypeOK From f65cf3901aa27394c506f4b05dff5253db945722 Mon Sep 17 00:00:00 2001 From: ethan Date: Wed, 16 Sep 2026 09:23:57 +0800 Subject: [PATCH 05/17] refactor: derive message identity from content --- Cargo.lock | 10 + Cargo.toml | 1 + crates/node-macros/Cargo.toml | 12 + crates/node-macros/src/lib.rs | 112 ++++++ crates/store/src/localdb.rs | 324 ++-------------- crates/store/src/schema.rs | 77 +--- node/Cargo.toml | 1 + node/src/action.rs | 361 +++++++++++------- node/src/bin/db_inject.rs | 72 +--- node/src/env.rs | 23 +- node/src/handle.rs | 62 --- node/src/main.rs | 8 +- node/src/rpc_service/bitvm.rs | 99 +++-- node/src/rpc_service/handler/bitvm_handler.rs | 9 - .../graph_maintenance_tasks.rs | 44 +-- .../instance_maintenance_tasks.rs | 98 ++--- node/src/scheduled_tasks/mod.rs | 70 +--- node/src/utils.rs | 119 +++--- 18 files changed, 576 insertions(+), 926 deletions(-) create mode 100644 crates/node-macros/Cargo.toml create mode 100644 crates/node-macros/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d45f8b19..8707e89c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3098,6 +3098,7 @@ dependencies = [ "libp2p-metrics", "libp2p-swarm-derive", "musig2", + "node-macros", "once_cell", "p3-bn254-fr", "p3-field", @@ -8148,6 +8149,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65" +[[package]] +name = "node-macros" +version = "0.4.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "nohash-hasher" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index dcecddcf..3c129ed0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/cbft-rpc", "crates/bitvm-gc", "crates/store", + "crates/node-macros", "crates/client", "crates/util", "crates/header-chain", diff --git a/crates/node-macros/Cargo.toml b/crates/node-macros/Cargo.toml new file mode 100644 index 00000000..370ac51c --- /dev/null +++ b/crates/node-macros/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "node-macros" +version.workspace = true +edition.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full"] } diff --git a/crates/node-macros/src/lib.rs b/crates/node-macros/src/lib.rs new file mode 100644 index 00000000..06854b5e --- /dev/null +++ b/crates/node-macros/src/lib.rs @@ -0,0 +1,112 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::spanned::Spanned; +use syn::{Attribute, Data, DeriveInput, Fields, Ident, Meta, Result, Variant, parse_macro_input}; + +#[proc_macro_derive(MessageBusinessRef, attributes(business_ref))] +pub fn derive_message_business_ref(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + match expand_message_business_ref(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} + +fn expand_message_business_ref(input: DeriveInput) -> Result { + let enum_name = input.ident; + let Data::Enum(data) = input.data else { + return Err(syn::Error::new( + enum_name.span(), + "MessageBusinessRef can only be derived for enums", + )); + }; + + let match_arms = data.variants.iter().map(expand_variant).collect::>>()?; + + Ok(quote! { + impl HasBusinessRef for #enum_name { + fn business_ref(&self) -> BusinessRef { + match self { + #(#match_arms),* + } + } + } + }) +} + +fn expand_variant(variant: &Variant) -> Result { + let scope = business_ref_scope(&variant.attrs)?; + let variant_name = &variant.ident; + + match scope.as_str() { + "graph" => { + let binding = tuple_payload_binding(variant)?; + Ok(quote! { + Self::#variant_name(#binding) => BusinessRef::Graph { + instance_id: #binding.instance_id, + graph_id: #binding.graph_id, + } + }) + } + "instance" => { + let binding = tuple_payload_binding(variant)?; + Ok(quote! { + Self::#variant_name(#binding) => BusinessRef::Instance { + instance_id: #binding.instance_id, + } + }) + } + "unscoped" => match &variant.fields { + Fields::Unit => Ok(quote! { + Self::#variant_name => BusinessRef::Unscoped + }), + Fields::Unnamed(_) => Ok(quote! { + Self::#variant_name(..) => BusinessRef::Unscoped + }), + Fields::Named(_) => Ok(quote! { + Self::#variant_name { .. } => BusinessRef::Unscoped + }), + }, + _ => unreachable!("business_ref_scope validates accepted values"), + } +} + +fn tuple_payload_binding(variant: &Variant) -> Result { + match &variant.fields { + Fields::Unnamed(fields) if fields.unnamed.len() == 1 => { + Ok(Ident::new("message", variant.span())) + } + _ => Err(syn::Error::new( + variant.span(), + "graph and instance business references require exactly one payload field", + )), + } +} + +fn business_ref_scope(attributes: &[Attribute]) -> Result { + let mut matching = + attributes.iter().filter(|attribute| attribute.path().is_ident("business_ref")); + let Some(attribute) = matching.next() else { + return Err(syn::Error::new( + proc_macro2::Span::call_site(), + "each message variant must declare #[business_ref(graph)], #[business_ref(instance)], or #[business_ref(unscoped)]", + )); + }; + if matching.next().is_some() { + return Err(syn::Error::new(attribute.span(), "duplicate business_ref attribute")); + } + + let Meta::List(list) = &attribute.meta else { + return Err(syn::Error::new( + attribute.span(), + "business_ref must be written as #[business_ref(graph)], #[business_ref(instance)], or #[business_ref(unscoped)]", + )); + }; + let scope: Ident = list.parse_args()?; + let scope = scope.to_string(); + if matches!(scope.as_str(), "graph" | "instance" | "unscoped") { + Ok(scope) + } else { + Err(syn::Error::new(attribute.span(), "business_ref must be graph, instance, or unscoped")) + } +} diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index 8a70b889..649b990c 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -1315,29 +1315,6 @@ impl<'a> StorageProcessor<'a> { count_query.fetch_one(self.conn()).await?.get::("total_instances"), )) } - /// Get network type by instance ID - /// - /// Retrieves the network type (e.g., "mainnet", "testnet") for a specific instance. - /// - /// Parameters: - /// - instance_id: The UUID of the instance - /// - /// Returns: - /// - Ok(network_string) if the instance was found - /// - Ok("") if no instance with the given ID exists - /// - Err if the query failed - pub async fn get_network_by_instance(&mut self, instance_id: &Uuid) -> anyhow::Result { - if let Some(raw) = - sqlx::query!(r#"SELECT network FROM instance WHERE instance_id = ?"#, instance_id) - .fetch_optional(self.conn()) - .await? - { - Ok(raw.network) - } else { - Ok("".to_string()) - } - } - /// Insert a swap escrow only when its escrow hash is not already present. /// /// The chain-event watcher is the sole writer for Initialize records; @@ -1476,28 +1453,6 @@ impl<'a> StorageProcessor<'a> { Ok(row.rows_affected()) } - /// Update instance status - /// - /// A concise method specifically for updating instance status - pub async fn update_instance_status( - &mut self, - instance_id: &Uuid, - new_status: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let result = sqlx::query!( - "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? WHERE instance_id = ?", - new_status, - current_time, - current_time, - instance_id - ) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected() > 0) - } - /// Transition an instance only when it is still in the expected status. pub async fn update_instance_status_if_current( &mut self, @@ -1521,48 +1476,6 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } - /// Update instance pegin confirmation information - /// - /// Method specifically for updating pegin confirmation transaction ID and fee - pub async fn update_instance_pegin_confirm( - &mut self, - instance_id: &Uuid, - pegin_confirm_txid: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let result = sqlx::query!( - "UPDATE instance SET pegin_confirm_txid = ?, updated_at = ? WHERE instance_id = ?", - pegin_confirm_txid, - current_time, - instance_id - ) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected() > 0) - } - - /// Update instance pegin data transaction ID - /// - /// Method specifically for updating pegin data transaction ID - pub async fn update_instance_pegin_data_txid( - &mut self, - instance_id: &Uuid, - pegin_data_tx_hash: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let result = sqlx::query!( - "UPDATE instance SET pegin_data_tx_hash = ?, updated_at = ? WHERE instance_id = ?", - pegin_data_tx_hash, - current_time, - instance_id - ) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected() > 0) - } - /// Update instance fields using builder pattern /// /// This is the most elegant update method, using the InstanceUpdate builder pattern @@ -1607,72 +1520,6 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } - /// Remove a committee answer from an instance - /// - /// This method removes a specific committee's answer from the committees_answers HashMap. - pub async fn remove_instance_committee_answer( - &mut self, - instance_id: &Uuid, - committee: &str, - ) -> anyhow::Result { - // JSON merge-patch removes object members with a null value, so this - // stays atomic with concurrent single-answer additions. - let committee_patch = - serde_json::json!({ (committee): serde_json::Value::Null }).to_string(); - let current_time = get_current_timestamp_secs(); - let result = sqlx::query( - "UPDATE instance \ - SET committees_answers = json_patch(COALESCE(committees_answers, '{}'), json(?)), \ - updated_at = ? \ - WHERE instance_id = ?", - ) - .bind(committee_patch) - .bind(current_time) - .bind(instance_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Get committees answers for an instance - /// - /// Returns the committees_answers HashMap for a specific instance. - pub async fn get_instance_committees_answers( - &mut self, - instance_id: &Uuid, - ) -> anyhow::Result>>> { - let current_instance = self.find_instance(instance_id).await?; - if let Some(instance) = current_instance { - Ok(Some(instance.committees_answers)) - } else { - Ok(None) - } - } - - /// Replace the complete committee-answer map. - /// - /// Callers that add a single answer should use - /// `update_instance_committee_answer` instead, which merges atomically. - pub async fn update_instance_committees_answers_map( - &mut self, - instance_id: &Uuid, - committees_answers: &IndexMap>, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let committees_answers_json = serde_json::to_string(&committees_answers)?; - - let res = sqlx::query!( - "UPDATE instance SET committees_answers = ?, updated_at = ? WHERE instance_id = ?", - committees_answers_json, - current_time, - instance_id - ) - .execute(self.conn()) - .await?; - - Ok(res.rows_affected() > 0) - } - pub async fn update_instance_parameters( &mut self, instance_id: &Uuid, @@ -2017,25 +1864,6 @@ impl<'a> StorageProcessor<'a> { Ok(row) } - pub async fn get_graph_operator(&mut self, graph_id: &Uuid) -> anyhow::Result> { - #[derive(sqlx::FromRow)] - struct OperatorRow { - operator_pubkey: String, - } - if let Some(operator_raw) = sqlx::query_as!( - OperatorRow, - "SELECT operator_pubkey FROM graph WHERE graph_id = ?", - graph_id - ) - .fetch_optional(self.conn()) - .await? - { - Ok(Some(operator_raw.operator_pubkey)) - } else { - Ok(None) - } - } - pub async fn find_graphs(&mut self, params: GraphQuery) -> anyhow::Result<(Vec, i64)> { // Build base query let mut count_params = params.clone(); @@ -2164,31 +1992,6 @@ impl<'a> StorageProcessor<'a> { Ok(res.map(|v| (v.graph_id, v.instance_id, v.cur_prekickoff_txid, v.next_prekickoff))) } - pub async fn get_graphs_ids_and_operator_by_instance_ids( - &mut self, - ids: &[Uuid], - ) -> anyhow::Result> { - #[derive(sqlx::FromRow)] - struct GraphIdRow { - pub graph_id: Uuid, - pub instance_id: Uuid, - pub operator: String, - } - let query_str = format!( - "SELECT graph_id, instance_id, operator - FROM graph - WHERE hex(instance_id) - COLLATE NOCASE IN ({})", - create_place_holders(ids) - ); - let mut update_query = sqlx::query_as::<_, GraphIdRow>(&query_str); - for id in ids { - update_query = update_query.bind(hex::encode(id)); - } - let graph_ids = update_query.fetch_all(self.conn()).await?; - Ok(graph_ids.into_iter().map(|v| (v.graph_id, v.instance_id, v.operator)).collect()) - } - pub async fn get_operator_graphs(&mut self, params: GraphQuery) -> anyhow::Result> { let graph_query_builder = params.get_query_builder("SELECT * FROM graph"); let operator_graph_sql = graph_query_builder.get_sql(); @@ -2197,31 +2000,6 @@ impl<'a> StorageProcessor<'a> { Ok(operator_graphs_query.fetch_all(self.conn()).await?) } - pub async fn get_operator_max_kickoff_index( - &mut self, - operator_pubkey: &str, - ) -> anyhow::Result<(Option, i64)> { - #[derive(sqlx::FromRow)] - struct MaxPreKickoffIndexRow { - pub graph_id: Uuid, - pub kickoff_index: i64, - } - - let record = sqlx::query_as!( - MaxPreKickoffIndexRow, - "SELECT graph_id AS \"graph_id:Uuid\", kickoff_index - FROM graph - WHERE operator_pubkey = ? - ORDER BY kickoff_index DESC - limit 1", - operator_pubkey - ) - .fetch_optional(self.conn()) - .await?; - - Ok(record.map_or((None, 0), |v| (Some(v.graph_id), v.kickoff_index))) - } - pub async fn update_node_timestamp( &mut self, peer_id: &str, @@ -3503,22 +3281,6 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } - pub async fn has_graph_compensation_marker( - &mut self, - graph_id: Uuid, - message_id: &str, - ) -> anyhow::Result { - let exists: i64 = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM graph_compensation_marker \ - WHERE graph_id = ? AND message_id = ?)", - ) - .bind(graph_id) - .bind(message_id) - .fetch_one(self.conn()) - .await?; - Ok(exists != 0) - } - pub async fn upsert_pegin_instance_process_data( &mut self, pegin_instance_process_data: &PeginInstanceProcessData, @@ -3607,23 +3369,6 @@ impl<'a> StorageProcessor<'a> { Ok(row) } - pub async fn update_pegin_graph_endorsed( - &mut self, - graph_id: &Uuid, - is_endorsed: bool, - ) -> anyhow::Result<()> { - sqlx::query!( - r#"UPDATE - pegin_graph_process_data - SET is_endorsed = ? - WHERE graph_id = ?"#, - is_endorsed, - graph_id - ) - .execute(self.conn()) - .await?; - Ok(()) - } pub async fn get_pegin_graph_endorsed_len_by_instance_id( &mut self, instance_id: &Uuid, @@ -3959,28 +3704,6 @@ impl<'a> StorageProcessor<'a> { Ok(row) } - pub async fn update_graph_btc_tx_vout_monitor_data( - &mut self, - graph_id: &Uuid, - txid: &SerializableTxid, - monitor_data: String, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE graph_btc_tx_vout_monitor - SET monitor_data = ?, - updated_at = ? - WHERE graph_id = ? AND txid = ?", - monitor_data, - current_time, - graph_id, - txid - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - pub async fn create_long_running_task_proof( &mut self, long_running_task_proof: &LongRunningTaskProof, @@ -4295,26 +4018,6 @@ impl<'a> StorageProcessor<'a> { Ok(res.rows_affected()) } - pub async fn update_operator_proof_state( - &mut self, - id: i64, - proof_state: i64, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE operator_proof - SET proof_state = ?, - updated_at = ? - WHERE id = ?", - proof_state, - current_time, - id, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - pub async fn find_operator_proof_by_instance_and_graph( &mut self, instance_id: &Uuid, @@ -4824,6 +4527,33 @@ mod tests { assert_eq!(stored.status, "CommitteesAnswered"); } + #[tokio::test] + async fn test_instance_update_rejects_stale_status_transition() { + let db = setup_db().await; + let instance_id = Uuid::new_v4(); + let mut storage = db.acquire().await.unwrap(); + assert!( + storage + .insert_instance_if_absent(&pegin_instance(instance_id, "RelayerL2Minted")) + .await + .unwrap() + ); + + let updated = storage + .update_instance( + &InstanceUpdate::new_with_instance_id(instance_id) + .with_status("Timeout".to_string()) + .with_only_if_status_in(vec!["Presigned".to_string()]), + ) + .await + .unwrap(); + assert!(!updated); + assert_eq!( + storage.find_instance(&instance_id).await.unwrap().unwrap().status, + "RelayerL2Minted" + ); + } + #[tokio::test] async fn test_node_metrics_state_counts() { let db = setup_db().await; diff --git a/crates/store/src/schema.rs b/crates/store/src/schema.rs index 40f9f431..de2451e6 100644 --- a/crates/store/src/schema.rs +++ b/crates/store/src/schema.rs @@ -208,15 +208,6 @@ pub enum InstanceBridgeInStatus { UserCanceled, // user broadcast Pegin-cancel tx NoEnoughCommitteesAnswered, // no enough committee responsed & window expired UserDiscarded, // pegin prepare tx input uxto been spent in other tx - - // for front end display - Initiated, // UserInited - Verified, // CommitteesAnswered - Submitted, // UserBroadcastPeginPrepare - Failed, // PresignedFailed, RelayerL2MintedFailed, NoEnoughCommitteesAnswered, UserDiscarded - Processing, // Presigned, RelayerL1Broadcasted - Success, // RelayerL2Minted - Canceled, // UserCanceled } /// Lifecycle of a swap-based bridge-out escrow. @@ -325,14 +316,6 @@ pub enum GraphStatus { Skipped, OperatorTake1, OperatorTake2, - - /// frontend use only - Created, - Presigned, - L2Recorded, - OperatorKickOffing, - Challenging, - Disproving, } /// The evidence that authorizes a graph status transition. @@ -477,8 +460,7 @@ impl GraphStatus { OperatorTake1 => TAKE1, OperatorTake2 | Disprove => TAKE2_OR_DISPROVE, Skipped => SKIPPED, - OperatorPresigned | Created | Presigned | L2Recorded | OperatorKickOffing - | Challenging | Disproving => &[], + OperatorPresigned => &[], }, } } @@ -702,53 +684,6 @@ pub struct PendingGraphInit { pub created_at: i64, } -#[derive(Debug, Clone, PartialEq, Display, EnumString)] -pub enum MessageType { - None, - PeginRequest, - CreateGraph, - ConfirmInstance, - InitGraph, - GenCircuits, - CutCircuits, - SolderingProof, - VerifierGraphParamsEndorsement, - NonceGeneration, - AggNonceConsensus, - CommitteePresign, - GraphFinalize, - EndorseGraph, - PeginConfirmNonce, - PeginConfirmNonceConsensus, - PeginConfirmPartialSig, - PostReady, - KickoffReady, - KickoffSent, - PreKickoffSent, - ChallengeSent, - WatchtowerChallengeInitSent, - WatchtowerChallengeSent, - WatchtowerChallengeTimeout, - NackReady, - OperatorCommitPubinReady, - OperatorCommitPubinTimeout, - AssertReady, - AssertSent, - ChallengeAssertSent, - WronglyChallengeTimeout, - DisproveSent, - Take1Ready, - Take1Sent, - Take2Ready, - Take2Sent, - RequestNodeInfo, - ResponseNodeInfo, - SyncGraphRequest, - SyncGraph, - InstanceDiscarded, - Tick, -} - #[derive(Clone, Debug, Serialize, Deserialize, Default, Display, EnumString)] pub enum WatchContractStatus { #[default] @@ -992,17 +927,16 @@ mod tests { #[test] fn test_graph_status_from_str() { - assert_eq!(GraphStatus::from_str("Created").unwrap(), GraphStatus::Created); assert_eq!( GraphStatus::from_str("OperatorPresigned").unwrap(), GraphStatus::OperatorPresigned ); + assert!(GraphStatus::from_str("Created").is_err()); assert!(GraphStatus::from_str("Invalid").is_err()); } #[test] fn test_graph_status_display() { - assert_eq!(GraphStatus::Created.to_string(), "Created"); assert_eq!(GraphStatus::OperatorPresigned.to_string(), "OperatorPresigned"); } @@ -1015,13 +949,6 @@ mod tests { assert!(InstanceBridgeInStatus::from_str("Invalid").is_err()); } - #[test] - fn test_message_type_from_str() { - assert_eq!(MessageType::from_str("PeginRequest").unwrap(), MessageType::PeginRequest); - assert_eq!(MessageType::from_str("CreateGraph").unwrap(), MessageType::CreateGraph); - assert!(MessageType::from_str("Invalid").is_err()); - } - #[test] fn test_byte_array_macro() { let bytes = ByteArray32([1u8; 32]); diff --git a/node/Cargo.toml b/node/Cargo.toml index 9a072d34..1fd58269 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -76,6 +76,7 @@ rand = { workspace = true } base64 = { workspace = true } dotenv = { workspace = true } strum = { workspace = true } +node-macros = { path = "../crates/node-macros" } tendermint = { workspace = true } bincode = { workspace = true } serde = { workspace = true } diff --git a/node/src/action.rs b/node/src/action.rs index 79c772dc..506748e6 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -8,7 +8,7 @@ use crate::env::{ }; use crate::handle::{ HandlerContext, HeavyTaskContext, dispatch as handle_dispatch, heavy_task_from_content, - is_heavy_task_message_type, run_heavy_task, + run_heavy_task, }; use crate::metrics_service::MetricsState; use crate::middleware::AllBehaviours; @@ -31,6 +31,7 @@ use futures::FutureExt; use libp2p::gossipsub::MessageId; use libp2p::{PeerId, Swarm, gossipsub}; use musig2::{PartialSignature, PubNonce}; +use node_macros::MessageBusinessRef; use secp256k1::{ Keypair, Message as SecpMessage, SECP256K1, schnorr::Signature as SchnorrSignature, }; @@ -42,6 +43,7 @@ use std::sync::{Arc, LazyLock, Mutex}; use std::time::{Duration, Instant}; use store::localdb::LocalDB; use store::{MessageState, P2pInboxMessage}; +use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -363,54 +365,212 @@ impl MessageDeferReason { } } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BusinessRef { + Instance { instance_id: Uuid }, + Graph { instance_id: Uuid, graph_id: Uuid }, + Unscoped, +} + +impl BusinessRef { + pub const fn primary_id(self) -> Option { + match self { + Self::Instance { instance_id } => Some(instance_id), + Self::Graph { graph_id, .. } => Some(graph_id), + Self::Unscoped => None, + } + } + + fn key_part(self) -> String { + match self { + Self::Instance { instance_id } => format!("instance:{instance_id}"), + Self::Graph { instance_id, graph_id } => format!("graph:{instance_id}:{graph_id}"), + Self::Unscoped => "unscoped".to_owned(), + } + } +} + +pub trait HasBusinessRef { + fn business_ref(&self) -> BusinessRef; +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum MessageQualifier { + Singleton, + Watchtower(usize), + Verifier(usize), + Disprove { kind: DisproveTxType, index: usize }, +} + +impl MessageQualifier { + fn key_part(&self) -> String { + match self { + Self::Singleton => "singleton".to_owned(), + Self::Watchtower(index) => format!("watchtower:{index}"), + Self::Verifier(index) => format!("verifier:{index}"), + Self::Disprove { kind, index } => format!("disprove:{kind}:{index}"), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalMessageKey { + actor: Actor, + kind: MessageKind, + business_ref: BusinessRef, + qualifier: MessageQualifier, +} + +impl LocalMessageKey { + pub fn from_content(actor: Actor, content: &GOATMessageContent) -> Result { + let business_ref = content.business_ref(); + if business_ref.primary_id().is_none() { + bail!("cannot persist unscoped {} as a local message", content.event_type()); + } + Ok(Self { actor, kind: content.kind(), business_ref, qualifier: content.qualifier() }) + } + + pub fn business_id(&self) -> Uuid { + match self.business_ref { + BusinessRef::Instance { instance_id } => instance_id, + BusinessRef::Graph { graph_id, .. } => graph_id, + BusinessRef::Unscoped => unreachable!("unscoped messages cannot have a local key"), + } + } + + pub fn message_id(&self) -> String { + format!( + "local:v1:{}:{}:{}:{}", + self.actor, + self.business_ref.key_part(), + self.kind, + self.qualifier.key_part(), + ) + } +} + +#[derive(Serialize, Deserialize, Clone, EnumDiscriminants, MessageBusinessRef)] +#[strum_discriminants(name(MessageKind))] +#[strum_discriminants(derive(Hash, Display, EnumString, EnumIter, IntoStaticStr))] pub enum GOATMessageContent { + #[business_ref(instance)] PeginRequest(PeginRequest), + #[business_ref(graph)] CreateGraph(CreateGraph), + #[business_ref(instance)] ConfirmInstance(ConfirmInstance), + #[business_ref(graph)] InitGraph(InitGraph), + #[business_ref(graph)] GenCircuits(GenCircuits), + #[business_ref(graph)] CutCircuits(CutCircuits), + #[business_ref(graph)] SolderingProofReady(SolderingProofReady), + #[business_ref(graph)] GraphSetupAck(GraphSetupAck), + #[business_ref(graph)] VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement), + #[business_ref(graph)] NonceGeneration(NonceGeneration), + #[business_ref(graph)] AggNonceConsensus(AggNonceConsensus), + #[business_ref(graph)] CommitteePresign(CommitteePresign), + #[business_ref(graph)] EndorseGraph(EndorseGraph), + #[business_ref(graph)] GraphFinalize(GraphFinalize), + #[business_ref(instance)] PeginConfirmNonce(PeginConfirmNonce), + #[business_ref(instance)] PeginConfirmNonceConsensus(PeginConfirmNonceConsensus), + #[business_ref(instance)] PeginConfirmPartialSig(PeginConfirmPartialSig), + #[business_ref(instance)] PostReady(PostReady), + #[business_ref(graph)] KickoffReady(KickoffReady), + #[business_ref(graph)] KickoffSent(KickoffSent), + #[business_ref(graph)] PreKickoffSent(PreKickoffSent), + #[business_ref(graph)] ChallengeSent(ChallengeSent), + #[business_ref(graph)] WatchtowerChallengeInitSent(WatchtowerChallengeInitSent), + #[business_ref(graph)] WatchtowerChallengeSent(WatchtowerChallengeSent), + #[business_ref(graph)] WatchtowerChallengeTimeout(WatchtowerChallengeTimeout), + #[business_ref(graph)] NackReady(NackReady), + #[business_ref(graph)] OperatorCommitPubinReady(OperatorCommitPubinReady), + #[business_ref(graph)] OperatorCommitPubinTimeout(OperatorCommitPubinTimeout), + #[business_ref(graph)] AssertReady(AssertReady), + #[business_ref(graph)] AssertSent(AssertSent), + #[business_ref(graph)] ChallengeAssertSent(ChallengeAssertSent), + #[business_ref(graph)] WronglyChallengeTimeout(WronglyChallengeTimeout), + #[business_ref(graph)] DisproveSent(DisproveSent), + #[business_ref(graph)] Take1Ready(Take1Ready), + #[business_ref(graph)] Take1Sent(Take1Sent), + #[business_ref(graph)] Take2Ready(Take2Ready), + #[business_ref(graph)] Take2Sent(Take2Sent), + #[business_ref(unscoped)] RequestNodeInfo(NodeInfo), + #[business_ref(unscoped)] ResponseNodeInfo(NodeInfo), + #[business_ref(graph)] SyncGraphRequest(SyncGraphRequest), + #[business_ref(graph)] SyncGraph(SyncGraph), + #[business_ref(unscoped)] InstanceDiscarded(InstanceDiscarded), + #[business_ref(unscoped)] Tick, } +impl MessageKind { + const fn is_pegin(self) -> bool { + matches!( + self, + Self::PeginRequest + | Self::ConfirmInstance + | Self::CreateGraph + | Self::InitGraph + | Self::GenCircuits + | Self::CutCircuits + | Self::SolderingProofReady + | Self::VerifierGraphParamsEndorsement + | Self::NonceGeneration + | Self::AggNonceConsensus + | Self::CommitteePresign + | Self::EndorseGraph + | Self::GraphFinalize + | Self::PeginConfirmNonce + | Self::PeginConfirmNonceConsensus + | Self::PeginConfirmPartialSig + | Self::PostReady + ) + } +} + impl GOATMessageContent { + pub fn kind(&self) -> MessageKind { + self.into() + } + /// New messages default to the durable inbox and must explicitly opt into /// immediate processing when they are safe to drop. pub const fn p2p_delivery(&self) -> P2PMessageDelivery { @@ -424,76 +584,27 @@ impl GOATMessageContent { } } - /// Stable message name for logs/metrics. Keep this independent of `Debug`, whose + /// Stable message name for logs/metrics. Keep this independent of `Debug`, whose /// output can include protocol payloads (and, for proofs, be very large). pub fn event_type(&self) -> &'static str { - match self { - Self::PeginRequest(_) => "PeginRequest", - Self::CreateGraph(_) => "CreateGraph", - Self::ConfirmInstance(_) => "ConfirmInstance", - Self::InitGraph(_) => "InitGraph", - Self::GenCircuits(_) => "GenCircuits", - Self::CutCircuits(_) => "CutCircuits", - Self::SolderingProofReady(_) => "SolderingProofReady", - Self::GraphSetupAck(_) => "GraphSetupAck", - Self::VerifierGraphParamsEndorsement(_) => "VerifierGraphParamsEndorsement", - Self::NonceGeneration(_) => "NonceGeneration", - Self::AggNonceConsensus(_) => "AggNonceConsensus", - Self::CommitteePresign(_) => "CommitteePresign", - Self::EndorseGraph(_) => "EndorseGraph", - Self::GraphFinalize(_) => "GraphFinalize", - Self::PeginConfirmNonce(_) => "PeginConfirmNonce", - Self::PeginConfirmNonceConsensus(_) => "PeginConfirmNonceConsensus", - Self::PeginConfirmPartialSig(_) => "PeginConfirmPartialSig", - Self::PostReady(_) => "PostReady", - Self::KickoffReady(_) => "KickoffReady", - Self::KickoffSent(_) => "KickoffSent", - Self::PreKickoffSent(_) => "PreKickoffSent", - Self::ChallengeSent(_) => "ChallengeSent", - Self::WatchtowerChallengeInitSent(_) => "WatchtowerChallengeInitSent", - Self::WatchtowerChallengeSent(_) => "WatchtowerChallengeSent", - Self::WatchtowerChallengeTimeout(_) => "WatchtowerChallengeTimeout", - Self::NackReady(_) => "NackReady", - Self::OperatorCommitPubinReady(_) => "OperatorCommitPubinReady", - Self::OperatorCommitPubinTimeout(_) => "OperatorCommitPubinTimeout", - Self::AssertReady(_) => "AssertReady", - Self::AssertSent(_) => "AssertSent", - Self::ChallengeAssertSent(_) => "ChallengeAssertSent", - Self::WronglyChallengeTimeout(_) => "WronglyChallengeTimeout", - Self::DisproveSent(_) => "DisproveSent", - Self::Take1Ready(_) => "Take1Ready", - Self::Take1Sent(_) => "Take1Sent", - Self::Take2Ready(_) => "Take2Ready", - Self::Take2Sent(_) => "Take2Sent", - Self::RequestNodeInfo(_) => "RequestNodeInfo", - Self::ResponseNodeInfo(_) => "ResponseNodeInfo", - Self::SyncGraphRequest(_) => "SyncGraphRequest", - Self::SyncGraph(_) => "SyncGraph", - Self::InstanceDiscarded(_) => "InstanceDiscarded", - Self::Tick => "Tick", - } + self.kind().into() } - fn pegin_retry_business_id(&self) -> Option { + pub fn qualifier(&self) -> MessageQualifier { match self { - Self::PeginRequest(message) => Some(message.instance_id), - Self::ConfirmInstance(message) => Some(message.instance_id), - Self::CreateGraph(message) => Some(message.graph_id), - Self::InitGraph(message) => Some(message.graph_id), - Self::GenCircuits(message) => Some(message.graph_id), - Self::CutCircuits(message) => Some(message.graph_id), - Self::SolderingProofReady(message) => Some(message.graph_id), - Self::VerifierGraphParamsEndorsement(message) => Some(message.graph_id), - Self::NonceGeneration(message) => Some(message.graph_id), - Self::AggNonceConsensus(message) => Some(message.graph_id), - Self::CommitteePresign(message) => Some(message.graph_id), - Self::EndorseGraph(message) => Some(message.graph_id), - Self::GraphFinalize(message) => Some(message.graph_id), - Self::PeginConfirmNonce(message) => Some(message.instance_id), - Self::PeginConfirmNonceConsensus(message) => Some(message.instance_id), - Self::PeginConfirmPartialSig(message) => Some(message.instance_id), - Self::PostReady(message) => Some(message.instance_id), - _ => None, + Self::WatchtowerChallengeSent(message) => { + MessageQualifier::Watchtower(message.watchtower_index) + } + Self::ChallengeAssertSent(message) => { + MessageQualifier::Verifier(message.verifier_index) + } + Self::WronglyChallengeTimeout(message) => { + MessageQualifier::Verifier(message.verifier_index) + } + Self::DisproveSent(message) => { + MessageQualifier::Disprove { kind: message.disprove_type, index: message.index } + } + _ => MessageQualifier::Singleton, } } } @@ -507,30 +618,6 @@ fn is_retryable_sqlite_error(error: &anyhow::Error) -> bool { }) } -fn is_pegin_message_type(message_type: &str) -> bool { - matches!( - message_type, - "PeginRequest" - | "ConfirmInstance" - | "CreateGraph" - | "InitGraph" - | "GenCircuits" - | "CutCircuits" - | "SolderingProof" - | "SolderingProofReady" - | "VerifierGraphParamsEndorsement" - | "NonceGeneration" - | "AggNonceConsensus" - | "CommitteePresign" - | "EndorseGraph" - | "GraphFinalize" - | "PeginConfirmNonce" - | "PeginConfirmNonceConsensus" - | "PeginConfirmPartialSig" - | "PostReady" - ) -} - /// Pegin #[derive(Serialize, Deserialize, Clone)] @@ -1076,7 +1163,7 @@ async fn enqueue_p2p_message( let message_id = hex::encode(&id.0); let inbox_message = P2pInboxMessage { message_id: message_id.clone(), - business_id: decoded.content.pegin_retry_business_id(), + business_id: decoded.content.business_ref().primary_id(), actor: actor.to_string(), from_peer: from_peer_id.to_string(), msg_type: decoded.content.event_type().to_owned(), @@ -1352,35 +1439,20 @@ async fn handle_p2p_inbox_messages( continue; } }; - let is_heavy_task_message = is_heavy_task_message_type(&message.msg_type, &actor); - let heavy_task = if is_heavy_task_message { - let decoded = match GOATMessage::deserialize_message(&message.content).await { - Ok(message) => message, - Err(error) => { - fail_p2p_inbox_without_aborting_batch( - local_db, - &message.message_id, - &message.lease_token, - &error.to_string(), - ) - .await; - continue; - } - }; - let Some(task) = heavy_task_from_content(decoded.content(), &actor) else { + let decoded = match GOATMessage::deserialize_message(&message.content).await { + Ok(message) => message, + Err(error) => { fail_p2p_inbox_without_aborting_batch( local_db, &message.message_id, &message.lease_token, - &format!("inbox message type does not match {} content", message.msg_type), + &error.to_string(), ) .await; continue; - }; - Some(task) - } else { - None + } }; + let heavy_task = heavy_task_from_content(decoded.content(), &actor); if let Some(heavy_task) = heavy_task { let local_db = local_db.clone(); @@ -1391,7 +1463,8 @@ async fn handle_p2p_inbox_messages( let message_id = message.message_id.clone(); let lease_token = message.lease_token.clone(); let attempt_count = message.attempt_count; - let task_type = heavy_task.message_type(); + let task_type = message.msg_type.clone(); + let task_type_for_task = task_type.clone(); let task_kind = heavy_task.kind(); let graph_id = heavy_task.graph_id(); let Some(permit) = try_acquire_heavy_task_permit(&message_id, &lease_token) else { @@ -1450,7 +1523,7 @@ async fn handle_p2p_inbox_messages( return; } metrics_state.record_message_dispatch( - task_type, + &task_type_for_task, if result.is_ok() { "success" } else { "failed" }, ); if let Err(error) = finish_p2p_inbox_attempt( @@ -1458,7 +1531,7 @@ async fn handle_p2p_inbox_messages( &metrics_state, &message_id, &lease_token, - task_type, + &task_type_for_task, attempt_count, result, ) @@ -1480,13 +1553,13 @@ async fn handle_p2p_inbox_messages( } } DispatchExecution::Panicked(detail) => { - metrics_state.record_message_dispatch(task_type, "failed"); + metrics_state.record_message_dispatch(&task_type_for_task, "failed"); tracing::error!( event = "heavy_task_panic", outcome = "node_shutdown", message_id, graph_id = %graph_id, - message_type = task_type, + message_type = %task_type_for_task, task_kind, detail, "heavy task panicked; recorded an abandon and stopping the node" @@ -1507,7 +1580,7 @@ async fn handle_p2p_inbox_messages( outcome = "heavy_task_started", message_id = %message.message_id, graph_id = %graph_id, - message_type = task_type, + message_type = %task_type, task_kind, "started background heavy task" ); @@ -1529,7 +1602,7 @@ async fn handle_p2p_inbox_messages( // Keep the deeply nested dispatch future out of the enclosing task's // inline state machine. - let dispatch: BoxedDispatch<'_> = Box::pin(recv_and_dispatch( + let dispatch: BoxedDispatch<'_> = Box::pin(dispatch_decoded_p2p_message( swarm, local_db, btc_client, @@ -1539,7 +1612,7 @@ async fn handle_p2p_inbox_messages( actor.clone(), from_peer_id, raw_message_id, - &message.content, + decoded, metrics_state, )); let result = match supervise_dispatch(dispatch, shutdown).await { @@ -1934,7 +2007,9 @@ pub async fn handle_self_p2p_msg( let requested_retry_delay = p2p_retryable_dispatch_error(&err).and_then(|(_, delay)| delay); let lock_time: i64 = requested_retry_delay.unwrap_or_else(|| { - if is_transient && is_pegin_message_type(&message.msg_type) { + if is_transient + && MessageKind::from_str(&message.msg_type).is_ok_and(MessageKind::is_pegin) + { TRANSIENT_PEGIN_RETRY_DELAY_SECS as i64 } else { LOCAL_MESSAGE_RETRY_DELAY_SECS @@ -2272,7 +2347,6 @@ pub async fn send_to_peer( pub async fn push_local_unhandled_messages_with_reason( local_db: &LocalDB, - business_id: Uuid, message: &GOATMessage, delay_secs: usize, reason: MessageDeferReason, @@ -2281,8 +2355,10 @@ pub async fn push_local_unhandled_messages_with_reason( let mut storage_processor = local_db.start_immediate_transaction().await?; let actor = message.actor.clone(); let content: GOATMessageContent = message.content().clone(); + let key = LocalMessageKey::from_content(actor.clone(), &content)?; + let business_id = key.business_id(); let message_type = message.content.event_type(); - let target_message_id = generate_message_id(business_id, message_type.to_owned(), None); + let target_message_id = key.message_id(); let active_claim = ACTIVE_LOCAL_MESSAGE_CLAIM .try_with(|claim| (claim.message_id.clone(), claim.message_version)) .ok(); @@ -2318,8 +2394,6 @@ pub async fn push_local_unhandled_messages_with_reason( let upserted = upsert_message( &mut storage_processor, true, - business_id, - None, SELF_SENDER.to_string(), actor, content, @@ -2406,7 +2480,6 @@ pub(crate) async fn get_graph_or_defer( let delay_secs: usize = 60; // 1 min default retry if let Err(error) = push_local_unhandled_messages_with_reason( local_db, - graph_id, message, delay_secs, MessageDeferReason::GraphSyncPending, @@ -2519,11 +2592,13 @@ mod tests { #[tokio::test] async fn non_owner_requeue_reports_processing_conflict() { let local_db = store::create_local_db("sqlite::memory:").await; - let business_id = Uuid::new_v4(); - let message = GOATMessage::new(Actor::Operator, GOATMessageContent::Tick); + let instance_id = Uuid::new_v4(); + let message = GOATMessage::new( + Actor::Operator, + GOATMessageContent::PostReady(PostReady { instance_id }), + ); push_local_unhandled_messages_with_reason( &local_db, - business_id, &message, 0, MessageDeferReason::HandlerError, @@ -2549,7 +2624,6 @@ mod tests { let error = push_local_unhandled_messages_with_reason( &local_db, - business_id, &message, 30, MessageDeferReason::HandlerError, @@ -2571,23 +2645,22 @@ mod tests { } #[tokio::test] - async fn owner_requeue_uses_exact_subtyped_message_id() { + async fn owner_requeue_uses_content_derived_message_id() { let local_db = store::create_local_db("sqlite::memory:").await; - let business_id = Uuid::new_v4(); - let message = GOATMessage::new(Actor::Operator, GOATMessageContent::Tick); - let subtyped_message_id = generate_message_id( - business_id, - message.content.event_type().to_owned(), - Some("7".to_owned()), + let instance_id = Uuid::new_v4(); + let message = GOATMessage::new( + Actor::Operator, + GOATMessageContent::PostReady(PostReady { instance_id }), ); + let message_id = LocalMessageKey::from_content(message.actor.clone(), &message.content) + .unwrap() + .message_id(); { let mut storage = local_db.acquire().await.unwrap(); assert!( upsert_message( &mut storage, false, - business_id, - Some("7".to_owned()), SELF_SENDER.to_owned(), message.actor.clone(), message.content.clone(), @@ -2613,7 +2686,7 @@ mod tests { .pop() .unwrap() }; - assert_eq!(claimed.message_id, subtyped_message_id); + assert_eq!(claimed.message_id, message_id); ACTIVE_LOCAL_MESSAGE_CLAIM .scope( @@ -2623,7 +2696,6 @@ mod tests { }, push_local_unhandled_messages_with_reason( &local_db, - business_id, &message, 30, MessageDeferReason::HandlerError, @@ -2634,10 +2706,7 @@ mod tests { .unwrap(); let mut storage = local_db.acquire().await.unwrap(); - let stored = storage.find_messages_by_id(&subtyped_message_id).await.unwrap().unwrap(); + let stored = storage.find_messages_by_id(&message_id).await.unwrap().unwrap(); assert_eq!(stored.state, MessageState::Pending.to_string()); - let base_message_id = - generate_message_id(business_id, message.content.event_type().to_owned(), None); - assert!(storage.find_messages_by_id(&base_message_id).await.unwrap().is_none()); } } diff --git a/node/src/bin/db_inject.rs b/node/src/bin/db_inject.rs index 0dc8b997..f7c08600 100644 --- a/node/src/bin/db_inject.rs +++ b/node/src/bin/db_inject.rs @@ -8,7 +8,6 @@ //! - --db-path: local SQLite path (e.g., sqlite:/tmp/bitvm-node.db) //! - --actor: Committee | Operator | Verifier | Watchtower | All //! - --message-json or --message-file (one required) -//! - --business-id (optional; inferred from content when unambiguous) //! //! Example: //! - cargo run -p bitvm-noded --bin update-db -- \ @@ -21,7 +20,6 @@ use std::str::FromStr; use anyhow::{Context, Result, anyhow}; use clap::Parser; -use uuid::Uuid; use bitvm_lib::actors::Actor; use bitvm_noded::action::*; @@ -39,14 +37,6 @@ struct Args { #[arg(long, value_parser = parse_actor)] actor: Actor, - /// Business id used for message_id (graph_id or instance_id). If omitted, infer it when unambiguous. - #[arg(long)] - business_id: Option, - - /// Optional sub-type used when generating message_id - #[arg(long)] - sub_type: Option, - /// from_peer column, defaults to "Manual" #[arg(long, default_value = "Manual")] from_peer: String, @@ -76,55 +66,6 @@ fn parse_actor(raw: &str) -> std::result::Result { Actor::from_str(raw).map_err(|_| format!("invalid actor: {raw}")) } -fn infer_business_id(content: &GOATMessageContent) -> Option { - match content { - GOATMessageContent::PeginRequest(v) => Some(v.instance_id), - GOATMessageContent::ConfirmInstance(v) => Some(v.instance_id), - GOATMessageContent::InitGraph(v) => Some(v.graph_id), - GOATMessageContent::GenCircuits(v) => Some(v.graph_id), - GOATMessageContent::CutCircuits(v) => Some(v.graph_id), - GOATMessageContent::SolderingProofReady(v) => Some(v.graph_id), - GOATMessageContent::GraphSetupAck(_) => None, - GOATMessageContent::VerifierGraphParamsEndorsement(v) => Some(v.graph_id), - GOATMessageContent::CreateGraph(v) => Some(v.graph_id), - GOATMessageContent::NonceGeneration(v) => Some(v.graph_id), - GOATMessageContent::AggNonceConsensus(v) => Some(v.graph_id), - GOATMessageContent::CommitteePresign(v) => Some(v.graph_id), - GOATMessageContent::EndorseGraph(v) => Some(v.graph_id), - GOATMessageContent::GraphFinalize(v) => Some(v.graph_id), - GOATMessageContent::PeginConfirmNonce(v) => Some(v.instance_id), - GOATMessageContent::PeginConfirmNonceConsensus(v) => Some(v.instance_id), - GOATMessageContent::PeginConfirmPartialSig(v) => Some(v.instance_id), - GOATMessageContent::PostReady(v) => Some(v.instance_id), - GOATMessageContent::KickoffReady(v) => Some(v.graph_id), - GOATMessageContent::KickoffSent(v) => Some(v.graph_id), - GOATMessageContent::PreKickoffSent(v) => Some(v.graph_id), - GOATMessageContent::ChallengeSent(v) => Some(v.graph_id), - GOATMessageContent::WatchtowerChallengeInitSent(v) => Some(v.graph_id), - GOATMessageContent::WatchtowerChallengeSent(v) => Some(v.graph_id), - GOATMessageContent::WatchtowerChallengeTimeout(v) => Some(v.graph_id), - GOATMessageContent::NackReady(v) => Some(v.graph_id), - GOATMessageContent::OperatorCommitPubinReady(v) => Some(v.graph_id), - GOATMessageContent::OperatorCommitPubinTimeout(v) => Some(v.graph_id), - GOATMessageContent::AssertReady(v) => Some(v.graph_id), - GOATMessageContent::AssertSent(v) => Some(v.graph_id), - GOATMessageContent::ChallengeAssertSent(v) => Some(v.graph_id), - GOATMessageContent::WronglyChallengeTimeout(v) => Some(v.graph_id), - GOATMessageContent::DisproveSent(v) => Some(v.graph_id), - GOATMessageContent::Take1Ready(v) => Some(v.graph_id), - GOATMessageContent::Take1Sent(v) => Some(v.graph_id), - GOATMessageContent::Take2Ready(v) => Some(v.graph_id), - GOATMessageContent::Take2Sent(v) => Some(v.graph_id), - GOATMessageContent::SyncGraphRequest(v) => Some(v.graph_id), - GOATMessageContent::SyncGraph(v) => Some(v.graph_id), - // This payload may refer to multiple graphs, so it has no canonical - // business id. Require callers to provide --business-id explicitly. - GOATMessageContent::InstanceDiscarded(_) => None, - GOATMessageContent::RequestNodeInfo(_) | GOATMessageContent::ResponseNodeInfo(_) => None, - GOATMessageContent::Tick => None, - } -} - fn load_message_json(args: &Args) -> Result { if let Some(ref inline) = args.message_json { return Ok(inline.clone()); @@ -143,13 +84,8 @@ async fn main() -> Result<()> { let content: GOATMessageContent = serde_json::from_str(&raw_json).context("parse GOATMessageContent JSON")?; - let business_id = match &args.business_id { - Some(raw) => Uuid::parse_str(raw).context("parse business_id as UUID")?, - None => infer_business_id(&content) - .ok_or_else(|| anyhow!("business_id not provided and cannot be inferred"))?, - }; - let actor = args.actor; + let message_type = content.event_type(); let local_db = create_local_db(&args.db_path).await; let mut storage_processor = local_db.acquire().await?; let is_update = !args.skip_if_exists; @@ -157,8 +93,6 @@ async fn main() -> Result<()> { upsert_message( &mut storage_processor, is_update, - business_id, - args.sub_type.clone(), args.from_peer.clone(), actor.clone(), content, @@ -168,8 +102,8 @@ async fn main() -> Result<()> { .await?; println!( - "Inserted message for actor={actor} business_id={business_id} db_path={} update={} lock_secs={} weight={}", - args.db_path, is_update, args.lock_secs, args.weight + "Inserted message for actor={actor} message_type={} db_path={} update={} lock_secs={} weight={}", + message_type, args.db_path, is_update, args.lock_secs, args.weight ); Ok(()) } diff --git a/node/src/env.rs b/node/src/env.rs index adb399bc..8d4abb55 100644 --- a/node/src/env.rs +++ b/node/src/env.rs @@ -108,10 +108,9 @@ pub const ENV_SEQUENCER_SET_MONITOR_START_COSMOS_BLOCK: &str = pub const ENV_COSMOS_RPC_URL: &str = "COSMOS_RPC_URL"; pub const DEFAULT_COSMOS_RPC_URL: &str = "https://rpc.testnet3.goat.network/goat-rpc"; -// fee estimate -// TODO: more precise fee estimation -pub const CHEKSIG_P2WSH_INPUT_VBYTES: u64 = 100; -pub const CHEKSIG_P2TR_INPUT_VBYTES: u64 = 100; +// Conservative fee-reserve estimates, not exact serialized transaction sizes. +pub const CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE: u64 = 100; +pub const CHECKSIG_P2TR_INPUT_VBYTES_ESTIMATE: u64 = 100; pub const P2WSH_OUTPUT_VBYTES: u64 = 50; pub const P2TR_OUTPUT_VBYTES: u64 = 50; pub const P2A_OUTPUT_VBYTES: u64 = 50; @@ -126,8 +125,6 @@ pub const MIN_CHALLENGE_AMOUNT: u64 = 1_000_000; // 0.01 BTC pub const STAKE_RATE: u64 = 0; // 0% pub const CHALLENGE_RATE: u64 = 0; // 0% -pub const RATE_MULTIPLIER: u64 = 10000; - const COMMITTEE_MEMBER_NUMBER: usize = 2; pub const MESSAGE_BROADCAST_MAX_TIMES: i64 = 3; @@ -144,8 +141,6 @@ pub const SYNC_GRAPH_MAX_WAIT_SECS: u64 = 30; // use to judge load history event thread is dead pub const LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS: i64 = 600; -pub const GATEWAY_RATE_MULTIPLIER: u64 = 10000; - pub const HEARTBEAT_INTERVAL_SECOND: u64 = 60 * 5; pub const REGULAR_TASK_INTERVAL_SECOND: u64 = 20; pub const SEQUENCER_SET_MONITOR_INTERVAL_SECS: u64 = 5; @@ -639,8 +634,16 @@ pub fn get_soldering_proof_payload_store_path() -> anyhow::Result { Ok(value.to_string()) } +pub const fn actor_needs_soldering_builder(actor: &Actor) -> bool { + matches!(actor, Actor::Verifier | Actor::Operator) +} + +pub const fn actor_runs_babe_setup_state_cleanup(actor: &Actor) -> bool { + actor_needs_soldering_builder(actor) +} + pub fn validate_soldering_proof_payload_store_config(actor: &Actor) -> anyhow::Result<()> { - if matches!(actor, Actor::Verifier | Actor::Operator | Actor::All) { + if actor_needs_soldering_builder(actor) { get_soldering_proof_payload_store_path() .map(|_| ()) .map_err(|err| anyhow::anyhow!("{err}; required for actor {actor}")) @@ -772,7 +775,7 @@ mod tests { assert!(validate_soldering_proof_payload_store_config(&Actor::Verifier).is_err()); assert!(validate_soldering_proof_payload_store_config(&Actor::Operator).is_err()); - assert!(validate_soldering_proof_payload_store_config(&Actor::All).is_err()); + assert!(validate_soldering_proof_payload_store_config(&Actor::All).is_ok()); assert!(validate_soldering_proof_payload_store_config(&Actor::Committee).is_ok()); assert!(validate_soldering_proof_payload_store_config(&Actor::Watchtower).is_ok()); assert!(validate_soldering_proof_payload_store_config(&Actor::Publisher).is_ok()); diff --git a/node/src/handle.rs b/node/src/handle.rs index 89d3a202..79833f90 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -95,15 +95,6 @@ impl HeavyTask { } } - pub(crate) fn message_type(&self) -> &'static str { - match self { - Self::GenerateVerifierSetup(_) => "InitGraph", - Self::GenerateSolderingProof(_) => "CutCircuits", - Self::ValidateVerifierGraph(_) => "CreateGraph", - Self::VerifySolderingProof(_) => "SolderingProofReady", - } - } - pub(crate) fn graph_id(&self) -> Uuid { match self { Self::GenerateVerifierSetup(message) => message.graph_id, @@ -135,15 +126,6 @@ pub(crate) fn heavy_task_from_content( } } -pub(crate) fn is_heavy_task_message_type(message_type: &str, actor: &Actor) -> bool { - matches!( - (message_type, actor), - ("SolderingProofReady", Actor::Operator) - | ("InitGraph" | "CutCircuits", Actor::Verifier) - | ("CreateGraph", Actor::Verifier) - ) -} - pub(crate) async fn run_heavy_task(context: &HeavyTaskContext, task: HeavyTask) -> Result<()> { match task { HeavyTask::GenerateVerifierSetup(message) => { @@ -1544,7 +1526,6 @@ async fn defer_confirm_instance_until_previous_graph_presigned( else { push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &retry_message, 60, MessageDeferReason::PreviousGraphPending, @@ -1573,7 +1554,6 @@ async fn defer_confirm_instance_until_previous_graph_presigned( } push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &retry_message, 60, MessageDeferReason::PreviousGraphPending, @@ -1591,7 +1571,6 @@ async fn defer_confirm_instance_until_previous_graph_presigned( push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &retry_message, 60, MessageDeferReason::PreviousGraphPending, @@ -3195,7 +3174,6 @@ async fn handle_create_graph_committee( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, 60, MessageDeferReason::PreviousGraphPending, @@ -3211,7 +3189,6 @@ async fn handle_create_graph_committee( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, 60, MessageDeferReason::PreviousGraphPending, @@ -3228,7 +3205,6 @@ async fn handle_create_graph_committee( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, 60, MessageDeferReason::PreviousGraphPending, @@ -3439,7 +3415,6 @@ async fn handle_agg_nonce_consensus_committee( else { push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, 30, MessageDeferReason::CommitteeNoncesPending, @@ -3597,7 +3572,6 @@ async fn validate_committee_presign_for_graph( if pub_nonces_unchecked.len() != committee_pubkeys.len() { push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, 30, MessageDeferReason::CommitteeNoncesPending, @@ -4362,7 +4336,6 @@ async fn handle_pegin_confirm_nonce_consensus_committee( else { push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, 30, MessageDeferReason::CommitteeNoncesPending, @@ -4434,7 +4407,6 @@ async fn handle_pegin_confirm_partial_sig_committee( if pub_nonces_unchecked.len() != committee_pubkeys.len() { push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, 30, MessageDeferReason::CommitteeNoncesPending, @@ -4490,7 +4462,6 @@ async fn handle_pegin_confirm_partial_sig_committee( { push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, 30, MessageDeferReason::CommitteeNonceConsensusPending, @@ -4538,7 +4509,6 @@ async fn handle_pegin_confirm_partial_sig_committee( Err(e) => { push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, 30, MessageDeferReason::ValidationRetry, @@ -4602,7 +4572,6 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ); push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, delay_secs as usize, MessageDeferReason::BitcoinTransactionPending, @@ -4628,7 +4597,6 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ); push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, delay_secs as usize, MessageDeferReason::CommitteeEndorsementsPending, @@ -4651,7 +4619,6 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ); push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, delay_secs as usize, MessageDeferReason::BitcoinConfirmationPending, @@ -4674,7 +4641,6 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ); push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, delay_secs as usize, MessageDeferReason::GoatSpvPending, @@ -4735,7 +4701,6 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ); push_local_unhandled_messages_with_reason( ctx.local_db, - instance_id, &message, delay_secs as usize, MessageDeferReason::CommitteeEndorsementsPending, @@ -4866,7 +4831,6 @@ async fn handle_kickoff_ready_operator( let delay_secs = min_pegout_time_secs * nonce_interval; push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::PreviousGraphPending, @@ -4882,7 +4846,6 @@ async fn handle_kickoff_ready_operator( let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 blocks push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::ChainStatePending, @@ -4907,7 +4870,6 @@ async fn handle_kickoff_ready_operator( let delay_secs = min_pegout_time_secs * nonce_interval; push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::PreviousGraphPending, @@ -4923,7 +4885,6 @@ async fn handle_kickoff_ready_operator( let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 blocks push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::ChainStatePending, @@ -4983,7 +4944,6 @@ async fn handle_kickoff_sent_committee( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinConfirmationPending, @@ -5003,7 +4963,6 @@ async fn handle_kickoff_sent_committee( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::GoatSpvPending, @@ -5077,7 +5036,6 @@ async fn handle_kickoff_sent_verifier( * (kickoff_height - goat_confirmed_btc_height) as u64; push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::GoatSpvPending, @@ -5402,7 +5360,6 @@ async fn handle_watchtower_challenge_init_sent_watchtower( ); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, wait_secs, MessageDeferReason::ProofPending, @@ -5721,7 +5678,6 @@ async fn handle_operator_commit_pubin_ready_operator( ); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, wait_secs, MessageDeferReason::ProtocolInputsPending, @@ -5746,7 +5702,6 @@ async fn handle_operator_commit_pubin_ready_operator( ); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, wait_secs, MessageDeferReason::ProtocolInputsPending, @@ -5877,7 +5832,6 @@ async fn handle_assert_ready_operator( ); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, wait_secs, MessageDeferReason::ProofPending, @@ -6042,7 +5996,6 @@ async fn handle_assert_sent_verifier( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::ProtocolInputsPending, @@ -6062,7 +6015,6 @@ async fn handle_assert_sent_verifier( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::ProtocolInputsPending, @@ -6076,7 +6028,6 @@ async fn handle_assert_sent_verifier( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::ProtocolInputsPending, @@ -6107,7 +6058,6 @@ async fn handle_assert_sent_verifier( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::ProtocolInputsPending, @@ -6318,7 +6268,6 @@ async fn handle_challenge_assert_sent_operator( let message = make_message(ctx, content); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinTransactionPending, @@ -6486,7 +6435,6 @@ async fn handle_wrongly_challenge_timeout_verifier( if ctx.btc_client.get_tx(&challenge_assert_txid).await?.is_none() { push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinTransactionPending, @@ -6509,7 +6457,6 @@ async fn handle_wrongly_challenge_timeout_verifier( None => { push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinConfirmationPending, @@ -6534,7 +6481,6 @@ async fn handle_wrongly_challenge_timeout_verifier( avg_block_time_secs(ctx.btc_client.network()) * (disprove_height - bitcoin_height); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, retry_secs as usize, MessageDeferReason::TimelockPending, @@ -6635,7 +6581,6 @@ async fn handle_disprove_sent_committee( let delay_secs = avg_block_time_secs(ctx.btc_client.network()); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinConfirmationPending, @@ -6654,7 +6599,6 @@ async fn handle_disprove_sent_committee( * (challenge_finish_height - goat_confirmed_height); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::GoatSpvPending, @@ -6802,7 +6746,6 @@ async fn handle_take1_sent_committee( let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * 6; // wait for 6 blocks push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::WithdrawKickoffPending, @@ -6826,7 +6769,6 @@ async fn handle_take1_sent_committee( let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 block push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinConfirmationPending, @@ -6845,7 +6787,6 @@ async fn handle_take1_sent_committee( avg_block_time_secs(ctx.btc_client.network()) * (take1_height - goat_confirmed_height); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::GoatSpvPending, @@ -7017,7 +6958,6 @@ async fn handle_take2_sent_committee( let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * 6; // wait for 6 blocks push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::WithdrawKickoffPending, @@ -7041,7 +6981,6 @@ async fn handle_take2_sent_committee( let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 block push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::BitcoinConfirmationPending, @@ -7060,7 +6999,6 @@ async fn handle_take2_sent_committee( avg_block_time_secs(ctx.btc_client.network()) * (take2_height - goat_confirmed_height); push_local_unhandled_messages_with_reason( ctx.local_db, - graph_id, &message, delay_secs as usize, MessageDeferReason::GoatSpvPending, diff --git a/node/src/main.rs b/node/src/main.rs index 0528c4d7..b52afce6 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -3,9 +3,9 @@ use base64::Engine; use bitvm_lib::actors::Actor; use bitvm_lib::babe_adapter::BabeBundleBuilder; use bitvm_noded::env::{ - self, ENV_PEER_KEY, SEQUENCER_SET_MONITOR_INTERVAL_SECS, check_node_info, get_btc_url_from_env, - get_goat_network, get_network, get_node_pubkey, goat_config_from_env, - validate_soldering_proof_payload_store_config, + self, ENV_PEER_KEY, SEQUENCER_SET_MONITOR_INTERVAL_SECS, actor_needs_soldering_builder, + check_node_info, get_btc_url_from_env, get_goat_network, get_network, get_node_pubkey, + goat_config_from_env, validate_soldering_proof_payload_store_config, }; use clap::{Parser, Subcommand}; use client::{btc_chain::BTCClient, goat_chain::GOATClient}; @@ -207,7 +207,7 @@ async fn main() -> Result<(), Box> { env::get_goat_network(), )), http_client: HttpAsyncClient::new(None), - soldering_builder: matches!(actor, Actor::Verifier | Actor::Operator) + soldering_builder: actor_needs_soldering_builder(&actor) .then(|| Arc::new(BabeBundleBuilder::new())), metrics_state: metrics_state.clone(), shutdown_token: cancellation_token.clone(), diff --git a/node/src/rpc_service/bitvm.rs b/node/src/rpc_service/bitvm.rs index 26dde07c..a4977e11 100644 --- a/node/src/rpc_service/bitvm.rs +++ b/node/src/rpc_service/bitvm.rs @@ -41,6 +41,25 @@ const GRAPH_OPERATOR_KICKOFFING_STATUS_DURATION_SECS: i64 = 1800; const GRAPH_OPERATOR_KICKOFF_STATUS_DURATION_SECS: i64 = 3600 * 3; const GRAPH_OPERATOR_CHALLENGE_STATUS_DURATION_SECS: i64 = 3600 * 9; +#[derive(Clone, Copy, Display, EnumString)] +enum GraphDisplayStatus { + Created, + Presigned, + L2Recorded, + OperatorKickOffing, +} + +#[derive(Clone, Copy, Display, EnumString)] +enum InstanceDisplayStatus { + Initiated, + Verified, + Submitted, + Failed, + Processing, + Success, + Canceled, +} + #[derive(Debug, Deserialize, Serialize)] pub struct InstanceSettingResponse { pub bridge_in_amount: Vec, @@ -311,21 +330,17 @@ fn get_bridge_in_status_time_window_secs(status: &str, response_window_blocks: i + INSTANCE_RELAYER_L1_BROADCAST_STATUS_DURATION_SECS; match InstanceBridgeInStatus::from_str(status) { - Ok(InstanceBridgeInStatus::UserIniting) - | Ok(InstanceBridgeInStatus::Initiated) - | Ok(InstanceBridgeInStatus::UserInited) => { + Ok(InstanceBridgeInStatus::UserIniting) | Ok(InstanceBridgeInStatus::UserInited) => { (response_window_blocks_with_margin * GOAT_BLOCK_INTERVAL_SECS, total_time) } - Ok(InstanceBridgeInStatus::CommitteesAnswered) | Ok(InstanceBridgeInStatus::Verified) => { + Ok(InstanceBridgeInStatus::CommitteesAnswered) => { (0, total_time - response_window_blocks_with_margin * GOAT_BLOCK_INTERVAL_SECS) } - Ok(InstanceBridgeInStatus::Submitted) - | Ok(InstanceBridgeInStatus::UserBroadcastPeginPrepare) => ( + Ok(InstanceBridgeInStatus::UserBroadcastPeginPrepare) => ( INSTANCE_USER_BROADCAST_PREPARE_STATUS_DURATION_SECS, total_time - response_window_blocks_with_margin * GOAT_BLOCK_INTERVAL_SECS, ), - Ok(InstanceBridgeInStatus::Processing) - | Ok(InstanceBridgeInStatus::Presigned) + Ok(InstanceBridgeInStatus::Presigned) | Ok(InstanceBridgeInStatus::RelayerL1Broadcasted) => ( INSTANCE_RELAYER_L1_BROADCAST_STATUS_DURATION_SECS, total_time @@ -469,7 +484,7 @@ impl From for GraphQuery { let mut is_init_withdraw_not_null = value .status .as_ref() - .map(|status| status == &GraphStatus::OperatorKickOffing.to_string()) + .map(|status| status == &GraphDisplayStatus::OperatorKickOffing.to_string()) .unwrap_or(false); is_init_withdraw_not_null = is_init_withdraw_not_null || value.is_pegout_started; let mut statuses = vec![]; @@ -647,28 +662,29 @@ trait DisplayStatusConvert { impl DisplayStatusConvert for Graph { fn convert_to_display_status(&self) -> String { match GraphStatus::from_str(&self.status) { - Ok(GraphStatus::OperatorPresigned) => GraphStatus::Created.to_string(), - Ok(GraphStatus::CommitteePresigned) => GraphStatus::Presigned.to_string(), + Ok(GraphStatus::OperatorPresigned) => GraphDisplayStatus::Created.to_string(), + Ok(GraphStatus::CommitteePresigned) => GraphDisplayStatus::Presigned.to_string(), Ok(GraphStatus::OperatorDataPushed) => { if self.init_withdraw_tx_hash.is_some() { - GraphStatus::OperatorKickOffing.to_string() + GraphDisplayStatus::OperatorKickOffing.to_string() } else { - GraphStatus::L2Recorded.to_string() + GraphDisplayStatus::L2Recorded.to_string() } } Ok(_) | Err(_) => self.status.clone(), } } fn parse_display_status(ori_status: &str) -> Vec { - match GraphStatus::from_str(ori_status) { - Ok(GraphStatus::Created) => vec![GraphStatus::OperatorPresigned.to_string()], - Ok(GraphStatus::Presigned) => vec![GraphStatus::CommitteePresigned.to_string()], - Ok(GraphStatus::L2Recorded) => vec![GraphStatus::OperatorDataPushed.to_string()], - Ok(GraphStatus::OperatorKickOffing) => { + match GraphDisplayStatus::from_str(ori_status) { + Ok(GraphDisplayStatus::Created) => vec![GraphStatus::OperatorPresigned.to_string()], + Ok(GraphDisplayStatus::Presigned) => vec![GraphStatus::CommitteePresigned.to_string()], + Ok(GraphDisplayStatus::L2Recorded) => vec![GraphStatus::OperatorDataPushed.to_string()], + Ok(GraphDisplayStatus::OperatorKickOffing) => { vec![GraphStatus::OperatorDataPushed.to_string()] } - Ok(v) => vec![v.to_string()], - Err(_) => vec![], + Err(_) => { + GraphStatus::from_str(ori_status).map(|v| vec![v.to_string()]).unwrap_or_default() + } } } } @@ -676,62 +692,63 @@ impl DisplayStatusConvert for Graph { impl DisplayStatusConvert for Instance { fn convert_to_display_status(&self) -> String { match InstanceBridgeInStatus::from_str(&self.status) { - Ok(InstanceBridgeInStatus::UserInited) => InstanceBridgeInStatus::Initiated.to_string(), + Ok(InstanceBridgeInStatus::UserInited) => InstanceDisplayStatus::Initiated.to_string(), Ok(InstanceBridgeInStatus::CommitteesAnswered) => { - InstanceBridgeInStatus::Verified.to_string() + InstanceDisplayStatus::Verified.to_string() } Ok(InstanceBridgeInStatus::UserBroadcastPeginPrepare) => { - InstanceBridgeInStatus::Submitted.to_string() + InstanceDisplayStatus::Submitted.to_string() } - Ok(InstanceBridgeInStatus::Presigned) => InstanceBridgeInStatus::Processing.to_string(), + Ok(InstanceBridgeInStatus::Presigned) => InstanceDisplayStatus::Processing.to_string(), Ok(InstanceBridgeInStatus::RelayerL1Broadcasted) => { - InstanceBridgeInStatus::Processing.to_string() + InstanceDisplayStatus::Processing.to_string() } Ok(InstanceBridgeInStatus::RelayerL2Minted) => { - InstanceBridgeInStatus::Success.to_string() + InstanceDisplayStatus::Success.to_string() } Ok(InstanceBridgeInStatus::PresignedFailed) | Ok(InstanceBridgeInStatus::RelayerL2MintedFailed) - | Ok(InstanceBridgeInStatus::NoEnoughCommitteesAnswered) => { - InstanceBridgeInStatus::Failed.to_string() - } - Ok(InstanceBridgeInStatus::UserCanceled) => { - InstanceBridgeInStatus::Canceled.to_string() + | Ok(InstanceBridgeInStatus::NoEnoughCommitteesAnswered) + | Ok(InstanceBridgeInStatus::UserDiscarded) => { + InstanceDisplayStatus::Failed.to_string() } + Ok(InstanceBridgeInStatus::UserCanceled) => InstanceDisplayStatus::Canceled.to_string(), Ok(_) | Err(_) => self.status.clone(), } } fn parse_display_status(ori_status: &str) -> Vec { - match InstanceBridgeInStatus::from_str(ori_status) { - Ok(InstanceBridgeInStatus::Initiated) => { + match InstanceDisplayStatus::from_str(ori_status) { + Ok(InstanceDisplayStatus::Initiated) => { vec![InstanceBridgeInStatus::UserInited.to_string()] } - Ok(InstanceBridgeInStatus::Verified) => { + Ok(InstanceDisplayStatus::Verified) => { vec![InstanceBridgeInStatus::CommitteesAnswered.to_string()] } - Ok(InstanceBridgeInStatus::Submitted) => { + Ok(InstanceDisplayStatus::Submitted) => { vec![InstanceBridgeInStatus::UserBroadcastPeginPrepare.to_string()] } - Ok(InstanceBridgeInStatus::Processing) => { + Ok(InstanceDisplayStatus::Processing) => { vec![ InstanceBridgeInStatus::RelayerL1Broadcasted.to_string(), InstanceBridgeInStatus::Presigned.to_string(), ] } - Ok(InstanceBridgeInStatus::Success) => { + Ok(InstanceDisplayStatus::Success) => { vec![InstanceBridgeInStatus::RelayerL2Minted.to_string()] } - Ok(InstanceBridgeInStatus::Canceled) => { + Ok(InstanceDisplayStatus::Canceled) => { vec![InstanceBridgeInStatus::UserCanceled.to_string()] } - Ok(InstanceBridgeInStatus::Failed) => vec![ + Ok(InstanceDisplayStatus::Failed) => vec![ InstanceBridgeInStatus::PresignedFailed.to_string(), InstanceBridgeInStatus::RelayerL2MintedFailed.to_string(), InstanceBridgeInStatus::NoEnoughCommitteesAnswered.to_string(), + InstanceBridgeInStatus::UserDiscarded.to_string(), ], - Ok(v) => vec![v.to_string()], - Err(_) => vec![], + Err(_) => InstanceBridgeInStatus::from_str(ori_status) + .map(|v| vec![v.to_string()]) + .unwrap_or_default(), } } } diff --git a/node/src/rpc_service/handler/bitvm_handler.rs b/node/src/rpc_service/handler/bitvm_handler.rs index 1095775d..b216e2ed 100644 --- a/node/src/rpc_service/handler/bitvm_handler.rs +++ b/node/src/rpc_service/handler/bitvm_handler.rs @@ -1632,15 +1632,6 @@ pub async fn pegout( ), ); } - _ => { - return error_response( - "PEGOUT_ERROR".to_string(), - format!( - "graph {} not ready: previous graph {} has unexpected status {}", - graph.graph_id, previous_graph.graph_id, previous_graph.status - ), - ); - } } } } diff --git a/node/src/scheduled_tasks/graph_maintenance_tasks.rs b/node/src/scheduled_tasks/graph_maintenance_tasks.rs index ed3a9600..83a48966 100644 --- a/node/src/scheduled_tasks/graph_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/graph_maintenance_tasks.rs @@ -66,16 +66,11 @@ pub struct ChallengeSubStatus { struct DetectedGraphMessage { actor: Actor, content: GOATMessageContent, - sub_type: Option, } impl DetectedGraphMessage { fn new(actor: Actor, content: GOATMessageContent) -> Self { - Self { actor, content, sub_type: None } - } - - fn with_sub_type(actor: Actor, content: GOATMessageContent, sub_type: String) -> Self { - Self { actor, content, sub_type: Some(sub_type) } + Self { actor, content } } } @@ -115,7 +110,6 @@ impl ChallengeSubStatus { async fn upsert_detected_messages( local_db: &LocalDB, - graph_id: Uuid, messages: Vec, ) -> anyhow::Result<()> { if messages.is_empty() { @@ -127,8 +121,6 @@ async fn upsert_detected_messages( upsert_message( &mut storage_processor, false, - graph_id, - message.sub_type, SELF_SENDER.to_string(), message.actor, message.content, @@ -189,7 +181,7 @@ async fn detect_watchtower_flow_disprove( ) -> anyhow::Result> { for (index, txid) in graph.operator_challenge_nack_txids.iter().enumerate() { if btc_client.get_tx_status(&txid.0).await?.confirmed { - return Ok(Some(DetectedGraphMessage::with_sub_type( + return Ok(Some(DetectedGraphMessage::new( Actor::Committee, GOATMessageContent::DisproveSent(DisproveSent { instance_id: graph.instance_id, @@ -199,7 +191,6 @@ async fn detect_watchtower_flow_disprove( challenge_start_txid: None, challenge_finish_txid: txid.0, }), - index.to_string(), ))); } } @@ -260,8 +251,6 @@ pub async fn detect_init_withdraw_call(local_db: &LocalDB) -> anyhow::Result<()> upsert_message( &mut tx, false, - graph_id, - None, SELF_SENDER.to_string(), Actor::Operator, GOATMessageContent::KickoffReady(KickoffReady { instance_id, graph_id }), @@ -291,8 +280,6 @@ async fn enqueue_kickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::Resu upsert_message( &mut storage_processor, false, - graph.graph_id, - None, SELF_SENDER.to_string(), Actor::All, GOATMessageContent::KickoffSent(KickoffSent { @@ -311,8 +298,6 @@ async fn enqueue_prekickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::R upsert_message( &mut storage_processor, false, - graph.graph_id, - None, SELF_SENDER.to_string(), Actor::Verifier, GOATMessageContent::PreKickoffSent(PreKickoffSent { @@ -535,8 +520,6 @@ async fn detect_take1_or_challenge_for_graph( upsert_message( &mut storage_processor, false, - graph.graph_id, - None, SELF_SENDER.to_string(), actor, message_content, @@ -590,7 +573,7 @@ async fn process_graph_challenge_for_graph( "process_graph_challenge detected {} watchtower/pubin flow messages", watchtower_flow_messages.len() ); - upsert_detected_messages(local_db, graph.graph_id, watchtower_flow_messages).await?; + upsert_detected_messages(local_db, watchtower_flow_messages).await?; } let assert_sent_messages = detect_assert_sent_flow(btc_client, local_db, &graph).await?; @@ -599,10 +582,10 @@ async fn process_graph_challenge_for_graph( "process_graph_challenge detected {} assert/challenge-assert messages", assert_sent_messages.len() ); - upsert_detected_messages(local_db, graph.graph_id, assert_sent_messages).await?; + upsert_detected_messages(local_db, assert_sent_messages).await?; } - if let Some((actor, message_content, sub_type)) = + if let Some((actor, message_content)) = detect_assert_disprove_ready(btc_client, local_db, &graph, current_height).await? { info!("process_graph_challenge detect assert disprove ready"); @@ -610,8 +593,6 @@ async fn process_graph_challenge_for_graph( upsert_message( &mut storage_processor, false, - graph.graph_id, - sub_type, SELF_SENDER.to_string(), actor, message_content, @@ -630,8 +611,6 @@ async fn process_graph_challenge_for_graph( upsert_message( &mut storage_processor, false, - graph.graph_id, - None, SELF_SENDER.to_string(), actor, message_content, @@ -759,14 +738,13 @@ async fn detect_watchtower_flow( all_watchtower_branches_resolved = false; } Some(_) if !watchtower_timeout_spent => { - messages.push(DetectedGraphMessage::with_sub_type( + messages.push(DetectedGraphMessage::new( Actor::Operator, GOATMessageContent::WatchtowerChallengeSent(WatchtowerChallengeSent { instance_id: graph.instance_id, graph_id: graph.graph_id, watchtower_index, }), - watchtower_index.to_string(), )); if !ack_spend_confirmed { @@ -922,7 +900,7 @@ async fn detect_assert_sent_flow( continue; }; - messages.push(DetectedGraphMessage::with_sub_type( + messages.push(DetectedGraphMessage::new( Actor::Operator, GOATMessageContent::ChallengeAssertSent(ChallengeAssertSent { instance_id: graph.instance_id, @@ -930,7 +908,6 @@ async fn detect_assert_sent_flow( challenge_assert_txid, verifier_index, }), - verifier_index.to_string(), )); } @@ -943,7 +920,7 @@ async fn detect_assert_disprove_ready( local_db: &LocalDB, graph: &Graph, current_height: i64, -) -> anyhow::Result)>> { +) -> anyhow::Result> { let operator_assert_txid = match graph.operator_assert_txid.clone() { Some(operator_assert_txid) => operator_assert_txid.into(), None => { @@ -1026,7 +1003,6 @@ async fn detect_assert_disprove_ready( challenge_assert_txid: verifier_assert_txid, verifier_index: index, }), - Some(index.to_string()), ))); } } @@ -1251,8 +1227,6 @@ async fn detect_kickoff_ref_disprove_tx( upsert_message( &mut storage_processor, false, - graph.graph_id, - None, SELF_SENDER.to_string(), Actor::Committee, GOATMessageContent::DisproveSent(DisproveSent { @@ -1453,8 +1427,6 @@ async fn check_pre_kickoff_sent( upsert_message( &mut storage_processor, false, - graph_id, - None, SELF_SENDER.to_string(), Actor::Verifier, GOATMessageContent::PreKickoffSent(PreKickoffSent { instance_id, graph_id }), diff --git a/node/src/scheduled_tasks/instance_maintenance_tasks.rs b/node/src/scheduled_tasks/instance_maintenance_tasks.rs index 4ee932c7..6f94bc06 100644 --- a/node/src/scheduled_tasks/instance_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/instance_maintenance_tasks.rs @@ -140,15 +140,18 @@ async fn find_one_instance_page( async fn update_instance<'a>( storage_processor: &mut StorageProcessor<'a>, params: &InstanceUpdate, -) -> anyhow::Result<()> { +) -> anyhow::Result { match storage_processor.update_instance(params).await { - Ok(true) => info!("update instance with input: {:?}", params), - Ok(false) => info!("skip stale instance update with input: {:?}", params), - Err(err) => { - warn!("update_instance_status with input: {:?} failed {}, will try later", params, err); + Ok(updated) => { + if updated { + info!("update instance with input: {:?}", params); + } else { + info!("skip stale instance update with input: {:?}", params); + } + Ok(updated) } + Err(err) => Err(err.context("update instance")), } - Ok(()) } /// for committee @@ -233,8 +236,6 @@ pub async fn instance_answers_monitor( upsert_message( &mut tx, false, - tx_record.instance_id, - None, SELF_SENDER.to_string(), Actor::All, GOATMessageContent::PeginRequest(PeginRequest { @@ -406,7 +407,8 @@ pub async fn instance_expiration_monitor( update_instance( &mut storage_processor, &InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(InstanceBridgeInStatus::Timeout.to_string()), + .with_status(InstanceBridgeInStatus::Timeout.to_string()) + .with_only_if_status_in(vec![instance.status.clone()]), ) .await?; } else { @@ -470,45 +472,46 @@ pub async fn instance_btc_tx_monitor( { let mut tx = local_db.start_transaction().await?; let mut instance_update = InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(next_status.to_string()); - match next_status { - InstanceBridgeInStatus::UserBroadcastPeginPrepare => { - instance_update = instance_update - .with_btc_height(status.block_height.unwrap_or_default() as i64); - upsert_message( - &mut tx, - false, - instance.instance_id, - None, - SELF_SENDER.to_string(), - Actor::All, - GOATMessageContent::ConfirmInstance(ConfirmInstance { - instance_id: instance.instance_id, - }), - 0, - 0, - ) - .await?; - } - InstanceBridgeInStatus::RelayerL1Broadcasted => { - upsert_message( - &mut tx, - false, - instance.instance_id, - None, - SELF_SENDER.to_string(), - Actor::All, - GOATMessageContent::PostReady(PostReady { - instance_id: instance.instance_id, - }), - 0, - 0, - ) - .await?; + .with_status(next_status.to_string()) + .with_only_if_status_in(vec![instance.status.clone()]); + if next_status == InstanceBridgeInStatus::UserBroadcastPeginPrepare { + instance_update = + instance_update.with_btc_height(status.block_height.unwrap_or_default() as i64); + } + + if update_instance(&mut tx, &instance_update).await? { + match next_status { + InstanceBridgeInStatus::UserBroadcastPeginPrepare => { + upsert_message( + &mut tx, + false, + SELF_SENDER.to_string(), + Actor::All, + GOATMessageContent::ConfirmInstance(ConfirmInstance { + instance_id: instance.instance_id, + }), + 0, + 0, + ) + .await?; + } + InstanceBridgeInStatus::RelayerL1Broadcasted => { + upsert_message( + &mut tx, + false, + SELF_SENDER.to_string(), + Actor::All, + GOATMessageContent::PostReady(PostReady { + instance_id: instance.instance_id, + }), + 0, + 0, + ) + .await?; + } + _ => {} } - _ => {} } - update_instance(&mut tx, &instance_update).await?; tx.commit().await?; } else { warn!( @@ -624,7 +627,6 @@ pub async fn pegin_confirm_recovery_monitor( if !finish_recovery_enqueue( push_local_unhandled_messages_with_reason( local_db, - instance_id, &message, 0, MessageDeferReason::RecoveryRepublish, @@ -685,7 +687,6 @@ pub async fn pegin_confirm_recovery_monitor( if !finish_recovery_enqueue( push_local_unhandled_messages_with_reason( local_db, - instance_id, &message, 0, MessageDeferReason::RecoveryRepublish, @@ -721,7 +722,6 @@ pub async fn pegin_confirm_recovery_monitor( if !finish_recovery_enqueue( push_local_unhandled_messages_with_reason( local_db, - instance_id, &message, 0, MessageDeferReason::RecoveryRepublish, diff --git a/node/src/scheduled_tasks/mod.rs b/node/src/scheduled_tasks/mod.rs index 8be12207..719415d9 100644 --- a/node/src/scheduled_tasks/mod.rs +++ b/node/src/scheduled_tasks/mod.rs @@ -6,10 +6,10 @@ mod node_maintenance_tasks; mod sequencer_set_hash_monitor_task; mod spv_maintenance_tasks; -use crate::action::GOATMessageContent; use crate::env::{ - get_maintenance_run_timeout_secs, get_network, get_node_goat_address, get_node_pubkey, - is_enable_babe_setup_state_cleanup, is_enable_update_spv_contract, is_relayer, + actor_runs_babe_setup_state_cleanup, get_maintenance_run_timeout_secs, get_network, + get_node_goat_address, get_node_pubkey, is_enable_babe_setup_state_cleanup, + is_enable_update_spv_contract, is_relayer, }; use crate::metrics_service::MetricsState; use crate::rpc_service::current_time_secs; @@ -33,8 +33,8 @@ pub use sequencer_set_hash_monitor_task::run_sequencer_set_hash_monitor_task; use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; +use store::Graph; use store::localdb::{LocalDB, StorageProcessor}; -use store::{Graph, MessageType}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -233,9 +233,7 @@ async fn run( let btc_client = btc_client.as_ref(); let goat_client = goat_client.as_ref(); - if is_enable_babe_setup_state_cleanup() - && matches!(&actor, Actor::Verifier | Actor::Operator | Actor::All) - { + if is_enable_babe_setup_state_cleanup() && actor_runs_babe_setup_state_cleanup(&actor) { run_maintenance_subtask( metrics_state, "babe_setup_state_cleanup_monitor", @@ -487,64 +485,6 @@ pub async fn run_maintenance_tasks( } } -pub fn get_goat_message_content_type(content: &GOATMessageContent) -> MessageType { - match content { - GOATMessageContent::PeginRequest(_) => MessageType::PeginRequest, - GOATMessageContent::CreateGraph(_) => MessageType::CreateGraph, - GOATMessageContent::ConfirmInstance(_) => MessageType::ConfirmInstance, - GOATMessageContent::InitGraph(_) => MessageType::InitGraph, - GOATMessageContent::GenCircuits(_) => MessageType::GenCircuits, - GOATMessageContent::CutCircuits(_) => MessageType::CutCircuits, - GOATMessageContent::SolderingProofReady(_) => MessageType::SolderingProof, - GOATMessageContent::GraphSetupAck(_) => MessageType::None, - GOATMessageContent::VerifierGraphParamsEndorsement(_) => { - MessageType::VerifierGraphParamsEndorsement - } - GOATMessageContent::NonceGeneration(_) => MessageType::NonceGeneration, - GOATMessageContent::AggNonceConsensus(_) => MessageType::AggNonceConsensus, - GOATMessageContent::CommitteePresign(_) => MessageType::CommitteePresign, - GOATMessageContent::GraphFinalize(_) => MessageType::GraphFinalize, - GOATMessageContent::EndorseGraph(_) => MessageType::EndorseGraph, - GOATMessageContent::PeginConfirmNonce(_) => MessageType::PeginConfirmNonce, - GOATMessageContent::PeginConfirmNonceConsensus(_) => { - MessageType::PeginConfirmNonceConsensus - } - GOATMessageContent::PeginConfirmPartialSig(_) => MessageType::PeginConfirmPartialSig, - GOATMessageContent::PostReady(_) => MessageType::PostReady, - GOATMessageContent::KickoffReady(_) => MessageType::KickoffReady, - GOATMessageContent::KickoffSent(_) => MessageType::KickoffSent, - GOATMessageContent::PreKickoffSent(_) => MessageType::PreKickoffSent, - GOATMessageContent::ChallengeSent(_) => MessageType::ChallengeSent, - GOATMessageContent::WatchtowerChallengeInitSent(_) => { - MessageType::WatchtowerChallengeInitSent - } - GOATMessageContent::WatchtowerChallengeSent(_) => MessageType::WatchtowerChallengeSent, - GOATMessageContent::WatchtowerChallengeTimeout(_) => { - MessageType::WatchtowerChallengeTimeout - } - GOATMessageContent::NackReady(_) => MessageType::NackReady, - GOATMessageContent::OperatorCommitPubinReady(_) => MessageType::OperatorCommitPubinReady, - GOATMessageContent::OperatorCommitPubinTimeout(_) => { - MessageType::OperatorCommitPubinTimeout - } - GOATMessageContent::AssertReady(_) => MessageType::AssertReady, - GOATMessageContent::AssertSent(_) => MessageType::AssertSent, - GOATMessageContent::ChallengeAssertSent(_) => MessageType::ChallengeAssertSent, - GOATMessageContent::WronglyChallengeTimeout(_) => MessageType::WronglyChallengeTimeout, - GOATMessageContent::DisproveSent(_) => MessageType::DisproveSent, - GOATMessageContent::Take1Ready(_) => MessageType::Take1Ready, - GOATMessageContent::Take1Sent(_) => MessageType::Take1Sent, - GOATMessageContent::Take2Ready(_) => MessageType::Take2Ready, - GOATMessageContent::Take2Sent(_) => MessageType::Take2Sent, - GOATMessageContent::RequestNodeInfo(_) => MessageType::RequestNodeInfo, - GOATMessageContent::ResponseNodeInfo(_) => MessageType::ResponseNodeInfo, - GOATMessageContent::SyncGraphRequest(_) => MessageType::SyncGraphRequest, - GOATMessageContent::SyncGraph(_) => MessageType::SyncGraph, - GOATMessageContent::InstanceDiscarded(_) => MessageType::InstanceDiscarded, - GOATMessageContent::Tick => MessageType::Tick, - } -} - fn get_timestamp_from_contract_data(input: &[u8; 32]) -> i64 { let mut timestamp_bytes = [0u8; 8]; timestamp_bytes.copy_from_slice(&input[24..32]); diff --git a/node/src/utils.rs b/node/src/utils.rs index b3493e3b..0aec93a9 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -1,6 +1,7 @@ use crate::action::{ - ChallengeSent, DisproveSent, GOATMessage, GOATMessageContent, KickoffSent, NodeInfo, - PreKickoffSent, SolderingProofReady, Take1Sent, Take2Sent, send_to_peer, + BusinessRef, ChallengeSent, DisproveSent, GOATMessage, GOATMessageContent, HasBusinessRef, + KickoffSent, LocalMessageKey, MessageKind, NodeInfo, PreKickoffSent, SolderingProofReady, + Take1Sent, Take2Sent, send_to_peer, }; use crate::env::*; use crate::error::SpecialError; @@ -78,7 +79,6 @@ use crate::rpc_service::routes::v1::{ NODES_OPERATOR_BASE, NODES_WATCHTOWER_BASE, PROOFS_WATCHTOWER_PROOF_TIMEOUT, }; -use crate::scheduled_tasks::get_goat_message_content_type; use crate::scheduled_tasks::graph_maintenance_tasks::{ ChallengeSubStatus, VerifierChallengeStatus, }; @@ -98,9 +98,8 @@ use proof_builder::{ }; use store::{ BridgeOutGlobalStats, ByteArray32, Graph, GraphRawData, GraphStatus, GraphStatusSource, - GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, Message, MessageState, - MessageType, Node, PeginGraphProcessData, PeginInstanceProcessData, SerializableTxid, - UInt64Array3, + GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, Message, MessageState, Node, + PeginGraphProcessData, PeginInstanceProcessData, SerializableTxid, UInt64Array3, }; use stun_client::{Attribute, Class, Client}; use tracing::{error, info, warn}; @@ -713,8 +712,8 @@ pub fn challenge_amount() -> Amount { Amount::from_sat(20000) } pub fn prekickoff_fee_amount(replenish_fee_inputs_num: usize) -> Amount { - let tx_vbytes = - PRE_KICKOFF_BASE_VBYTES + (replenish_fee_inputs_num as u64 * CHEKSIG_P2WSH_INPUT_VBYTES); + let tx_vbytes = PRE_KICKOFF_BASE_VBYTES + + (replenish_fee_inputs_num as u64 * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE); Amount::from_sat(tx_vbytes) } pub mod evm_swap_utils { @@ -1754,8 +1753,7 @@ fn compensation_previous_status(status: GraphStatus) -> Option { OperatorKickOff => Some(PreKickoff), OperatorTake1 | Challenge => Some(OperatorKickOff), Disprove | OperatorTake2 => Some(Challenge), - OperatorPresigned | Created | Presigned | L2Recorded | OperatorKickOffing | Challenging - | Disproving => None, + OperatorPresigned => None, } } @@ -1789,13 +1787,15 @@ fn compensation_events_from( async fn upsert_graph_compensate_message( local_db: &LocalDB, - graph_id: Uuid, - sub_type: Option, actor: Actor, message_content: GOATMessageContent, ) -> Result<()> { - let message_type = get_goat_message_content_type(&message_content); - let message_id = generate_message_id(graph_id, message_type.to_string(), sub_type.clone()); + let key = LocalMessageKey::from_content(actor.clone(), &message_content)?; + let graph_id = match message_content.business_ref() { + BusinessRef::Graph { graph_id, .. } => graph_id, + _ => bail!("graph compensation message must be graph-scoped"), + }; + let message_id = key.message_id(); let mut storage_processor = local_db.start_transaction().await?; if !storage_processor.insert_graph_compensation_marker(graph_id, &message_id).await? { storage_processor.commit().await?; @@ -1805,8 +1805,6 @@ async fn upsert_graph_compensate_message( upsert_message( &mut storage_processor, false, - graph_id, - sub_type, SELF_SENDER.to_string(), actor, message_content, @@ -1819,14 +1817,13 @@ async fn upsert_graph_compensate_message( async fn push_graph_compensate_message( local_db: &LocalDB, - graph_id: Uuid, actor: Actor, message_content: GOATMessageContent, ) -> Result<()> { // Unlike an action retry, an inferred chain event must not reset an // existing queued message. This makes compensation safe to retry when the // status write committed before the message was persisted. - upsert_graph_compensate_message(local_db, graph_id, None, actor, message_content).await + upsert_graph_compensate_message(local_db, actor, message_content).await } #[allow(dead_code)] @@ -1875,7 +1872,6 @@ pub(crate) async fn compensate_graph_events( GraphCompensateEventKind::PreKickoffSent => { push_graph_compensate_message( local_db, - graph_id, Actor::Verifier, GOATMessageContent::PreKickoffSent(PreKickoffSent { instance_id, graph_id }), ) @@ -1884,7 +1880,6 @@ pub(crate) async fn compensate_graph_events( GraphCompensateEventKind::KickoffSent => { push_graph_compensate_message( local_db, - graph_id, Actor::All, GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), ) @@ -1893,7 +1888,6 @@ pub(crate) async fn compensate_graph_events( GraphCompensateEventKind::Take1Sent => { push_graph_compensate_message( local_db, - graph_id, Actor::Committee, GOATMessageContent::Take1Sent(Take1Sent { instance_id, graph_id }), ) @@ -1903,7 +1897,6 @@ pub(crate) async fn compensate_graph_events( if let Some(challenge_txid) = scan.challenge_txid { push_graph_compensate_message( local_db, - graph_id, Actor::Operator, GOATMessageContent::ChallengeSent(ChallengeSent { instance_id, @@ -1922,8 +1915,6 @@ pub(crate) async fn compensate_graph_events( })?; upsert_graph_compensate_message( local_db, - graph_id, - Some(disprove.index.to_string()), Actor::Committee, GOATMessageContent::DisproveSent(DisproveSent { instance_id, @@ -1939,7 +1930,6 @@ pub(crate) async fn compensate_graph_events( GraphCompensateEventKind::Take2Sent => { push_graph_compensate_message( local_db, - graph_id, Actor::Committee, GOATMessageContent::Take2Sent(Take2Sent { instance_id, graph_id }), ) @@ -3066,7 +3056,7 @@ pub async fn get_proper_utxo_set( fn estimate_tx_vbytes(base_vbytes: u64, extra_inputs: usize, extra_outputs: usize) -> u64 { // p2wsh inputs/outputs base_vbytes - + (extra_inputs as u64 * CHEKSIG_P2WSH_INPUT_VBYTES) + + (extra_inputs as u64 * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE) + (extra_outputs as u64 * P2WSH_OUTPUT_VBYTES) } fn to_input(utxos: Vec) -> Vec { @@ -3172,8 +3162,9 @@ pub async fn get_proper_utxo_sets( let n_inputs = tx_ins.len() as u64; let n_outputs = base_outputs.len() as u64 + 1; - let est_vbytes = - 100u64 + n_inputs * CHEKSIG_P2WSH_INPUT_VBYTES + n_outputs * P2WSH_OUTPUT_VBYTES; + let est_vbytes = 100u64 + + n_inputs * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE + + n_outputs * P2WSH_OUTPUT_VBYTES; let est_fee_sat = (est_vbytes as f64 * fee_rate).ceil() as u64; if total_available_sat < total_target_sat + est_fee_sat { @@ -3751,37 +3742,23 @@ pub async fn outpoint_spent_txin( } } -pub(crate) fn generate_message_id( - business_id: Uuid, - msg_type: String, - sub_type: Option, -) -> String { - match sub_type { - Some(sub_type) => { - format!("{business_id}_{msg_type}_{sub_type}") - } - None => format!("{business_id}_{msg_type}"), - } -} - -#[allow(clippy::too_many_arguments)] pub async fn upsert_message( storage_processor: &mut StorageProcessor<'_>, is_update: bool, - business_id: Uuid, - sub_type: Option, from_peer: String, actor: Actor, message_content: GOATMessageContent, weight: i64, lock_time: i64, ) -> Result { + let key = LocalMessageKey::from_content(actor.clone(), &message_content)?; + let business_id = key.business_id(); + let message_id = key.message_id(); + let msg_type = message_content.kind(); let message = GOATMessage::new(actor.clone(), message_content.clone()); - let msg_type = get_goat_message_content_type(&message_content); - let message_id = generate_message_id(business_id, msg_type.to_string().clone(), sub_type); if is_update || storage_processor.find_messages_by_id(&message_id).await?.is_none() { if let Some(cancel_msg_type) = match msg_type { - MessageType::AssertSent => Some(MessageType::WatchtowerChallengeInitSent), + MessageKind::AssertSent => Some(MessageKind::WatchtowerChallengeInitSent), _ => None, } { notify_to_cancel_proof_task(storage_processor, business_id, cancel_msg_type).await?; @@ -3815,10 +3792,10 @@ pub async fn upsert_message( pub async fn notify_to_cancel_proof_task( storage_processor: &mut StorageProcessor<'_>, business_id: Uuid, - msg_type: MessageType, + msg_type: MessageKind, ) -> Result<()> { // AssertInitSent is removed; update related logic if needed; - if !matches!(msg_type, MessageType::WatchtowerChallengeInitSent) { + if !matches!(msg_type, MessageKind::WatchtowerChallengeInitSent) { warn!("notify_to_cancel_proof_task: input wrong message type:{msg_type}"); return Ok(()); } @@ -3849,7 +3826,7 @@ pub async fn notify_to_cancel_proof_task( // It will only be called a few times under limited conditions, so we just create a new object let http_client = HttpAsyncClient::new(None); let notify_result = match msg_type { - MessageType::WatchtowerChallengeInitSent => { + MessageKind::WatchtowerChallengeInitSent => { let url = host.join(PROOFS_WATCHTOWER_PROOF_TIMEOUT)?; let payload = WatchtowerProofTimeoutUpdateRequest { instance_id: graph.instance_id.to_string(), @@ -4098,7 +4075,7 @@ pub async fn save_node_info(local_db: &LocalDB, node_info: &NodeInfo) -> Result< info!("save_node_info for {}", node_info.peer_id); let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; let mut storage_process = local_db.acquire().await?; - let _ = storage_process + storage_process .upsert_node(&Node { peer_id: node_info.peer_id.clone(), actor: node_info.actor.clone(), @@ -4112,7 +4089,7 @@ pub async fn save_node_info(local_db: &LocalDB, node_info: &NodeInfo) -> Result< updated_at: current_time, created_at: current_time, }) - .await; + .await?; Ok(()) } @@ -4506,15 +4483,19 @@ pub async fn get_instance_parameters( instance_id: Uuid, ) -> Result> { let mut storage_processor = local_db.acquire().await?; - if let Some(instance) = storage_processor.find_instance(&instance_id).await? { - Ok(if let Some(parameters) = instance.parameters { - Some(serde_json::from_str(¶meters)?) - } else { - gen_instance_parameters_local(&instance).ok() - }) - } else { - Ok(None) + let Some(instance) = storage_processor.find_instance(&instance_id).await? else { + return Ok(None); + }; + + if let Some(parameters) = instance.parameters { + return Ok(Some(serde_json::from_str(¶meters)?)); } + + Ok(Some( + gen_instance_parameters_local(&instance).with_context(|| { + format!("failed to reconstruct parameters for instance {instance_id}") + })?, + )) } fn convert_graph( @@ -5903,8 +5884,15 @@ pub fn gen_instance_parameters_local( let committee_pubkeys: Vec = instance .committees_answers .iter() - .map(|(_k, v)| PublicKey::from_slice(v).unwrap()) - .collect(); + .map(|(committee, pubkey)| { + PublicKey::from_slice(pubkey).with_context(|| { + format!( + "invalid committee public key for committee {committee} in instance {}", + instance.instance_id + ) + }) + }) + .collect::>()?; let committee_agg_pubkey = generate_n_of_n_public_key(&committee_pubkeys).0; let utxos: Vec = serde_json::from_str(&instance.input_utxos)?; @@ -6033,7 +6021,12 @@ pub async fn get_largest_watchtower_challenge_block( if let Some(block_height) = tx_status.block_height { if block_height > largest_watchtower_challenge_block_height { largest_watchtower_challenge_block_height = block_height; - largest_watchtower_challenge_block_hash = tx_status.block_hash.unwrap(); + largest_watchtower_challenge_block_hash = tx_status.block_hash.ok_or_else(|| { + anyhow!( + "Watchtower challenge tx {txid} for graph {}, index: {watchtower_index} is confirmed at height {block_height} without a block hash", + graph.parameters.graph_id + ) + })?; } } else { anyhow::bail!( From de9c7a86470459f2e304d92988e212f08d6f6ab0 Mon Sep 17 00:00:00 2001 From: ethan Date: Wed, 16 Sep 2026 20:38:37 +0800 Subject: [PATCH 06/17] fix: prevent batch abandon and preserve poison-message detection --- crates/node-macros/src/lib.rs | 14 +- crates/store/src/localdb.rs | 650 +++++++++++++++--- deployment/regtest/bitvm-noded/stop_nodes.sh | 12 +- deployment/testnet4/bitvm-noded/stop_nodes.sh | 12 +- node/src/action.rs | 329 +++++++-- node/src/main.rs | 70 +- node/src/rpc_service/bitvm.rs | 24 +- node/src/utils.rs | 23 +- 8 files changed, 947 insertions(+), 187 deletions(-) diff --git a/crates/node-macros/src/lib.rs b/crates/node-macros/src/lib.rs index 06854b5e..58162f73 100644 --- a/crates/node-macros/src/lib.rs +++ b/crates/node-macros/src/lib.rs @@ -35,7 +35,7 @@ fn expand_message_business_ref(input: DeriveInput) -> Result Result { - let scope = business_ref_scope(&variant.attrs)?; + let scope = business_ref_scope(variant)?; let variant_name = &variant.ident; match scope.as_str() { @@ -83,12 +83,16 @@ fn tuple_payload_binding(variant: &Variant) -> Result { } } -fn business_ref_scope(attributes: &[Attribute]) -> Result { - let mut matching = - attributes.iter().filter(|attribute| attribute.path().is_ident("business_ref")); +fn business_ref_scope(variant: &Variant) -> Result { + let mut matching = variant + .attrs + .iter() + .filter(|attribute: &&Attribute| attribute.path().is_ident("business_ref")); let Some(attribute) = matching.next() else { + // Point at the offending variant rather than the derive site, so the + // compiler error names the variant that lacks an attribute. return Err(syn::Error::new( - proc_macro2::Span::call_site(), + variant.ident.span(), "each message variant must declare #[business_ref(graph)], #[business_ref(instance)], or #[business_ref(unscoped)]", )); }; diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index 649b990c..f93ff7d6 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -2,11 +2,11 @@ use crate::utils::{QueryBuilder, QueryParam, create_place_holders}; use crate::{ BridgeOutGlobalStats, EventWatchMetricsSnapshot, GoatTxRecord, Graph, GraphBtcTxVoutMonitor, GraphRawData, GraphStatus, GraphStatusSource, GraphStatusTransitionOutcome, Instance, - LongRunningTaskProof, Message, MessageDebugOverview, MessageDebugReason, MessageState, - MetricsStateCount, Node, NodeAlertMetricsSnapshot, NodesOverview, OperatorProof, - P2pInboxMessage, P2pOutboxMessage, PeginGraphProcessData, PeginInstanceProcessData, - PendingGraphInit, SequencerSetHashChange, SequencerSetScanState, SerializableTxid, SwapEscrow, - SwapEscrowStatus, WatchContract, WatchtowerProof, + LongRunningTaskProof, Message, MessageDebugOverview, MessageDebugReason, MetricsStateCount, + Node, NodeAlertMetricsSnapshot, NodesOverview, OperatorProof, P2pInboxMessage, + P2pOutboxMessage, PeginGraphProcessData, PeginInstanceProcessData, PendingGraphInit, + SequencerSetHashChange, SequencerSetScanState, SerializableTxid, SwapEscrow, SwapEscrowStatus, + WatchContract, WatchtowerProof, }; use indexmap::IndexMap; @@ -28,6 +28,11 @@ const MESSAGE_COLUMNS: &str = "message_id, business_id, from_peer, actor, msg_ty message_version, state, weight, lock_time_until, attempt_count, abandon_count, last_error, \ created_at"; +/// Columns every `p2p_inbox` SELECT must fetch. +const P2P_INBOX_COLUMNS: &str = "message_id, business_id, actor, from_peer, msg_type, content, \ + content_size, state, attempt_count, abandon_count, next_retry_at, lease_until, lease_token, \ + last_error, created_at, updated_at"; + fn message_from_row(row: &SqliteRow) -> Result { Ok(Message { message_id: row.try_get("message_id")?, @@ -2370,18 +2375,16 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } - /// Claim a batch of local messages for dispatch. + /// Rows the local dispatcher may attempt right now, oldest first. /// - /// This replaces the previous select-only pop, which wrote nothing before - /// handing work to the dispatcher. Without a durable claim, a handler that - /// panicked left the row `Pending` with its lock untouched, so the very next - /// tick re-read it and panicked again — a crash loop with no backoff at all. - /// Charging the claim up front means the record survives an abort or a kill, - /// not just an unwinding panic. - pub async fn claim_local_messages( + /// Nothing is written here: the dispatcher claims each row with + /// [`Self::claim_local_message`] immediately before dispatching it. Claiming + /// a whole batch up front meant that an abort or kill mid-dispatch charged + /// an unfinished attempt to every row in the batch, so healthy messages + /// followed the poison message into quarantine. + pub async fn list_claimable_local_messages( &mut self, now: i64, - lease_until: i64, expired: i64, limit: i64, max_abandons: i64, @@ -2401,35 +2404,75 @@ impl<'a> StorageProcessor<'a> { .bind(limit) .fetch_all(self.conn()) .await?; + rows.iter().map(message_from_row).collect::, _>>().map_err(Into::into) + } - let mut claimed = Vec::with_capacity(rows.len()); - for row in rows { - let mut message = message_from_row(&row)?; - let was_abandoned = message.state == MessageState::Processing.to_string(); - // `message_version` is deliberately left alone: callers guard their - // completion writes with the version they were handed, and bumping - // it here would make every one of those writes miss. - let result = sqlx::query( - "UPDATE message \ - SET state = 'Processing', \ - abandon_count = abandon_count + ?, \ - lock_time_until = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? \ - AND state IN ('Pending', 'Processing') \ - AND lock_time_until <= ?", - ) - .bind(i64::from(was_abandoned)) - .bind(lease_until) - .bind(now) - .bind(&message.message_id) - .bind(message.message_version) - .bind(now) - .execute(self.conn()) - .await?; - if result.rows_affected() > 0 { - message.state = MessageState::Processing.to_string(); - message.abandon_count += i64::from(was_abandoned); - message.lock_time_until = lease_until; + /// Claim exactly one local message for dispatch. + /// + /// Without a durable claim, a handler that panicked left the row `Pending` + /// with its lock untouched, so the very next tick re-read it and panicked + /// again with no backoff at all. Charging the claim before dispatch means + /// the record survives an abort or a kill, not just an unwinding panic. + /// + /// Re-claiming a row that is still `Processing` means the previous attempt + /// never reported an outcome, which is charged as an abandon. + /// `message_version` is deliberately left alone: callers guard their + /// completion writes with the version they were handed, and bumping it here + /// would make every one of those writes miss. Returns `None` when the row + /// is no longer claimable: cancelled, re-armed under a new version, or + /// locked since it was listed. + pub async fn claim_local_message( + &mut self, + message_id: &str, + message_version: i64, + now: i64, + lease_until: i64, + ) -> anyhow::Result> { + let row = sqlx::query(&format!( + "UPDATE message \ + SET state = 'Processing', \ + abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ + lock_time_until = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? \ + AND state IN ('Pending', 'Processing') \ + AND lock_time_until <= ? \ + RETURNING {MESSAGE_COLUMNS}" + )) + .bind(lease_until) + .bind(now) + .bind(message_id) + .bind(message_version) + .bind(now) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| message_from_row(&row)).transpose()?) + } + + /// List and claim up to `limit` rows in one call. + /// + /// Production dispatchers claim one row at a time; this convenience exists + /// for tests and tooling that need a whole batch held under a lease. + pub async fn claim_local_messages( + &mut self, + now: i64, + lease_until: i64, + expired: i64, + limit: i64, + max_abandons: i64, + ) -> anyhow::Result> { + let candidates = + self.list_claimable_local_messages(now, expired, limit, max_abandons).await?; + let mut claimed = Vec::with_capacity(candidates.len()); + for candidate in candidates { + if let Some(message) = self + .claim_local_message( + &candidate.message_id, + candidate.message_version, + now, + lease_until, + ) + .await? + { claimed.push(message); } } @@ -2439,7 +2482,9 @@ impl<'a> StorageProcessor<'a> { /// Record a failed dispatch attempt and reschedule it with backoff. /// /// `attempt_count` is observability only. Only an unfinished claim increments - /// `abandon_count` and contributes to quarantine. + /// `abandon_count` and contributes to quarantine. This is for a handler + /// error reported to the dispatcher, which is a completed attempt; a handler + /// rescheduling its own row uses [`Self::self_defer_local_message`]. pub async fn defer_local_message( &mut self, message_id: &str, @@ -2463,22 +2508,89 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Reschedule the row a handler is currently running, at that handler's + /// own request. + /// + /// Unlike [`Self::defer_local_message`], the consecutive-abandon counter is + /// left untouched: the handler is still running and may yet panic, and + /// resetting here let a handler that reschedules itself and then panics + /// start every round from zero, so it never reached quarantine. The + /// dispatcher resets the counter with + /// [`Self::confirm_local_message_self_defer`] once the handler returned. + pub async fn self_defer_local_message( + &mut self, + message_id: &str, + message_version: i64, + lock_time_until: i64, + reason: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Pending', attempt_count = attempt_count + 1, \ + lock_time_until = ?, last_error = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state = 'Processing'", + ) + .bind(lock_time_until) + .bind(reason.chars().take(1024).collect::()) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .bind(message_version) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + /// Acknowledge a self-deferred row once its handler has returned. + /// + /// Only at this point is the attempt known to have reported an outcome, so + /// only here does the consecutive-abandon counter reset. Returns `false` + /// when the row is not `Pending` under this claim version, which means it + /// was not self-deferred by the caller. + pub async fn confirm_local_message_self_defer( + &mut self, + message_id: &str, + message_version: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message SET abandon_count = 0, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state = 'Pending'", + ) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .bind(message_version) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + /// Record a handler panic before shutting down the process. Unlike a normal - /// defer, this increments the consecutive unfinished-attempt counter. + /// defer, this increments the consecutive unfinished-attempt counter and + /// pushes the next attempt out by `backoff_secs` per recorded abandon, so a + /// supervisor restart cannot replay the payload at full speed. + /// + /// A handler may reschedule its own row and then panic, leaving the row + /// `Pending` already. The version guard still identifies the claim, so the + /// abandon is charged either way and the longer of the two delays wins. pub async fn abandon_local_message( &mut self, message_id: &str, message_version: i64, + now: i64, + backoff_secs: i64, error: &str, ) -> anyhow::Result { let result = sqlx::query( "UPDATE message \ SET state = 'Pending', abandon_count = abandon_count + 1, \ - lock_time_until = 0, last_error = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state = 'Processing'", + lock_time_until = MAX(lock_time_until, ? + ? * (abandon_count + 1)), \ + last_error = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? \ + AND state IN ('Processing', 'Pending')", ) + .bind(now) + .bind(backoff_secs) .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) + .bind(now) .bind(message_id) .bind(message_version) .execute(self.conn()) @@ -2522,6 +2634,34 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } + /// Startup sweep for claims left behind by a process that no longer exists. + /// + /// The database is process-local, so at startup every `Processing` row is + /// an attempt that never reported an outcome. Charging it now, rather than + /// when the lease lapses, keeps the row from sitting locked for the whole + /// lease while every producer that touches it backs off with + /// ResourceLocked. The same per-abandon backoff as a panic applies. + pub async fn reclaim_processing_local_messages( + &mut self, + now: i64, + backoff_secs: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE message \ + SET state = 'Pending', abandon_count = abandon_count + 1, \ + lock_time_until = ? + ? * (abandon_count + 1), \ + last_error = 'reclaimed at startup: previous process exited mid-dispatch', \ + updated_at = ? \ + WHERE state = 'Processing'", + ) + .bind(now) + .bind(backoff_secs) + .bind(now) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + pub async fn get_message_queue_stats( &mut self, actor: &str, @@ -2767,10 +2907,12 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } - pub async fn claim_p2p_inbox_messages( + /// Inbox rows the dispatcher may attempt right now, oldest first. Nothing + /// is written; see [`Self::list_claimable_local_messages`] for why rows are + /// claimed one at a time instead of as a batch. + pub async fn list_claimable_p2p_inbox_messages( &mut self, now: i64, - lease_until: i64, limit: i64, max_abandons: i64, excluded_message_ids: &[String], @@ -2781,8 +2923,7 @@ impl<'a> StorageProcessor<'a> { format!(" AND message_id NOT IN ({})", create_place_holders(excluded_message_ids)) }; let query = format!( - "SELECT message_id, business_id, actor, from_peer, msg_type, content, content_size, \ - state, attempt_count, abandon_count, next_retry_at, lease_until, lease_token, last_error, created_at, updated_at \ + "SELECT {P2P_INBOX_COLUMNS} \ FROM p2p_inbox \ WHERE ((state = 'Pending' AND next_retry_at <= ?) \ OR (state = 'Processing' AND lease_until <= ?)) \ @@ -2795,42 +2936,65 @@ impl<'a> StorageProcessor<'a> { query = query.bind(message_id); } let rows = query.bind(limit).fetch_all(self.conn()).await?; + rows.iter() + .map(p2p_inbox_message_from_row) + .collect::, _>>() + .map_err(Into::into) + } - let mut claimed = Vec::with_capacity(rows.len()); - for row in rows { - let mut message = p2p_inbox_message_from_row(&row)?; - // Re-claiming a row that is still `Processing` means the previous - // attempt never reported an outcome: the worker panicked, the - // process died, or it hung past the lease. That is charged - // separately from an ordinary handler error so that a transient - // outage cannot push healthy messages toward quarantine. - let was_abandoned = message.state == "Processing"; - let lease_token = Uuid::new_v4().to_string(); - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Processing', attempt_count = attempt_count + 1, \ - abandon_count = abandon_count + ?, \ - lease_until = ?, lease_token = ?, updated_at = ? \ - WHERE message_id = ? \ - AND ((state = 'Pending' AND next_retry_at <= ?) \ - OR (state = 'Processing' AND lease_until <= ?))", - ) - .bind(i64::from(was_abandoned)) - .bind(lease_until) - .bind(&lease_token) - .bind(now) - .bind(&message.message_id) - .bind(now) - .bind(now) - .execute(self.conn()) + /// Claim exactly one inbox row under a fresh lease token. + /// + /// Re-claiming a row that is still `Processing` means the previous attempt + /// never reported an outcome: the worker panicked, the process died, or it + /// hung past the lease. That is charged separately from an ordinary handler + /// error so that a transient outage cannot push healthy messages toward + /// quarantine. Returns `None` when the row is no longer claimable. + pub async fn claim_p2p_inbox_message( + &mut self, + message_id: &str, + now: i64, + lease_until: i64, + ) -> anyhow::Result> { + let lease_token = Uuid::new_v4().to_string(); + let row = sqlx::query(&format!( + "UPDATE p2p_inbox \ + SET state = 'Processing', attempt_count = attempt_count + 1, \ + abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ + lease_until = ?, lease_token = ?, updated_at = ? \ + WHERE message_id = ? \ + AND ((state = 'Pending' AND next_retry_at <= ?) \ + OR (state = 'Processing' AND lease_until <= ?)) \ + RETURNING {P2P_INBOX_COLUMNS}" + )) + .bind(lease_until) + .bind(&lease_token) + .bind(now) + .bind(message_id) + .bind(now) + .bind(now) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| p2p_inbox_message_from_row(&row)).transpose()?) + } + + /// List and claim up to `limit` inbox rows in one call. Production + /// dispatchers claim one row at a time; this is for tests and tooling. + pub async fn claim_p2p_inbox_messages( + &mut self, + now: i64, + lease_until: i64, + limit: i64, + max_abandons: i64, + excluded_message_ids: &[String], + ) -> anyhow::Result> { + let candidates = self + .list_claimable_p2p_inbox_messages(now, limit, max_abandons, excluded_message_ids) .await?; - if result.rows_affected() > 0 { - message.state = "Processing".to_owned(); - message.attempt_count += 1; - message.abandon_count += i64::from(was_abandoned); - message.lease_until = lease_until; - message.lease_token = lease_token; - message.updated_at = now; + let mut claimed = Vec::with_capacity(candidates.len()); + for candidate in candidates { + if let Some(message) = + self.claim_p2p_inbox_message(&candidate.message_id, now, lease_until).await? + { claimed.push(message); } } @@ -2880,20 +3044,27 @@ impl<'a> StorageProcessor<'a> { } /// Record a panic from the current lease before terminating the process. + /// The next attempt is pushed out by `backoff_secs` per recorded abandon so + /// a supervisor restart cannot replay the payload at full speed. pub async fn abandon_p2p_inbox_message( &mut self, message_id: &str, lease_token: &str, + now: i64, + backoff_secs: i64, error: &str, ) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_inbox \ SET state = 'Pending', abandon_count = abandon_count + 1, lease_until = 0, \ - lease_token = '', next_retry_at = 0, last_error = ?, updated_at = ? \ + lease_token = '', next_retry_at = ? + ? * (abandon_count + 1), \ + last_error = ?, updated_at = ? \ WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", ) + .bind(now) + .bind(backoff_secs) .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) + .bind(now) .bind(message_id) .bind(lease_token) .execute(self.conn()) @@ -3000,6 +3171,29 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } + /// Startup sweep for inbox claims left behind by a process that no longer + /// exists. See [`Self::reclaim_processing_local_messages`]. + pub async fn reclaim_processing_p2p_inbox_messages( + &mut self, + now: i64, + backoff_secs: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE p2p_inbox \ + SET state = 'Pending', abandon_count = abandon_count + 1, lease_until = 0, \ + lease_token = '', next_retry_at = ? + ? * (abandon_count + 1), \ + last_error = 'reclaimed at startup: previous process exited mid-dispatch', \ + updated_at = ? \ + WHERE state = 'Processing'", + ) + .bind(now) + .bind(backoff_secs) + .bind(now) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + pub async fn requeue_p2p_inbox_message(&mut self, message_id: &str) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_inbox \ @@ -4401,6 +4595,7 @@ pub async fn create_local_db(db_path: &str) -> LocalDB { #[cfg(test)] mod tests { use super::*; + use crate::MessageState; async fn setup_db() -> LocalDB { create_local_db("sqlite::memory:").await @@ -4723,9 +4918,14 @@ mod tests { .unwrap(); let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); assert!( - s.defer_local_message("self-defer-1", claimed[0].message_version, 150, "not ready") - .await - .unwrap() + s.self_defer_local_message( + "self-defer-1", + claimed[0].message_version, + 150, + "not ready" + ) + .await + .unwrap() ); assert!( !s.complete_local_message("self-defer-1", claimed[0].message_version).await.unwrap() @@ -5059,6 +5259,276 @@ mod tests { assert_eq!(row.get::, _>("content"), vec![1, 2]); } + /// Rows are claimed one at a time immediately before dispatch, so an abort + /// mid-dispatch charges only the row that was actually running. + #[tokio::test] + async fn local_claims_are_taken_per_message_not_per_batch() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + for (message_id, created_at) in [("first", 10), ("second", 20)] { + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES (?, ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, ?, ?)", + ) + .bind(message_id) + .bind(business_id) + .bind(created_at) + .bind(created_at) + .execute(s.conn()) + .await + .unwrap(); + } + + let candidates = s.list_claimable_local_messages(100, 0, 10, 3).await.unwrap(); + assert_eq!( + candidates.iter().map(|message| message.message_id.as_str()).collect::>(), + ["first", "second"] + ); + assert!( + candidates.iter().all(|message| message.state == "Pending"), + "listing must not write" + ); + + let claimed = s + .claim_local_message("first", candidates[0].message_version, 100, 200) + .await + .unwrap() + .expect("first claim"); + assert_eq!(claimed.state, "Processing"); + assert_eq!(claimed.lock_time_until, 200); + assert_eq!(s.find_messages_by_id("second").await.unwrap().unwrap().state, "Pending"); + + // Inside the lease the same row is not claimable again. + assert!( + s.claim_local_message("first", claimed.message_version, 150, 250) + .await + .unwrap() + .is_none() + ); + // A row re-armed under a new version since it was listed is skipped too. + assert!( + s.claim_local_message("second", candidates[1].message_version + 1, 100, 200) + .await + .unwrap() + .is_none() + ); + // Once the lease lapses, the reclaim charges the unfinished attempt. + let reclaimed = s + .claim_local_message("first", claimed.message_version, 300, 400) + .await + .unwrap() + .expect("reclaim"); + assert_eq!(reclaimed.abandon_count, 1); + } + + #[tokio::test] + async fn inbox_claims_are_taken_per_message_not_per_batch() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for message_id in ["inbox-first", "inbox-second"] { + let message = P2pInboxMessage { + message_id: message_id.to_owned(), + actor: "Operator".to_owned(), + from_peer: "peer".to_owned(), + msg_type: "CreateGraph".to_owned(), + content: vec![1], + content_size: 1, + ..Default::default() + }; + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + } + let candidates = s.list_claimable_p2p_inbox_messages(100, 10, 3, &[]).await.unwrap(); + assert_eq!(candidates.len(), 2); + assert!( + candidates + .iter() + .all(|message| message.state == "Pending" && message.lease_token.is_empty()), + "listing must not write" + ); + + let claimed = + s.claim_p2p_inbox_message("inbox-first", 100, 200).await.unwrap().expect("claim"); + assert_eq!(claimed.state, "Processing"); + assert_eq!(claimed.attempt_count, 1); + assert_eq!(claimed.lease_until, 200); + assert!(!claimed.lease_token.is_empty()); + assert!(s.claim_p2p_inbox_message("inbox-first", 150, 250).await.unwrap().is_none()); + let second = sqlx::query("SELECT state FROM p2p_inbox WHERE message_id = 'inbox-second'") + .fetch_one(s.conn()) + .await + .unwrap(); + assert_eq!(second.get::("state"), "Pending"); + } + + /// An unclean exit leaves claims behind. At startup they are provably + /// abandoned, so they are charged and released immediately with a backoff + /// instead of sitting locked until the lease lapses. + #[tokio::test] + async fn startup_reclaim_charges_abandon_and_applies_backoff() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES ('startup-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 10, 10)", + ) + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + let inbox = P2pInboxMessage { + message_id: "startup-inbox".to_owned(), + actor: "Operator".to_owned(), + from_peer: "peer".to_owned(), + msg_type: "CreateGraph".to_owned(), + content: vec![1], + content_size: 1, + ..Default::default() + }; + assert!(s.insert_p2p_inbox_message(&inbox).await.unwrap()); + assert_eq!(s.claim_local_messages(100, 700, 0, 10, 3).await.unwrap().len(), 1); + assert_eq!(s.claim_p2p_inbox_messages(100, 400, 10, 3, &[]).await.unwrap().len(), 1); + + assert_eq!(s.reclaim_processing_local_messages(1000, 60).await.unwrap(), 1); + assert_eq!(s.reclaim_processing_p2p_inbox_messages(1000, 60).await.unwrap(), 1); + + let local = s.find_messages_by_id("startup-local").await.unwrap().unwrap(); + assert_eq!(local.state, "Pending"); + assert_eq!(local.abandon_count, 1); + assert_eq!(local.lock_time_until, 1060, "the first abandon backs off by one interval"); + let inbox_row = sqlx::query( + "SELECT state, abandon_count, next_retry_at, lease_token FROM p2p_inbox WHERE message_id = 'startup-inbox'", + ) + .fetch_one(s.conn()) + .await + .unwrap(); + assert_eq!(inbox_row.get::("state"), "Pending"); + assert_eq!(inbox_row.get::("abandon_count"), 1); + assert_eq!(inbox_row.get::("next_retry_at"), 1060); + assert!(inbox_row.get::("lease_token").is_empty()); + + // Nothing is claimable until the backoff has passed. + assert!(s.list_claimable_local_messages(1030, 0, 10, 3).await.unwrap().is_empty()); + assert_eq!(s.list_claimable_local_messages(1060, 0, 10, 3).await.unwrap().len(), 1); + assert!(s.list_claimable_p2p_inbox_messages(1030, 10, 3, &[]).await.unwrap().is_empty()); + assert_eq!(s.list_claimable_p2p_inbox_messages(1060, 10, 3, &[]).await.unwrap().len(), 1); + } + + /// A handler that reschedules its own row and then panics must accumulate + /// abandons across restarts. The self-defer must not reset the counter, + /// because the handler is still running when it happens; only the + /// dispatcher's confirmation after a normal return may reset it. + #[tokio::test] + async fn panic_after_self_defer_accumulates_abandons_until_quarantine() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ + VALUES ('panic-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 10, 10)", + ) + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + + let mut now = 100; + for round in 1..=3 { + // The row is Pending, so the restart's reclaim sweep leaves it alone + // and the next tick claims it without a charge. + let candidate = s.find_messages_by_id("panic-local").await.unwrap().unwrap(); + let claimed = s + .claim_local_message("panic-local", candidate.message_version, now, now + 600) + .await + .unwrap() + .expect("claim"); + assert_eq!(claimed.abandon_count, round - 1); + // The handler reschedules itself, then panics. + assert!( + s.self_defer_local_message( + "panic-local", + claimed.message_version, + now + 5, + "not ready" + ) + .await + .unwrap() + ); + assert!( + s.abandon_local_message( + "panic-local", + claimed.message_version, + now, + 60, + "panicked" + ) + .await + .unwrap() + ); + let row = s.find_messages_by_id("panic-local").await.unwrap().unwrap(); + assert_eq!(row.state, "Pending"); + assert_eq!(row.abandon_count, round, "round {round} adds to the preserved count"); + assert_eq!(row.lock_time_until, now + 60 * round, "backoff grows with the count"); + now = row.lock_time_until + 1; + } + + // The sweep on the next tick retires it instead of dispatching again. + assert_eq!(s.quarantine_local_messages(now, 3).await.unwrap(), 1); + assert_eq!( + s.find_messages_by_id("panic-local").await.unwrap().unwrap().state, + "Quarantined" + ); + } + + /// A self-defer is only a reported outcome once the handler has returned, + /// so the counter survives the self-defer and resets on confirmation. + #[tokio::test] + async fn self_defer_keeps_abandons_until_the_dispatcher_confirms_it() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let business_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, abandon_count, created_at, updated_at) \ + VALUES ('confirm-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 2, 10, 10)", + ) + .bind(business_id) + .execute(s.conn()) + .await + .unwrap(); + let candidate = s.find_messages_by_id("confirm-local").await.unwrap().unwrap(); + let claimed = s + .claim_local_message("confirm-local", candidate.message_version, 100, 700) + .await + .unwrap() + .expect("claim"); + assert_eq!(claimed.abandon_count, 2, "a claim from Pending is not charged"); + + assert!( + s.self_defer_local_message("confirm-local", claimed.message_version, 150, "not ready") + .await + .unwrap() + ); + let row = s.find_messages_by_id("confirm-local").await.unwrap().unwrap(); + assert_eq!(row.state, "Pending"); + assert_eq!(row.attempt_count, 1); + assert_eq!(row.abandon_count, 2, "the self-defer must not reset the counter"); + + // The dispatcher confirms once the handler returned normally. + assert!( + s.confirm_local_message_self_defer("confirm-local", claimed.message_version) + .await + .unwrap() + ); + assert_eq!(s.find_messages_by_id("confirm-local").await.unwrap().unwrap().abandon_count, 0); + // A different claim generation, or a row that is not Pending, is not confirmed. + assert!( + !s.confirm_local_message_self_defer("confirm-local", claimed.message_version + 1) + .await + .unwrap() + ); + } + #[tokio::test] async fn test_message_debug_reasons_are_deduplicated() { let db = setup_db().await; diff --git a/deployment/regtest/bitvm-noded/stop_nodes.sh b/deployment/regtest/bitvm-noded/stop_nodes.sh index 48bc8e01..59225d6b 100644 --- a/deployment/regtest/bitvm-noded/stop_nodes.sh +++ b/deployment/regtest/bitvm-noded/stop_nodes.sh @@ -1 +1,11 @@ -killall -TERM bitvm-noded +#!/bin/sh +# SIGTERM lets the node release its queue claims and finish in-flight writes. +# Wait for the processes to exit before returning, so a start_nodes.sh that +# follows does not race a node that is still shutting down. +killall -TERM bitvm-noded 2>/dev/null +for _ in $(seq 1 60); do + pgrep -x bitvm-noded >/dev/null 2>&1 || exit 0 + sleep 1 +done +echo "bitvm-noded did not exit within 60s; sending SIGKILL" >&2 +killall -KILL bitvm-noded 2>/dev/null diff --git a/deployment/testnet4/bitvm-noded/stop_nodes.sh b/deployment/testnet4/bitvm-noded/stop_nodes.sh index 48bc8e01..59225d6b 100644 --- a/deployment/testnet4/bitvm-noded/stop_nodes.sh +++ b/deployment/testnet4/bitvm-noded/stop_nodes.sh @@ -1 +1,11 @@ -killall -TERM bitvm-noded +#!/bin/sh +# SIGTERM lets the node release its queue claims and finish in-flight writes. +# Wait for the processes to exit before returning, so a start_nodes.sh that +# follows does not race a node that is still shutting down. +killall -TERM bitvm-noded 2>/dev/null +for _ in $(seq 1 60); do + pgrep -x bitvm-noded >/dev/null 2>&1 || exit 0 + sleep 1 +done +echo "bitvm-noded did not exit within 60s; sending SIGKILL" >&2 +killall -KILL bitvm-noded 2>/dev/null diff --git a/node/src/action.rs b/node/src/action.rs index 506748e6..290501aa 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -70,6 +70,10 @@ const LOCAL_MESSAGE_LEASE_SECS: i64 = 10 * 60; /// Backoff applied to a local message whose handler returned a non-transient error. const LOCAL_MESSAGE_RETRY_DELAY_SECS: i64 = 600; const LOCAL_MESSAGE_BATCH_SIZE: i64 = 50; +/// Delay applied per recorded abandon before a payload may run again, so a +/// supervisor restart after a panic or an unclean exit does not replay it at +/// full speed. +const QUEUE_ABANDON_BACKOFF_SECS: i64 = 60; /// A dispatch future erased behind a box to keep the enclosing task's state /// machine reasonably small. type BoxedDispatch<'a> = std::pin::Pin> + 'a>>; @@ -384,7 +388,7 @@ impl BusinessRef { fn key_part(self) -> String { match self { Self::Instance { instance_id } => format!("instance:{instance_id}"), - Self::Graph { instance_id, graph_id } => format!("graph:{instance_id}:{graph_id}"), + Self::Graph { graph_id, .. } => format!("graph:{graph_id}"), Self::Unscoped => "unscoped".to_owned(), } } @@ -1293,7 +1297,15 @@ async fn abandon_p2p_inbox_after_panic( let error = format!("handler panicked: {detail}"); let result = async { let mut storage = local_db.acquire().await?; - storage.abandon_p2p_inbox_message(message_id, lease_token, &error).await + storage + .abandon_p2p_inbox_message( + message_id, + lease_token, + current_time_secs(), + QUEUE_ABANDON_BACKOFF_SECS, + &error, + ) + .await } .await; match result { @@ -1312,7 +1324,15 @@ async fn abandon_local_message_after_panic( let error = format!("handler panicked: {detail}"); let result = async { let mut storage = local_db.acquire().await?; - storage.abandon_local_message(message_id, message_version, &error).await + storage + .abandon_local_message( + message_id, + message_version, + current_time_secs(), + QUEUE_ABANDON_BACKOFF_SECS, + &error, + ) + .await } .await; match result { @@ -1331,6 +1351,105 @@ async fn abandon_local_message_after_panic( } } +/// Claim one listed inbox row immediately before dispatching it. +/// +/// `None` means the row is no longer claimable or the claim could not be +/// persisted; either way it is skipped this tick and listed again on the next. +async fn claim_p2p_inbox_candidate( + local_db: &LocalDB, + message_id: &str, +) -> Option { + let now = current_time_secs(); + let result = async { + let mut storage = local_db.acquire().await?; + storage.claim_p2p_inbox_message(message_id, now, now + P2P_INBOX_LEASE_SECS).await + } + .await; + match result { + Ok(Some(message)) => Some(message), + Ok(None) => { + tracing::debug!( + event = "p2p_inbox", + outcome = "claim_skipped", + message_id, + "listed inbox message is no longer claimable" + ); + None + } + Err(error) => { + tracing::error!( + event = "p2p_inbox", + outcome = "claim_failed", + message_id, + error = %error, + "failed to claim a listed inbox message; it stays queued for the next tick" + ); + None + } + } +} + +/// Claim one listed local message immediately before dispatching it. See +/// [`claim_p2p_inbox_candidate`]. +async fn claim_local_candidate( + local_db: &LocalDB, + candidate: &store::Message, +) -> Option { + let now = current_time_secs(); + let result = async { + let mut storage = local_db.acquire().await?; + storage + .claim_local_message( + &candidate.message_id, + candidate.message_version, + now, + now + LOCAL_MESSAGE_LEASE_SECS, + ) + .await + } + .await; + match result { + Ok(Some(message)) => Some(message), + Ok(None) => { + tracing::debug!( + event = "local_message_queue", + outcome = "claim_skipped", + queued_message_id = %candidate.message_id, + message_version = candidate.message_version, + "listed local message is no longer claimable" + ); + None + } + Err(error) => { + tracing::error!( + event = "local_message_queue", + outcome = "claim_failed", + queued_message_id = %candidate.message_id, + error = %error, + "failed to claim a listed local message; it stays queued for the next tick" + ); + None + } + } +} + +/// Charge and release every queue claim left behind by a previous process. +/// +/// Run once at startup, before any dispatcher starts. The database is +/// process-local, so a `Processing` row at this point is an attempt that never +/// reported an outcome. Waiting for its lease to lapse instead kept the row +/// locked for minutes while every producer touching it backed off with +/// ResourceLocked; a graceful shutdown never leaves such rows behind. +pub async fn reclaim_stale_queue_claims(local_db: &LocalDB) -> Result<(u64, u64)> { + let now = current_time_secs(); + let mut storage = local_db.start_immediate_transaction().await?; + let local = storage.reclaim_processing_local_messages(now, QUEUE_ABANDON_BACKOFF_SECS).await?; + let inbox = + storage.reclaim_processing_p2p_inbox_messages(now, QUEUE_ABANDON_BACKOFF_SECS).await?; + storage.commit().await?; + Ok((local, inbox)) +} + async fn renew_p2p_inbox_lease_until_cancelled( local_db: LocalDB, message_id: String, @@ -1396,10 +1515,11 @@ async fn handle_p2p_inbox_messages( // Bound terminal metadata and the temporary payload retained for manual // inspection of quarantined rows. let purged = storage.purge_terminal_p2p_inbox_messages(now - MESSAGE_EXPIRE_TIME).await?; - let messages = storage - .claim_p2p_inbox_messages( + // Only list here. Each row is claimed right before its own dispatch so a + // crash mid-dispatch is charged to that row alone. + let candidates = storage + .list_claimable_p2p_inbox_messages( now, - now + P2P_INBOX_LEASE_SECS, get_p2p_inbox_batch_size(), QUEUE_MAX_ABANDONS, &active_heavy_task_ids, @@ -1425,7 +1545,10 @@ async fn handle_p2p_inbox_messages( ); } - for message in messages { + for candidate in candidates { + let Some(message) = claim_p2p_inbox_candidate(local_db, &candidate.message_id).await else { + continue; + }; let from_peer_id = match PeerId::from_str(&message.from_peer) { Ok(peer_id) => peer_id, Err(error) => { @@ -1856,13 +1979,8 @@ pub async fn handle_self_p2p_msg( "received local queue trigger" ); - let (messages, quarantined) = claim_batch_local_msg( - local_db, - LOCAL_MESSAGE_LEASE_SECS, - QUEUE_MAX_ABANDONS, - LOCAL_MESSAGE_BATCH_SIZE, - ) - .await?; + let (candidates, quarantined) = + list_batch_local_msg(local_db, QUEUE_MAX_ABANDONS, LOCAL_MESSAGE_BATCH_SIZE).await?; if quarantined > 0 { tracing::warn!( event = "local_message_queue", @@ -1874,12 +1992,18 @@ pub async fn handle_self_p2p_msg( } tracing::info!( event = "local_message_queue", - outcome = "batch_loaded", + outcome = "batch_listed", role = %actor, - batch_size = messages.len(), - "claimed pending local messages" + batch_size = candidates.len(), + "listed claimable local messages" ); - for message in messages { + for candidate in candidates { + // Claim right before dispatch so a crash mid-dispatch is charged to + // this row alone, and so a producer that re-armed the row since it was + // listed wins: the stale version is skipped until the next tick. + let Some(message) = claim_local_candidate(local_db, &candidate).await else { + continue; + }; let queue_wait_secs = current_time_secs().saturating_sub(message.created_at); let started_at = Instant::now(); let claim = LocalMessageClaim { @@ -1968,37 +2092,56 @@ pub async fn handle_self_p2p_msg( "processed local message" ); } else { - let current_state = storage_processor - .find_messages_by_id(&message.message_id) + // A row that is Pending under our claim version was + // rescheduled by the handler itself. Only now, after the + // handler returned, is that a reported outcome, so only now + // does its consecutive-abandon counter reset. + match storage_processor + .confirm_local_message_self_defer( + &message.message_id, + message.message_version, + ) .await - .ok() - .flatten() - .map(|message| message.state); - if current_state.as_deref() == Some("Pending") { - tracing::debug!( - event = "local_message_queue", - outcome = "self_deferred", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "local message handler rescheduled its own queue entry" - ); - } else { - tracing::warn!( - event = "local_message_queue", - outcome = "state_update_conflict", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - current_state = ?current_state, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "local message handler completed but its processed state was not persisted" - ); + { + Ok(true) => { + tracing::debug!( + event = "local_message_queue", + outcome = "self_deferred", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "local message handler rescheduled its own queue entry" + ); + } + Ok(false) => { + let current_state = storage_processor + .find_messages_by_id(&message.message_id) + .await + .ok() + .flatten() + .map(|message| message.state); + tracing::warn!( + event = "local_message_queue", + outcome = "state_update_conflict", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + current_state = ?current_state, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "local message handler completed but its processed state was not persisted" + ); + } + Err(error) => log_queue_bookkeeping_failure( + "local_message_queue", + &message.message_id, + "confirm_self_defer", + &error, + ), } } } @@ -2054,14 +2197,37 @@ pub async fn handle_self_p2p_msg( match deferred { Ok(true) => {} Ok(false) => { - tracing::warn!( - event = "local_message_queue", - outcome = "stale_claim", - queued_message_id = %message.message_id, - message_version = message.message_version, - operation = "defer", - "ignored local message update from a stale claim" - ); + // The handler may have rescheduled its own row before + // returning the error; that is still a reported outcome. + match storage_processor + .confirm_local_message_self_defer( + &message.message_id, + message.message_version, + ) + .await + { + Ok(true) => tracing::debug!( + event = "local_message_queue", + outcome = "self_deferred", + queued_message_id = %message.message_id, + error = %err, + "local message handler rescheduled its own queue entry before failing" + ), + Ok(false) => tracing::warn!( + event = "local_message_queue", + outcome = "stale_claim", + queued_message_id = %message.message_id, + message_version = message.message_version, + operation = "defer", + "ignored local message update from a stale claim" + ), + Err(error) => log_queue_bookkeeping_failure( + "local_message_queue", + &message.message_id, + "confirm_self_defer", + &error, + ), + } continue; } Err(error) => { @@ -2380,7 +2546,7 @@ pub async fn push_local_unhandled_messages_with_reason( && owns_requeued_message { storage_processor - .defer_local_message( + .self_defer_local_message( &existing.message_id, existing.message_version, current_time_secs() + delay_secs as i64, @@ -2688,6 +2854,42 @@ mod tests { }; assert_eq!(claimed.message_id, message_id); + // Two earlier attempts died mid-dispatch; the row carries their abandons + // into this claim, and the handler's own reschedule must not erase them. + { + let mut storage = local_db.acquire().await.unwrap(); + for _ in 0..2 { + assert!( + storage + .abandon_local_message( + &message_id, + claimed.message_version, + current_time_secs(), + 0, + "died mid-dispatch", + ) + .await + .unwrap() + ); + } + } + let claimed = { + let mut storage = local_db.acquire().await.unwrap(); + storage + .claim_local_messages( + current_time_secs() + 301, + current_time_secs() + 600, + 0, + 1, + QUEUE_MAX_ABANDONS, + ) + .await + .unwrap() + .pop() + .unwrap() + }; + assert_eq!(claimed.abandon_count, 2); + ACTIVE_LOCAL_MESSAGE_CLAIM .scope( LocalMessageClaim { @@ -2708,5 +2910,18 @@ mod tests { let mut storage = local_db.acquire().await.unwrap(); let stored = storage.find_messages_by_id(&message_id).await.unwrap().unwrap(); assert_eq!(stored.state, MessageState::Pending.to_string()); + assert_eq!(stored.abandon_count, 2, "a self-defer must keep the abandon count"); + + // The dispatcher confirms the self-defer once the handler has returned. + assert!( + storage + .confirm_local_message_self_defer(&message_id, claimed.message_version) + .await + .unwrap() + ); + assert_eq!( + storage.find_messages_by_id(&message_id).await.unwrap().unwrap().abandon_count, + 0 + ); } } diff --git a/node/src/main.rs b/node/src/main.rs index b52afce6..d56312e6 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -13,7 +13,7 @@ use libp2p::PeerId; use libp2p_metrics::Registry; use std::error::Error; use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::time::{Duration, Instant}; use tracing::Instrument; use tracing_subscriber::EnvFilter; @@ -25,6 +25,7 @@ use bitvm_noded::{ }; use anyhow::Result; +use bitvm_noded::action::reclaim_stale_queue_claims; use bitvm_noded::metrics_service::{MetricsState, set_node_metrics_state}; use bitvm_noded::middleware::swarm::{BitvmNetworkManager, BitvmSwarmConfig}; use bitvm_noded::p2p_msg_handler::BitvmNodeProcessor; @@ -196,6 +197,18 @@ async fn main() -> Result<(), Box> { ); let _node_span_guard = node_span.enter(); let local_db = store::create_local_db(&opt.db_path).await; + // Claims left by a previous process are provably abandoned: charge and + // release them before any dispatcher can wait on their leases. + let (reclaimed_local, reclaimed_inbox) = reclaim_stale_queue_claims(&local_db).await?; + if reclaimed_local + reclaimed_inbox > 0 { + tracing::warn!( + event = "message_queue_startup", + outcome = "claims_reclaimed", + reclaimed_local, + reclaimed_inbox, + "reclaimed queue claims left behind by a previous process" + ); + } let metric_registry = Arc::new(Mutex::new(metric_registry)); let metrics_state = MetricsState::new(metric_registry); set_node_metrics_state(metrics_state.clone()); @@ -514,8 +527,9 @@ async fn main() -> Result<(), Box> { "all node background tasks have been started" ); + let mut core_tasks = future::select_all(task_handles); let fatal_error = tokio::select! { - (result, index, remaining_handles) = future::select_all(task_handles) => { + (result, index, remaining_handles) = &mut core_tasks => { let task_name = task_names[index]; // Log the specific failure let failure_reason = match &result { @@ -564,14 +578,10 @@ async fn main() -> Result<(), Box> { "triggering node shutdown after background task result" ); - // Initiate graceful shutdown + // Initiate graceful shutdown and let the tasks release their queue + // claims; anything still running after the grace period is aborted. cancellation_token.cancel(); - - // Wait a moment for graceful shutdown, then force abort remaining tasks - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - - // Force abort any tasks that didn't respond to cancellation - remaining_handles.into_iter().for_each(|handle| handle.abort()); + wait_for_task_shutdown(remaining_handles).await; tracing::info!( event = "service_shutdown", @@ -595,9 +605,7 @@ async fn main() -> Result<(), Box> { "received shutdown signal; initiating graceful shutdown" ); cancellation_token.cancel(); - - // Give tasks some time to shutdown gracefully - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + wait_for_task_shutdown(core_tasks.into_inner()).await; tracing::info!( event = "service_shutdown", service = "bitvm-noded", @@ -615,6 +623,44 @@ async fn main() -> Result<(), Box> { Ok(()) } +/// Upper bound on how long shutdown waits for the background tasks to exit on +/// their own before aborting them. +const SHUTDOWN_GRACE_SECS: u64 = 30; + +/// Wait for the background tasks to exit after cancellation. +/// +/// The swarm task releases the queue claims of in-flight messages during this +/// window. A fixed two-second sleep was not enough for that release when a +/// heavy task still held the SQLite write lock, which left the rows to be +/// reclaimed and charged as abandoned on the next start. +async fn wait_for_task_shutdown(mut handles: Vec>>) { + let started_at = Instant::now(); + let joined = tokio::time::timeout( + Duration::from_secs(SHUTDOWN_GRACE_SECS), + future::join_all(handles.iter_mut()), + ) + .await; + match joined { + Ok(_) => tracing::info!( + event = "service_shutdown", + outcome = "tasks_exited", + elapsed_ms = started_at.elapsed().as_millis() as u64, + "all node background tasks exited" + ), + Err(_) => { + tracing::warn!( + event = "service_shutdown", + outcome = "grace_period_exceeded", + grace_secs = SHUTDOWN_GRACE_SECS, + "aborting node background tasks that did not exit within the grace period" + ); + for handle in &handles { + handle.abort(); + } + } + } +} + /// Listen for shutdown signals (Ctrl+C, SIGTERM, etc.) async fn shutdown_signal() { let ctrl_c = async { diff --git a/node/src/rpc_service/bitvm.rs b/node/src/rpc_service/bitvm.rs index a4977e11..9e57ea82 100644 --- a/node/src/rpc_service/bitvm.rs +++ b/node/src/rpc_service/bitvm.rs @@ -659,6 +659,22 @@ trait DisplayStatusConvert { fn parse_display_status(ori_status: &str) -> Vec; } +/// Map a status that is not a display alias onto the stored lifecycle status. +/// +/// An empty filter means "no status filter" downstream, so an unknown status +/// name must not collapse into an empty list: that turned a stale or mistyped +/// status into a full listing. It matches nothing instead. Only an empty +/// parameter still means "no filter". +fn raw_status_filter(ori_status: &str) -> Vec { + if ori_status.trim().is_empty() { + return vec![]; + } + match S::from_str(ori_status) { + Ok(status) => vec![status.to_string()], + Err(_) => vec![ori_status.to_owned()], + } +} + impl DisplayStatusConvert for Graph { fn convert_to_display_status(&self) -> String { match GraphStatus::from_str(&self.status) { @@ -682,9 +698,7 @@ impl DisplayStatusConvert for Graph { Ok(GraphDisplayStatus::OperatorKickOffing) => { vec![GraphStatus::OperatorDataPushed.to_string()] } - Err(_) => { - GraphStatus::from_str(ori_status).map(|v| vec![v.to_string()]).unwrap_or_default() - } + Err(_) => raw_status_filter::(ori_status), } } } @@ -746,9 +760,7 @@ impl DisplayStatusConvert for Instance { InstanceBridgeInStatus::NoEnoughCommitteesAnswered.to_string(), InstanceBridgeInStatus::UserDiscarded.to_string(), ], - Err(_) => InstanceBridgeInStatus::from_str(ori_status) - .map(|v| vec![v.to_string()]) - .unwrap_or_default(), + Err(_) => raw_status_filter::(ori_status), } } } diff --git a/node/src/utils.rs b/node/src/utils.rs index 0aec93a9..dd57ee58 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -4209,14 +4209,14 @@ pub fn reflect_goat_address(addr_op: Option) -> (bool, Option) { (false, None) } -/// Claim a batch of local messages for dispatch, retiring exhausted ones first. +/// Sweep the local queue and list the messages the dispatcher may attempt now. /// -/// Returns `(claimed, quarantined)`. Unlike the select-only pop this replaces, -/// every returned message carries a durable claim, so an attempt that never -/// reports an outcome is visible to the next tick instead of replaying forever. -pub async fn claim_batch_local_msg( +/// Returns `(candidates, quarantined)`. Nothing returned here is claimed yet: +/// the dispatcher claims each row immediately before dispatching it, so a +/// crash mid-dispatch is charged to that one row rather than to the whole +/// batch. Expired rows are retired and exhausted rows quarantined first. +pub async fn list_batch_local_msg( local_db: &LocalDB, - lease_secs: i64, max_abandons: i64, limit: i64, ) -> Result<(Vec, u64)> { @@ -4226,15 +4226,8 @@ pub async fn claim_batch_local_msg( tx.set_messages_expired(expired_before).await?; tx.delete_old_messages(expired_before).await?; let quarantined = tx.quarantine_local_messages(current_time, max_abandons).await?; - let messages = tx - .claim_local_messages( - current_time, - current_time + lease_secs, - expired_before, - limit, - max_abandons, - ) - .await?; + let messages = + tx.list_claimable_local_messages(current_time, expired_before, limit, max_abandons).await?; tx.commit().await?; Ok((messages, quarantined)) } From 0a8f755e04f4d8e289c9317ceac7cc3456b4e95f Mon Sep 17 00:00:00 2001 From: ethan Date: Thu, 17 Sep 2026 22:25:49 +0800 Subject: [PATCH 07/17] add tests for BusinessRef derivation & helper for bip68 check --- .gitignore | 1 + crates/bitvm-gc/src/timelocks.rs | 28 ++++-- crates/node-macros/src/lib.rs | 94 +++++++++++++++++++ .../node-macros/tests/message_business_ref.rs | 47 ++++++++++ 4 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 crates/node-macros/tests/message_business_ref.rs diff --git a/.gitignore b/.gitignore index a222e211..3968c89a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ proof-builder-rpc/*.ckpt node/tla/states/ local_docs/ +local_scripts/ scripts/testnet/ scripts/devnet/ *.DS_Store \ No newline at end of file diff --git a/crates/bitvm-gc/src/timelocks.rs b/crates/bitvm-gc/src/timelocks.rs index 917a404f..dd38dae6 100644 --- a/crates/bitvm-gc/src/timelocks.rs +++ b/crates/bitvm-gc/src/timelocks.rs @@ -155,12 +155,7 @@ pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Re ("operator_commit", config.operator_commit), ("connector_f", config.connector_f), ] { - const BIP68_BLOCKS_MASK: u32 = 0x0000_ffff; - if value & !BIP68_BLOCKS_MASK != 0 { - bail!( - "timelock_config.{name} must use a BIP68 height-based sequence value, got {value:#010x}" - ); - } + ensure_bip68_height_based_sequence(name, value)?; if value < budget.reaction_blocks { bail!( "timelock_config.{name} must be at least {} reaction blocks, got {value}", @@ -216,6 +211,17 @@ pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Re Ok(()) } +const BIP68_BLOCKS_MASK: u32 = 0x0000_ffff; + +fn ensure_bip68_height_based_sequence(name: &str, value: u32) -> Result<()> { + if value & !BIP68_BLOCKS_MASK != 0 { + bail!( + "timelock_config.{name} must use a BIP68 height-based sequence value, got {value:#010x}" + ); + } + Ok(()) +} + fn ensure_gap_at_least( left_name: &str, left: u32, @@ -297,4 +303,14 @@ mod tests { validate_timelock_config(network, &config).unwrap(); } } + + #[test] + fn rejects_non_height_based_bip68_sequences() { + assert!(ensure_bip68_height_based_sequence("test", BIP68_BLOCKS_MASK).is_ok()); + + for invalid in [1 << 16, 1 << 22, 1 << 31] { + let error = ensure_bip68_height_based_sequence("test", invalid).unwrap_err(); + assert!(error.to_string().contains("BIP68 height-based sequence")); + } + } } diff --git a/crates/node-macros/src/lib.rs b/crates/node-macros/src/lib.rs index 58162f73..6869b4ea 100644 --- a/crates/node-macros/src/lib.rs +++ b/crates/node-macros/src/lib.rs @@ -114,3 +114,97 @@ fn business_ref_scope(variant: &Variant) -> Result { Err(syn::Error::new(attribute.span(), "business_ref must be graph, instance, or unscoped")) } } + +#[cfg(test)] +mod tests { + use super::*; + use syn::parse_quote; + + fn expand_error(input: DeriveInput) -> String { + expand_message_business_ref(input).unwrap_err().to_string() + } + + #[test] + fn rejects_non_enum_input() { + let input = parse_quote! { + struct Message; + }; + + assert_eq!(expand_error(input), "MessageBusinessRef can only be derived for enums"); + } + + #[test] + fn requires_a_business_ref_scope_for_every_variant() { + let input = parse_quote! { + enum Message { + Missing(Payload), + } + }; + + assert_eq!( + expand_error(input), + "each message variant must declare #[business_ref(graph)], #[business_ref(instance)], or #[business_ref(unscoped)]" + ); + } + + #[test] + fn rejects_invalid_or_duplicate_scopes() { + let invalid = parse_quote! { + enum Message { + #[business_ref(other)] + Invalid(Payload), + } + }; + assert_eq!(expand_error(invalid), "business_ref must be graph, instance, or unscoped"); + + let duplicate = parse_quote! { + enum Message { + #[business_ref(graph)] + #[business_ref(instance)] + Duplicate(Payload), + } + }; + assert_eq!(expand_error(duplicate), "duplicate business_ref attribute"); + } + + #[test] + fn graph_and_instance_scopes_require_one_tuple_payload() { + let named = parse_quote! { + enum Message { + #[business_ref(graph)] + Graph { payload: Payload }, + } + }; + assert_eq!( + expand_error(named), + "graph and instance business references require exactly one payload field" + ); + + let multiple = parse_quote! { + enum Message { + #[business_ref(instance)] + Instance(Payload, Payload), + } + }; + assert_eq!( + expand_error(multiple), + "graph and instance business references require exactly one payload field" + ); + } + + #[test] + fn accepts_all_unscoped_variant_shapes() { + let input = parse_quote! { + enum Message { + #[business_ref(unscoped)] + Unit, + #[business_ref(unscoped)] + Tuple(u8, u16), + #[business_ref(unscoped)] + Named { value: u8 }, + } + }; + + assert!(expand_message_business_ref(input).is_ok()); + } +} diff --git a/crates/node-macros/tests/message_business_ref.rs b/crates/node-macros/tests/message_business_ref.rs new file mode 100644 index 00000000..33d65ca8 --- /dev/null +++ b/crates/node-macros/tests/message_business_ref.rs @@ -0,0 +1,47 @@ +use node_macros::MessageBusinessRef; + +#[derive(Debug, Eq, PartialEq)] +enum BusinessRef { + Instance { instance_id: u64 }, + Graph { instance_id: u64, graph_id: u64 }, + Unscoped, +} + +trait HasBusinessRef { + fn business_ref(&self) -> BusinessRef; +} + +struct Payload { + instance_id: u64, + graph_id: u64, +} + +#[allow(dead_code)] +#[derive(MessageBusinessRef)] +enum Message { + #[business_ref(graph)] + Graph(Payload), + #[business_ref(instance)] + Instance(Payload), + #[business_ref(unscoped)] + Unit, + #[business_ref(unscoped)] + Tuple(u8, u16), + #[business_ref(unscoped)] + Named { value: u8 }, +} + +#[test] +fn derives_the_expected_business_reference_for_every_scope() { + assert_eq!( + Message::Graph(Payload { instance_id: 7, graph_id: 11 }).business_ref(), + BusinessRef::Graph { instance_id: 7, graph_id: 11 } + ); + assert_eq!( + Message::Instance(Payload { instance_id: 7, graph_id: 11 }).business_ref(), + BusinessRef::Instance { instance_id: 7 } + ); + assert_eq!(Message::Unit.business_ref(), BusinessRef::Unscoped); + assert_eq!(Message::Tuple(1, 2).business_ref(), BusinessRef::Unscoped); + assert_eq!(Message::Named { value: 1 }.business_ref(), BusinessRef::Unscoped); +} From a46fccabe94f56195ecdad5c54459069f49357c2 Mon Sep 17 00:00:00 2001 From: ethan Date: Fri, 18 Sep 2026 14:23:55 +0800 Subject: [PATCH 08/17] fix sequencer-set-publish --- node/src/bin/sequencer-set-publish.rs | 240 ++++++++++++++++++++++++-- 1 file changed, 223 insertions(+), 17 deletions(-) diff --git a/node/src/bin/sequencer-set-publish.rs b/node/src/bin/sequencer-set-publish.rs index 8e1ce96b..acd1f088 100644 --- a/node/src/bin/sequencer-set-publish.rs +++ b/node/src/bin/sequencer-set-publish.rs @@ -759,6 +759,107 @@ async fn update_sequencer_set_on_goat( Ok(()) } +/// Collect the threshold signatures for the update connector in redeem-script order. +/// +/// Witnesses are permissionlessly appended on Goat, so every field must be treated as +/// untrusted. Invalid, stale, duplicate, or unauthorized witnesses are ignored rather +/// than allowing one entry to abort publishing for the whole height. +fn collect_ordered_publisher_signatures( + witnesses: &[SequencerSetUpdateWitness], + btc_public_keys: &[secp256k1::PublicKey], + expected_sighash: [u8; 32], + threshold: usize, + goat_block_number: u64, +) -> anyhow::Result>> { + let secp = Secp256k1::verification_only(); + let message = Message::from_digest_slice(&expected_sighash) + .expect("a Bitcoin sighash is always exactly 32 bytes"); + let mut signatures_by_member = vec![None; btc_public_keys.len()]; + + for (witness_index, witness) in witnesses.iter().enumerate() { + let public_key = match secp256k1::PublicKey::from_slice(&witness.btc_pub_key) { + Ok(public_key) => public_key, + Err(error) => { + tracing::warn!( + goat_block_number, + witness_index, + error = %error, + "Skipping sequencer set witness with an invalid Bitcoin public key" + ); + continue; + } + }; + + let Some(member_index) = btc_public_keys.iter().position(|key| key == &public_key) else { + tracing::warn!( + goat_block_number, + witness_index, + public_key = %public_key, + "Skipping sequencer set witness from an unauthorized Bitcoin public key" + ); + continue; + }; + + if witness.sig_hash != expected_sighash { + tracing::warn!( + goat_block_number, + witness_index, + public_key = %public_key, + "Skipping sequencer set witness for a different transaction sighash" + ); + continue; + } + + let signature = match secp256k1::ecdsa::Signature::from_compact(&witness.btc_sig) { + Ok(signature) => signature, + Err(error) => { + tracing::warn!( + goat_block_number, + witness_index, + public_key = %public_key, + error = %error, + "Skipping sequencer set witness with an invalid compact signature" + ); + continue; + } + }; + + if let Err(error) = secp.verify_ecdsa(&message, &signature, &public_key) { + tracing::warn!( + goat_block_number, + witness_index, + public_key = %public_key, + error = %error, + "Skipping sequencer set witness with a signature that does not match the transaction" + ); + continue; + } + + if signatures_by_member[member_index].is_some() { + tracing::warn!( + goat_block_number, + witness_index, + public_key = %public_key, + "Skipping duplicate sequencer set witness" + ); + continue; + } + + let mut signature_bytes = signature.serialize_der().to_vec(); + signature_bytes.push(EcdsaSighashType::AllPlusAnyoneCanPay as u8); + signatures_by_member[member_index] = Some(signature_bytes); + } + + let mut signatures: Vec<_> = signatures_by_member.into_iter().flatten().collect(); + anyhow::ensure!( + signatures.len() >= threshold, + "only {} valid publisher signatures for Goat height {goat_block_number}; need {threshold}", + signatures.len() + ); + signatures.truncate(threshold); + Ok(signatures) +} + /// Submit sequencer set commitment #[allow(clippy::too_many_arguments)] async fn action_push_sequencer_set_update( @@ -778,23 +879,9 @@ async fn action_push_sequencer_set_update( output_file: &str, ) -> Result<(), Box> { let witnesses = goat_client.ss_get_sequencer_set_update_witness(goat_block_number).await?; - let mut sigs: Vec<_> = witnesses - .iter() - .filter(|x| { - btc_public_keys.contains(&secp256k1::PublicKey::from_slice(&x.btc_pub_key).unwrap()) - }) - .map(|x| { - let sig = secp256k1::ecdsa::Signature::from_compact(&x.btc_sig).expect("Invalid sig"); - let mut sig_bytes = sig.serialize_der().to_vec(); - sig_bytes.push(EcdsaSighashType::AllPlusAnyoneCanPay as u8); - sig_bytes - }) - .collect(); let total = btc_public_keys.len(); let threshold = (2 * total).div_ceil(3); - sigs.resize(threshold, vec![]); - let total = next_btc_public_keys.len(); let next_threshold = (2 * total).div_ceil(3); @@ -802,8 +889,6 @@ async fn action_push_sequencer_set_update( let next_redeem_script = create_sequencer_update_script(&next_btc_public_keys, next_threshold); let next_update_connector_address = Address::p2wsh(&next_redeem_script, btc_client.network()); - println!("sigs: {sigs:?}"); - // update the sequencer set publish tx with multisig signatures let (update_connector, update_connector_value) = match &update_connector_outpoint { Some(update_connector) => { @@ -850,6 +935,24 @@ async fn action_push_sequencer_set_update( Amount::from_sat(RELAYER_FEE), )?; + let publisher_sigs = if let Some(update_connector_value) = update_connector_value { + let sighash = SighashCache::new(&mut sequencer_set_publish_tx).p2wsh_signature_hash( + 0, + &redeem_script, + update_connector_value, + EcdsaSighashType::AllPlusAnyoneCanPay, + )?; + collect_ordered_publisher_signatures( + &witnesses, + &btc_public_keys, + sighash.to_byte_array(), + threshold, + goat_block_number, + )? + } else { + Vec::new() + }; + let secp = secp256k1::Secp256k1::new(); let owner_private_key = PrivateKey::from_wif(owner_btc_key_wif.as_ref().unwrap())?; let owner_p2wpkh = Address::p2wpkh( @@ -865,7 +968,7 @@ async fn action_push_sequencer_set_update( btc_client, &mut sequencer_set_publish_tx, &redeem_script, - sigs, + publisher_sigs, ) .await?; @@ -1119,3 +1222,106 @@ async fn fund_publishers( ); Ok((tx.compute_txid(), current_tx_vout as u32)) } + +#[cfg(test)] +mod tests { + use super::*; + use bitcoin::secp256k1::SecretKey; + + const SIGHASH: [u8; 32] = [42; 32]; + + fn secret_key(value: u8) -> SecretKey { + SecretKey::from_slice(&[value; 32]).expect("valid test secret key") + } + + fn public_key(secret_key: &SecretKey) -> secp256k1::PublicKey { + secp256k1::PublicKey::from_secret_key(&Secp256k1::new(), secret_key) + } + + fn signed_witness(secret_key: &SecretKey, sighash: [u8; 32]) -> SequencerSetUpdateWitness { + let secp = Secp256k1::new(); + let signature = secp.sign_ecdsa( + &Message::from_digest_slice(&sighash).expect("test sighash is 32 bytes"), + secret_key, + ); + SequencerSetUpdateWitness { + sig_hash: sighash, + btc_pub_key: public_key(secret_key).serialize().to_vec(), + btc_sig: signature.serialize_compact().to_vec(), + } + } + + fn expected_signature(secret_key: &SecretKey) -> Vec { + let secp = Secp256k1::new(); + let signature = secp.sign_ecdsa( + &Message::from_digest_slice(&SIGHASH).expect("test sighash is 32 bytes"), + secret_key, + ); + let mut encoded = signature.serialize_der().to_vec(); + encoded.push(EcdsaSighashType::AllPlusAnyoneCanPay as u8); + encoded + } + + #[test] + fn publisher_witnesses_are_filtered_deduplicated_and_reordered() { + let first = secret_key(1); + let second = secret_key(2); + let third = secret_key(3); + let stranger = secret_key(4); + let publishers = vec![public_key(&first), public_key(&second), public_key(&third)]; + + let mut malformed_key = signed_witness(&first, SIGHASH); + malformed_key.btc_pub_key[0] = 0x01; + let mut malformed_signature = signed_witness(&second, SIGHASH); + malformed_signature.btc_sig.clear(); + + let witnesses = vec![ + malformed_key, + signed_witness(&stranger, SIGHASH), + malformed_signature, + signed_witness(&third, SIGHASH), + signed_witness(&first, SIGHASH), + signed_witness(&first, SIGHASH), + ]; + + let signatures = + collect_ordered_publisher_signatures(&witnesses, &publishers, SIGHASH, 2, 123) + .expect("two valid publisher signatures"); + + assert_eq!(signatures, vec![expected_signature(&first), expected_signature(&third)]); + } + + #[test] + fn publisher_witnesses_keep_exactly_threshold_signatures_in_script_order() { + let first = secret_key(1); + let second = secret_key(2); + let third = secret_key(3); + let publishers = vec![public_key(&first), public_key(&second), public_key(&third)]; + let witnesses = vec![ + signed_witness(&third, SIGHASH), + signed_witness(&second, SIGHASH), + signed_witness(&first, SIGHASH), + ]; + + let signatures = + collect_ordered_publisher_signatures(&witnesses, &publishers, SIGHASH, 2, 123) + .expect("three valid signatures meet the threshold of two"); + + assert_eq!(signatures.len(), 2); + assert_eq!(signatures, vec![expected_signature(&first), expected_signature(&second)]); + } + + #[test] + fn publisher_witnesses_require_a_valid_threshold() { + let first = secret_key(1); + let second = secret_key(2); + let third = secret_key(3); + let publishers = vec![public_key(&first), public_key(&second), public_key(&third)]; + + let witnesses = vec![signed_witness(&first, SIGHASH), signed_witness(&second, [7; 32])]; + + let error = collect_ordered_publisher_signatures(&witnesses, &publishers, SIGHASH, 2, 123) + .expect_err("one valid signature is below threshold"); + assert!(error.to_string().contains("only 1 valid publisher signatures")); + } +} From a64f1927a31dc3ee2e6cc3b3dba63501a92e38aa Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 21 Sep 2026 22:52:59 +0800 Subject: [PATCH 09/17] fix: reuse genesis UTXOs and preserve retryable RPC errors --- node/src/action.rs | 24 ++++++++++++++++++- node/src/utils.rs | 59 ++++++++++++++++++++++++++-------------------- 2 files changed, 56 insertions(+), 27 deletions(-) diff --git a/node/src/action.rs b/node/src/action.rs index 290501aa..f34bdff2 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -279,8 +279,14 @@ fn is_retryable_reqwest_error(error: &reqwest::Error) -> bool { /// Only classify transport-level RPC failures. Contract reverts, malformed /// responses and application errors are intentionally left terminal. -fn is_retryable_external_rpc_error(error: &anyhow::Error) -> bool { +pub(crate) fn is_retryable_external_rpc_error(error: &anyhow::Error) -> bool { error.chain().any(|cause| { + // Inspect contract transport errors before traversing their transparent sources. + if let Some(alloy::contract::Error::TransportError(error)) = + cause.downcast_ref::() + { + return error.is_transport_error(); + } if let Some(error) = cause.downcast_ref::() { return is_retryable_reqwest_error(error); } @@ -2732,6 +2738,22 @@ mod tests { ); } + #[test] + fn contract_revert_is_not_a_retryable_transport_failure() { + let response = + serde_json::from_str(r#"{"code":3,"message":"execution reverted"}"#).unwrap(); + let error = anyhow::Error::new(alloy::contract::Error::TransportError( + alloy::transports::TransportError::ErrorResp(response), + )) + .context("validate SyncGraph instance parameters"); + assert!(!is_retryable_external_rpc_error(&error)); + assert!( + classify_retryable_dispatch_error(error) + .downcast_ref::() + .is_none() + ); + } + #[tokio::test] async fn dispatch_supervisor_distinguishes_shutdown_and_panic() { let shutdown = CancellationToken::new(); diff --git a/node/src/utils.rs b/node/src/utils.rs index dd57ee58..55858e0e 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -229,12 +229,9 @@ pub async fn validate_graph_instance_parameters( goat_client: &GOATClient, parameters: &BitvmGcInstanceParameters, ) -> Result<()> { - let expected = - read_instance_info_from_goat(goat_client, parameters.instance_id).await.map_err(|e| { - SpecialError::InvalidGraph(format!( - "failed to load instance parameters from GoatChain: {e}" - )) - })?; + let expected = read_instance_info_from_goat(goat_client, parameters.instance_id) + .await + .context("failed to load instance parameters from GoatChain")?; if parameters != &expected { bail!(SpecialError::InvalidGraph( "instance parameters mismatch with GoatChain peg-in data".to_string() @@ -242,11 +239,8 @@ pub async fn validate_graph_instance_parameters( } for input in &expected.user_info.inputs { - let funding_tx = btc_client.get_tx(&input.outpoint.txid).await.map_err(|e| { - SpecialError::InvalidGraph(format!( - "failed to load peg-in funding transaction {}: {e}", - input.outpoint.txid - )) + let funding_tx = btc_client.get_tx(&input.outpoint.txid).await.with_context(|| { + format!("failed to load peg-in funding transaction {}", input.outpoint.txid) })?; let Some(funding_tx) = funding_tx else { bail!(SpecialError::InvalidGraph(format!( @@ -283,11 +277,7 @@ pub async fn validate_graph_instance_parameters( if btc_client .get_tx(&pegin_deposit_txid) .await - .map_err(|e| { - SpecialError::InvalidGraph(format!( - "failed to load peg-in deposit transaction {pegin_deposit_txid}: {e}" - )) - })? + .with_context(|| format!("failed to load peg-in deposit transaction {pegin_deposit_txid}"))? .is_none() { bail!(SpecialError::InvalidGraph(format!( @@ -3248,16 +3238,13 @@ pub async fn build_genesis_prekickoff_tx( let next_kickoff_connector = KickoffConnector::new(network, &operator_taproot_public_key); let next_prekickoff_connector = PrekickoffConnector::new(network, &operator_taproot_public_key); let init_amount = prekickoff_replenishment_amount(); - let cur_prekickoff_connector_input = Input { - outpoint: fund_address( - btc_client, - node_keypair, - cur_prekickoff_connector.generate_taproot_address(), - init_amount, - ) - .await?, - amount: init_amount, - }; + let cur_prekickoff_connector_input = genesis_funding_input( + btc_client, + node_keypair, + cur_prekickoff_connector.generate_taproot_address(), + init_amount, + ) + .await?; let fee_amount = prekickoff_fee_amount(0); PrekickoffTransaction::new_for_validation( &cur_prekickoff_connector, @@ -3274,6 +3261,26 @@ pub async fn build_genesis_prekickoff_tx( .map_err(|e| anyhow::anyhow!("failed to create pre-kickoff txn: {e}")) } +/// Reuse a matching unspent output, or fund the genesis address. +async fn genesis_funding_input( + btc_client: &BTCClient, + node_keypair: Keypair, + address: Address, + amount: Amount, +) -> Result { + let existing = btc_client + .get_address_utxo(address.clone()) + .await? + .into_iter() + .filter(|utxo| utxo.value == amount) + .min_by_key(|utxo| (utxo.txid, utxo.vout)); + let outpoint = match existing { + Some(utxo) => OutPoint { txid: utxo.txid, vout: utxo.vout }, + None => fund_address(btc_client, node_keypair, address, amount).await?, + }; + Ok(Input { outpoint, amount }) +} + pub async fn build_prekickoff_params( btc_client: &BTCClient, graph_nonce: u64, From 58174203a566e0f8ef3ed65a1d24c572ea981994 Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 21 Sep 2026 22:56:44 +0800 Subject: [PATCH 10/17] feat(store): persist P2P admission, identity and delivery state --- ...a9244b2039b01d37b4f44177c6509a47ca762.json | 12 - ...e88b134680a10d296d3dfa0bae4123d18cf79.json | 12 + ...76bec1ebc16fca59511a8e9ddde8dfa7dd50b.json | 12 - ...e5183ab5186b8646c12d0fc9d67cf056d1a37.json | 12 - ...2c334ce0cfe6c6a77a976f24df8c09d0cae77.json | 12 - ...172542d25b667277ec137a232868f8499b628.json | 12 - ...27c61387d003fc88772b35e4dcb4f23d1b93c.json | 12 - ...05de750a66de24a10768808471d8219186695.json | 20 - ...6ec87546965ff35877dff9d8cc394a4ac89e1.json | 12 - ...7eee05465d9dd4b5196243d745e3f1471047c.json | 20 - ...0f22e533b821267451fa27ca7bcfd3a8a069f.json | 12 - ...0a2240d1ac7ac98e37d9a2f12d83d3c2740d5.json | 12 - ...dc59aeb48e932b907e7bdd415e5c9f652daf9.json | 12 - ...55a3e30fdc1fe9a927a9a98be1d4a6dd0cb4a.json | 12 + ...0019bede7fb6329fde8ea1e547fc88a14090a.json | 12 + ...7201faaa223c9cdd4fb829ed7ca11ab750a55.json | 26 - .../20260918000000_add_p2p_admission.sql | 29 + ...20260918000001_add_p2p_registered_peer.sql | 35 + ...918000002_add_p2p_outbox_publish_count.sql | 5 + crates/store/src/localdb.rs | 1092 ++++++++++++++++- crates/store/src/schema.rs | 43 + node/src/action.rs | 57 + node/src/bin/mock_rpc.rs | 2 + node/src/env.rs | 17 + node/src/p2p_msg_handler.rs | 1 + node/src/rpc_service/mod.rs | 4 + node/src/utils.rs | 8 +- 27 files changed, 1299 insertions(+), 216 deletions(-) delete mode 100644 crates/store/.sqlx/query-0d089d8a580c440ac9dda18b624a9244b2039b01d37b4f44177c6509a47ca762.json delete mode 100644 crates/store/.sqlx/query-3a04d39b58be5a083e5ee1b2c5676bec1ebc16fca59511a8e9ddde8dfa7dd50b.json delete mode 100644 crates/store/.sqlx/query-4dd64d5bd882832e9edf8d0458ae5183ab5186b8646c12d0fc9d67cf056d1a37.json delete mode 100644 crates/store/.sqlx/query-54bf64e1979c461c2756634c5d42c334ce0cfe6c6a77a976f24df8c09d0cae77.json delete mode 100644 crates/store/.sqlx/query-78a3ed37ee823fd7fb10d17a943172542d25b667277ec137a232868f8499b628.json delete mode 100644 crates/store/.sqlx/query-877f2c1dd5c11493f63209165f427c61387d003fc88772b35e4dcb4f23d1b93c.json delete mode 100644 crates/store/.sqlx/query-89968da14614dd5c31cac25905305de750a66de24a10768808471d8219186695.json delete mode 100644 crates/store/.sqlx/query-8c69b8b08d7321de8160a9a61a66ec87546965ff35877dff9d8cc394a4ac89e1.json delete mode 100644 crates/store/.sqlx/query-92b2237700e5203200e44978cb37eee05465d9dd4b5196243d745e3f1471047c.json delete mode 100644 crates/store/.sqlx/query-9976812510d073d012a0ca76e4e0f22e533b821267451fa27ca7bcfd3a8a069f.json delete mode 100644 crates/store/.sqlx/query-b13ab16c6227e223d4cafc1090a0a2240d1ac7ac98e37d9a2f12d83d3c2740d5.json delete mode 100644 crates/store/.sqlx/query-cdfcead2319141ce6d0a42ef5cbdc59aeb48e932b907e7bdd415e5c9f652daf9.json create mode 100644 crates/store/.sqlx/query-eb0125dd27e3d2ae8a6eb9a65540019bede7fb6329fde8ea1e547fc88a14090a.json delete mode 100644 crates/store/.sqlx/query-f6f86624bdb191f28d36d0f044a7201faaa223c9cdd4fb829ed7ca11ab750a55.json create mode 100644 crates/store/migrations/20260918000000_add_p2p_admission.sql create mode 100644 crates/store/migrations/20260918000001_add_p2p_registered_peer.sql create mode 100644 crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql diff --git a/crates/store/.sqlx/query-0d089d8a580c440ac9dda18b624a9244b2039b01d37b4f44177c6509a47ca762.json b/crates/store/.sqlx/query-0d089d8a580c440ac9dda18b624a9244b2039b01d37b4f44177c6509a47ca762.json deleted file mode 100644 index b7da187c..00000000 --- a/crates/store/.sqlx/query-0d089d8a580c440ac9dda18b624a9244b2039b01d37b4f44177c6509a47ca762.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE operator_proof\n SET proof_state = ?,\n updated_at = ?\n WHERE id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "0d089d8a580c440ac9dda18b624a9244b2039b01d37b4f44177c6509a47ca762" -} diff --git a/crates/store/.sqlx/query-31eb2bac6c69f321c1ec0522972e88b134680a10d296d3dfa0bae4123d18cf79.json b/crates/store/.sqlx/query-31eb2bac6c69f321c1ec0522972e88b134680a10d296d3dfa0bae4123d18cf79.json index 408071ec..3e90f0e4 100644 --- a/crates/store/.sqlx/query-31eb2bac6c69f321c1ec0522972e88b134680a10d296d3dfa0bae4123d18cf79.json +++ b/crates/store/.sqlx/query-31eb2bac6c69f321c1ec0522972e88b134680a10d296d3dfa0bae4123d18cf79.json @@ -57,6 +57,16 @@ "name": "updated_at", "ordinal": 10, "type_info": "Integer" + }, + { + "name": "binding_sig", + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "binding_issued_at", + "ordinal": 12, + "type_info": "Integer" } ], "parameters": { @@ -73,6 +83,8 @@ false, false, false, + false, + false, false ] }, diff --git a/crates/store/.sqlx/query-3a04d39b58be5a083e5ee1b2c5676bec1ebc16fca59511a8e9ddde8dfa7dd50b.json b/crates/store/.sqlx/query-3a04d39b58be5a083e5ee1b2c5676bec1ebc16fca59511a8e9ddde8dfa7dd50b.json deleted file mode 100644 index 5409ee32..00000000 --- a/crates/store/.sqlx/query-3a04d39b58be5a083e5ee1b2c5676bec1ebc16fca59511a8e9ddde8dfa7dd50b.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE instance SET pegin_confirm_txid = ?, updated_at = ? WHERE instance_id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "3a04d39b58be5a083e5ee1b2c5676bec1ebc16fca59511a8e9ddde8dfa7dd50b" -} diff --git a/crates/store/.sqlx/query-4dd64d5bd882832e9edf8d0458ae5183ab5186b8646c12d0fc9d67cf056d1a37.json b/crates/store/.sqlx/query-4dd64d5bd882832e9edf8d0458ae5183ab5186b8646c12d0fc9d67cf056d1a37.json deleted file mode 100644 index 4e447fcb..00000000 --- a/crates/store/.sqlx/query-4dd64d5bd882832e9edf8d0458ae5183ab5186b8646c12d0fc9d67cf056d1a37.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE instance SET pegin_data_tx_hash = ?, updated_at = ? WHERE instance_id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "4dd64d5bd882832e9edf8d0458ae5183ab5186b8646c12d0fc9d67cf056d1a37" -} diff --git a/crates/store/.sqlx/query-54bf64e1979c461c2756634c5d42c334ce0cfe6c6a77a976f24df8c09d0cae77.json b/crates/store/.sqlx/query-54bf64e1979c461c2756634c5d42c334ce0cfe6c6a77a976f24df8c09d0cae77.json deleted file mode 100644 index 5bc37feb..00000000 --- a/crates/store/.sqlx/query-54bf64e1979c461c2756634c5d42c334ce0cfe6c6a77a976f24df8c09d0cae77.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE\n pegin_graph_process_data\n SET is_endorsed = ?\n WHERE graph_id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - }, - "hash": "54bf64e1979c461c2756634c5d42c334ce0cfe6c6a77a976f24df8c09d0cae77" -} diff --git a/crates/store/.sqlx/query-78a3ed37ee823fd7fb10d17a943172542d25b667277ec137a232868f8499b628.json b/crates/store/.sqlx/query-78a3ed37ee823fd7fb10d17a943172542d25b667277ec137a232868f8499b628.json deleted file mode 100644 index d4317d72..00000000 --- a/crates/store/.sqlx/query-78a3ed37ee823fd7fb10d17a943172542d25b667277ec137a232868f8499b628.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE graph_btc_tx_vout_monitor\n SET monitor_data = ?,\n updated_at = ?\n WHERE graph_id = ? AND txid = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "78a3ed37ee823fd7fb10d17a943172542d25b667277ec137a232868f8499b628" -} diff --git a/crates/store/.sqlx/query-877f2c1dd5c11493f63209165f427c61387d003fc88772b35e4dcb4f23d1b93c.json b/crates/store/.sqlx/query-877f2c1dd5c11493f63209165f427c61387d003fc88772b35e4dcb4f23d1b93c.json deleted file mode 100644 index a8833cd2..00000000 --- a/crates/store/.sqlx/query-877f2c1dd5c11493f63209165f427c61387d003fc88772b35e4dcb4f23d1b93c.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n INSERT INTO node (peer_id, node_name, actor, goat_addr, btc_pub_key, socket_addr, service_fee_rate, available_peg_btc,\n created_at, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT (peer_id) DO UPDATE SET actor = excluded.actor,\n node_name = excluded.node_name,\n goat_addr = excluded.goat_addr,\n btc_pub_key = excluded.btc_pub_key,\n service_fee_rate = excluded.service_fee_rate,\n available_peg_btc = excluded.available_peg_btc,\n socket_addr = excluded.socket_addr,\n updated_at = excluded.updated_at\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - }, - "hash": "877f2c1dd5c11493f63209165f427c61387d003fc88772b35e4dcb4f23d1b93c" -} diff --git a/crates/store/.sqlx/query-89968da14614dd5c31cac25905305de750a66de24a10768808471d8219186695.json b/crates/store/.sqlx/query-89968da14614dd5c31cac25905305de750a66de24a10768808471d8219186695.json deleted file mode 100644 index 1f7ef3db..00000000 --- a/crates/store/.sqlx/query-89968da14614dd5c31cac25905305de750a66de24a10768808471d8219186695.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT network FROM instance WHERE instance_id = ?", - "describe": { - "columns": [ - { - "name": "network", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false - ] - }, - "hash": "89968da14614dd5c31cac25905305de750a66de24a10768808471d8219186695" -} diff --git a/crates/store/.sqlx/query-8c69b8b08d7321de8160a9a61a66ec87546965ff35877dff9d8cc394a4ac89e1.json b/crates/store/.sqlx/query-8c69b8b08d7321de8160a9a61a66ec87546965ff35877dff9d8cc394a4ac89e1.json deleted file mode 100644 index 006fde06..00000000 --- a/crates/store/.sqlx/query-8c69b8b08d7321de8160a9a61a66ec87546965ff35877dff9d8cc394a4ac89e1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT OR\n REPLACE INTO instance (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, status, goat_tx_hash, goat_tx_height,\n user_xonly_pubkey, user_change_addr, user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers,\n pegin_data_tx_hash, btc_height, parameters, status_updated_at, post_pegin_txhash, created_at, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - "describe": { - "columns": [], - "parameters": { - "Right": 24 - }, - "nullable": [] - }, - "hash": "8c69b8b08d7321de8160a9a61a66ec87546965ff35877dff9d8cc394a4ac89e1" -} diff --git a/crates/store/.sqlx/query-92b2237700e5203200e44978cb37eee05465d9dd4b5196243d745e3f1471047c.json b/crates/store/.sqlx/query-92b2237700e5203200e44978cb37eee05465d9dd4b5196243d745e3f1471047c.json deleted file mode 100644 index b1afb2bc..00000000 --- a/crates/store/.sqlx/query-92b2237700e5203200e44978cb37eee05465d9dd4b5196243d745e3f1471047c.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT operator_pubkey FROM graph WHERE graph_id = ?", - "describe": { - "columns": [ - { - "name": "operator_pubkey", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false - ] - }, - "hash": "92b2237700e5203200e44978cb37eee05465d9dd4b5196243d745e3f1471047c" -} diff --git a/crates/store/.sqlx/query-9976812510d073d012a0ca76e4e0f22e533b821267451fa27ca7bcfd3a8a069f.json b/crates/store/.sqlx/query-9976812510d073d012a0ca76e4e0f22e533b821267451fa27ca7bcfd3a8a069f.json deleted file mode 100644 index 168c8f39..00000000 --- a/crates/store/.sqlx/query-9976812510d073d012a0ca76e4e0f22e533b821267451fa27ca7bcfd3a8a069f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "Update message Set lock_time_until = ?, updated_at = ? WHERE message_id = ? AND message_version = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "9976812510d073d012a0ca76e4e0f22e533b821267451fa27ca7bcfd3a8a069f" -} diff --git a/crates/store/.sqlx/query-b13ab16c6227e223d4cafc1090a0a2240d1ac7ac98e37d9a2f12d83d3c2740d5.json b/crates/store/.sqlx/query-b13ab16c6227e223d4cafc1090a0a2240d1ac7ac98e37d9a2f12d83d3c2740d5.json deleted file mode 100644 index a5b1d01c..00000000 --- a/crates/store/.sqlx/query-b13ab16c6227e223d4cafc1090a0a2240d1ac7ac98e37d9a2f12d83d3c2740d5.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE instance SET committees_answers = ?, updated_at = ? WHERE instance_id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "b13ab16c6227e223d4cafc1090a0a2240d1ac7ac98e37d9a2f12d83d3c2740d5" -} diff --git a/crates/store/.sqlx/query-cdfcead2319141ce6d0a42ef5cbdc59aeb48e932b907e7bdd415e5c9f652daf9.json b/crates/store/.sqlx/query-cdfcead2319141ce6d0a42ef5cbdc59aeb48e932b907e7bdd415e5c9f652daf9.json deleted file mode 100644 index fd562482..00000000 --- a/crates/store/.sqlx/query-cdfcead2319141ce6d0a42ef5cbdc59aeb48e932b907e7bdd415e5c9f652daf9.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? WHERE instance_id = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "cdfcead2319141ce6d0a42ef5cbdc59aeb48e932b907e7bdd415e5c9f652daf9" -} diff --git a/crates/store/.sqlx/query-e4bd68a486be21ff33af14a577155a3e30fdc1fe9a927a9a98be1d4a6dd0cb4a.json b/crates/store/.sqlx/query-e4bd68a486be21ff33af14a577155a3e30fdc1fe9a927a9a98be1d4a6dd0cb4a.json index 1f6ffd82..0149ae6f 100644 --- a/crates/store/.sqlx/query-e4bd68a486be21ff33af14a577155a3e30fdc1fe9a927a9a98be1d4a6dd0cb4a.json +++ b/crates/store/.sqlx/query-e4bd68a486be21ff33af14a577155a3e30fdc1fe9a927a9a98be1d4a6dd0cb4a.json @@ -57,6 +57,16 @@ "name": "updated_at", "ordinal": 10, "type_info": "Integer" + }, + { + "name": "binding_sig", + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "binding_issued_at", + "ordinal": 12, + "type_info": "Integer" } ], "parameters": { @@ -73,6 +83,8 @@ false, false, false, + false, + false, false ] }, diff --git a/crates/store/.sqlx/query-eb0125dd27e3d2ae8a6eb9a65540019bede7fb6329fde8ea1e547fc88a14090a.json b/crates/store/.sqlx/query-eb0125dd27e3d2ae8a6eb9a65540019bede7fb6329fde8ea1e547fc88a14090a.json new file mode 100644 index 00000000..33a13f6b --- /dev/null +++ b/crates/store/.sqlx/query-eb0125dd27e3d2ae8a6eb9a65540019bede7fb6329fde8ea1e547fc88a14090a.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO node (peer_id, node_name, actor, goat_addr, btc_pub_key, socket_addr, service_fee_rate, available_peg_btc,\n binding_sig, binding_issued_at, created_at, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT (peer_id) DO UPDATE SET actor = excluded.actor,\n node_name = excluded.node_name,\n goat_addr = excluded.goat_addr,\n btc_pub_key = excluded.btc_pub_key,\n service_fee_rate = excluded.service_fee_rate,\n available_peg_btc = excluded.available_peg_btc,\n socket_addr = excluded.socket_addr,\n binding_sig = excluded.binding_sig,\n binding_issued_at = excluded.binding_issued_at,\n updated_at = excluded.updated_at\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + }, + "hash": "eb0125dd27e3d2ae8a6eb9a65540019bede7fb6329fde8ea1e547fc88a14090a" +} diff --git a/crates/store/.sqlx/query-f6f86624bdb191f28d36d0f044a7201faaa223c9cdd4fb829ed7ca11ab750a55.json b/crates/store/.sqlx/query-f6f86624bdb191f28d36d0f044a7201faaa223c9cdd4fb829ed7ca11ab750a55.json deleted file mode 100644 index a3cfd123..00000000 --- a/crates/store/.sqlx/query-f6f86624bdb191f28d36d0f044a7201faaa223c9cdd4fb829ed7ca11ab750a55.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT graph_id AS \"graph_id:Uuid\", kickoff_index\n FROM graph\n WHERE operator_pubkey = ?\n ORDER BY kickoff_index DESC\n limit 1", - "describe": { - "columns": [ - { - "name": "graph_id:Uuid", - "ordinal": 0, - "type_info": "Text" - }, - { - "name": "kickoff_index", - "ordinal": 1, - "type_info": "Integer" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - }, - "hash": "f6f86624bdb191f28d36d0f044a7201faaa223c9cdd4fb829ed7ca11ab750a55" -} diff --git a/crates/store/migrations/20260918000000_add_p2p_admission.sql b/crates/store/migrations/20260918000000_add_p2p_admission.sql new file mode 100644 index 00000000..3ea18e53 --- /dev/null +++ b/crates/store/migrations/20260918000000_add_p2p_admission.sql @@ -0,0 +1,29 @@ +-- Admission control for externally received gossip. +-- +-- Inbound P2P messages are persisted before any business validation, so the +-- node must bound what an arbitrary peer can make it store, and must be able to +-- tell an authenticated sender apart from an anonymous one. + +-- p2p_inbox: how the sender was classified when a row was admitted, and a digest +-- used to collapse an identical payload the same sender re-publishes with a fresh +-- gossip id. `admission_class` is `Committee`, `Registered`, or `Unregistered`. +ALTER TABLE p2p_inbox ADD COLUMN admission_class TEXT NOT NULL DEFAULT 'Unregistered'; +ALTER TABLE p2p_inbox ADD COLUMN content_hash BLOB; + +-- Covering index for the per-sender quota query. `content` is a large blob and +-- `content_size` follows it, so reading usage from the table would walk every +-- overflow page of every queued row. `state` leads so the partial working set is +-- scanned directly. +CREATE INDEX IF NOT EXISTS idx_p2p_inbox_admission + ON p2p_inbox (state, admission_class, from_peer, content_size); + +CREATE INDEX IF NOT EXISTS idx_p2p_inbox_content_hash + ON p2p_inbox (from_peer, content_hash); + +-- node: the sender's proof that the on-chain-registered key `btc_pub_key` +-- authorised this `peer_id`. gossipsub authenticates the peer id; this Schnorr +-- signature (by the node master key over the peer id) is what lets a receiver +-- trust the mapping and grant the registered sender class. `binding_issued_at` +-- orders re-bindings so one key maps to a single peer id at a time. +ALTER TABLE node ADD COLUMN binding_sig TEXT NOT NULL DEFAULT ''; +ALTER TABLE node ADD COLUMN binding_issued_at BIGINT NOT NULL DEFAULT 0; diff --git a/crates/store/migrations/20260918000001_add_p2p_registered_peer.sql b/crates/store/migrations/20260918000001_add_p2p_registered_peer.sql new file mode 100644 index 00000000..8b0864b5 --- /dev/null +++ b/crates/store/migrations/20260918000001_add_p2p_registered_peer.sql @@ -0,0 +1,35 @@ +-- Peers this node confirmed on chain as registered senders: committee members +-- (peer id registry) and operators (staked master key bound to the peer id). +-- +-- The admission registry is an in-memory cache filled by chain lookups, and +-- first-time lookups are budgeted so a flood of fresh identities cannot turn +-- into a flood of RPC calls. Without this table a restart would send every real +-- member back through that budget, competing with the flood. +-- +-- `pubkey` is the hex x-only master key the confirmation was made for (empty for +-- committee rows). A registration is only as good as the key that was checked: +-- a peer that re-binds to another key must not inherit it, and a key that moves +-- to another peer id takes its row along. Rows are written only after a positive +-- chain answer, removed on a negative one, and an operator key holds at most one +-- row, so the table is bounded by the on-chain registries, not by who connects. +CREATE TABLE IF NOT EXISTS p2p_registered_peer +( + peer_id TEXT NOT NULL, + kind TEXT NOT NULL, + pubkey TEXT NOT NULL DEFAULT '', + verified_at BIGINT NOT NULL, + PRIMARY KEY (peer_id, kind) +); + +CREATE INDEX IF NOT EXISTS idx_p2p_registered_peer_pubkey + ON p2p_registered_peer (kind, pubkey); + +-- Highest gossipsub sequence number admitted per registered author. Replay +-- protection keeps a window per author in memory; this mark is what survives a +-- restart, so messages captured before it cannot be replayed into a node that +-- has just come back up. One row per registered author. +CREATE TABLE IF NOT EXISTS p2p_replay_mark +( + peer_id TEXT NOT NULL PRIMARY KEY, + highest BIGINT NOT NULL +); diff --git a/crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql b/crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql new file mode 100644 index 00000000..2480d0c4 --- /dev/null +++ b/crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql @@ -0,0 +1,5 @@ +-- Successful publishes of an outbox row. Signing-round rows are re-published for +-- as long as their round is open, and back off by this count. `attempt_count` +-- cannot serve: it also counts claims whose publish failed, so a node that came +-- up without peers would reach the slowest tier before delivering anything. +ALTER TABLE p2p_outbox ADD COLUMN publish_count BIGINT NOT NULL DEFAULT 0; diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index f93ff7d6..3b6c2dfe 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -3,10 +3,10 @@ use crate::{ BridgeOutGlobalStats, EventWatchMetricsSnapshot, GoatTxRecord, Graph, GraphBtcTxVoutMonitor, GraphRawData, GraphStatus, GraphStatusSource, GraphStatusTransitionOutcome, Instance, LongRunningTaskProof, Message, MessageDebugOverview, MessageDebugReason, MetricsStateCount, - Node, NodeAlertMetricsSnapshot, NodesOverview, OperatorProof, P2pInboxMessage, - P2pOutboxMessage, PeginGraphProcessData, PeginInstanceProcessData, PendingGraphInit, - SequencerSetHashChange, SequencerSetScanState, SerializableTxid, SwapEscrow, SwapEscrowStatus, - WatchContract, WatchtowerProof, + Node, NodeAlertMetricsSnapshot, NodesOverview, OperatorProof, P2pInboxAdmissionClass, + P2pInboxClassUsage, P2pInboxMessage, P2pOutboxMessage, PeginGraphProcessData, + PeginInstanceProcessData, PendingGraphInit, SequencerSetHashChange, SequencerSetScanState, + SerializableTxid, SwapEscrow, SwapEscrowStatus, WatchContract, WatchtowerProof, }; use indexmap::IndexMap; @@ -17,7 +17,6 @@ use sqlx::types::Uuid; use sqlx::{Row, Sqlite, SqliteConnection, SqlitePool, Transaction, migrate::MigrateDatabase}; use std::str::FromStr; use std::time::{SystemTime, UNIX_EPOCH}; -use tracing::warn; fn get_current_timestamp_secs() -> i64 { SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 @@ -31,7 +30,7 @@ const MESSAGE_COLUMNS: &str = "message_id, business_id, from_peer, actor, msg_ty /// Columns every `p2p_inbox` SELECT must fetch. const P2P_INBOX_COLUMNS: &str = "message_id, business_id, actor, from_peer, msg_type, content, \ content_size, state, attempt_count, abandon_count, next_retry_at, lease_until, lease_token, \ - last_error, created_at, updated_at"; + last_error, admission_class, content_hash, created_at, updated_at"; fn message_from_row(row: &SqliteRow) -> Result { Ok(Message { @@ -68,6 +67,8 @@ fn p2p_inbox_message_from_row(row: &SqliteRow) -> Result Result StorageProcessor<'a> { .await?; if result.rows_affected() == 0 { - warn!("Node {peer_id} not found in DB, no rows updated"); + // Any peer that never announced itself lands here, once per message + // burst, so this must not be louder than debug. + tracing::debug!("Node {peer_id} not found in DB, no rows updated"); } Ok(()) @@ -2059,8 +2063,8 @@ impl<'a> StorageProcessor<'a> { let res = sqlx::query!( r#" INSERT INTO node (peer_id, node_name, actor, goat_addr, btc_pub_key, socket_addr, service_fee_rate, available_peg_btc, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + binding_sig, binding_issued_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (peer_id) DO UPDATE SET actor = excluded.actor, node_name = excluded.node_name, goat_addr = excluded.goat_addr, @@ -2068,6 +2072,8 @@ impl<'a> StorageProcessor<'a> { service_fee_rate = excluded.service_fee_rate, available_peg_btc = excluded.available_peg_btc, socket_addr = excluded.socket_addr, + binding_sig = excluded.binding_sig, + binding_issued_at = excluded.binding_issued_at, updated_at = excluded.updated_at "#, node.peer_id, @@ -2078,6 +2084,8 @@ impl<'a> StorageProcessor<'a> { node.socket_addr, node.service_fee_rate, node.available_peg_btc, + node.binding_sig, + node.binding_issued_at, node.created_at, node.updated_at, ) @@ -2853,11 +2861,17 @@ impl<'a> StorageProcessor<'a> { message: &P2pInboxMessage, ) -> anyhow::Result { let now = get_current_timestamp_secs(); + let admission_class = if message.admission_class.is_empty() { + P2pInboxAdmissionClass::Unregistered.to_string() + } else { + message.admission_class.clone() + }; let result = sqlx::query( "INSERT INTO p2p_inbox \ (message_id, business_id, actor, from_peer, msg_type, content, content_size, \ - state, attempt_count, next_retry_at, lease_until, lease_token, created_at, updated_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?, 'Pending', 0, 0, 0, '', ?, ?) \ + state, attempt_count, next_retry_at, lease_until, lease_token, \ + admission_class, content_hash, created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, 'Pending', 0, 0, 0, '', ?, ?, ?, ?) \ ON CONFLICT(message_id) DO NOTHING", ) .bind(&message.message_id) @@ -2867,6 +2881,8 @@ impl<'a> StorageProcessor<'a> { .bind(&message.msg_type) .bind(&message.content) .bind(message.content_size) + .bind(admission_class) + .bind(&message.content_hash) .bind(now) .bind(now) .execute(self.conn()) @@ -2874,6 +2890,137 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Test oracle for class totals and per-peer usage, including quarantined payloads. + #[cfg(test)] + async fn p2p_inbox_usage( + &mut self, + from_peer: &str, + ) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT admission_class, \ + COUNT(*) AS total_rows, \ + COALESCE(SUM(content_size), 0) AS total_bytes, \ + COALESCE(SUM(CASE WHEN from_peer = ? THEN 1 ELSE 0 END), 0) AS peer_rows, \ + COALESCE(SUM(CASE WHEN from_peer = ? THEN content_size ELSE 0 END), 0) \ + AS peer_bytes \ + FROM p2p_inbox \ + WHERE state IN ('Pending', 'Processing', 'Quarantined') \ + GROUP BY admission_class", + ) + .bind(from_peer) + .bind(from_peer) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(P2pInboxClassUsage { + admission_class: row.try_get("admission_class")?, + rows: row.try_get("total_rows")?, + bytes: row.try_get("total_bytes")?, + peer_rows: row.try_get("peer_rows")?, + peer_bytes: row.try_get("peer_bytes")?, + }) + }) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// Per-class queued payload totals, without per-peer usage. + pub async fn p2p_inbox_class_totals(&mut self) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT admission_class, \ + COUNT(*) AS total_rows, \ + COALESCE(SUM(content_size), 0) AS total_bytes \ + FROM p2p_inbox \ + WHERE state IN ('Pending', 'Processing', 'Quarantined') \ + GROUP BY admission_class", + ) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(P2pInboxClassUsage { + admission_class: row.try_get("admission_class")?, + rows: row.try_get("total_rows")?, + bytes: row.try_get("total_bytes")?, + ..Default::default() + }) + }) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// Per-peer payload usage by class; `rows` and `bytes` remain zero. + pub async fn p2p_inbox_peer_usage( + &mut self, + from_peer: &str, + ) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT admission_class, \ + COUNT(*) AS peer_rows, \ + COALESCE(SUM(content_size), 0) AS peer_bytes \ + FROM p2p_inbox \ + WHERE state IN ('Pending', 'Processing', 'Quarantined') \ + AND admission_class IN ('Committee', 'Registered', 'Unregistered') \ + AND from_peer = ? \ + GROUP BY admission_class", + ) + .bind(from_peer) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(P2pInboxClassUsage { + admission_class: row.try_get("admission_class")?, + peer_rows: row.try_get("peer_rows")?, + peer_bytes: row.try_get("peer_bytes")?, + ..Default::default() + }) + }) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// Whether the sender already has this payload in a non-terminal row. + pub async fn has_queued_p2p_inbox_payload( + &mut self, + from_peer: &str, + content_hash: &[u8], + ) -> anyhow::Result { + let row = sqlx::query( + "SELECT 1 FROM p2p_inbox \ + WHERE from_peer = ? AND content_hash = ? AND state IN ('Pending', 'Processing') \ + LIMIT 1", + ) + .bind(from_peer) + .bind(content_hash) + .fetch_optional(self.conn()) + .await?; + Ok(row.is_some()) + } + + /// Expire Pending rows by `created_at`; `None` selects every admission class. + pub async fn expire_pending_p2p_inbox_messages( + &mut self, + admission_class: Option, + created_before: i64, + ) -> anyhow::Result { + let class_predicate = + if admission_class.is_some() { " AND admission_class = ?" } else { "" }; + let query = format!( + "UPDATE p2p_inbox \ + SET state = 'Failed', content = X'', lease_until = 0, next_retry_at = 0, \ + last_error = 'expired: still pending past the inbox retention window', \ + updated_at = ? \ + WHERE state = 'Pending' AND created_at < ?{class_predicate}" + ); + let mut query = sqlx::query(&query).bind(get_current_timestamp_secs()).bind(created_before); + if let Some(admission_class) = admission_class { + query = query.bind(admission_class.to_string()); + } + Ok(query.execute(self.conn()).await?.rows_affected()) + } + /// Move inbox rows that repeatedly abandoned a claim out of the work set. /// /// `attempt_count` remains a pure diagnostic counter. Only `abandon_count` @@ -2917,6 +3064,85 @@ impl<'a> StorageProcessor<'a> { max_abandons: i64, excluded_message_ids: &[String], ) -> anyhow::Result> { + self.list_claimable_p2p_inbox_messages_in( + now, + limit, + 0, + max_abandons, + excluded_message_ids, + "", + ) + .await + } + + /// List each class's reserved share, then lend unused capacity in C/R/U order. + pub async fn list_claimable_p2p_inbox_messages_by_class( + &mut self, + now: i64, + limit: i64, + registered_reserve: i64, + unregistered_reserve: i64, + max_abandons: i64, + excluded_message_ids: &[String], + ) -> anyhow::Result> { + const COMMITTEE: &str = " AND admission_class = 'Committee'"; + const REGISTERED: &str = " AND admission_class = 'Registered'"; + const UNREGISTERED: &str = " AND admission_class NOT IN ('Committee', 'Registered')"; + + let limit = limit.max(0); + let unregistered_reserve = unregistered_reserve.clamp(0, limit); + let registered_reserve = registered_reserve.clamp(0, limit - unregistered_reserve); + let shares = [ + (COMMITTEE, limit - registered_reserve - unregistered_reserve), + (REGISTERED, registered_reserve), + (UNREGISTERED, unregistered_reserve), + ]; + + // Guaranteed shares first ... + let mut rows: [Vec; 3] = Default::default(); + for (index, (class, share)) in shares.iter().enumerate() { + rows[index] = self + .list_claimable_p2p_inbox_messages_in( + now, + *share, + 0, + max_abandons, + excluded_message_ids, + class, + ) + .await?; + } + // ... then whatever a class left of its share is lent out, in priority + // order, to the classes that have more to claim. + for (index, (class, _)) in shares.iter().enumerate() { + let free = limit - rows.iter().map(|rows| rows.len() as i64).sum::(); + let more = self + .list_claimable_p2p_inbox_messages_in( + now, + free, + rows[index].len() as i64, + max_abandons, + excluded_message_ids, + class, + ) + .await?; + rows[index].extend(more); + } + Ok(rows.into_iter().flatten().collect()) + } + + async fn list_claimable_p2p_inbox_messages_in( + &mut self, + now: i64, + limit: i64, + offset: i64, + max_abandons: i64, + excluded_message_ids: &[String], + class_predicate: &'static str, + ) -> anyhow::Result> { + if limit <= 0 { + return Ok(Vec::new()); + } let excluded_predicate = if excluded_message_ids.is_empty() { String::new() } else { @@ -2927,15 +3153,15 @@ impl<'a> StorageProcessor<'a> { FROM p2p_inbox \ WHERE ((state = 'Pending' AND next_retry_at <= ?) \ OR (state = 'Processing' AND lease_until <= ?)) \ - AND abandon_count < ?{excluded_predicate} \ - ORDER BY created_at ASC \ - LIMIT ?" + AND abandon_count < ?{excluded_predicate}{class_predicate} \ + ORDER BY created_at ASC, message_id ASC \ + LIMIT ? OFFSET ?" ); let mut query = sqlx::query(&query).bind(now).bind(now).bind(max_abandons); for message_id in excluded_message_ids { query = query.bind(message_id); } - let rows = query.bind(limit).fetch_all(self.conn()).await?; + let rows = query.bind(limit).bind(offset).fetch_all(self.conn()).await?; rows.iter() .map(p2p_inbox_message_from_row) .collect::, _>>() @@ -3137,6 +3363,240 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } + /// Delete the oldest terminal rows beyond `max_rows`, up to `limit` per call. + pub async fn purge_p2p_inbox_over_terminal_cap( + &mut self, + max_rows: i64, + limit: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "DELETE FROM p2p_inbox WHERE message_id IN ( \ + SELECT message_id FROM p2p_inbox \ + WHERE state IN ('Processed', 'Failed', 'Quarantined') \ + ORDER BY updated_at ASC \ + LIMIT MIN(?, MAX( \ + (SELECT COUNT(*) FROM p2p_inbox \ + WHERE state IN ('Processed', 'Failed', 'Quarantined')) - ?, 0)))", + ) + .bind(limit) + .bind(max_rows) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + + /// Clear the oldest Quarantined payloads until retained bytes are within `max_bytes`. + pub async fn trim_quarantined_p2p_inbox_payloads( + &mut self, + max_bytes: i64, + ) -> anyhow::Result { + let result = sqlx::query( + "UPDATE p2p_inbox SET content = X'', content_size = 0, updated_at = ? \ + WHERE state = 'Quarantined' AND length(content) > 0 AND message_id IN ( \ + SELECT message_id FROM ( \ + SELECT message_id, \ + SUM(content_size) OVER ( \ + ORDER BY updated_at DESC \ + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running \ + FROM p2p_inbox WHERE state = 'Quarantined' AND length(content) > 0 \ + ) WHERE running - content_size >= ?)", + ) + .bind(get_current_timestamp_secs()) + .bind(max_bytes) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + + /// Load up to `limit` operator bindings, confirmed operators first. + /// The caller re-verifies signatures. + pub async fn load_p2p_peer_bindings(&mut self, limit: i64) -> anyhow::Result> { + let rows = sqlx::query_as::<_, Node>( + "SELECT * FROM node \ + WHERE binding_sig != '' AND btc_pub_key != '' AND actor = 'Operator' \ + ORDER BY peer_id IN (SELECT peer_id FROM p2p_registered_peer \ + WHERE kind = 'Operator') DESC, \ + updated_at DESC \ + LIMIT ?", + ) + .bind(limit) + .fetch_all(self.conn()) + .await?; + Ok(rows) + } + + /// Persist confirmed registration; an operator key may belong to only one peer. + pub async fn upsert_p2p_registered_peer( + &mut self, + peer_id: &str, + kind: &str, + pubkey: &str, + ) -> anyhow::Result<()> { + if !pubkey.is_empty() { + sqlx::query( + "DELETE FROM p2p_registered_peer WHERE kind = ? AND pubkey = ? AND peer_id != ?", + ) + .bind(kind) + .bind(pubkey) + .bind(peer_id) + .execute(self.conn()) + .await?; + } + sqlx::query( + "INSERT INTO p2p_registered_peer (peer_id, kind, pubkey, verified_at) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (peer_id, kind) DO UPDATE SET pubkey = excluded.pubkey, \ + verified_at = excluded.verified_at", + ) + .bind(peer_id) + .bind(kind) + .bind(pubkey) + .bind(get_current_timestamp_secs()) + .execute(self.conn()) + .await?; + Ok(()) + } + + pub async fn delete_p2p_registered_peer( + &mut self, + peer_id: &str, + kind: &str, + ) -> anyhow::Result<()> { + sqlx::query("DELETE FROM p2p_registered_peer WHERE peer_id = ? AND kind = ?") + .bind(peer_id) + .bind(kind) + .execute(self.conn()) + .await?; + Ok(()) + } + + /// `(peer_id, kind, pubkey)` of every peer confirmed in an earlier session. + pub async fn load_p2p_registered_peers( + &mut self, + ) -> anyhow::Result> { + let rows = sqlx::query("SELECT peer_id, kind, pubkey FROM p2p_registered_peer") + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| Ok((row.try_get("peer_id")?, row.try_get("kind")?, row.try_get("pubkey")?))) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// Raise the persisted replay marks to `marks` (`peer_id -> highest sequence + /// number admitted`). A mark never moves down. + pub async fn raise_p2p_replay_marks(&mut self, marks: &[(String, i64)]) -> anyhow::Result<()> { + for (peer_id, highest) in marks { + sqlx::query( + "INSERT INTO p2p_replay_mark (peer_id, highest) VALUES (?, ?) \ + ON CONFLICT (peer_id) DO UPDATE SET highest = MAX(highest, excluded.highest)", + ) + .bind(peer_id) + .bind(highest) + .execute(self.conn()) + .await?; + } + Ok(()) + } + + pub async fn load_p2p_replay_marks(&mut self) -> anyhow::Result> { + let rows = sqlx::query("SELECT peer_id, highest FROM p2p_replay_mark") + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| Ok((row.try_get("peer_id")?, row.try_get("highest")?))) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// Remove marks for unregistered peers, retaining verifier peer IDs in `keep`. + pub async fn prune_p2p_replay_marks(&mut self, keep: &[String]) -> anyhow::Result { + let keep_predicate = if keep.is_empty() { + String::new() + } else { + format!(" AND peer_id NOT IN ({})", create_place_holders(keep)) + }; + let query = format!( + "DELETE FROM p2p_replay_mark \ + WHERE peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer){keep_predicate}" + ); + let mut query = sqlx::query(&query); + for peer_id in keep { + query = query.bind(peer_id); + } + Ok(query.execute(self.conn()).await?.rows_affected()) + } + + /// Allow existing peers or a new unregistered row below `max_rows`. + /// At capacity, evict the oldest unregistered row older than `evict_stale_before`. + pub async fn node_row_admissible( + &mut self, + peer_id: &str, + max_rows: i64, + evict_stale_before: i64, + local_peer_id: &str, + ) -> anyhow::Result { + let row = sqlx::query( + "SELECT EXISTS(SELECT 1 FROM node WHERE peer_id = ?) AS known, \ + (SELECT COUNT(*) FROM node \ + WHERE peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer)) AS total", + ) + .bind(peer_id) + .fetch_one(self.conn()) + .await?; + let known: i64 = row.try_get("known")?; + let total: i64 = row.try_get("total")?; + if known != 0 || total < max_rows { + return Ok(true); + } + let evicted = sqlx::query( + "DELETE FROM node WHERE peer_id = ( \ + SELECT peer_id FROM node \ + WHERE updated_at < ? AND peer_id != ? \ + AND peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer) \ + ORDER BY updated_at ASC LIMIT 1)", + ) + .bind(evict_stale_before) + .bind(local_peer_id) + .execute(self.conn()) + .await?; + Ok(evicted.rows_affected() > 0) + } + + /// Drop the persisted replay marks of `peer_ids`. + pub async fn delete_p2p_replay_marks(&mut self, peer_ids: &[String]) -> anyhow::Result { + if peer_ids.is_empty() { + return Ok(0); + } + let query = format!( + "DELETE FROM p2p_replay_mark WHERE peer_id IN ({})", + create_place_holders(peer_ids) + ); + let mut query = sqlx::query(&query); + for peer_id in peer_ids { + query = query.bind(peer_id); + } + Ok(query.execute(self.conn()).await?.rows_affected()) + } + + /// Delete stale node rows except this node and confirmed registered peers. + pub async fn purge_stale_unregistered_nodes( + &mut self, + updated_before: i64, + local_peer_id: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "DELETE FROM node \ + WHERE updated_at < ? AND peer_id != ? \ + AND peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer)", + ) + .bind(updated_before) + .bind(local_peer_id) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + pub async fn renew_p2p_inbox_lease( &mut self, message_id: &str, @@ -3264,6 +3724,8 @@ impl<'a> StorageProcessor<'a> { /// A non-empty `ack_peer_id` is the only ACK that may stop this retry /// loop early. Broadcast messages without an authenticated recipient keep /// publishing until their window expires. + /// `first_publish_at`: first outbox delivery time; zero makes the row immediately due. + #[allow(clippy::too_many_arguments)] pub async fn enqueue_p2p_outbox_retry_message( &mut self, message_id: &str, @@ -3272,17 +3734,19 @@ impl<'a> StorageProcessor<'a> { retry_until: i64, retry_interval_secs: i64, ack_peer_id: Option<&str>, + first_publish_at: i64, ) -> anyhow::Result { let now = get_current_timestamp_secs(); let result = sqlx::query( "INSERT INTO p2p_outbox \ (message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, retry_until, retry_interval_secs, ack_peer_id, created_at, updated_at) \ - VALUES (?, ?, ?, 'Pending', 0, 0, 0, ?, ?, ?, ?, ?) \ + VALUES (?, ?, ?, 'Pending', 0, ?, 0, ?, ?, ?, ?, ?) \ ON CONFLICT(message_id) DO NOTHING", ) .bind(message_id) .bind(msg_type) .bind(content) + .bind(first_publish_at) .bind(retry_until) .bind(retry_interval_secs) .bind(ack_peer_id.unwrap_or_default()) @@ -3293,6 +3757,7 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Claim due outbox rows in ascending `next_retry_at` order. pub async fn claim_p2p_outbox_messages( &mut self, now: i64, @@ -3300,12 +3765,12 @@ impl<'a> StorageProcessor<'a> { limit: i64, ) -> anyhow::Result> { let rows = sqlx::query( - "SELECT message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, last_error, retry_until, retry_interval_secs, ack_peer_id, created_at \ + "SELECT message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, last_error, retry_until, retry_interval_secs, ack_peer_id, publish_count, created_at \ FROM p2p_outbox \ WHERE ((state = 'Pending' AND next_retry_at <= ?) \ OR (state = 'Processing' AND lease_until <= ?) \ ) AND (retry_until = 0 OR retry_until > ?) \ - ORDER BY created_at ASC LIMIT ?", + ORDER BY next_retry_at ASC, created_at ASC LIMIT ?", ) .bind(now) .bind(now) @@ -3353,13 +3818,15 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } + /// Record a publish that went through and set when the row is next due. pub async fn schedule_p2p_outbox_retry( &mut self, message_id: &str, next_retry_at: i64, ) -> anyhow::Result { let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Pending', lease_until = 0, next_retry_at = ?, updated_at = ? \ + "UPDATE p2p_outbox SET state = 'Pending', lease_until = 0, next_retry_at = ?, \ + publish_count = publish_count + 1, updated_at = ? \ WHERE message_id = ? AND state = 'Processing' AND retry_until > 0", ) .bind(next_retry_at) @@ -3370,6 +3837,35 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Make a waiting row due at once. For a caller whose own publish of the + /// message failed: the row was scheduled on the assumption that it would not. + pub async fn expedite_p2p_outbox_message(&mut self, message_id: &str) -> anyhow::Result { + let result = sqlx::query( + "UPDATE p2p_outbox SET next_retry_at = 0, updated_at = ? \ + WHERE message_id = ? AND state = 'Pending' AND next_retry_at > 0", + ) + .bind(get_current_timestamp_secs()) + .bind(message_id) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + /// State and stored payload length of one outbox entry, for diagnostics and + /// tests. A terminal entry has dropped its payload. + pub async fn p2p_outbox_entry_state( + &mut self, + message_id: &str, + ) -> anyhow::Result> { + let row = sqlx::query( + "SELECT state, length(content) AS content_len FROM p2p_outbox WHERE message_id = ?", + ) + .bind(message_id) + .fetch_optional(self.conn()) + .await?; + row.map(|row| Ok((row.try_get("state")?, row.try_get("content_len")?))).transpose() + } + pub async fn acknowledge_p2p_outbox_message( &mut self, message_id: &str, @@ -3650,6 +4146,25 @@ impl<'a> StorageProcessor<'a> { Ok(row) } + /// Check graph data exists under both IDs without reading the blob. + pub async fn has_graph_of_instance( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + ) -> anyhow::Result { + let row = sqlx::query( + "SELECT 1 FROM graph \ + WHERE graph_id = ? AND instance_id = ? \ + AND EXISTS (SELECT 1 FROM graph_raw_data WHERE graph_raw_data.graph_id = graph.graph_id) \ + LIMIT 1", + ) + .bind(graph_id) + .bind(instance_id) + .fetch_optional(self.conn()) + .await?; + Ok(row.is_some()) + } + pub async fn find_watch_contract( &mut self, addr: &str, @@ -5015,9 +5530,534 @@ mod tests { assert_eq!(claimed[0].message_id, "hydrate-1"); } - /// A message is only charged an abandon when its previous claim never - /// reported an outcome. An ordinary deferred retry must not count, otherwise - /// a transient outage would drive healthy work into quarantine. + fn admission_inbox_message( + message_id: &str, + from_peer: &str, + admission_class: P2pInboxAdmissionClass, + content: Vec, + ) -> P2pInboxMessage { + P2pInboxMessage { + message_id: message_id.to_string(), + actor: "Committee".to_string(), + from_peer: from_peer.to_string(), + msg_type: "KickoffSent".to_string(), + content_size: content.len() as i64, + content_hash: Some(content.clone()), + content, + admission_class: admission_class.to_string(), + ..Default::default() + } + } + + /// Quota accounting must see what a sender currently makes the node store: + /// queued rows only, split by class, with the sender's own share. + #[tokio::test] + async fn inbox_usage_counts_queued_rows_per_class_and_sender() { + use P2pInboxAdmissionClass::{Registered, Unregistered}; + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for (message_id, from_peer, class, size) in [ + ("usage-1", "peer-a", Registered, 10), + ("usage-2", "peer-a", Unregistered, 20), + ("usage-3", "peer-b", Unregistered, 30), + ("usage-4", "peer-b", Unregistered, 40), + ] { + let message = admission_inbox_message(message_id, from_peer, class, vec![7; size]); + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + } + // A terminal row has dropped its payload and must stop counting. + let claimed = s.claim_p2p_inbox_message("usage-4", 100, 200).await.unwrap().unwrap(); + assert!(s.complete_p2p_inbox_message("usage-4", &claimed.lease_token).await.unwrap()); + // A claimed row still holds its payload. + assert!(s.claim_p2p_inbox_message("usage-3", 100, 200).await.unwrap().is_some()); + + let mut usage = s.p2p_inbox_usage("peer-b").await.unwrap(); + usage.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); + assert_eq!( + usage, + vec![ + P2pInboxClassUsage { + admission_class: "Registered".to_string(), + rows: 1, + bytes: 10, + peer_rows: 0, + peer_bytes: 0, + }, + P2pInboxClassUsage { + admission_class: "Unregistered".to_string(), + rows: 2, + bytes: 50, + peer_rows: 1, + peer_bytes: 30, + }, + ] + ); + } + + /// The usage query runs for every inbound message; it must be answered from + /// the covering index, never by reading rows that carry multi-megabyte blobs. + #[tokio::test] + async fn inbox_usage_is_answered_from_the_covering_index() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let plan: Vec = sqlx::query( + "EXPLAIN QUERY PLAN \ + SELECT admission_class, COUNT(*), COALESCE(SUM(content_size), 0), \ + COALESCE(SUM(CASE WHEN from_peer = 'p' THEN content_size ELSE 0 END), 0) \ + FROM p2p_inbox WHERE state IN ('Pending', 'Processing', 'Quarantined') \ + GROUP BY admission_class", + ) + .fetch_all(s.conn()) + .await + .unwrap() + .iter() + .map(|row| row.get::("detail")) + .collect(); + assert!( + plan.iter().any(|step| step.contains("COVERING INDEX idx_p2p_inbox_admission")), + "unexpected plan: {plan:?}" + ); + } + + /// A row written without a classification defaults to `Unregistered` rather + /// than silently joining the registered pool. + #[tokio::test] + async fn unclassified_inbox_rows_default_to_unregistered() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let message = P2pInboxMessage { + message_id: "unclassified-1".to_string(), + actor: "Operator".to_string(), + from_peer: "peer".to_string(), + msg_type: "CreateGraph".to_string(), + content: vec![1, 2, 3], + content_size: 3, + ..Default::default() + }; + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + let usage = s.p2p_inbox_usage("peer").await.unwrap(); + assert_eq!(usage.len(), 1); + assert_eq!(usage[0].admission_class, "Unregistered"); + assert!(!s.has_queued_p2p_inbox_payload("peer", &[1, 2, 3]).await.unwrap()); + } + + /// Verify quota accounting and cleanup of quarantined payloads and terminal rows. + #[tokio::test] + async fn quarantined_payloads_count_and_are_capped() { + use P2pInboxAdmissionClass::Unregistered; + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for (message_id, updated_at) in [("q-old", 100), ("q-new", 200)] { + let message = admission_inbox_message(message_id, "peer", Unregistered, vec![9; 1000]); + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + sqlx::query( + "UPDATE p2p_inbox SET state = 'Quarantined', updated_at = ? WHERE message_id = ?", + ) + .bind(updated_at) + .bind(message_id) + .execute(s.conn()) + .await + .unwrap(); + } + let usage = s.p2p_inbox_usage("peer").await.unwrap(); + assert_eq!(usage.iter().map(|u| u.bytes).sum::(), 2000, "quarantine counts"); + + // Keep only ~1000 bytes: the oldest quarantined payload is cleared. + assert_eq!(s.trim_quarantined_p2p_inbox_payloads(1000).await.unwrap(), 1); + let cleared: i64 = + sqlx::query("SELECT length(content) AS len FROM p2p_inbox WHERE message_id = 'q-old'") + .fetch_one(s.conn()) + .await + .unwrap() + .get("len"); + assert_eq!(cleared, 0, "the oldest payload was cleared"); + let kept: i64 = + sqlx::query("SELECT length(content) AS len FROM p2p_inbox WHERE message_id = 'q-new'") + .fetch_one(s.conn()) + .await + .unwrap() + .get("len"); + assert_eq!(kept, 1000, "the newest payload is retained"); + + // The row cap deletes the oldest terminal rows beyond the ceiling. + assert_eq!(s.purge_p2p_inbox_over_terminal_cap(1, 10).await.unwrap(), 1); + let remaining: i64 = sqlx::query("SELECT COUNT(*) AS n FROM p2p_inbox") + .fetch_one(s.conn()) + .await + .unwrap() + .get("n"); + assert_eq!(remaining, 1); + } + + /// Verify split usage queries match the combined query and use the covering index. + #[tokio::test] + async fn split_inbox_usage_matches_and_seeks_the_sender() { + use P2pInboxAdmissionClass::{Registered, Unregistered}; + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for (message_id, from_peer, class, size) in [ + ("split-1", "peer-a", Registered, 10), + ("split-2", "peer-a", Unregistered, 20), + ("split-3", "peer-b", Unregistered, 30), + ] { + let message = admission_inbox_message(message_id, from_peer, class, vec![7; size]); + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + } + let mut combined = s.p2p_inbox_usage("peer-a").await.unwrap(); + combined.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); + let mut totals = s.p2p_inbox_class_totals().await.unwrap(); + totals.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); + for peer in s.p2p_inbox_peer_usage("peer-a").await.unwrap() { + let total = + totals.iter_mut().find(|t| t.admission_class == peer.admission_class).unwrap(); + total.peer_rows = peer.peer_rows; + total.peer_bytes = peer.peer_bytes; + } + assert_eq!(totals, combined); + + let plan: Vec = sqlx::query( + "EXPLAIN QUERY PLAN \ + SELECT admission_class, COUNT(*), COALESCE(SUM(content_size), 0) \ + FROM p2p_inbox \ + WHERE state IN ('Pending', 'Processing', 'Quarantined') \ + AND admission_class IN ('Registered', 'Unregistered') \ + AND from_peer = 'p' \ + GROUP BY admission_class", + ) + .fetch_all(s.conn()) + .await + .unwrap() + .iter() + .map(|row| row.get::("detail")) + .collect(); + assert!( + plan.iter().any(|step| step.contains("COVERING INDEX idx_p2p_inbox_admission") + && step.contains("from_peer=?")), + "the sender's rows must be reached by an index seek: {plan:?}" + ); + } + + /// Registered rows take the batch ahead of older unregistered ones, but a + /// reserved share keeps the unregistered queue moving. + #[tokio::test] + async fn claimable_listing_drains_registered_rows_first() { + use P2pInboxAdmissionClass::{Committee, Registered, Unregistered}; + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let mut created_at = 100; + let mut insert = async |s: &mut StorageProcessor<'_>, id: String, class| { + let message = admission_inbox_message(&id, "peer", class, id.clone().into_bytes()); + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + sqlx::query("UPDATE p2p_inbox SET created_at = ? WHERE message_id = ?") + .bind(created_at) + .bind(&id) + .execute(s.conn()) + .await + .unwrap(); + created_at += 1; + }; + // The flood arrived first ... + for index in 0..8 { + insert(&mut s, format!("flood-{index}"), Unregistered).await; + } + // ... and the registered messages queue up behind it, the committee's + // last of all. + for index in 0..4 { + insert(&mut s, format!("member-{index}"), Registered).await; + } + for index in 4..6 { + insert(&mut s, format!("member-{index}"), Committee).await; + } + let ids = |rows: Vec| -> Vec { + rows.into_iter().map(|row| row.message_id).collect() + }; + + let plain = ids(s.list_claimable_p2p_inbox_messages(1000, 4, 3, &[]).await.unwrap()); + assert_eq!(plain, ["flood-0", "flood-1", "flood-2", "flood-3"], "oldest first"); + + let fair = + ids(s.list_claimable_p2p_inbox_messages_by_class(1000, 4, 1, 1, 3, &[]).await.unwrap()); + assert_eq!(fair, ["member-4", "member-5", "member-0", "flood-0"], "committee first"); + + // With fewer registered rows than the batch, the rest goes to the others. + let fair = ids(s + .list_claimable_p2p_inbox_messages_by_class(1000, 10, 2, 2, 3, &[]) + .await + .unwrap()); + assert_eq!(fair.len(), 10); + assert_eq!(fair.iter().filter(|id| id.starts_with("member-")).count(), 6); + assert_eq!(&fair[6..], ["flood-0", "flood-1", "flood-2", "flood-3"]); + + // A committee backlog that would fill every batch does not starve the + // other registered senders, nor the unregistered ones. + for index in 0..8 { + insert(&mut s, format!("committee-backlog-{index}"), Committee).await; + } + let fair = + ids(s.list_claimable_p2p_inbox_messages_by_class(1000, 4, 1, 1, 3, &[]).await.unwrap()); + assert_eq!(fair, ["member-4", "member-5", "member-0", "flood-0"]); + + // Lend unused reserved capacity to committee first. + let fair = ids(s + .list_claimable_p2p_inbox_messages_by_class(1000, 16, 6, 4, 3, &[]) + .await + .unwrap()); + assert_eq!(fair.len(), 16); + assert_eq!(fair.iter().filter(|id| id.starts_with("flood-")).count(), 4); + assert_eq!(fair.iter().filter(|id| id.starts_with("committee-backlog-")).count(), 6); + let mut unique = fair.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + unique.len(), + 16, + "borrowing continues a class's listing, it does not repeat it" + ); + } + + /// Verify confirmed operator bindings are loaded before unconfirmed identities. + #[tokio::test] + async fn confirmed_operator_bindings_are_loaded_first() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let operator = |peer_id: &str, updated_at: i64| Node { + peer_id: peer_id.to_string(), + actor: "Operator".to_string(), + btc_pub_key: format!("key-of-{peer_id}"), + binding_sig: "sig".to_string(), + binding_issued_at: 1, + updated_at, + created_at: updated_at, + ..Default::default() + }; + s.upsert_node(&operator("confirmed", 10)).await.unwrap(); + for index in 0..4 { + s.upsert_node(&operator(&format!("recent-{index}"), 900 + index)).await.unwrap(); + } + // Not an operator, and an operator without a binding: never loaded. + s.upsert_node(&Node { actor: "Committee".to_string(), ..operator("member", 999) }) + .await + .unwrap(); + s.upsert_node(&Node { binding_sig: String::new(), ..operator("unbound", 999) }) + .await + .unwrap(); + s.upsert_p2p_registered_peer("confirmed", "Operator", "aa").await.unwrap(); + + let loaded: Vec = s + .load_p2p_peer_bindings(2) + .await + .unwrap() + .into_iter() + .map(|node| node.peer_id) + .collect(); + assert_eq!(loaded, ["confirmed", "recent-3"]); + } + + /// A sync request names both ids. A real graph under another instance's id + /// is not a graph this node holds for that request. + #[tokio::test] + async fn graph_is_held_only_under_its_own_instance() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let (instance, graph, other) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + sqlx::query( + "INSERT INTO graph (graph_id, instance_id, status, created_at, updated_at) \ + VALUES (?, ?, 'Created', 1, 1)", + ) + .bind(graph) + .bind(instance) + .execute(s.conn()) + .await + .unwrap(); + assert!(!s.has_graph_of_instance(&instance, &graph).await.unwrap(), "no data stored yet"); + sqlx::query("INSERT INTO graph_raw_data (graph_id, raw_data) VALUES (?, 'x')") + .bind(graph) + .execute(s.conn()) + .await + .unwrap(); + assert!(s.has_graph_of_instance(&instance, &graph).await.unwrap()); + assert!(!s.has_graph_of_instance(&other, &graph).await.unwrap(), "wrong instance"); + assert!(!s.has_graph_of_instance(&instance, &other).await.unwrap(), "unknown graph"); + } + + #[tokio::test] + async fn registered_peers_persist_and_shield_their_node_rows() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + s.upsert_p2p_registered_peer("member", "Committee", "").await.unwrap(); + s.upsert_p2p_registered_peer("member", "Committee", "").await.unwrap(); + s.upsert_p2p_registered_peer("other-member", "Committee", "").await.unwrap(); + s.upsert_p2p_registered_peer("operator", "Operator", "aa").await.unwrap(); + let load = async |s: &mut StorageProcessor<'_>| { + let mut peers = s.load_p2p_registered_peers().await.unwrap(); + peers.sort(); + peers + }; + let row = |peer: &str, kind: &str, key: &str| { + (peer.to_string(), kind.to_string(), key.to_string()) + }; + assert_eq!( + load(&mut s).await, + [ + row("member", "Committee", ""), + row("operator", "Operator", "aa"), + row("other-member", "Committee", "") + ], + "keyless kinds never displace each other" + ); + // Move the key's registration to the new peer. + s.upsert_p2p_registered_peer("operator-moved", "Operator", "aa").await.unwrap(); + // A peer confirmed for another key keeps one row, for the new key. + s.upsert_p2p_registered_peer("operator-moved", "Operator", "bb").await.unwrap(); + assert_eq!( + load(&mut s).await, + [ + row("member", "Committee", ""), + row("operator-moved", "Operator", "bb"), + row("other-member", "Committee", "") + ] + ); + s.delete_p2p_registered_peer("operator-moved", "Operator").await.unwrap(); + s.delete_p2p_registered_peer("other-member", "Committee").await.unwrap(); + assert_eq!(load(&mut s).await.len(), 1); + + // Replay marks only move up, and are kept for registered peers and the + // verifiers the caller names. + s.raise_p2p_replay_marks(&[("member".into(), 50), ("verifier".into(), 7)]).await.unwrap(); + s.raise_p2p_replay_marks(&[("member".into(), 40), ("gone".into(), 9)]).await.unwrap(); + assert_eq!(s.prune_p2p_replay_marks(&["verifier".to_string()]).await.unwrap(), 1); + let mut marks = s.load_p2p_replay_marks().await.unwrap(); + marks.sort(); + assert_eq!(marks, [("member".to_string(), 50), ("verifier".to_string(), 7)]); + + for (peer_id, updated_at) in [("member", 10), ("sybil", 10), ("local", 10), ("live", 900)] { + s.upsert_node(&Node { + peer_id: peer_id.to_string(), + actor: "Operator".to_string(), + updated_at, + created_at: updated_at, + ..Default::default() + }) + .await + .unwrap(); + } + // Only unregistered rows count toward capacity; stale rows may be evicted. + assert!(s.node_row_admissible("sybil", 3, 0, "local").await.unwrap()); + assert!(s.node_row_admissible("newcomer", 4, 0, "local").await.unwrap()); + assert!( + !s.node_row_admissible("newcomer", 3, 5, "local").await.unwrap(), + "full, and every row was refreshed after the eviction horizon" + ); + assert!( + s.node_row_admissible("newcomer", 3, 500, "local").await.unwrap(), + "the stalest unregistered row makes room" + ); + let known = async |s: &mut StorageProcessor<'_>, peer_id: &str| { + s.node_row_admissible(peer_id, 0, 0, "local").await.unwrap() + }; + assert!(!known(&mut s, "sybil").await, "evicted: stale, unregistered, not this node"); + for kept in ["member", "local", "live"] { + assert!(known(&mut s, kept).await, "{kept}"); + } + + s.upsert_node(&Node { + peer_id: "left".to_string(), + actor: "Watchtower".to_string(), + updated_at: 20, + created_at: 20, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(s.purge_stale_unregistered_nodes(500, "local").await.unwrap(), 1); + assert!(!known(&mut s, "left").await); + assert!(known(&mut s, "member").await && known(&mut s, "local").await); + + s.raise_p2p_replay_marks(&[("locked-out".into(), 99)]).await.unwrap(); + assert_eq!(s.delete_p2p_replay_marks(&["locked-out".to_string()]).await.unwrap(), 1); + assert_eq!(s.delete_p2p_replay_marks(&[]).await.unwrap(), 0); + } + + /// Only a copy that is still queued suppresses a re-publish; once it reached + /// a terminal state the same payload may be delivered again. + #[tokio::test] + async fn queued_payload_lookup_ignores_other_senders_and_terminal_rows() { + use P2pInboxAdmissionClass::Registered; + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let message = admission_inbox_message("dup-1", "peer-a", Registered, vec![9; 8]); + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + + assert!(s.has_queued_p2p_inbox_payload("peer-a", &[9; 8]).await.unwrap()); + assert!(!s.has_queued_p2p_inbox_payload("peer-b", &[9; 8]).await.unwrap()); + assert!(!s.has_queued_p2p_inbox_payload("peer-a", &[8; 8]).await.unwrap()); + + let claimed = s.claim_p2p_inbox_message("dup-1", 100, 200).await.unwrap().unwrap(); + assert!( + s.has_queued_p2p_inbox_payload("peer-a", &[9; 8]).await.unwrap(), + "a claimed row is still in flight" + ); + assert!(s.complete_p2p_inbox_message("dup-1", &claimed.lease_token).await.unwrap()); + assert!(!s.has_queued_p2p_inbox_payload("peer-a", &[9; 8]).await.unwrap()); + } + + /// Verify Pending expiry uses `created_at`, not the last retry time. + #[tokio::test] + async fn pending_inbox_rows_expire_by_class_and_age() { + use P2pInboxAdmissionClass::{Registered, Unregistered}; + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for (message_id, class, created_at) in [ + ("expire-old-unregistered", Unregistered, 100), + ("expire-new-unregistered", Unregistered, 900), + ("expire-old-registered", Registered, 100), + ("expire-claimed-unregistered", Unregistered, 100), + ] { + let message = admission_inbox_message(message_id, "peer", class, vec![5; 16]); + assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); + sqlx::query( + "UPDATE p2p_inbox SET created_at = ?, updated_at = 5000 WHERE message_id = ?", + ) + .bind(created_at) + .bind(message_id) + .execute(s.conn()) + .await + .unwrap(); + } + let now = get_current_timestamp_secs(); + assert!( + s.claim_p2p_inbox_message("expire-claimed-unregistered", now, now + 300) + .await + .unwrap() + .is_some() + ); + + assert_eq!(s.expire_pending_p2p_inbox_messages(Some(Unregistered), 500).await.unwrap(), 1); + let states: Vec<(String, String, i64)> = sqlx::query( + "SELECT message_id, state, length(content) AS len FROM p2p_inbox ORDER BY message_id", + ) + .fetch_all(s.conn()) + .await + .unwrap() + .iter() + .map(|row| (row.get("message_id"), row.get("state"), row.get("len"))) + .collect(); + assert_eq!( + states, + vec![ + ("expire-claimed-unregistered".to_string(), "Processing".to_string(), 16), + ("expire-new-unregistered".to_string(), "Pending".to_string(), 16), + ("expire-old-registered".to_string(), "Pending".to_string(), 16), + ("expire-old-unregistered".to_string(), "Failed".to_string(), 0), + ] + ); + + // The class-independent ceiling reaches every remaining pending row. + assert_eq!(s.expire_pending_p2p_inbox_messages(None, 1000).await.unwrap(), 2); + assert_eq!(s.p2p_inbox_usage("peer").await.unwrap().iter().map(|u| u.rows).sum::(), 1); + } + + /// Verify only unfinished claims increment abandon_count. #[tokio::test] async fn test_inbox_abandon_is_charged_only_for_unfinished_claims() { let db = setup_db().await; @@ -5076,9 +6116,7 @@ mod tests { assert_eq!(claimed[0].abandon_count, 0, "a reported outcome resets consecutive abandons"); } - /// A payload that keeps taking the node down is quarantined rather than - /// dispatched again. Its content remains available for manual requeue until - /// terminal-row cleanup removes it. + /// Verify repeated abandonment quarantines the retained payload. #[tokio::test] async fn test_inbox_quarantines_repeatedly_abandoned_message() { let db = setup_db().await; @@ -5131,9 +6169,7 @@ mod tests { assert_eq!(requeued[0].abandon_count, 0); } - /// The local queue used to hand out work without writing anything, so a - /// handler that panicked left the row immediately claimable again. A claim - /// must hold the message for the length of its lease. + /// Verify local claims hold a lease. #[tokio::test] async fn test_local_claim_holds_lease_and_charges_abandon() { let db = setup_db().await; diff --git a/crates/store/src/schema.rs b/crates/store/src/schema.rs index de2451e6..3fdd0acb 100644 --- a/crates/store/src/schema.rs +++ b/crates/store/src/schema.rs @@ -166,6 +166,11 @@ pub struct Node { pub reward: String, pub service_fee_rate: f64, pub available_peg_btc: String, + /// Hex master-key signature authorizing this peer binding; empty when absent. + pub binding_sig: String, + /// When the binding was issued. Orders re-bindings so one key maps to a + /// single peer id at a time. + pub binding_issued_at: i64, pub updated_at: i64, pub created_at: i64, } @@ -585,10 +590,46 @@ pub struct P2pInboxMessage { pub lease_until: i64, pub lease_token: String, pub last_error: Option, + /// How the sender was classified when the row was admitted. See + /// [`P2pInboxAdmissionClass`]; empty means the caller did not classify it. + pub admission_class: String, + /// Digest of `content`, used to collapse an identical payload re-published + /// by the same sender while the first copy is still queued. + pub content_hash: Option>, pub created_at: i64, pub updated_at: i64, } +/// Sender classification recorded on an inbox row for quota accounting. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Display, EnumString)] +pub enum P2pInboxAdmissionClass { + /// Committee member registered by peer ID on chain. + Committee, + /// A verifier (on-chain peer id registry) or an operator (staked master key + /// bound to the peer id by a NodeInfo binding proof). + Registered, + /// Anyone else, including anonymous peers. + Unregistered, +} + +impl P2pInboxAdmissionClass { + /// Whether the sender's identity is backed by an on-chain registration. + pub const fn is_registered(self) -> bool { + !matches!(self, Self::Unregistered) + } +} + +/// Queued (`Pending` + `Processing` + `Quarantined`) inbox usage for one +/// admission class, with the share attributable to the sender being admitted. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct P2pInboxClassUsage { + pub admission_class: String, + pub rows: i64, + pub bytes: i64, + pub peer_rows: i64, + pub peer_bytes: i64, +} + #[derive(Clone, FromRow, Debug, Serialize, Deserialize, Default)] pub struct P2pOutboxMessage { pub message_id: String, @@ -602,6 +643,8 @@ pub struct P2pOutboxMessage { pub retry_until: i64, pub retry_interval_secs: i64, pub ack_peer_id: String, + /// Publishes that went through, unlike `attempt_count`, which counts claims. + pub publish_count: i64, pub created_at: i64, } diff --git a/node/src/action.rs b/node/src/action.rs index f34bdff2..84af9f4a 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -798,6 +798,8 @@ pub async fn enqueue_graph_setup_outbox_message( now + get_p2p_graph_setup_retry_window_secs(), get_p2p_graph_setup_retry_interval_secs(), ack_peer_id, + // Only the outbox ever sends these, so the row is due at once. + 0, ) .await?; Ok(outbox_id) @@ -1007,6 +1009,61 @@ pub struct NodeInfo { pub node_name: String, pub service_fee_rate: f64, pub available_peg_btc: String, + /// Hex master-key Schnorr signature over peer ID, public key and issuance time. + #[serde(default)] + pub binding_sig: String, + /// Unix seconds when the binding was signed. Orders re-bindings so one key + /// maps to a single peer id at a time. + #[serde(default)] + pub binding_issued_at: i64, +} + +const NODE_INFO_BINDING_DOMAIN: &[u8] = b"bitvm2-node/node-info-binding/v1"; + +fn node_info_binding_message(peer_id: &str, btc_pub_key: &str, issued_at: i64) -> SecpMessage { + let mut hasher = Sha256::new(); + hasher.update(NODE_INFO_BINDING_DOMAIN); + hasher.update((peer_id.len() as u32).to_be_bytes()); + hasher.update(peer_id.as_bytes()); + hasher.update((btc_pub_key.len() as u32).to_be_bytes()); + hasher.update(btc_pub_key.as_bytes()); + hasher.update(issued_at.to_be_bytes()); + SecpMessage::from_digest(hasher.finalize().into()) +} + +/// Sign the binding that proves `master_keypair`'s key authorised `peer_id`. +pub fn sign_node_info_binding( + peer_id: &str, + btc_pub_key: &str, + issued_at: i64, + master_keypair: &Keypair, +) -> String { + let signature = SECP256K1 + .sign_schnorr(&node_info_binding_message(peer_id, btc_pub_key, issued_at), master_keypair); + hex::encode(signature.serialize()) +} + +/// Verify the NodeInfo binding and return its parsed public key. +pub fn verify_node_info_binding(node_info: &NodeInfo) -> Option { + if node_info.binding_sig.is_empty() { + return None; + } + let pubkey = PublicKey::from_str(&node_info.btc_pub_key).ok()?; + let xonly = XOnlyPublicKey::from(pubkey); + let signature_bytes = hex::decode(&node_info.binding_sig).ok()?; + let signature = SchnorrSignature::from_slice(&signature_bytes).ok()?; + SECP256K1 + .verify_schnorr( + &signature, + &node_info_binding_message( + &node_info.peer_id, + &node_info.btc_pub_key, + node_info.binding_issued_at, + ), + &xonly, + ) + .ok() + .map(|()| xonly) } #[derive(Serialize, Deserialize, Clone)] diff --git a/node/src/bin/mock_rpc.rs b/node/src/bin/mock_rpc.rs index cff02fca..21cd0526 100644 --- a/node/src/bin/mock_rpc.rs +++ b/node/src/bin/mock_rpc.rs @@ -131,6 +131,8 @@ fn seeded_node( reward: "0".to_string(), service_fee_rate: 0.001, available_peg_btc: U256::from(4_700_000_000_000_000_000_000_000_u128).to_string(), + binding_sig: String::new(), + binding_issued_at: 0, updated_at: now, created_at: now, } diff --git a/node/src/env.rs b/node/src/env.rs index 8d4abb55..73ec519d 100644 --- a/node/src/env.rs +++ b/node/src/env.rs @@ -18,6 +18,7 @@ use reqwest::Url; use sha2::{Digest, Sha256}; use std::path::PathBuf; use std::str::FromStr; +use std::time::SystemTime; use strum::{Display, EnumString}; use tracing::{info, warn}; use util::hex_parse; @@ -348,6 +349,20 @@ pub fn get_local_node_info() -> NodeInfo { addr_op }; let socket_addr = std::env::var(ENV_EXTERNAL_SOCKET_ADDR).unwrap_or("".to_string()); + // Sign this node's peer ID binding with its master key. + let issued_at = SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs() as i64) + .unwrap_or_default(); + let binding_sig = match get_bitvm_key() { + Ok(keypair) => { + crate::action::sign_node_info_binding(&peer_key, &pubkey_str, issued_at, &keypair) + } + Err(error) => { + tracing::error!("failed to sign node info binding: {error}"); + String::new() + } + }; NodeInfo { peer_id: peer_key, actor: actor.to_string(), @@ -357,6 +372,8 @@ pub fn get_local_node_info() -> NodeInfo { node_name: get_node_name(), service_fee_rate: get_operator_node_service_fee_rate(), available_peg_btc: "0".to_string(), + binding_sig, + binding_issued_at: issued_at, } } pub fn get_committee_member_num() -> usize { diff --git a/node/src/p2p_msg_handler.rs b/node/src/p2p_msg_handler.rs index d8ccdd86..55bca8fa 100644 --- a/node/src/p2p_msg_handler.rs +++ b/node/src/p2p_msg_handler.rs @@ -266,6 +266,7 @@ mod tests { node_name: "".to_string(), service_fee_rate: 0.0, available_peg_btc: "0".to_string(), + ..Default::default() }, ) .await diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index 94fdfc6c..621f1799 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -816,6 +816,8 @@ mod tests { .to_string(), updated_at: current_time_secs(), created_at: current_time_secs(), + binding_sig: String::new(), + binding_issued_at: 0, }); let goat_addr = get_rand_goat_address(); nodes.push(Node { @@ -832,6 +834,8 @@ mod tests { .to_string(), updated_at: current_time_secs(), created_at: current_time_secs(), + binding_sig: String::new(), + binding_issued_at: 0, }); let local_db = create_local_db(&temp_sqlite_db_path()).await; diff --git a/node/src/utils.rs b/node/src/utils.rs index 55858e0e..3827ace6 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -4079,7 +4079,8 @@ fn validate_node_socket_addr(socket_addr: &str) -> std::result::Result<(), Strin } pub async fn save_node_info(local_db: &LocalDB, node_info: &NodeInfo) -> Result<()> { - info!("save_node_info for {}", node_info.peer_id); + // Reachable once per announcement from any peer: not louder than debug. + tracing::debug!("save_node_info for {}", node_info.peer_id); let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; let mut storage_process = local_db.acquire().await?; storage_process @@ -4093,6 +4094,8 @@ pub async fn save_node_info(local_db: &LocalDB, node_info: &NodeInfo) -> Result< reward: "0".to_string(), service_fee_rate: node_info.service_fee_rate, available_peg_btc: node_info.available_peg_btc.clone(), + binding_sig: node_info.binding_sig.clone(), + binding_issued_at: node_info.binding_issued_at, updated_at: current_time, created_at: current_time, }) @@ -4117,7 +4120,7 @@ pub async fn save_local_info(local_db: &LocalDB) { } pub async fn update_node_timestamp(local_db: &LocalDB, peer_id: &str) -> Result<()> { - tracing::info!("update timestamp for {peer_id}"); + tracing::debug!("update timestamp for {peer_id}"); let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; let mut storage_process = local_db.acquire().await?; match storage_process.update_node_timestamp(peer_id, current_time).await { @@ -6573,6 +6576,7 @@ mod node_info_tests { node_name: "zkm".to_string(), service_fee_rate: 0.001, available_peg_btc: "1000".to_string(), + ..Default::default() } } From d917fbd795ffb92120f3e3d18a3e318c5a546947 Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 21 Sep 2026 22:57:28 +0800 Subject: [PATCH 11/17] feat(p2p): harden admission and isolate message workers --- node/src/action.rs | 1978 ++++++++++-- node/src/env.rs | 164 +- node/src/handle.rs | 255 +- node/src/lib.rs | 1 + node/src/main.rs | 28 +- node/src/middleware/behaviour.rs | 57 +- node/src/middleware/connection_gate.rs | 246 ++ node/src/middleware/mod.rs | 2 + node/src/middleware/publisher.rs | 114 + node/src/middleware/swarm.rs | 27 +- node/src/p2p_admission.rs | 4087 ++++++++++++++++++++++++ node/src/p2p_msg_handler.rs | 580 +++- node/src/utils.rs | 37 +- 13 files changed, 7275 insertions(+), 301 deletions(-) create mode 100644 node/src/middleware/connection_gate.rs create mode 100644 node/src/middleware/publisher.rs create mode 100644 node/src/p2p_admission.rs diff --git a/node/src/action.rs b/node/src/action.rs index 84af9f4a..f6061bf6 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -3,8 +3,8 @@ #![allow(clippy::collapsible_else_if)] use crate::env::{ - MESSAGE_EXPIRE_TIME, get_local_node_info, get_p2p_graph_setup_retry_interval_secs, - get_p2p_graph_setup_retry_window_secs, get_p2p_inbox_batch_size, get_p2p_outbox_batch_size, + MESSAGE_EXPIRE_TIME, get_p2p_graph_setup_retry_interval_secs, + get_p2p_graph_setup_retry_window_secs, get_p2p_inbox_batch_size, }; use crate::handle::{ HandlerContext, HeavyTaskContext, dispatch as handle_dispatch, heavy_task_from_content, @@ -12,6 +12,7 @@ use crate::handle::{ }; use crate::metrics_service::MetricsState; use crate::middleware::AllBehaviours; +use crate::middleware::publisher::MessagePublisher; use crate::rpc_service::current_time_secs; use crate::utils::*; use alloy::primitives::Address as EvmAddress; @@ -42,7 +43,7 @@ use std::str::FromStr; use std::sync::{Arc, LazyLock, Mutex}; use std::time::{Duration, Instant}; use store::localdb::LocalDB; -use store::{MessageState, P2pInboxMessage}; +use store::{GraphStatus, InstanceBridgeInStatus, MessageState, P2pInboxMessage}; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -74,6 +75,32 @@ const LOCAL_MESSAGE_BATCH_SIZE: i64 = 50; /// supervisor restart after a panic or an unclean exit does not replay it at /// full speed. const QUEUE_ABANDON_BACKOFF_SECS: i64 = 60; +/// Maximum retained Quarantined payload bytes. +const P2P_INBOX_QUARANTINE_MAX_BYTES: i64 = 256 * 1024 * 1024; +/// Ceiling on terminal (`Processed`/`Failed`/`Quarantined`) inbox rows kept for +/// diagnostics; the oldest beyond it are deleted. +const P2P_INBOX_TERMINAL_MAX_ROWS: i64 = 100_000; +/// Maximum terminal rows removed per cleanup pass. +const P2P_INBOX_TERMINAL_PURGE_BATCH: i64 = 4_096; +/// Minimum listed-batch shares for Registered and Unregistered senders. +const P2P_INBOX_UNREGISTERED_BATCH_DIVISOR: i64 = 4; +const P2P_INBOX_REGISTERED_BATCH_DIVISOR: i64 = 4; +/// A `node` row of an unregistered peer may be evicted to make room for a new +/// one once it has gone this long without a refresh: three missed heartbeats. +pub const NODE_TABLE_EVICTABLE_AFTER_SECS: i64 = 3 * crate::env::HEARTBEAT_INTERVAL_SECOND as i64; +/// Maximum unregistered node rows. +pub const NODE_TABLE_MAX_UNREGISTERED_ROWS: i64 = 4_096; +/// A `node` row of a peer that is not registered is deleted once it has not been +/// refreshed for this long; live nodes refresh theirs on every heartbeat. +const NODE_TABLE_STALE_SECS: i64 = 7 * 24 * 60 * 60; +const NODE_TABLE_PURGE_INTERVAL: Duration = Duration::from_secs(60 * 60); +/// How long each durable queue may start work in a business-worker tick. +/// Individual handlers have a separate timeout; swarm runs on another thread. +const P2P_INBOX_DRAIN_BUDGET: Duration = Duration::from_secs(4); +/// Initial signing-round retry delay; doubles after successful publishes up to the configured ceiling. +const PROTOCOL_RETRY_BASE_SECS: i64 = 30; +/// Maximum accepted binding timestamp lead. +pub const NODE_INFO_BINDING_MAX_FUTURE_SECS: i64 = 10 * 60; /// A dispatch future erased behind a box to keep the enclosing task's state /// machine reasonably small. type BoxedDispatch<'a> = std::pin::Pin> + 'a>>; @@ -89,8 +116,58 @@ struct LocalMessageClaim { message_version: i64, } +/// Identify the current inbox message by kind, business reference and qualifier. +#[derive(Clone, Debug, Eq, PartialEq)] +struct DispatchFingerprint { + kind: MessageKind, + business_ref: BusinessRef, + qualifier: MessageQualifier, +} + +impl DispatchFingerprint { + fn of(content: &GOATMessageContent) -> Self { + Self { + kind: content.kind(), + business_ref: content.business_ref(), + qualifier: content.qualifier(), + } + } +} + +struct InboxDispatchContext { + fingerprint: DispatchFingerprint, + retry: std::cell::RefCell>, +} + tokio::task_local! { static ACTIVE_LOCAL_MESSAGE_CLAIM: LocalMessageClaim; + static ACTIVE_INBOX_DISPATCH: InboxDispatchContext; +} + +async fn track_inbox_retry( + fingerprint: DispatchFingerprint, + dispatch: impl std::future::Future>, +) -> Result<()> { + ACTIVE_INBOX_DISPATCH + .scope(InboxDispatchContext { fingerprint, retry: Default::default() }, async { + let result = dispatch.await; + // Apply self-deferral after handler compensation, including when compensation returns an error. + match ACTIVE_INBOX_DISPATCH.with(|active| active.retry.borrow_mut().take()) { + Some(retry) => { + if let Err(error) = &result { + tracing::warn!( + event = "p2p_inbox", + outcome = "deferred_despite_error", + error = %error, + "handler failed after asking to be retried; keeping the retry" + ); + } + Err(retry) + } + None => result, + } + }) + .await } fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String { @@ -576,6 +653,78 @@ impl MessageKind { } } +impl MessageKind { + /// Receiver roles handled by `dispatch` and `heavy_task_from_content`. + pub fn handled_by(self, actor: &Actor) -> bool { + use Actor::{Committee, Operator, Verifier, Watchtower}; + match self { + // Every role has an arm (a role-specific one or the default). + Self::PeginRequest + | Self::ConfirmInstance + | Self::GraphSetupAck + | Self::GraphFinalize + | Self::KickoffSent + | Self::PreKickoffSent + | Self::ChallengeSent + | Self::DisproveSent + | Self::Take1Sent + | Self::Take2Sent + | Self::RequestNodeInfo + | Self::ResponseNodeInfo + | Self::SyncGraph + // Not dispatched through the role table; keep them as they are. + | Self::InstanceDiscarded + | Self::Tick => true, + Self::InitGraph | Self::CutCircuits => matches!(actor, Verifier), + Self::CreateGraph => matches!(actor, Verifier | Committee), + Self::GenCircuits | Self::SolderingProofReady | Self::EndorseGraph => { + matches!(actor, Operator) + } + Self::NonceGeneration | Self::CommitteePresign => matches!(actor, Committee | Operator), + Self::VerifierGraphParamsEndorsement + | Self::AggNonceConsensus + | Self::PeginConfirmNonce + | Self::PeginConfirmNonceConsensus + | Self::PeginConfirmPartialSig + | Self::PostReady + | Self::SyncGraphRequest => matches!(actor, Committee), + Self::KickoffReady + | Self::WatchtowerChallengeSent + | Self::WatchtowerChallengeTimeout + | Self::OperatorCommitPubinReady + | Self::AssertReady + | Self::ChallengeAssertSent + | Self::Take1Ready + | Self::Take2Ready => matches!(actor, Operator), + Self::WatchtowerChallengeInitSent => matches!(actor, Watchtower), + Self::NackReady + | Self::OperatorCommitPubinTimeout + | Self::AssertSent + | Self::WronglyChallengeTimeout => matches!(actor, Verifier), + } + } +} + +impl MessageKind { + /// Sender roles verified against chain registration; other message kinds use `Any`. + pub fn sender_role(self) -> crate::p2p_admission::SenderRole { + use crate::p2p_admission::SenderRole; + match self { + Self::NonceGeneration + | Self::AggNonceConsensus + | Self::CommitteePresign + | Self::EndorseGraph + | Self::PeginConfirmNonce + | Self::PeginConfirmNonceConsensus + | Self::PeginConfirmPartialSig + | Self::SyncGraph => SenderRole::Committee, + Self::GenCircuits | Self::VerifierGraphParamsEndorsement => SenderRole::Verifier, + Self::InitGraph => SenderRole::Operator, + _ => SenderRole::Any, + } + } +} + impl GOATMessageContent { pub fn kind(&self) -> MessageKind { self.into() @@ -585,11 +734,9 @@ impl GOATMessageContent { /// immediate processing when they are safe to drop. pub const fn p2p_delivery(&self) -> P2PMessageDelivery { match self { - Self::RequestNodeInfo(_) - | Self::ResponseNodeInfo(_) - | Self::SyncGraphRequest(_) - | Self::SyncGraph(_) - | Self::GraphSetupAck(_) => P2PMessageDelivery::Immediate, + Self::RequestNodeInfo(_) | Self::ResponseNodeInfo(_) | Self::GraphSetupAck(_) => { + P2PMessageDelivery::Immediate + } _ => P2PMessageDelivery::Inbox, } } @@ -775,7 +922,7 @@ fn graph_setup_ack(content: &GOATMessageContent) -> Option { instance_id, graph_id, stage, - acknowledger_peer_id: get_local_node_info().peer_id, + acknowledger_peer_id: crate::env::get_peer_id(), }) } @@ -1115,6 +1262,11 @@ impl GOATMessage { .await? } + /// Whether `message` uses the bincode envelope, which only `GenCircuits` does. + pub fn is_binary_envelope(message: &[u8]) -> bool { + message.starts_with(GOAT_MESSAGE_BIN_PREFIX) + } + pub async fn deserialize_message(message: &[u8]) -> Result { let cloned = message.to_vec(); tokio::task::spawn_blocking(move || { @@ -1129,96 +1281,269 @@ impl GOATMessage { } } -/// Decode an externally received P2P message and route it by delivery semantics. +/// Report exactly one forwarding verdict for each received gossipsub message. +fn report_gossip_validation( + swarm: &mut Swarm, + id: &MessageId, + propagation_source: &PeerId, + acceptance: gossipsub::MessageAcceptance, +) { + let cached = swarm.behaviour_mut().gossipsub.report_message_validation_result( + id, + propagation_source, + acceptance, + ); + if !cached { + tracing::debug!( + event = "p2p_message", + outcome = "validation_report_missed", + message_id = %hex::encode(&id.0), + "gossipsub had already evicted the message awaiting validation" + ); + } +} + +/// Apply admission before routing; a dropped message is not an error. #[allow(clippy::too_many_arguments)] pub async fn handle_inbound_p2p_message( swarm: &mut Swarm, local_db: &LocalDB, - btc_client: &Arc, + _btc_client: &Arc, goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, + _http_client: &HttpAsyncClient, + _soldering_builder: &Option>, actor: Actor, from_peer_id: PeerId, + propagation_source: PeerId, + sequence_number: Option, id: MessageId, message: &[u8], metrics_state: &MetricsState, + worker: &crate::p2p_msg_handler::WorkerControl, ) -> Result<()> { - let decoded = match GOATMessage::deserialize_message(message).await { - Ok(message) => { - metrics_state.record_p2p_receive(true); - message - } + use crate::p2p_admission::{self, InboundVerdict}; + use gossipsub::MessageAcceptance; + + let now = Instant::now(); + p2p_admission::refresh_registry_in_background(local_db, goat_client, Some(from_peer_id)); + let verdict = p2p_admission::evaluate_inbound_message( + local_db, + &p2p_admission::AdmissionGates::global(&actor), + &p2p_admission::InboundGossip { + direct_peer: &propagation_source, + source: &from_peer_id, + sequence_number, + data: message, + }, + now, + ) + .await; + let verdict = match verdict { + Ok(verdict) => verdict, Err(error) => { - metrics_state.record_p2p_receive(false); - return Err(error).context("decode inbound P2P message"); + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Ignore); + return Err(error).context("evaluate inbound P2P message admission"); } }; - if let Err(error) = update_node_timestamp(local_db, &from_peer_id.to_string()).await { - tracing::warn!( - event = "p2p_message", - outcome = "peer_timestamp_update_failed", - message_id = %hex::encode(&id.0), - from_peer_id = %from_peer_id, - error = %error, - "received inbound P2P message but failed to update peer timestamp" - ); - } - - match decoded.content.p2p_delivery() { - P2PMessageDelivery::Inbox => { - enqueue_p2p_message(local_db, actor, from_peer_id, id, message, &decoded).await?; - if let Some(ack) = graph_setup_ack(&decoded.content) { - // ACKs are deliberately ephemeral. A duplicate delivery is - // acknowledged again, which lets the sender recover when its - // previous ACK was dropped. - if let Err(error) = send_to_peer( - swarm, - GOATMessage::new(Actor::All, GOATMessageContent::GraphSetupAck(ack)), - ) - .await - { - tracing::debug!( - event = "p2p_graph_setup_ack", - outcome = "publish_failed", - error = %error, - "inbound graph-setup message remains durable; sender will retry" - ); - } + match verdict { + InboundVerdict::Drop(reason) => { + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Ignore); + if reason.blames_direct_peer() { + strike_direct_peer(swarm, local_db, goat_client, propagation_source, now); + } + if reason == p2p_admission::DropReason::Undecodable { + metrics_state.record_p2p_receive(false); + } else if reason.after_decode() { + metrics_state.record_p2p_receive(true); } + p2p_admission::record_drop(reason); Ok(()) } - P2PMessageDelivery::Immediate => { - tracing::debug!( - event = "p2p_message", - delivery = "immediate", - message_id = %hex::encode(&id.0), - message_type = decoded.content.event_type(), - from_peer_id = %from_peer_id, - content_size = message.len(), - "dispatching ephemeral P2P message" - ); - dispatch_decoded_p2p_message( - swarm, + // Forward messages for other roles without persistence or ACK. + InboundVerdict::Forward => { + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + Ok(()) + } + // Persist durable messages before allowing forwarding. + InboundVerdict::Enqueue { message: decoded, class, content_hash } => { + if let Err(error) = enqueue_p2p_message( local_db, - btc_client, - goat_client, - http_client, - soldering_builder, actor, from_peer_id, - id, - decoded, - metrics_state, + id.clone(), + message, + &decoded, + class, + content_hash, ) .await + { + report_gossip_validation( + swarm, + &id, + &propagation_source, + MessageAcceptance::Ignore, + ); + return Err(error); + } + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, now).await; + Ok(()) + } + // Forward duplicates with rate-limited ACKs and logs. + InboundVerdict::Duplicate(decoded) => { + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + p2p_admission::record_drop(p2p_admission::DropReason::DuplicatePayload); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, now).await; + Ok(()) + } + InboundVerdict::Immediate(decoded) => { + if !worker.enqueue(from_peer_id, id.clone(), decoded, message.len()) { + report_gossip_validation( + swarm, + &id, + &propagation_source, + MessageAcceptance::Ignore, + ); + p2p_admission::record_drop(p2p_admission::DropReason::ControlQueueFull); + return Ok(()); + } + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + Ok(()) + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn dispatch_immediate_message( + publisher: &mut dyn MessagePublisher, + local_db: &LocalDB, + btc_client: &Arc, + goat_client: &Arc, + http_client: &HttpAsyncClient, + soldering_builder: &Option>, + actor: Actor, + from_peer_id: PeerId, + id: MessageId, + message: GOATMessage, + metrics_state: &MetricsState, +) -> Result<()> { + let kind = message.content.event_type(); + match tokio::time::timeout( + crate::env::get_p2p_handler_timeout(), + dispatch_decoded_p2p_message( + publisher, + local_db, + btc_client, + goat_client, + http_client, + soldering_builder, + actor, + from_peer_id, + id, + message, + metrics_state, + ), + ) + .await + { + Ok(result) => result, + Err(_) => { + tracing::warn!( + event = "p2p_message", + outcome = "handler_timeout", + message_type = kind, + "ephemeral message exceeded business worker budget" + ); + Ok(()) } } } +/// Count relay strikes and temporarily ban repeat offenders. +/// Registered peers and peers with unresolved registration are exempt. +fn strike_direct_peer( + swarm: &mut Swarm, + local_db: &LocalDB, + goat_client: &Arc, + direct_peer: PeerId, + now: Instant, +) { + let registry = crate::p2p_admission::peer_registry(); + if !registry.is_known_unregistered(&direct_peer, now) { + crate::p2p_admission::refresh_neighbour_in_background(local_db, goat_client, direct_peer); + return; + } + if !crate::p2p_admission::direct_peer_strikes().strike(&direct_peer, now) { + return; + } + // The strike table now lists the peer as banned, and the connection gate + // consults it for every connection: closing this one is all that is left. + let _ = swarm.disconnect_peer_id(direct_peer); + tracing::warn!( + event = "p2p_admission", + outcome = "direct_peer_banned", + peer_id = %direct_peer, + "temporarily ignoring a neighbour that keeps relaying invalid messages" + ); +} + +/// Refresh peer liveness at most once per cooldown. +async fn refresh_peer_timestamp_throttled(local_db: &LocalDB, from_peer_id: PeerId, now: Instant) { + let peer = from_peer_id.to_string(); + if !crate::p2p_admission::peer_timestamp_gate().allow(&peer, now) { + return; + } + if let Err(error) = update_node_timestamp(local_db, &peer).await { + tracing::debug!( + event = "p2p_message", + outcome = "peer_timestamp_update_failed", + from_peer_id = %peer, + error = %error, + "failed to refresh peer liveness timestamp" + ); + } +} + +/// Rate-limit graph-setup ACKs by sender and outbox slot. +async fn maybe_send_graph_setup_ack( + swarm: &mut dyn MessagePublisher, + decoded: &GOATMessage, + from_peer_id: PeerId, + now: Instant, +) { + let Some(ack) = graph_setup_ack(&decoded.content) else { + return; + }; + let key = format!("{from_peer_id}:{}", ack.outbox_id); + if !crate::p2p_admission::dedup_ack_gate().allow(&key, now) { + return; + } + if let Err(error) = + send_to_peer(swarm, GOATMessage::new(Actor::All, GOATMessageContent::GraphSetupAck(ack))) + .await + { + tracing::debug!( + event = "p2p_graph_setup_ack", + outcome = "publish_failed", + error = %error, + "inbound graph-setup message remains durable; sender will retry" + ); + } +} + /// Persist a decoded durable P2P message. Protocol work runs from the inbox on /// a regular tick. +#[allow(clippy::too_many_arguments)] async fn enqueue_p2p_message( local_db: &LocalDB, actor: Actor, @@ -1226,6 +1551,8 @@ async fn enqueue_p2p_message( id: MessageId, message: &[u8], decoded: &GOATMessage, + admission_class: store::P2pInboxAdmissionClass, + content_hash: [u8; 32], ) -> Result<()> { let message_id = hex::encode(&id.0); let inbox_message = P2pInboxMessage { @@ -1236,6 +1563,8 @@ async fn enqueue_p2p_message( msg_type: decoded.content.event_type().to_owned(), content: message.to_vec(), content_size: message.len() as i64, + admission_class: admission_class.to_string(), + content_hash: Some(content_hash.to_vec()), ..Default::default() }; let mut inserted = None; @@ -1267,15 +1596,29 @@ async fn enqueue_p2p_message( } } let inserted = inserted.expect("P2P inbox insert loop exits only after success or error"); - tracing::info!( - event = "p2p_inbox", - outcome = if inserted { "enqueued" } else { "duplicate" }, - message_id = %message_id, - message_type = %inbox_message.msg_type, - from_peer_id = %from_peer_id, - content_size = message.len(), - "received P2P message" - ); + // Aggregate unregistered admission logs; log registered messages individually. + if admission_class.is_registered() { + tracing::info!( + event = "p2p_inbox", + outcome = if inserted { "enqueued" } else { "duplicate" }, + message_id = %message_id, + message_type = %inbox_message.msg_type, + from_peer_id = %from_peer_id, + content_size = message.len(), + "received P2P message" + ); + } else { + crate::p2p_admission::record_admitted_unregistered(); + tracing::debug!( + event = "p2p_inbox", + outcome = if inserted { "enqueued" } else { "duplicate" }, + message_id = %message_id, + message_type = %inbox_message.msg_type, + from_peer_id = %from_peer_id, + content_size = message.len(), + "received P2P message" + ); + } Ok(()) } @@ -1513,6 +1856,182 @@ pub async fn reclaim_stale_queue_claims(local_db: &LocalDB) -> Result<(u64, u64) Ok((local, inbox)) } +/// Restore persisted operator bindings, re-verify signatures and refresh stake asynchronously. +pub async fn load_persisted_peer_bindings( + local_db: &LocalDB, + goat_client: &Arc, +) -> Result { + let bindings = local_db + .acquire() + .await? + .load_p2p_peer_bindings(crate::p2p_admission::OPERATOR_BINDING_LOAD_LIMIT) + .await?; + let registry = crate::p2p_admission::peer_registry(); + let mut loaded = 0; + let mut verified_bindings = std::collections::HashMap::new(); + for node in bindings { + let node_info = NodeInfo { + peer_id: node.peer_id.clone(), + btc_pub_key: node.btc_pub_key.clone(), + binding_sig: node.binding_sig.clone(), + binding_issued_at: node.binding_issued_at, + ..Default::default() + }; + let Some(operator_key) = verify_node_info_binding(&node_info) else { + continue; + }; + if node.binding_issued_at > current_time_secs() + NODE_INFO_BINDING_MAX_FUTURE_SECS { + continue; + } + if let Ok(peer_id) = PeerId::from_str(&node.peer_id) { + let operator_key = crate::p2p_admission::operator_key(&operator_key); + registry.observe_operator_binding(&peer_id, &operator_key, node.binding_issued_at); + verified_bindings.insert(peer_id, (operator_key, node.binding_issued_at)); + loaded += 1; + } + } + // Apply configured bindings only where no signed binding exists; stake still requires confirmation. + for (peer_id, operator_key) in crate::env::get_p2p_trusted_operator_bindings() { + match registry.trust_operator_binding(&peer_id, &operator_key) { + Ok(()) => loaded += 1, + Err(conflict) => tracing::warn!( + event = "p2p_admission", + outcome = "trusted_binding_ignored", + peer_id = %peer_id, + configured_key = %hex::encode(operator_key), + conflict = ?conflict, + env = crate::env::ENV_P2P_TRUSTED_OPERATOR_BINDINGS, + "configured operator binding disagrees with one the operator signed; keeping \ + the signed binding — update or remove the configured entry" + ), + } + } + // Apply configured replay resets, then restore persisted marks before admission. + let reset_peers = crate::env::get_p2p_replay_mark_reset_peers(); + if !reset_peers.is_empty() { + let dropped = local_db.acquire().await?.delete_p2p_replay_marks(&reset_peers).await?; + tracing::warn!( + event = "p2p_admission", + outcome = "replay_marks_reset", + requested = reset_peers.len(), + dropped, + "dropped persisted replay marks on operator request" + ); + } + for (peer_id, highest) in local_db.acquire().await?.load_p2p_replay_marks().await? { + if let Ok(peer_id) = PeerId::from_str(&peer_id) { + crate::p2p_admission::replay_guard().restore_mark(&peer_id, highest as u64); + } + } + // Peers confirmed on chain in an earlier session keep their class and are + // re-validated as known members, outside the discovery budget. + loaded += crate::p2p_admission::seed_registry_from_store( + local_db, + registry, + &verified_bindings, + Instant::now(), + ) + .await?; + // Everything loaded is due; the unconfirmed remainder is picked up by the + // regular tick as lookup slots free up. + crate::p2p_admission::refresh_due_registrations_in_background(local_db, goat_client); + Ok(loaded) +} + +/// A clock disagreement below this is not worth reporting. +const CLOCK_LEAD_REPORT_THRESHOLD: Duration = Duration::from_secs(60); + +/// Warn about registered authors whose message numbering is ahead of the local clock. +fn report_clock_leads(leads: Vec<(PeerId, Duration)>) { + let ahead: Vec<&(PeerId, Duration)> = + leads.iter().filter(|(_, lead)| *lead >= CLOCK_LEAD_REPORT_THRESHOLD).collect(); + let max_lead = ahead.iter().map(|(_, lead)| *lead).max().unwrap_or_default(); + match ahead.as_slice() { + [] => {} + [(peer, lead)] => tracing::warn!( + event = "p2p_admission", + outcome = "author_clock_ahead", + peer_id = %peer, + lead_secs = lead.as_secs(), + "a registered author's clock is ahead of this node's; past the tolerance its \ + messages are refused as coming from the future" + ), + several => tracing::error!( + event = "p2p_admission", + outcome = "local_clock_behind_suspected", + authors = several.len(), + max_lead_secs = max_lead.as_secs(), + "several registered authors' clocks are ahead of this node's: check this node's \ + clock (NTP). Past the tolerance their messages are refused as coming from the future" + ), + } +} + +/// Refresh registrations, summarize drops and purge stale node rows. +pub async fn run_p2p_admission_maintenance(local_db: &LocalDB, goat_client: &Arc) { + let now = Instant::now(); + crate::p2p_admission::refresh_due_registrations_in_background(local_db, goat_client); + crate::p2p_admission::log_drop_summary(); + // Expired bans only need forgetting: the gate compares against the clock. + crate::p2p_admission::direct_peer_strikes().take_expired_bans(now); + report_clock_leads(crate::p2p_admission::replay_guard().take_clock_leads()); + for (peer, rejected) in crate::p2p_admission::replay_guard().take_lockout_suspects() { + tracing::warn!( + event = "p2p_admission", + outcome = "replay_lockout_suspected", + peer_id = %peer, + rejected, + reset_env = crate::env::ENV_P2P_REPLAY_MARK_RESET_PEERS, + "a registered author keeps being refused as a replay; if it is not under \ + attack, its clock stepped back or another instance shares its key" + ); + } + + // Persist replay marks and confirm them only after a successful write. + let owed = crate::p2p_admission::replay_guard().pending_marks(); + let marks: Vec<(String, i64)> = owed + .iter() + .map(|(peer, highest)| (peer.to_string(), (*highest).min(i64::MAX as u64) as i64)) + .collect(); + let purge_due = crate::p2p_admission::node_table_purge_due(NODE_TABLE_PURGE_INTERVAL, now); + if marks.is_empty() && !purge_due { + return; + } + let result = async { + let mut storage = local_db.acquire().await?; + storage.raise_p2p_replay_marks(&marks).await?; + crate::p2p_admission::replay_guard().confirm_persisted(&owed); + if !purge_due { + return Ok(0); + } + if let Some(verifiers) = crate::p2p_admission::peer_registry().verifier_peer_ids() { + storage.prune_p2p_replay_marks(&verifiers).await?; + } + storage + .purge_stale_unregistered_nodes( + current_time_secs() - NODE_TABLE_STALE_SECS, + &crate::env::get_peer_id(), + ) + .await + } + .await; + match result { + Ok(0) => {} + Ok(purged) => tracing::info!( + event = "p2p_admission", + outcome = "stale_nodes_purged", + purged, + "removed node rows of unregistered peers that stopped announcing themselves" + ), + Err(error) => tracing::warn!( + event = "p2p_admission", + outcome = "maintenance_write_failed", + error = %error, + "failed to persist replay marks or purge stale node rows; retried next tick" + ), + } +} + async fn renew_p2p_inbox_lease_until_cancelled( local_db: LocalDB, message_id: String, @@ -1557,9 +2076,10 @@ async fn renew_p2p_inbox_lease_until_cancelled( } } +/// Drain inbox batches until caught up or the message-start budget is exhausted. #[allow(clippy::too_many_arguments)] async fn handle_p2p_inbox_messages( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, btc_client: &Arc, goat_client: &Arc, @@ -1568,27 +2088,123 @@ async fn handle_p2p_inbox_messages( actor: Actor, metrics_state: &MetricsState, shutdown: &CancellationToken, -) -> Result<()> { - let now = current_time_secs(); - let active_heavy_task_ids = active_heavy_task_message_ids(); - let mut storage = local_db.start_immediate_transaction().await?; - // Quarantine rows whose dispatch repeatedly failed to report any outcome. - // Returned retryable errors do not consume this budget. - let quarantined = storage.quarantine_p2p_inbox_messages(now, QUEUE_MAX_ABANDONS).await?; - // Bound terminal metadata and the temporary payload retained for manual - // inspection of quarantined rows. - let purged = storage.purge_terminal_p2p_inbox_messages(now - MESSAGE_EXPIRE_TIME).await?; - // Only list here. Each row is claimed right before its own dispatch so a - // crash mid-dispatch is charged to that row alone. - let candidates = storage - .list_claimable_p2p_inbox_messages( - now, - get_p2p_inbox_batch_size(), - QUEUE_MAX_ABANDONS, - &active_heavy_task_ids, +) -> Result { + let started_at = Instant::now(); + let deadline = started_at + P2P_INBOX_DRAIN_BUDGET; + let mut first_batch = true; + // A backlogged worker comes straight back here; the retention sweeps keep + // the cadence of the tick they were written for. + let housekeeping = inbox_housekeeping_due(started_at); + let mut total_claimed = 0; + loop { + let batch = drain_p2p_inbox_batch( + swarm, + local_db, + btc_client, + goat_client, + http_client, + soldering_builder, + actor.clone(), + metrics_state, + shutdown, + first_batch, + first_batch && housekeeping, + deadline, ) .await?; - storage.commit().await?; + first_batch = false; + total_claimed += batch.claimed; + // Stop when fewer messages were claimed than requested. + let full_batch = (batch.claimed as i64) >= get_p2p_inbox_batch_size(); + if !full_batch || Instant::now() >= deadline || shutdown.is_cancelled() { + // Continue immediately only when a productive pass stopped on its budget. + return Ok(!shutdown.is_cancelled() + && total_claimed > 0 + && (batch.cut_short || full_batch)); + } + } +} + +/// Whether the inbox retention sweeps should run in this pass: once per regular +/// tick interval, however often a backlog brings the worker back. +fn inbox_housekeeping_due(now: Instant) -> bool { + static LAST_RUN: Mutex> = Mutex::new(None); + let mut last_run = LAST_RUN.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let interval = Duration::from_secs(crate::env::REGULAR_TASK_INTERVAL_SECOND); + if last_run.is_some_and(|last_run| now.saturating_duration_since(last_run) < interval) { + return false; + } + *last_run = Some(now); + true +} + +struct InboxBatch { + claimed: usize, + /// The budget ran out with listed rows still unserved. + cut_short: bool, +} + +/// List and dispatch one inbox batch. +#[allow(clippy::too_many_arguments)] +async fn drain_p2p_inbox_batch( + swarm: &mut dyn MessagePublisher, + local_db: &LocalDB, + btc_client: &Arc, + goat_client: &Arc, + http_client: &HttpAsyncClient, + soldering_builder: &Option>, + actor: Actor, + metrics_state: &MetricsState, + shutdown: &CancellationToken, + first_batch: bool, + housekeeping: bool, + deadline: Instant, +) -> Result { + let now = current_time_secs(); + let active_heavy_task_ids = active_heavy_task_message_ids(); + let mut storage = local_db.start_immediate_transaction().await?; + let (mut quarantined, mut purged, mut expired, mut trimmed, mut capped) = (0, 0, 0, 0, 0); + if housekeeping { + // Quarantine rows whose dispatch repeatedly failed to report any outcome. + // Returned retryable errors do not consume this budget. + quarantined = storage.quarantine_p2p_inbox_messages(now, QUEUE_MAX_ABANDONS).await?; + // Bound terminal metadata and the temporary payload retained for manual + // inspection of quarantined rows. + purged = storage.purge_terminal_p2p_inbox_messages(now - MESSAGE_EXPIRE_TIME).await?; + // Expire Pending rows by age, using the shorter unregistered retention. + expired = storage + .expire_pending_p2p_inbox_messages( + Some(store::P2pInboxAdmissionClass::Unregistered), + now - crate::p2p_admission::UNREGISTERED_PENDING_TTL_SECS, + ) + .await? + + storage.expire_pending_p2p_inbox_messages(None, now - MESSAGE_EXPIRE_TIME).await?; + // Cap quarantined payload bytes and terminal row count. + trimmed = + storage.trim_quarantined_p2p_inbox_payloads(P2P_INBOX_QUARANTINE_MAX_BYTES).await?; + capped = storage + .purge_p2p_inbox_over_terminal_cap( + P2P_INBOX_TERMINAL_MAX_ROWS, + P2P_INBOX_TERMINAL_PURGE_BATCH, + ) + .await?; + } + // Only list here. Each row is claimed right before its own dispatch so a + // crash mid-dispatch is charged to that row alone. + let batch_size = get_p2p_inbox_batch_size(); + let candidates = storage + .list_claimable_p2p_inbox_messages_by_class( + now, + batch_size, + (batch_size / P2P_INBOX_REGISTERED_BATCH_DIVISOR).max(1), + (batch_size / P2P_INBOX_UNREGISTERED_BATCH_DIVISOR).max(1), + QUEUE_MAX_ABANDONS, + &active_heavy_task_ids, + ) + .await?; + storage.commit().await?; + let mut claimed = 0; + let mut cut_short = false; if quarantined > 0 { tracing::warn!( @@ -1607,11 +2223,43 @@ async fn handle_p2p_inbox_messages( "removed terminal inbox rows past their retention window" ); } + if expired > 0 { + tracing::warn!( + event = "p2p_inbox", + outcome = "expired", + expired, + "retired inbox messages that stayed pending past their retention window" + ); + } + if trimmed > 0 || capped > 0 { + tracing::info!( + event = "p2p_inbox", + outcome = "capacity_trimmed", + trimmed_quarantine_payloads = trimmed, + deleted_terminal_rows = capped, + "trimmed retained inbox data over its capacity limits" + ); + } + // Serve listed classes using the persistent weighted rotation. + let mut queues: [std::collections::VecDeque; 3] = Default::default(); for candidate in candidates { + queues[crate::p2p_admission::InboxSchedule::queue_of(&candidate.admission_class)] + .push_back(candidate); + } + loop { + // Check the budget before each claim; always allow the first message to make progress. + if (claimed > 0 || !first_batch) && Instant::now() >= deadline { + cut_short = queues.iter().any(|queue| !queue.is_empty()); + break; + } + let Some(candidate) = crate::p2p_admission::inbox_schedule().next(&mut queues) else { + break; + }; let Some(message) = claim_p2p_inbox_candidate(local_db, &candidate.message_id).await else { continue; }; + claimed += 1; let from_peer_id = match PeerId::from_str(&message.from_peer) { Ok(peer_id) => peer_id, Err(error) => { @@ -1638,6 +2286,8 @@ async fn handle_p2p_inbox_messages( continue; } }; + // After persistence, sender authorization belongs to the handler, not the admission cache. + let fingerprint = DispatchFingerprint::of(decoded.content()); let heavy_task = heavy_task_from_content(decoded.content(), &actor); if let Some(heavy_task) = heavy_task { @@ -1693,8 +2343,12 @@ async fn handle_p2p_inbox_messages( metrics_state: metrics_state.clone(), from_peer_id, }; - let execution = - supervise_dispatch(run_heavy_task(&context, heavy_task), &shutdown).await; + // Set the inbox task-local context explicitly for the spawned heavy task. + let execution = supervise_dispatch( + track_inbox_retry(fingerprint, run_heavy_task(&context, heavy_task)), + &shutdown, + ) + .await; lease_cancellation.cancel(); let lease_is_current = match lease_renewal.await { Ok(lease_is_current) => lease_is_current, @@ -1708,6 +2362,8 @@ async fn handle_p2p_inbox_messages( if !lease_is_current { return; } + // Apply the same RPC retry classification as inline dispatch. + let result = result.map_err(classify_retryable_dispatch_error); metrics_state.record_message_dispatch( &task_type_for_task, if result.is_ok() { "success" } else { "failed" }, @@ -1801,8 +2457,33 @@ async fn handle_p2p_inbox_messages( decoded, metrics_state, )); + let handler_budget = crate::env::get_p2p_handler_timeout(); + let dispatch = track_inbox_retry(fingerprint, dispatch); + let dispatch = tokio::time::timeout(handler_budget, dispatch); let result = match supervise_dispatch(dispatch, shutdown).await { - DispatchExecution::Completed(result) => result, + DispatchExecution::Completed(Ok(result)) => result, + // Cancel an expired handler at its current await and retry its persisted work. + DispatchExecution::Completed(Err(_elapsed)) => { + tracing::warn!( + event = "p2p_inbox", + outcome = "handler_timeout", + message_id = %message.message_id, + message_type = %message.msg_type, + from_peer_id = %from_peer_id, + budget_secs = handler_budget.as_secs(), + attempt_count = message.attempt_count + 1, + "P2P message handler exceeded its execution budget; deferred for retry" + ); + defer_p2p_inbox_without_aborting_batch( + local_db, + &message.message_id, + &message.lease_token, + current_time_secs() + p2p_retry_delay_secs(message.attempt_count + 1), + "handler_timeout", + ) + .await; + continue; + } DispatchExecution::Shutdown => { defer_p2p_inbox_without_aborting_batch( local_db, @@ -1812,7 +2493,7 @@ async fn handle_p2p_inbox_messages( "graceful_shutdown", ) .await; - return Ok(()); + return Ok(InboxBatch { claimed, cut_short: false }); } DispatchExecution::Panicked(detail) => { tracing::error!( @@ -1852,7 +2533,7 @@ async fn handle_p2p_inbox_messages( log_queue_bookkeeping_failure("p2p_inbox", &message.message_id, "finish", &error); } } - Ok(()) + Ok(InboxBatch { claimed, cut_short }) } async fn finish_p2p_inbox_attempt( @@ -1921,16 +2602,15 @@ async fn finish_p2p_inbox_attempt( Ok(()) } +/// Publish due outbox rows until caught up, blocked or past the start budget. async fn handle_p2p_outbox_messages( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, ) -> Result<()> { + let deadline = Instant::now() + P2P_INBOX_DRAIN_BUDGET; let now = current_time_secs(); let mut storage = local_db.start_immediate_transaction().await?; let expired = storage.expire_p2p_outbox_retry_messages(now).await?; - let messages = storage - .claim_p2p_outbox_messages(now, now + P2P_INBOX_LEASE_SECS, get_p2p_outbox_batch_size()) - .await?; storage.commit().await?; if expired > 0 { @@ -1938,78 +2618,177 @@ async fn handle_p2p_outbox_messages( event = "p2p_outbox", outcome = "retry_window_expired", expired, - "graph-setup outbound messages reached their retry window without the expected ACK" + "outbound messages reached the end of their retry window: graph-setup messages without the expected ACK, signing-round messages without their round closing" ); } + drain_p2p_outbox(swarm, local_db, deadline).await +} - for message in messages { - let outbound = match GOATMessage::deserialize_message(&message.content).await { - Ok(message) => message, +async fn drain_p2p_outbox( + swarm: &mut dyn MessagePublisher, + local_db: &LocalDB, + deadline: Instant, +) -> Result<()> { + loop { + // Check the start budget before claiming the next row; let in-flight publishes finish. + if Instant::now() >= deadline { + break; + } + let now = current_time_secs(); + let mut storage = local_db.start_immediate_transaction().await?; + let message = + storage.claim_p2p_outbox_messages(now, now + P2P_INBOX_LEASE_SECS, 1).await?.pop(); + storage.commit().await?; + let Some(message) = message else { break }; + match publish_p2p_outbox_row(swarm, local_db, &message).await { + Ok(OutboxRowOutcome::PublishFailed) => break, + Ok(OutboxRowOutcome::Published | OutboxRowOutcome::Closed) => {} Err(error) => { - local_db - .acquire() - .await? - .fail_p2p_outbox_message(&message.message_id, &error.to_string()) - .await?; - tracing::error!( + tracing::warn!( event = "p2p_outbox", - outcome = "failed", + outcome = "bookkeeping_failed", message_id = %message.message_id, message_type = %message.msg_type, error = %error, - "discarded corrupt durable outbound P2P message" + "could not settle an outbound P2P message; releasing it for the next pass" ); - continue; - } - }; - let result = send_to_peer(swarm, outbound).await; - let mut storage = local_db.acquire().await?; - match result { - Ok(_) => { - if message.retry_until > 0 { - let next_retry_at = current_time_secs() + message.retry_interval_secs; - storage.schedule_p2p_outbox_retry(&message.message_id, next_retry_at).await?; - tracing::info!( - event = "p2p_outbox", - outcome = "published_retry_window", - message_id = %message.message_id, - message_type = %message.msg_type, - next_retry_at, - retry_until = message.retry_until, - "published graph-setup P2P message; awaiting ACK or retry window expiry" - ); - } else { - storage.complete_p2p_outbox_message(&message.message_id).await?; + if let Ok(mut storage) = local_db.acquire().await { + let _ = storage + .retry_p2p_outbox_message( + &message.message_id, + current_time_secs() + PROTOCOL_RETRY_BASE_SECS, + &error.to_string(), + ) + .await; } } - Err(error) => { - let retry_after_secs = p2p_retry_delay_secs(message.attempt_count); - storage - .retry_p2p_outbox_message( - &message.message_id, - current_time_secs() + retry_after_secs, - &error.to_string(), - ) - .await?; - tracing::warn!( + } + } + Ok(()) +} + +enum OutboxRowOutcome { + Published, + PublishFailed, + /// Settled without publishing: corrupt, or nobody needs it any more. + Closed, +} + +async fn publish_p2p_outbox_row( + swarm: &mut dyn MessagePublisher, + local_db: &LocalDB, + message: &store::P2pOutboxMessage, +) -> Result { + let outbound = match GOATMessage::deserialize_message(&message.content).await { + Ok(message) => message, + Err(error) => { + local_db + .acquire() + .await? + .fail_p2p_outbox_message(&message.message_id, &error.to_string()) + .await?; + tracing::error!( + event = "p2p_outbox", + outcome = "failed", + message_id = %message.message_id, + message_type = %message.msg_type, + error = %error, + "discarded corrupt durable outbound P2P message" + ); + return Ok(OutboxRowOutcome::Closed); + } + }; + let signing_round = if message.message_id.starts_with(PROTOCOL_OUTBOX_PREFIX) { + let round = SigningRound::of(&outbound.content) + .filter(|_| outbound.content.business_ref() != BusinessRef::Unscoped); + if round.is_none() { + local_db + .acquire() + .await? + .fail_p2p_outbox_message( + &message.message_id, + "signing outbox message has no round or business reference", + ) + .await?; + return Ok(OutboxRowOutcome::Closed); + } + round + } else { + None + }; + if signing_round.is_some() && protocol_delivery_finished(local_db, &outbound.content).await? { + local_db.acquire().await?.complete_p2p_outbox_message(&message.message_id).await?; + tracing::info!( + event = "p2p_outbox", + outcome = "round_closed", + message_id = %message.message_id, + message_type = %message.msg_type, + publish_count = message.publish_count, + "stopped re-publishing a signing-round message: its round is over" + ); + return Ok(OutboxRowOutcome::Closed); + } + let result = send_to_peer(swarm, outbound).await; + let mut storage = local_db.acquire().await?; + match result { + Ok(_) => { + if message.retry_until > 0 { + let interval = match signing_round { + Some(round) => protocol_retry_interval_secs( + message.publish_count + 1, + round.retry_ceiling_secs(crate::env::get_p2p_protocol_retry_max_secs()), + ), + None => message.retry_interval_secs, + }; + let next_retry_at = current_time_secs() + interval; + storage.schedule_p2p_outbox_retry(&message.message_id, next_retry_at).await?; + tracing::info!( event = "p2p_outbox", - outcome = "deferred", + outcome = "published_retry_window", message_id = %message.message_id, message_type = %message.msg_type, - attempt_count = message.attempt_count, - retry_after_secs, - error = %error, - "deferred outbound P2P message" + next_retry_at, + retry_until = message.retry_until, + "published durable P2P message; it stays due until acknowledged, closed or expired" ); + } else { + storage.complete_p2p_outbox_message(&message.message_id).await?; } + Ok(OutboxRowOutcome::Published) + } + Err(error) => { + // `attempt_count` counts claims, and a signing-round row is claimed + // for days: by it, one failed publish would cost the longest delay. + let retry_after_secs = if signing_round.is_some() { + PROTOCOL_RETRY_BASE_SECS + } else { + p2p_retry_delay_secs(message.attempt_count) + }; + storage + .retry_p2p_outbox_message( + &message.message_id, + current_time_secs() + retry_after_secs, + &error.to_string(), + ) + .await?; + tracing::warn!( + event = "p2p_outbox", + outcome = "deferred", + message_id = %message.message_id, + message_type = %message.msg_type, + attempt_count = message.attempt_count, + retry_after_secs, + error = %error, + "deferred outbound P2P message" + ); + Ok(OutboxRowOutcome::PublishFailed) } } - Ok(()) } #[allow(clippy::too_many_arguments)] pub async fn handle_self_p2p_msg( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, btc_client: &Arc, goat_client: &Arc, @@ -2021,7 +2800,7 @@ pub async fn handle_self_p2p_msg( message: &[u8], metrics_state: &MetricsState, shutdown: &CancellationToken, -) -> Result<()> { +) -> Result { if id != GOATMessage::default_message_id() { tracing::warn!( event = "local_message_queue", @@ -2029,7 +2808,7 @@ pub async fn handle_self_p2p_msg( message_id = ?id, "ignoring local queue trigger with an unexpected message id" ); - return Ok(()); + return Ok(false); } let message = GOATMessage::deserialize_message(message).await?; tracing::info!( @@ -2060,13 +2839,22 @@ pub async fn handle_self_p2p_msg( batch_size = candidates.len(), "listed claimable local messages" ); - for candidate in candidates { + let local_deadline = Instant::now() + P2P_INBOX_DRAIN_BUDGET; + let listed_full_batch = candidates.len() as i64 >= LOCAL_MESSAGE_BATCH_SIZE; + let mut local_dispatched = 0_usize; + let mut local_cut_short = false; + for (index, candidate) in candidates.into_iter().enumerate() { + if index > 0 && Instant::now() >= local_deadline { + local_cut_short = true; + break; + } // Claim right before dispatch so a crash mid-dispatch is charged to // this row alone, and so a producer that re-armed the row since it was // listed wins: the stale version is skipped until the next tick. let Some(message) = claim_local_candidate(local_db, &candidate).await else { continue; }; + local_dispatched += 1; let queue_wait_secs = current_time_secs().saturating_sub(message.created_at); let started_at = Instant::now(); let claim = LocalMessageClaim { @@ -2089,9 +2877,15 @@ pub async fn handle_self_p2p_msg( metrics_state, ), )); + let dispatch = tokio::time::timeout(crate::env::get_p2p_handler_timeout(), dispatch); let result = match supervise_dispatch(dispatch, shutdown).await { - DispatchExecution::Completed(result) => result, - DispatchExecution::Shutdown => return Ok(()), + DispatchExecution::Completed(Ok(result)) => result, + DispatchExecution::Completed(Err(_)) => Err(retryable_dispatch_error( + RetryableDispatchReason::ExternalRpcUnavailable, + None, + "local handler execution budget exceeded", + )), + DispatchExecution::Shutdown => return Ok(false), DispatchExecution::Panicked(detail) => { tracing::error!( event = "local_message_dispatch_panic", @@ -2321,12 +3115,14 @@ pub async fn handle_self_p2p_msg( } } } - // The three queues share a tick but must not share a failure: propagating - // here would let one stalled queue starve the other two every tick. + // Handle each queue's failure independently. if let Err(error) = handle_p2p_outbox_messages(swarm, local_db).await { tracing::error!(error = %error, "failed to drain the durable P2P outbox"); } - if let Err(error) = handle_p2p_inbox_messages( + // A queue that failed reports no backlog: asking for another pass at once + // would spin on whatever made it fail. + let local_backlog = local_dispatched > 0 && (local_cut_short || listed_full_batch); + let inbox_backlog = match handle_p2p_inbox_messages( swarm, local_db, btc_client, @@ -2339,12 +3135,16 @@ pub async fn handle_self_p2p_msg( ) .await { - tracing::error!(error = %error, "failed to drain the durable P2P inbox"); - if shutdown.is_cancelled() { - return Err(error); + Ok(backlog) => backlog, + Err(error) => { + tracing::error!(error = %error, "failed to drain the durable P2P inbox"); + if shutdown.is_cancelled() { + return Err(error); + } + false } - } - Ok(()) + }; + Ok(local_backlog || inbox_backlog) } /// Filter the message and dispatch message to different handlers, like rpc handler, or other peers @@ -2353,7 +3153,7 @@ pub async fn handle_self_p2p_msg( /// TODO: we should create a trait for all the actions of different roles to simplify this function. #[allow(clippy::too_many_arguments)] pub async fn recv_and_dispatch( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, btc_client: &Arc, goat_client: &Arc, @@ -2384,7 +3184,7 @@ pub async fn recv_and_dispatch( #[allow(clippy::too_many_arguments)] async fn dispatch_decoded_p2p_message( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, btc_client: &Arc, goat_client: &Arc, @@ -2397,7 +3197,9 @@ async fn dispatch_decoded_p2p_message( metrics_state: &MetricsState, ) -> Result<()> { // Determine whether the message comes from this node itself to optionally skip validations. - let is_self_peer = get_local_node_info().peer_id == from_peer_id.to_string(); + // Not `get_local_node_info()`: that signs a fresh binding on every call, and + // this runs for every dispatched message. + let is_self_peer = crate::env::get_peer_id() == from_peer_id.to_string(); let message_type = message.content.event_type(); let role = actor.to_string(); let from_peer_id_string = from_peer_id.to_string(); @@ -2447,7 +3249,7 @@ async fn dispatch_decoded_p2p_message( } pub(crate) async fn try_finalize_graph( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, goat_client: &GOATClient, instance_id: Uuid, @@ -2486,21 +3288,32 @@ pub(crate) async fn try_finalize_graph( graph.parameters.graph_id ); } - let pub_nonces = - order_committee_values(&committee_pubkeys, pub_nonoces, "graph committee pub nonces")?; - let agg_nonces = nonces_aggregation(&pub_nonces)?; - let partial_sigs = order_committee_values( - &committee_pubkeys, - partial_sigs, - "graph committee partial sigs", - )?; - let committee_sig_for_graph = signature_aggregation(&partial_sigs, &agg_nonces, &graph)?; - push_committee_pre_signatures(&mut graph, &committee_sig_for_graph)?; + // Reuse stored committee signatures when retrying graph finalization. + if !graph.committee_pre_signed() { + let pub_nonces = order_committee_values( + &committee_pubkeys, + pub_nonoces, + "graph committee pub nonces", + )?; + let agg_nonces = nonces_aggregation(&pub_nonces)?; + let partial_sigs = order_committee_values( + &committee_pubkeys, + partial_sigs, + "graph committee partial sigs", + )?; + let committee_sig_for_graph = + signature_aggregation(&partial_sigs, &agg_nonces, &graph)?; + push_committee_pre_signatures(&mut graph, &committee_sig_for_graph)?; + } let simplified_graph = graph.to_simplified()?; let store_outcome = store_finalized_graph_if_needed(local_db, &simplified_graph).await?; mark_graph_as_endorsed(local_db, instance_id, graph_id).await?; try_transition_instance_to_presigned(local_db, instance_id).await?; - if broadcast_graph_finalize { + // Throttle GraphFinalize recovery and stamp the gate only on success. + let gate = crate::p2p_admission::protocol_republish_gate(); + let gate_key = format!("graph-finalize:{graph_id}"); + let now = Instant::now(); + if broadcast_graph_finalize && !gate.is_cooling(&gate_key, now) { let message_content = GOATMessageContent::GraphFinalize(GraphFinalize { instance_id, graph_id, @@ -2509,15 +3322,224 @@ pub(crate) async fn try_finalize_graph( params_endorse_sigs: params_endorsements, graph: simplified_graph, }); - send_to_peer(swarm, GOATMessage::new(Actor::All, message_content)).await?; + send_protocol_message( + swarm, + local_db, + GOATMessage::new(Actor::All, message_content), + StoredValue::Fresh, + ) + .await?; + gate.allow(&gate_key, now); } return Ok(Some((graph, store_outcome))); } Ok(None) } +const PROTOCOL_OUTBOX_PREFIX: &str = "protocol:"; + +/// The rounds whose messages are kept in the outbox and re-published until the +/// round is over, whatever traffic arrives or does not. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SigningRound { + /// NonceGeneration, AggNonceConsensus, CommitteePresign and EndorseGraph: + /// what the operator needs to assemble GraphFinalize. + GraphSigning, + /// The operator's GraphFinalize, the only source of the finalized graph + /// until a relayer has posted it. + GraphFinalize, + /// The committee's PeginConfirm nonce, consensus and partial-signature rounds. + PeginConfirm, +} + +impl SigningRound { + pub(crate) fn of(content: &GOATMessageContent) -> Option { + match content { + GOATMessageContent::NonceGeneration(_) + | GOATMessageContent::AggNonceConsensus(_) + | GOATMessageContent::CommitteePresign(_) + | GOATMessageContent::EndorseGraph(_) => Some(Self::GraphSigning), + GOATMessageContent::GraphFinalize(_) => Some(Self::GraphFinalize), + GOATMessageContent::PeginConfirmNonce(_) + | GOATMessageContent::PeginConfirmNonceConsensus(_) + | GOATMessageContent::PeginConfirmPartialSig(_) => Some(Self::PeginConfirm), + _ => None, + } + } + + /// Retry ceiling; GraphFinalize uses six times the ordinary signing-round ceiling. + fn retry_ceiling_secs(self, configured_max_secs: i64) -> i64 { + match self { + Self::GraphFinalize => configured_max_secs.saturating_mul(6), + Self::GraphSigning | Self::PeginConfirm => configured_max_secs, + } + } + + /// Check delivery completion from local state; unknown statuses remain open. + /// Graph rounds close by graph status, not another operator's instance progress. + fn delivery_finished( + self, + graph: Option, + instance: Option<&InstanceBridgeInStatus>, + ) -> bool { + use InstanceBridgeInStatus as Instance; + let instance_failed = matches!( + instance, + Some( + Instance::PresignedFailed + | Instance::RelayerL2MintedFailed + | Instance::Timeout + | Instance::UserCanceled + | Instance::NoEnoughCommitteesAnswered + | Instance::UserDiscarded + ) + ); + if instance_failed { + return true; + } + match self { + Self::PeginConfirm => { + matches!(instance, Some(Instance::RelayerL1Broadcasted | Instance::RelayerL2Minted)) + } + // A finalized or posted graph proves every member's values reached + // the operator. An obsoleted one will never be posted or used. + Self::GraphSigning => graph.is_some_and(|status| match status { + GraphStatus::OperatorPresigned => false, + GraphStatus::CommitteePresigned + | GraphStatus::OperatorDataPushed + | GraphStatus::PreKickoff + | GraphStatus::OperatorKickOff + | GraphStatus::Challenge + | GraphStatus::Disprove + | GraphStatus::Obsoleted + | GraphStatus::Skipped + | GraphStatus::OperatorTake1 + | GraphStatus::OperatorTake2 => true, + }), + // Keep GraphFinalize delivery open until the graph is posted or retired. + Self::GraphFinalize => graph.is_some_and(|status| match status { + GraphStatus::OperatorPresigned | GraphStatus::CommitteePresigned => false, + GraphStatus::OperatorDataPushed + | GraphStatus::PreKickoff + | GraphStatus::OperatorKickOff + | GraphStatus::Challenge + | GraphStatus::Disprove + | GraphStatus::Obsoleted + | GraphStatus::Skipped + | GraphStatus::OperatorTake1 + | GraphStatus::OperatorTake2 => true, + }), + } + } +} + +/// [`SigningRound::delivery_finished`] over the local graph and instance rows. +/// Reads the rows only: loading the graph itself re-verifies its signatures. +pub(crate) async fn protocol_delivery_finished( + local_db: &LocalDB, + content: &GOATMessageContent, +) -> Result { + let Some(round) = SigningRound::of(content) else { + return Ok(false); + }; + let (instance_id, graph_id) = match content.business_ref() { + BusinessRef::Graph { instance_id, graph_id } => (instance_id, Some(graph_id)), + BusinessRef::Instance { instance_id } => (instance_id, None), + BusinessRef::Unscoped => return Ok(false), + }; + let mut storage = local_db.acquire().await?; + let instance = storage + .find_instance(&instance_id) + .await? + .and_then(|instance| InstanceBridgeInStatus::from_str(&instance.status).ok()); + let graph = match graph_id { + Some(graph_id) => storage + .find_graph(&graph_id) + .await? + .filter(|graph| graph.instance_id == instance_id) + .and_then(|graph| GraphStatus::from_str(&graph.status).ok()), + None => None, + }; + Ok(round.delivery_finished(graph, instance.as_ref())) +} + +/// Wait before the next re-publish of a signing-round row that has gone out of +/// the outbox `publish_count` times. +fn protocol_retry_interval_secs(publish_count: i64, max_secs: i64) -> i64 { + let doublings = publish_count.clamp(0, 20) as u32; + PROTOCOL_RETRY_BASE_SECS + .saturating_mul(1_i64 << doublings) + .min(max_secs.max(PROTOCOL_RETRY_BASE_SECS)) +} + +fn protocol_outbox_id(content: &GOATMessageContent) -> String { + format!( + "{PROTOCOL_OUTBOX_PREFIX}{}:{}", + content.event_type(), + content.business_ref().key_part() + ) +} + +/// Store a signing-round message for re-publication. Returns whether this call +/// created the row: an existing one, live or closed, is left as it is. +async fn store_protocol_message(local_db: &LocalDB, message: &GOATMessage) -> Result { + let bytes = message.serialize_message().await?; + let now = current_time_secs(); + local_db + .acquire() + .await? + .enqueue_p2p_outbox_retry_message( + &protocol_outbox_id(&message.content), + message.content.event_type(), + &bytes, + now + MESSAGE_EXPIRE_TIME, + PROTOCOL_RETRY_BASE_SECS, + None, + // The caller publishes right after this; a row due at once would + // have the outbox send a second copy seconds later. + now + PROTOCOL_RETRY_BASE_SECS, + ) + .await +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum StoredValue { + Fresh, + Found, +} + +/// Persist signing-round bytes; recovery leaves live outbox rows to the worker. +pub async fn send_protocol_message( + publisher: &mut dyn MessagePublisher, + local_db: &LocalDB, + message: GOATMessage, + origin: StoredValue, +) -> Result<()> { + let outbox_id = + SigningRound::of(&message.content).map(|_| protocol_outbox_id(&message.content)); + if let Some(id) = &outbox_id { + let inserted = store_protocol_message(local_db, &message).await?; + if origin == StoredValue::Found && !inserted { + let state = local_db.acquire().await?.p2p_outbox_entry_state(id).await?; + let live = state.is_some_and(|(state, _)| state == "Pending" || state == "Processing"); + if live || protocol_delivery_finished(local_db, &message.content).await? { + return Ok(()); + } + } + } + let result = send_to_peer(publisher, message).await; + if let (Err(_), Some(outbox_id)) = (&result, outbox_id) { + // Nothing went out, so there is no copy for the outbox to duplicate: + // let it publish on its next pass instead of a full interval from now. + if let Ok(mut storage) = local_db.acquire().await { + let _ = storage.expedite_p2p_outbox_message(&outbox_id).await; + } + } + result.map(|_| ()) +} + pub async fn send_to_peer( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, message: GOATMessage, ) -> Result { let target_actor = message.actor.to_string(); @@ -2533,12 +3555,29 @@ pub async fn send_to_peer( return Err(error); } }; - if serialized.len() > crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE - && let Some(metrics_state) = crate::metrics_service::node_metrics_state() - { - metrics_state.record_p2p_oversized_message(); + // Receivers drop what exceeds their envelope limit without telling anyone, + // so an outgrown message has to be loud on the sending side. + let limits = crate::p2p_admission::inbound_limits(); + let receiver_limit = if GOATMessage::is_binary_envelope(&serialized) { + limits.max_binary_bytes + } else { + limits.max_json_bytes + }; + if serialized.len() > receiver_limit { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_p2p_oversized_message(); + } + tracing::error!( + event = "p2p_message_publish", + outcome = "oversized", + target_actor, + message_type, + content_size = serialized.len(), + receiver_limit, + "outbound protocol message exceeds the inbound size limit peers enforce" + ); } - match swarm.behaviour_mut().gossipsub.publish(gossipsub_topic, serialized) { + match swarm.publish_message(gossipsub_topic, serialized).await { Ok(message_id) => { if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { metrics_state.record_p2p_publish(true); @@ -2581,6 +3620,23 @@ pub async fn push_local_unhandled_messages_with_reason( reason: MessageDeferReason, reason_detail: &str, ) -> Result<()> { + // Keep self-deferral in the original sender-specific inbox row. + let deferred_in_inbox = ACTIVE_INBOX_DISPATCH + .try_with(|active| { + if DispatchFingerprint::of(message.content()) != active.fingerprint { + return false; + } + *active.retry.borrow_mut() = Some(retryable_dispatch_error( + RetryableDispatchReason::DependencyPending, + Some(delay_secs.max(1) as i64), + reason_detail.to_owned(), + )); + true + }) + .unwrap_or(false); + if deferred_in_inbox { + return Ok(()); + } let mut storage_processor = local_db.start_immediate_transaction().await?; let actor = message.actor.clone(); let content: GOATMessageContent = message.content().clone(); @@ -2678,7 +3734,7 @@ pub async fn push_local_unhandled_messages_with_reason( /// Helper: try to get graph. If missing, send SyncGraphRequest and defer current handling. pub(crate) async fn get_graph_or_defer( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, local_db: &LocalDB, goat_client: &GOATClient, instance_id: Uuid, @@ -2745,12 +3801,18 @@ pub(crate) async fn get_graph_or_defer( } pub async fn try_send_sync_graph_request( - swarm: &mut Swarm, + swarm: &mut dyn MessagePublisher, goat_client: &GOATClient, instance_id: Uuid, graph_id: Uuid, ) -> Result<()> { validate_graph_id_on_goat(goat_client, instance_id, graph_id).await?; + // Record the graph request before publishing it. + if !crate::p2p_admission::requested_graphs().insert(&graph_id.to_string(), Instant::now()) { + bail!( + "too many graph sync requests outstanding; {instance_id}:{graph_id} is retried later" + ); + } let message_content = GOATMessageContent::SyncGraphRequest(SyncGraphRequest { instance_id, graph_id }); let message = GOATMessage::new(Actor::All, message_content); @@ -2762,6 +3824,532 @@ pub async fn try_send_sync_graph_request( mod tests { use super::*; + struct UnavailablePublisher; + impl MessagePublisher for UnavailablePublisher { + fn publish_message( + &mut self, + _: gossipsub::IdentTopic, + _: Vec, + ) -> futures::future::BoxFuture<'_, Result> { + Box::pin(async { bail!("network unavailable") }) + } + } + + #[tokio::test] + async fn signing_round_survives_publish_failure_without_reopening_completed_delivery() { + let db = store::create_local_db("sqlite::memory:").await; + let key = Keypair::from_secret_key( + SECP256K1, + &secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), + ); + let message = GOATMessage::new( + Actor::Committee, + GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + committee_pubkey: key.public_key().into(), + consensus_hash: [3; 32], + signature: SECP256K1.sign_schnorr(&SecpMessage::from_digest([3; 32]), &key), + }), + ); + assert!( + send_protocol_message( + &mut UnavailablePublisher, + &db, + message.clone(), + StoredValue::Fresh + ) + .await + .is_err() + ); + let now = current_time_secs(); + let mut storage = db.acquire().await.unwrap(); + let rows = storage.claim_p2p_outbox_messages(now, now + 300, 10).await.unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].content, message.serialize_message().await.unwrap()); + assert_eq!(rows[0].retry_interval_secs, 30); + storage.complete_p2p_outbox_message(&rows[0].message_id).await.unwrap(); + drop(storage); + let _ = send_protocol_message(&mut UnavailablePublisher, &db, message, StoredValue::Fresh) + .await; + assert!( + db.acquire() + .await + .unwrap() + .claim_p2p_outbox_messages(now + 301, now + 601, 10) + .await + .unwrap() + .is_empty() + ); + } + + #[derive(Default)] + struct RecordingPublisher(Vec>); + impl MessagePublisher for RecordingPublisher { + fn publish_message( + &mut self, + _: gossipsub::IdentTopic, + data: Vec, + ) -> futures::future::BoxFuture<'_, Result> { + self.0.push(data); + Box::pin(async { Ok(MessageId::from("published")) }) + } + } + + async fn queued_outbox_fixture() -> LocalDB { + let db = store::create_local_db("sqlite::memory:").await; + let content = + committee_vote(Uuid::new_v4(), Uuid::new_v4()).serialize_message().await.unwrap(); + for index in 0..3 { + db.acquire() + .await + .unwrap() + .enqueue_p2p_outbox_message( + &format!("budget-{index}"), + "AggNonceConsensus", + &content, + ) + .await + .unwrap(); + } + db + } + + async fn assert_unclaimed_outbox_rows(db: &LocalDB, expected: usize) { + let now = current_time_secs(); + let rows = db + .acquire() + .await + .unwrap() + .claim_p2p_outbox_messages(now, now + 300, 16) + .await + .unwrap(); + assert_eq!(rows.len(), expected, "untouched rows must remain immediately claimable"); + assert!(rows.iter().all(|row| row.attempt_count == 1)); + } + + #[tokio::test] + async fn outbox_expired_budget_claims_nothing() { + let db = queued_outbox_fixture().await; + let mut publisher = RecordingPublisher::default(); + drain_p2p_outbox(&mut publisher, &db, Instant::now()).await.unwrap(); + assert!(publisher.0.is_empty()); + assert_unclaimed_outbox_rows(&db, 3).await; + } + + #[tokio::test] + async fn outbox_publish_failure_does_not_claim_the_rest() { + let db = queued_outbox_fixture().await; + drain_p2p_outbox(&mut UnavailablePublisher, &db, Instant::now() + Duration::from_secs(60)) + .await + .unwrap(); + assert_unclaimed_outbox_rows(&db, 2).await; + } + + #[tokio::test] + async fn outbox_checks_budget_between_publishes() { + struct DeadlinePublisher(Instant); + impl MessagePublisher for DeadlinePublisher { + fn publish_message( + &mut self, + _: gossipsub::IdentTopic, + _: Vec, + ) -> futures::future::BoxFuture<'_, Result> { + Box::pin(async move { + tokio::time::sleep_until(tokio::time::Instant::from_std(self.0)).await; + Ok(MessageId::from("published")) + }) + } + } + let db = queued_outbox_fixture().await; + let deadline = Instant::now() + Duration::from_secs(2); + drain_p2p_outbox(&mut DeadlinePublisher(deadline), &db, deadline).await.unwrap(); + assert_unclaimed_outbox_rows(&db, 2).await; + } + + fn committee_vote(instance_id: Uuid, graph_id: Uuid) -> GOATMessage { + let key = Keypair::from_secret_key( + SECP256K1, + &secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), + ); + GOATMessage::new( + Actor::Committee, + GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + instance_id, + graph_id, + committee_pubkey: key.public_key().into(), + consensus_hash: [3; 32], + signature: SECP256K1.sign_schnorr(&SecpMessage::from_digest([3; 32]), &key), + }), + ) + } + + async fn store_graph_row(db: &LocalDB, instance_id: Uuid, graph_id: Uuid, status: GraphStatus) { + let mut storage = db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&store::Graph { + graph_id, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_owned(), + ..Default::default() + }) + .await + .unwrap(); + if status != GraphStatus::OperatorPresigned { + storage + .transition_graph_status( + instance_id, + graph_id, + status, + store::GraphStatusSource::Definition, + None, + ) + .await + .unwrap(); + } + } + + #[test] + fn signing_round_backoff_doubles_up_to_its_ceiling() { + let waits: Vec = (0..8).map(|n| protocol_retry_interval_secs(n, 600)).collect(); + assert_eq!(waits, [30, 60, 120, 240, 480, 600, 600, 600]); + // A short-timelock network asks for a ceiling below the base step. + assert_eq!(protocol_retry_interval_secs(5, 10), 30); + assert_eq!(protocol_retry_interval_secs(i64::MAX, 600), 600); + assert_eq!(protocol_retry_interval_secs(-1, 600), 30); + // The whole graph backs off further than the small round messages. + assert_eq!(SigningRound::GraphSigning.retry_ceiling_secs(600), 600); + assert_eq!(SigningRound::GraphFinalize.retry_ceiling_secs(600), 3_600); + assert_eq!(protocol_retry_interval_secs(9, 3_600), 3_600); + } + + #[test] + fn a_graph_round_is_closed_by_its_graph_and_not_by_the_instance() { + use GraphStatus as G; + use InstanceBridgeInStatus as I; + // The instance moves on with the first graph; a second operator's graph + // is still mid-round then, and must keep its retries. + for instance in [I::Presigned, I::RelayerL1Broadcasted, I::RelayerL2Minted] { + for round in [SigningRound::GraphSigning, SigningRound::GraphFinalize] { + assert!(!round.delivery_finished(Some(G::OperatorPresigned), Some(&instance))); + assert!(!round.delivery_finished(None, Some(&instance))); + } + } + // This node holding the finalized graph closes its own round values, but + // says nothing about whether a relayer has the GraphFinalize. + assert!(SigningRound::GraphSigning.delivery_finished(Some(G::CommitteePresigned), None)); + assert!(!SigningRound::GraphFinalize.delivery_finished(Some(G::CommitteePresigned), None)); + for posted in [G::OperatorDataPushed, G::PreKickoff, G::OperatorTake1, G::Obsoleted] { + assert!(SigningRound::GraphSigning.delivery_finished(Some(posted), None)); + assert!(SigningRound::GraphFinalize.delivery_finished(Some(posted), None)); + } + // PeginConfirm is the instance's round. + assert!(!SigningRound::PeginConfirm.delivery_finished(None, Some(&I::Presigned))); + assert!(!SigningRound::PeginConfirm.delivery_finished(None, None)); + assert!(SigningRound::PeginConfirm.delivery_finished(None, Some(&I::RelayerL1Broadcasted))); + // A failed instance closes everything. + for failed in [I::PresignedFailed, I::Timeout, I::UserCanceled, I::UserDiscarded] { + assert!( + SigningRound::GraphSigning + .delivery_finished(Some(G::OperatorPresigned), Some(&failed)) + ); + assert!(SigningRound::GraphFinalize.delivery_finished(None, Some(&failed))); + assert!(SigningRound::PeginConfirm.delivery_finished(None, Some(&failed))); + } + } + + #[tokio::test] + async fn a_published_signing_message_is_not_sent_again_until_its_first_interval() { + let db = store::create_local_db("sqlite::memory:").await; + let message = committee_vote(Uuid::new_v4(), Uuid::new_v4()); + let mut publisher = RecordingPublisher::default(); + send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Fresh) + .await + .unwrap(); + assert_eq!(publisher.0.len(), 1); + + // The handler has just published: the outbox must not follow with a copy. + handle_p2p_outbox_messages(&mut publisher, &db).await.unwrap(); + assert_eq!(publisher.0.len(), 1); + + // A later visit that finds the value stored leaves the live row alone... + send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Found) + .await + .unwrap(); + assert_eq!(publisher.0.len(), 1); + + // ...and the outbox re-publishes it on schedule, backing off. + let now = current_time_secs(); + let mut storage = db.acquire().await.unwrap(); + let due = storage + .claim_p2p_outbox_messages(now + PROTOCOL_RETRY_BASE_SECS, now + 600, 10) + .await + .unwrap(); + assert_eq!(due.len(), 1); + assert_eq!(due[0].publish_count, 0); + storage.schedule_p2p_outbox_retry(&due[0].message_id, now + 1_000).await.unwrap(); + let due = storage.claim_p2p_outbox_messages(now + 1_000, now + 2_000, 10).await.unwrap(); + assert_eq!(due[0].publish_count, 1, "only publishes that went through are counted"); + } + + #[tokio::test] + async fn a_value_stored_without_an_outbox_row_is_published_by_the_retry() { + // The first run stored its value and was cut off before the outbox row: + // nothing was ever published, and the retry is the only one who can. + let db = store::create_local_db("sqlite::memory:").await; + let message = committee_vote(Uuid::new_v4(), Uuid::new_v4()); + let mut publisher = RecordingPublisher::default(); + send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Found) + .await + .unwrap(); + assert_eq!(publisher.0.len(), 1); + let id = protocol_outbox_id(&message.content); + let state = db.acquire().await.unwrap().p2p_outbox_entry_state(&id).await.unwrap(); + assert_eq!(state.unwrap().0, "Pending"); + + // A failed publish leaves the row due at once rather than an interval away. + let other = committee_vote(Uuid::new_v4(), Uuid::new_v4()); + assert!( + send_protocol_message(&mut UnavailablePublisher, &db, other, StoredValue::Found) + .await + .is_err() + ); + let now = current_time_secs(); + let due = db + .acquire() + .await + .unwrap() + .claim_p2p_outbox_messages(now, now + 300, 10) + .await + .unwrap(); + assert_eq!(due.len(), 1); + } + + #[tokio::test] + async fn a_closed_outbox_row_is_published_past_only_while_the_round_is_open() { + let db = store::create_local_db("sqlite::memory:").await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + store_graph_row(&db, instance_id, graph_id, GraphStatus::OperatorPresigned).await; + let message = committee_vote(instance_id, graph_id); + assert!( + send_protocol_message( + &mut UnavailablePublisher, + &db, + message.clone(), + StoredValue::Fresh + ) + .await + .is_err() + ); + let now = current_time_secs(); + let mut storage = db.acquire().await.unwrap(); + let rows = storage.claim_p2p_outbox_messages(now, now + 300, 10).await.unwrap(); + storage.fail_p2p_outbox_message(&rows[0].message_id, "corrupt").await.unwrap(); + drop(storage); + + // The row is gone for good, the round is not over: silence would strand it. + let mut publisher = RecordingPublisher::default(); + send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Found) + .await + .unwrap(); + assert_eq!(publisher.0.len(), 1); + + // Once the graph is finalized here, nobody needs the vote any more. + store_graph_row(&db, instance_id, graph_id, GraphStatus::CommitteePresigned).await; + send_protocol_message(&mut publisher, &db, message, StoredValue::Found).await.unwrap(); + assert_eq!(publisher.0.len(), 1); + } + + #[tokio::test] + async fn the_outbox_stops_a_round_by_its_graph_and_survives_a_bad_row() { + let db = store::create_local_db("sqlite::memory:").await; + let (instance_id, open_graph, finalized_graph) = + (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + store_graph_row(&db, instance_id, open_graph, GraphStatus::OperatorPresigned).await; + store_graph_row(&db, instance_id, finalized_graph, GraphStatus::CommitteePresigned).await; + // A graph row of another instance must not close this one's round. + let foreign_graph = Uuid::new_v4(); + store_graph_row(&db, Uuid::new_v4(), foreign_graph, GraphStatus::CommitteePresigned).await; + for graph_id in [open_graph, finalized_graph, foreign_graph] { + let _ = send_protocol_message( + &mut UnavailablePublisher, + &db, + committee_vote(instance_id, graph_id), + StoredValue::Fresh, + ) + .await; + } + // A corrupt row sits first in the batch; the rows behind it are still served. + let now = current_time_secs(); + db.acquire() + .await + .unwrap() + .enqueue_p2p_outbox_retry_message( + "protocol:AggNonceConsensus:graph:corrupt", + "AggNonceConsensus", + b"not a message", + now + 600, + 30, + None, + 0, + ) + .await + .unwrap(); + + let mut publisher = RecordingPublisher::default(); + handle_p2p_outbox_messages(&mut publisher, &db).await.unwrap(); + assert_eq!(publisher.0.len(), 2, "the open round and the foreign-row round publish"); + + let mut storage = db.acquire().await.unwrap(); + let state = + |graph_id: Uuid| protocol_outbox_id(&committee_vote(instance_id, graph_id).content); + let open = storage.p2p_outbox_entry_state(&state(open_graph)).await.unwrap().unwrap(); + assert_eq!(open.0, "Pending"); + let closed = + storage.p2p_outbox_entry_state(&state(finalized_graph)).await.unwrap().unwrap(); + assert_eq!(closed, ("Processed".to_owned(), 0)); + let foreign = storage.p2p_outbox_entry_state(&state(foreign_graph)).await.unwrap().unwrap(); + assert_eq!(foreign.0, "Pending"); + } + + #[test] + fn inbox_retention_sweeps_keep_the_tick_cadence_under_backlog() { + let start = Instant::now(); + // Whatever an earlier test left behind, two calls inside one interval + // cannot both be due. + let first = inbox_housekeeping_due(start); + let second = inbox_housekeeping_due(start + Duration::from_secs(1)); + assert!(!(first && second)); + assert!(!second); + let interval = Duration::from_secs(crate::env::REGULAR_TASK_INTERVAL_SECOND); + assert!(inbox_housekeeping_due(start + interval + Duration::from_secs(2))); + } + + #[tokio::test] + async fn inbox_dependency_retry_never_collapses_into_local_queue() { + let local_db = store::create_local_db("sqlite::memory:").await; + let message = GOATMessage::new( + Actor::Committee, + GOATMessageContent::SyncGraphRequest(SyncGraphRequest { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + }), + ); + let mut compensated = false; + let result = track_inbox_retry(DispatchFingerprint::of(message.content()), async { + push_local_unhandled_messages_with_reason( + &local_db, + &message, + 30, + MessageDeferReason::PreviousGraphPending, + "wait for graph", + ) + .await?; + // Some handlers rebroadcast a prerequisite only after asking + // for deferral. An early Err would skip that recovery action. + compensated = true; + Ok(()) + }) + .await; + assert!(compensated); + assert!(result.unwrap_err().downcast_ref::().is_some()); + let key = LocalMessageKey::from_content(Actor::Committee, message.content()).unwrap(); + assert!( + local_db + .acquire() + .await + .unwrap() + .find_messages_by_id(&key.message_id()) + .await + .unwrap() + .is_none() + ); + } + + /// Verify self-deferral survives later handler errors; other events still use the local queue. + #[tokio::test] + async fn deferral_survives_a_later_error_and_other_events_stay_local() { + let local_db = store::create_local_db("sqlite::memory:").await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let dispatched = GOATMessage::new( + Actor::Committee, + GOATMessageContent::SyncGraphRequest(SyncGraphRequest { instance_id, graph_id }), + ); + // The handler builds its own copy, under its own role: same message. + let own_copy = GOATMessage::new(Actor::Operator, dispatched.content().clone()); + let other_event = GOATMessage::new( + Actor::Committee, + GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), + ); + + let result = track_inbox_retry(DispatchFingerprint::of(dispatched.content()), async { + push_local_unhandled_messages_with_reason( + &local_db, + &other_event, + 30, + MessageDeferReason::ChainStatePending, + "a different event", + ) + .await?; + push_local_unhandled_messages_with_reason( + &local_db, + &own_copy, + 45, + MessageDeferReason::PreviousGraphPending, + "wait for graph", + ) + .await?; + bail!("compensation failed after the deferral was recorded") + }) + .await; + let retry = result.unwrap_err(); + let retry = retry + .downcast_ref::() + .expect("the recorded deferral wins over the later, non-retryable error"); + assert_eq!(retry.retry_after_secs, Some(45)); + + let mut storage = local_db.acquire().await.unwrap(); + let own_key = LocalMessageKey::from_content(Actor::Operator, own_copy.content()).unwrap(); + assert!(storage.find_messages_by_id(&own_key.message_id()).await.unwrap().is_none()); + let other_key = + LocalMessageKey::from_content(Actor::Committee, other_event.content()).unwrap(); + assert!( + storage.find_messages_by_id(&other_key.message_id()).await.unwrap().is_some(), + "an event of another kind is not the dispatched message" + ); + } + + /// Verify handler timeout, completion and shutdown outcomes. + #[tokio::test] + async fn handler_budget_cuts_off_an_overrunning_dispatch() { + let shutdown = CancellationToken::new(); + let budget = Duration::from_millis(100); + + let overrunning = async { + tokio::time::sleep(Duration::from_secs(3600)).await; + Ok::<(), anyhow::Error>(()) + }; + let outcome = + supervise_dispatch(tokio::time::timeout(budget, overrunning), &shutdown).await; + assert!(matches!(outcome, DispatchExecution::Completed(Err(_))), "cut off at the budget"); + + let prompt = async { + tokio::time::sleep(Duration::from_millis(1)).await; + Ok::<(), anyhow::Error>(()) + }; + let outcome = supervise_dispatch(tokio::time::timeout(budget, prompt), &shutdown).await; + assert!(matches!(outcome, DispatchExecution::Completed(Ok(Ok(()))))); + + shutdown.cancel(); + let outcome = supervise_dispatch( + tokio::time::timeout(budget, std::future::pending::>()), + &shutdown, + ) + .await; + assert!(matches!(outcome, DispatchExecution::Shutdown)); + } + #[test] fn soldering_proof_ready_is_descriptor_only() { let ready = SolderingProofReady { diff --git a/node/src/env.rs b/node/src/env.rs index 73ec519d..66318658 100644 --- a/node/src/env.rs +++ b/node/src/env.rs @@ -151,8 +151,40 @@ pub const ENV_MAINTENANCE_RUN_TIMEOUT_SECS: &str = "MAINTENANCE_RUN_TIMEOUT_SECS pub const DEFAULT_MAINTENANCE_RUN_TIMEOUT_SECS: u64 = 60; pub const ENV_P2P_INBOX_BATCH_SIZE: &str = "P2P_INBOX_BATCH_SIZE"; pub const DEFAULT_P2P_INBOX_BATCH_SIZE: i64 = 16; -pub const ENV_P2P_OUTBOX_BATCH_SIZE: &str = "P2P_OUTBOX_BATCH_SIZE"; -pub const DEFAULT_P2P_OUTBOX_BATCH_SIZE: i64 = 16; +/// Maximum decoded JSON gossip size. +pub const ENV_P2P_MAX_JSON_MESSAGE_BYTES: &str = "P2P_MAX_JSON_MESSAGE_BYTES"; +pub const DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES: usize = 2 * 1024 * 1024; +/// Ceiling on the payload bytes queued in the durable P2P inbox. The per-class +/// and per-sender quotas are derived from it, see `p2p_admission::InboundLimits`. +pub const ENV_P2P_INBOX_MAX_QUEUED_BYTES: &str = "P2P_INBOX_MAX_QUEUED_BYTES"; +pub const DEFAULT_P2P_INBOX_MAX_QUEUED_BYTES: i64 = 2 * 1024 * 1024 * 1024; +/// Ceilings on libp2p connections. Inbound only: outgoing dials stay unlimited. +/// Keep the inbound ceiling well below the process file descriptor limit. +pub const ENV_P2P_MAX_INCOMING_CONNECTIONS: &str = "P2P_MAX_INCOMING_CONNECTIONS"; +pub const DEFAULT_P2P_MAX_INCOMING_CONNECTIONS: u32 = 512; +/// Inbound slots, out of `P2P_MAX_INCOMING_CONNECTIONS`, only registered peers +/// may take. +pub const ENV_P2P_INBOUND_REGISTERED_RESERVE: &str = "P2P_INBOUND_REGISTERED_RESERVE"; +pub const DEFAULT_P2P_INBOUND_REGISTERED_RESERVE: u32 = 128; +/// Comma-separated peer IDs eligible for reserved inbound capacity. +/// Does not grant registration, message quota or ban exemption. +pub const ENV_P2P_RESERVED_PEERS: &str = "P2P_RESERVED_PEERS"; +/// Comma-separated `=` bindings. +/// Stake is verified on chain at startup. +pub const ENV_P2P_TRUSTED_OPERATOR_BINDINGS: &str = "P2P_TRUSTED_OPERATOR_BINDINGS"; +/// Longest wait between two re-publishes of a signing-round outbox row. +pub const ENV_P2P_PROTOCOL_RETRY_MAX_SECS: &str = "P2P_PROTOCOL_RETRY_MAX_SECS"; +pub const DEFAULT_P2P_PROTOCOL_RETRY_MAX_SECS: i64 = 600; +/// Handler execution timeout on the business/control worker. +/// Timeout cancels at an await; durable messages are retried. +pub const ENV_P2P_HANDLER_TIMEOUT_SECS: &str = "P2P_HANDLER_TIMEOUT_SECS"; +pub const DEFAULT_P2P_HANDLER_TIMEOUT_SECS: u64 = 300; +/// Comma-separated peer IDs whose persisted replay marks are cleared at startup. +pub const ENV_P2P_REPLAY_MARK_RESET_PEERS: &str = "P2P_REPLAY_MARK_RESET_PEERS"; +pub const ENV_P2P_MAX_PENDING_INCOMING_CONNECTIONS: &str = "P2P_MAX_PENDING_INCOMING_CONNECTIONS"; +pub const DEFAULT_P2P_MAX_PENDING_INCOMING_CONNECTIONS: u32 = 128; +pub const ENV_P2P_MAX_CONNECTIONS_PER_PEER: &str = "P2P_MAX_CONNECTIONS_PER_PEER"; +pub const DEFAULT_P2P_MAX_CONNECTIONS_PER_PEER: u32 = 4; pub const ENV_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE: &str = "ENABLE_COMMITTEE_INSTANCE_KEY_DELETE"; pub const DEFAULT_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE: bool = false; pub const ENV_COMMITTEE_INSTANCE_KEY_DELETE_TIMELOCK_BLOCKS: &str = @@ -736,12 +768,132 @@ pub fn get_p2p_inbox_batch_size() -> i64 { .unwrap_or(DEFAULT_P2P_INBOX_BATCH_SIZE) } -pub fn get_p2p_outbox_batch_size() -> i64 { - std::env::var(ENV_P2P_OUTBOX_BATCH_SIZE) +pub fn get_p2p_max_json_message_bytes() -> usize { + std::env::var(ENV_P2P_MAX_JSON_MESSAGE_BYTES) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|size| { + size.clamp(64 * 1024, crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE) + }) + .unwrap_or(DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES) +} + +fn p2p_connection_limit(name: &str, default: u32) -> u32 { + std::env::var(name) + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|limit| *limit > 0) + .unwrap_or(default) +} + +pub fn get_p2p_max_incoming() -> u32 { + p2p_connection_limit(ENV_P2P_MAX_INCOMING_CONNECTIONS, DEFAULT_P2P_MAX_INCOMING_CONNECTIONS) +} + +pub fn get_p2p_inbound_registered_reserve() -> u32 { + p2p_connection_limit(ENV_P2P_INBOUND_REGISTERED_RESERVE, DEFAULT_P2P_INBOUND_REGISTERED_RESERVE) +} + +pub fn get_p2p_reserved_peers() -> std::collections::HashSet { + parse_reserved_peers(&std::env::var(ENV_P2P_RESERVED_PEERS).unwrap_or_default()) +} + +pub fn parse_reserved_peers(value: &str) -> std::collections::HashSet { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .filter_map(|entry| { + let parsed = libp2p::PeerId::from_str(entry).ok(); + if parsed.is_none() { + tracing::error!("ignoring malformed {ENV_P2P_RESERVED_PEERS} entry {entry:?}"); + } + parsed + }) + .collect() +} + +/// Signing-round retry ceiling; defaults to one quarter of the pre-signing window, clamped to 30–600 seconds. +pub fn get_p2p_protocol_retry_max_secs() -> i64 { + std::env::var(ENV_P2P_PROTOCOL_RETRY_MAX_SECS) .ok() .and_then(|value| value.parse::().ok()) - .map(|size| size.clamp(1, 128)) - .unwrap_or(DEFAULT_P2P_OUTBOX_BATCH_SIZE) + .map(|secs| secs.clamp(30, 3600)) + .unwrap_or_else(|| { + (get_instance_presigned_time_expired_secs() / 4) + .clamp(30, DEFAULT_P2P_PROTOCOL_RETRY_MAX_SECS) + }) +} + +pub fn get_p2p_handler_timeout() -> std::time::Duration { + let secs = std::env::var(ENV_P2P_HANDLER_TIMEOUT_SECS) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|secs| secs.clamp(5, 3600)) + .unwrap_or(DEFAULT_P2P_HANDLER_TIMEOUT_SECS); + std::time::Duration::from_secs(secs) +} + +/// Parse `P2P_TRUSTED_OPERATOR_BINDINGS`. A malformed entry is reported and +/// skipped: a typo must not stop the node, nor silently trust something else. +pub fn get_p2p_trusted_operator_bindings() -> Vec<(libp2p::PeerId, [u8; 32])> { + parse_trusted_operator_bindings( + &std::env::var(ENV_P2P_TRUSTED_OPERATOR_BINDINGS).unwrap_or_default(), + ) +} + +pub fn parse_trusted_operator_bindings(value: &str) -> Vec<(libp2p::PeerId, [u8; 32])> { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .filter_map(|entry| { + let parsed = entry.split_once('=').and_then(|(peer_id, pubkey)| { + let peer_id = libp2p::PeerId::from_str(peer_id.trim()).ok()?; + let pubkey = PublicKey::from_str(pubkey.trim()).ok()?; + Some((peer_id, bitcoin::XOnlyPublicKey::from(pubkey).serialize())) + }); + if parsed.is_none() { + tracing::error!( + "ignoring malformed {ENV_P2P_TRUSTED_OPERATOR_BINDINGS} entry {entry:?}; \ + expected =" + ); + } + parsed + }) + .collect() +} + +pub fn get_p2p_replay_mark_reset_peers() -> Vec { + std::env::var(ENV_P2P_REPLAY_MARK_RESET_PEERS) + .map(|peers| { + peers + .split(',') + .map(str::trim) + .filter(|peer| !peer.is_empty()) + .map(String::from) + .collect() + }) + .unwrap_or_default() +} + +pub fn get_p2p_max_pending_incoming() -> u32 { + p2p_connection_limit( + ENV_P2P_MAX_PENDING_INCOMING_CONNECTIONS, + DEFAULT_P2P_MAX_PENDING_INCOMING_CONNECTIONS, + ) +} + +pub fn get_p2p_max_per_peer() -> u32 { + p2p_connection_limit(ENV_P2P_MAX_CONNECTIONS_PER_PEER, DEFAULT_P2P_MAX_CONNECTIONS_PER_PEER) +} + +pub fn get_p2p_inbox_max_queued_bytes() -> i64 { + std::env::var(ENV_P2P_INBOX_MAX_QUEUED_BYTES) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|size| size.max(256 * 1024 * 1024)) + .unwrap_or(DEFAULT_P2P_INBOX_MAX_QUEUED_BYTES) } pub fn is_enable_committee_instance_key_delete() -> bool { diff --git a/node/src/handle.rs b/node/src/handle.rs index 79833f90..33e3d124 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -6,7 +6,7 @@ use crate::env::{ }; use crate::error::SpecialError; use crate::metrics_service::MetricsState; -use crate::middleware::AllBehaviours; +use crate::middleware::publisher::MessagePublisher; use crate::rpc_service::current_time_secs; use crate::scheduled_tasks::graph_maintenance_tasks::ChallengeSubStatus; use crate::soldering_payload_store::{ @@ -42,8 +42,8 @@ use goat::transactions::base::output_topology; use goat::transactions::pre_signed::PreSignedTransaction; use goat::transactions::pre_signed_musig2::verify_public_nonce; use goat::wots::{Wots, Wots96}; +use libp2p::PeerId; use libp2p::gossipsub::MessageId; -use libp2p::{PeerId, Swarm}; use secp256k1::{Message as SecpMessage, SECP256K1}; use std::str::FromStr; use std::sync::Arc; @@ -53,7 +53,7 @@ use store::{GoatTxType, GraphStatus, SerializableTxid}; use uuid::Uuid; pub struct HandlerContext<'a> { - pub swarm: &'a mut Swarm, + pub swarm: &'a mut dyn MessagePublisher, pub local_db: &'a LocalDB, pub btc_client: &'a Arc, pub goat_client: &'a Arc, @@ -7038,18 +7038,45 @@ async fn handle_sync_graph_request( // sent by other nodes when they find a graph is missing locally // 1. (Relayer) send SyncGraph response if have the graph if !is_relayer() { - tracing::warn!("Ignore SyncGraphRequest for {instance_id}:{graph_id}: not a relayer node"); + tracing::debug!("Ignore SyncGraphRequest for {instance_id}:{graph_id}: not a relayer node"); return Ok(()); } - if let Some(graph) = get_graph(ctx.local_db, instance_id, graph_id).await? { - let message_content = - GOATMessageContent::SyncGraph(SyncGraph { instance_id, graph_id, graph }); - let message = GOATMessage::new(Actor::All, message_content); - send_to_peer(ctx.swarm, message).await?; - } else { + // Check graph ownership before charging the response budget or recording cooldown. + let now = std::time::Instant::now(); + let graph_key = graph_id.to_string(); + if crate::p2p_admission::sync_graph_response_gate().is_cooling(&graph_key, now) { + return Ok(()); + } + // Require the graph to belong to the requested instance. + if !ctx.local_db.acquire().await?.has_graph_of_instance(&instance_id, &graph_id).await? { + tracing::debug!("Graph not found for SyncGraphRequest {instance_id}:{graph_id}"); + return Ok(()); + } + let requester_registered = + crate::p2p_admission::peer_registry().sender_class(&ctx.from_peer_id, now).is_registered(); + if !crate::p2p_admission::sync_graph_response_budget(requester_registered).allow(now) { + tracing::debug!( + "Defer SyncGraphRequest for {instance_id}:{graph_id}: sync-graph response budget spent" + ); + return Ok(()); + } + let Some(graph) = get_graph(ctx.local_db, instance_id, graph_id).await? else { // TODO: if no relayer has the graph, how to recover? - tracing::warn!("Graph not found for SyncGraphRequest {instance_id}:{graph_id}"); + tracing::warn!("Graph not loadable for SyncGraphRequest {instance_id}:{graph_id}"); + return Ok(()); + }; + // The cooldown is taken only for a response that is actually going out. + if !crate::p2p_admission::sync_graph_response_gate().allow(&graph_key, now) { + return Ok(()); } + let message_content = GOATMessageContent::SyncGraph(SyncGraph { instance_id, graph_id, graph }); + send_protocol_message( + ctx.swarm, + ctx.local_db, + GOATMessage::new(Actor::All, message_content), + StoredValue::Fresh, + ) + .await?; Ok(()) } @@ -7060,24 +7087,40 @@ async fn handle_sync_graph( graph_id: Uuid, graph: &SimplifiedBitvmGcGraph, ) -> Result<()> { - // sent by relayer nodes in response to SyncGraphRequest - if !ctx - .goat_client - .committee_mana_is_validate_peer_id(&ctx.from_peer_id.to_bytes()) - .await - .with_context(|| { - format!( - "failed to validate SyncGraph sender {} against the committee registry", - ctx.from_peer_id - ) - })? + // Accept only responses to a recorded local request. + if !crate::p2p_admission::requested_graphs() + .contains(&graph_id.to_string(), std::time::Instant::now()) { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: sender {} is not a registered committee peer", + tracing::debug!( + "Ignore SyncGraph for {instance_id}:{graph_id}: no matching local sync request" + ); + return Ok(()); + } + // Require a cached committee identity for the relayer. + if !crate::p2p_admission::peer_registry() + .is_committee(&ctx.from_peer_id, std::time::Instant::now()) + { + tracing::debug!( + "Ignore SyncGraph for {instance_id}:{graph_id}: sender {} is not a committee peer", ctx.from_peer_id ); return Ok(()); } + // Apply rejection cooldown per graph and sender; RPC failures leave the gate open. + let gate_key = format!("{graph_id}:{}", ctx.from_peer_id); + if crate::p2p_admission::sync_graph_validation_gate() + .is_cooling(&gate_key, std::time::Instant::now()) + { + tracing::debug!( + "Ignore SyncGraph for {instance_id}:{graph_id}: this sender's graph was rejected recently" + ); + return Ok(()); + } + let reject = |why: String| { + crate::p2p_admission::sync_graph_validation_gate() + .allow(&gate_key, std::time::Instant::now()); + tracing::warn!("Ignore SyncGraph for {instance_id}:{graph_id}: {why}"); + }; if !message_identity_matches( "SyncGraph", @@ -7088,13 +7131,12 @@ async fn handle_sync_graph( graph.parameters.graph_id, graph.parameters.graph_nonce, ) { + reject("envelope and graph parameters name different graphs".to_string()); return Ok(()); } - validate_graph_id_on_goat(ctx.goat_client, instance_id, graph_id).await.map_err(|e| { - anyhow!( - "Failed to validate graph_id on GoatChain for SyncGraph {instance_id}:{graph_id}: {e}" - ) + validate_graph_id_on_goat(ctx.goat_client, instance_id, graph_id).await.with_context(|| { + format!("Failed to validate graph_id on GoatChain for SyncGraph {instance_id}:{graph_id}") })?; let validation = validate_graph_instance_parameters( ctx.btc_client, @@ -7104,37 +7146,37 @@ async fn handle_sync_graph( .await; ctx.metrics_state.record_graph_validation(validation.is_ok()); if let Err(e) = validation { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: invalid instance parameters: {e}" - ); + // This check talks to both chains. "Could not ask" is not a verdict on + // the graph: let the inbox retry instead of rejecting a good answer. + if crate::action::is_retryable_external_rpc_error(&e) { + return Err(e).context("validate SyncGraph instance parameters"); + } + reject(format!("invalid instance parameters: {e}")); return Ok(()); } let graph = BitvmGcGraph::from_simplified(graph)?; let graph_data = build_graph_data(&graph)?; let graph_data_on_goat = ctx.goat_client.gateway_get_graph_data(&graph_id).await?; if graph_data != graph_data_on_goat { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: reconstructed graph data does not match GoatChain" - ); + reject("reconstructed graph data does not match GoatChain".to_string()); return Ok(()); } if let Err(e) = verify_graph_operator_pre_signatures(&graph) { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: invalid operator pre-signatures: {e}" - ); + reject(format!("invalid operator pre-signatures: {e}")); return Ok(()); } if let Err(e) = verify_graph_committee_pre_signatures(&graph) { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: invalid committee pre-signatures: {e}" - ); + reject(format!("invalid committee pre-signatures: {e}")); return Ok(()); } let simplified_graph = graph.to_simplified()?; let _ = store_finalized_graph_if_needed(ctx.local_db, &simplified_graph).await?; refresh_and_compensate(ctx, instance_id, graph_id, &graph, GraphStatus::OperatorPresigned) .await?; + // Close only after compensation succeeds. Otherwise a transient refresh + // failure would make this inbox row's retry look unsolicited and drop it. + crate::p2p_admission::requested_graphs().remove(&graph_id.to_string()); Ok(()) } @@ -7142,19 +7184,104 @@ async fn handle_sync_graph( /// sender's own record: the registry is keyed by `peer_id`, and without this /// check any peer could overwrite any other node's row. fn accept_node_info(ctx: &HandlerContext<'_>, node_info: &NodeInfo, message_kind: &str) -> bool { + let Err(reason) = check_node_info(ctx, node_info) else { + return true; + }; + // Log registered-peer failures individually and aggregate unregistered-peer failures. + let registered = crate::p2p_admission::peer_registry() + .sender_class(&ctx.from_peer_id, std::time::Instant::now()) + .is_registered(); + if registered { + tracing::warn!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); + } else { + crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::NodeInfoRejected); + tracing::debug!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); + } + false +} + +fn check_node_info( + ctx: &HandlerContext<'_>, + node_info: &NodeInfo, +) -> std::result::Result<(), String> { if !node_info_matches_sender(node_info, &ctx.from_peer_id) { - tracing::warn!( - "Ignore {message_kind} from {}: payload claims peer_id {}", - ctx.from_peer_id, - node_info.peer_id + return Err(format!("payload claims peer_id {}", node_info.peer_id)); + } + validate_node_info_payload(node_info)?; + if node_info.binding_sig.is_empty() { + return Ok(()); + } + // A present-but-invalid binding is a forgery attempt (claiming another key's + // authorisation), not a benign omission, so reject the whole payload. + if crate::action::verify_node_info_binding(node_info).is_none() { + return Err("node info binding signature is invalid".to_string()); + } + // `binding_issued_at` orders re-bindings, newest winning. One dated far ahead + // would win against every honest re-binding until that date. + if node_info.binding_issued_at + > current_time_secs() + crate::action::NODE_INFO_BINDING_MAX_FUTURE_SECS + { + return Err("node info binding is dated in the future".to_string()); + } + Ok(()) +} + +/// Record verified operator bindings by x-only key; confirm stake asynchronously. +async fn learn_node_info_binding( + local_db: &LocalDB, + goat_client: &Arc, + node_info: &NodeInfo, +) { + if node_info.actor != Actor::Operator.to_string() { + return; + } + let Some(operator_key) = crate::action::verify_node_info_binding(node_info) else { + return; + }; + let operator_key = crate::p2p_admission::operator_key(&operator_key); + if let Ok(peer_id) = PeerId::from_str(&node_info.peer_id) { + let released_confirmed_key = crate::p2p_admission::peer_registry() + .observe_operator_binding(&peer_id, &operator_key, node_info.binding_issued_at); + if released_confirmed_key { + // The peer moved to another key: what was confirmed for the old one + // must not be there to be restored after a restart. + crate::p2p_admission::revoke_persisted_operator(local_db, &peer_id).await; + } + crate::p2p_admission::refresh_operator_binding_in_background( + local_db, + goat_client, + peer_id, + operator_key, ); - return false; } - if let Err(reason) = validate_node_info_payload(node_info) { - tracing::warn!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); - return false; +} + +/// Store accepted NodeInfo within the unregistered-row limit and learn its binding. +async fn record_node_info(ctx: &HandlerContext<'_>, node_info: &NodeInfo) -> Result<()> { + let registered = crate::p2p_admission::peer_registry() + .sender_class(&ctx.from_peer_id, std::time::Instant::now()) + .is_registered(); + let admissible = registered + || ctx + .local_db + .acquire() + .await? + .node_row_admissible( + &node_info.peer_id, + crate::action::NODE_TABLE_MAX_UNREGISTERED_ROWS, + current_time_secs() - crate::action::NODE_TABLE_EVICTABLE_AFTER_SECS, + &crate::env::get_peer_id(), + ) + .await?; + if admissible { + save_node_info(ctx.local_db, node_info).await?; + } else { + crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::NodeTableFull); } - true + // The binding is learned either way: it lives in the bounded registry, and + // an operator must be able to become registered while the table is full. + learn_node_info_binding(ctx.local_db, ctx.goat_client, node_info).await; + Ok(()) } async fn handle_request_node_info( @@ -7162,21 +7289,45 @@ async fn handle_request_node_info( node_info: &NodeInfo, ) -> Result<()> { if accept_node_info(ctx, node_info, "RequestNodeInfo") { - save_node_info(ctx.local_db, node_info).await?; + record_node_info(ctx, node_info).await?; } // Answer regardless: the response only carries this node's own public info, // and staying silent would break discovery for a misconfigured peer. + // Coalesce recent requests into one broadcast response. + if crate::p2p_admission::node_info_response_gate().try_respond(std::time::Instant::now()) { + publish_node_info_response(ctx.swarm).await?; + } + Ok(()) +} + +async fn publish_node_info_response(swarm: &mut dyn MessagePublisher) -> Result<()> { let message_content = GOATMessageContent::ResponseNodeInfo(crate::env::get_local_node_info()); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::All, message_content)).await?; + send_to_peer(swarm, GOATMessage::new(Actor::All, message_content)).await?; Ok(()) } +/// Answer the `RequestNodeInfo`s that arrived while the previous response was +/// still cooling down. Called from the regular tick. +pub async fn flush_deferred_node_info_response(swarm: &mut dyn MessagePublisher) { + if !crate::p2p_admission::node_info_response_gate().take_pending(std::time::Instant::now()) { + return; + } + if let Err(error) = publish_node_info_response(swarm).await { + tracing::debug!( + event = "p2p_node_info", + outcome = "deferred_response_failed", + error = %error, + "failed to publish the deferred node info response; peers will ask again" + ); + } +} + async fn handle_response_node_info( ctx: &mut HandlerContext<'_>, node_info: &NodeInfo, ) -> Result<()> { if accept_node_info(ctx, node_info, "ResponseNodeInfo") { - save_node_info(ctx.local_db, node_info).await?; + record_node_info(ctx, node_info).await?; } Ok(()) } diff --git a/node/src/lib.rs b/node/src/lib.rs index fe677b09..78665b20 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -3,6 +3,7 @@ pub mod env; pub mod handle; pub mod metrics_service; pub mod middleware; +pub mod p2p_admission; pub mod p2p_msg_handler; pub mod rpc_service; diff --git a/node/src/main.rs b/node/src/main.rs index d56312e6..0227df49 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -25,7 +25,7 @@ use bitvm_noded::{ }; use anyhow::Result; -use bitvm_noded::action::reclaim_stale_queue_claims; +use bitvm_noded::action::{load_persisted_peer_bindings, reclaim_stale_queue_claims}; use bitvm_noded::metrics_service::{MetricsState, set_node_metrics_state}; use bitvm_noded::middleware::swarm::{BitvmNetworkManager, BitvmSwarmConfig}; use bitvm_noded::p2p_msg_handler::BitvmNodeProcessor; @@ -212,18 +212,34 @@ async fn main() -> Result<(), Box> { let metric_registry = Arc::new(Mutex::new(metric_registry)); let metrics_state = MetricsState::new(metric_registry); set_node_metrics_state(metrics_state.clone()); + let goat_client = + Arc::new(GOATClient::new(env::goat_config_from_env().await, env::get_goat_network())); + // Restore persisted operator bindings into the admission registry. + match load_persisted_peer_bindings(&local_db, &goat_client).await { + Ok(loaded) if loaded > 0 => tracing::info!( + event = "p2p_admission", + outcome = "bindings_loaded", + loaded, + "re-seeded operator bindings from previous sessions" + ), + Ok(_) => {} + Err(error) => tracing::warn!( + event = "p2p_admission", + outcome = "bindings_load_failed", + error = %error, + "failed to re-seed operator bindings; they will be relearned from gossip" + ), + } let handler = BitvmNodeProcessor { local_db: local_db.clone(), btc_client: Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())), - goat_client: Arc::new(GOATClient::new( - env::goat_config_from_env().await, - env::get_goat_network(), - )), - http_client: HttpAsyncClient::new(None), + goat_client, + http_client: Arc::new(HttpAsyncClient::new(None)), soldering_builder: actor_needs_soldering_builder(&actor) .then(|| Arc::new(BabeBundleBuilder::new())), metrics_state: metrics_state.clone(), shutdown_token: cancellation_token.clone(), + worker: Default::default(), }; tracing::info!( diff --git a/node/src/middleware/behaviour.rs b/node/src/middleware/behaviour.rs index 6e20aac4..e130bfc9 100644 --- a/node/src/middleware/behaviour.rs +++ b/node/src/middleware/behaviour.rs @@ -1,22 +1,49 @@ +use bitvm_lib::actors::Actor; use libp2p::identity::Keypair; -use libp2p::{gossipsub, kad, kad::store::MemoryStore, swarm::StreamProtocol}; +use libp2p::{connection_limits, gossipsub, kad, kad::store::MemoryStore, swarm::StreamProtocol}; use libp2p_swarm_derive::NetworkBehaviour; use std::time::Duration; use tokio::io::{self}; pub const MAX_GOSSIPSUB_TRANSMIT_SIZE: usize = 16 * 1024 * 1024; +/// Peers may only register interest in the role topics of this protocol; a +/// subscription to anything else is ignored instead of being tracked. +pub type TopicFilter = + gossipsub::MaxCountSubscriptionFilter; + +/// Every role topic, whether or not this node subscribes to it. +fn protocol_topics() -> std::collections::HashSet { + [ + Actor::Committee, + Actor::Operator, + Actor::Verifier, + Actor::Watchtower, + Actor::Publisher, + Actor::All, + ] + .iter() + .map(|actor| gossipsub::IdentTopic::new(get_topic_name(&actor.to_string())).hash()) + .collect() +} + // We create a custom network behaviour that combines Kademlia and mDNS. #[derive(NetworkBehaviour)] pub struct AllBehaviours { + /// Enforce bans and reserved inbound capacity before other behaviours. + pub connection_gate: super::connection_gate::ConnectionGate, + /// Ceilings on half-open inbound connections and on connections per peer. + pub connection_limits: connection_limits::Behaviour, pub kademlia: kad::Behaviour, //pub mdns: mdns::tokio::Behaviour, - pub gossipsub: gossipsub::Behaviour, + pub gossipsub: gossipsub::Behaviour, } impl AllBehaviours { pub fn new(key: &Keypair) -> Self { let mut cfg = kad::Config::new(get_proto_name()); cfg.set_query_timeout(Duration::from_secs(5 * 60)); + // Use Kademlia for discovery only; ignore remote record and provider writes. + cfg.set_record_filtering(kad::StoreInserts::FilterBoth); let store = kad::store::MemoryStore::new(key.public().to_peer_id()); let kademlia = kad::Behaviour::with_config(key.public().to_peer_id(), store, cfg); //let mdns = mdns::tokio::Behaviour::new(mdns::Config::default(), key.public().to_peer_id()) @@ -24,15 +51,37 @@ impl AllBehaviours { let gossipsub_config = gossipsub::ConfigBuilder::default() .max_transmit_size(MAX_GOSSIPSUB_TRANSMIT_SIZE) + // Require application admission before gossip forwarding. + .validate_messages() .build() .map_err(io::Error::other) .unwrap(); - let gossipsub = gossipsub::Behaviour::new( + let topics = protocol_topics(); + let subscription_filter = gossipsub::MaxCountSubscriptionFilter { + max_subscribed_topics: topics.len(), + max_subscriptions_per_request: 4 * topics.len(), + filter: gossipsub::WhitelistSubscriptionFilter(topics), + }; + let gossipsub = gossipsub::Behaviour::new_with_subscription_filter( gossipsub::MessageAuthenticity::Signed(key.clone()), gossipsub_config, + None, + subscription_filter, ) .expect("Valid configuration"); - Self { kademlia, gossipsub } + // Limit half-open inbound connections and connections per peer. + let connection_limits = connection_limits::Behaviour::new( + connection_limits::ConnectionLimits::default() + .with_max_pending_incoming(Some(crate::env::get_p2p_max_pending_incoming())) + .with_max_established_per_peer(Some(crate::env::get_p2p_max_per_peer())), + ); + // ConnectionGate enforces registered-aware inbound capacity. + let connection_gate = super::connection_gate::ConnectionGate::new( + crate::env::get_p2p_max_incoming() as usize, + crate::env::get_p2p_inbound_registered_reserve() as usize, + crate::env::get_p2p_reserved_peers(), + ); + Self { connection_gate, connection_limits, kademlia, gossipsub } } } diff --git a/node/src/middleware/connection_gate.rs b/node/src/middleware/connection_gate.rs new file mode 100644 index 00000000..6c0512f3 --- /dev/null +++ b/node/src/middleware/connection_gate.rs @@ -0,0 +1,246 @@ +//! Inbound connection quotas and bidirectional ban enforcement. +//! Reserved capacity covers registered identities and `P2P_RESERVED_PEERS`. +//! Unknown operators require a prior binding or explicit reserved-peer configuration. +//! Unlisted watchtowers and challengers use general capacity. + +use std::collections::HashMap; +use std::convert::Infallible; +use std::task::{Context, Poll}; +use std::time::Instant; + +use libp2p::PeerId; +use libp2p::core::{ConnectedPoint, Endpoint, Multiaddr, transport::PortUse}; +use libp2p::swarm::behaviour::{ConnectionClosed, ConnectionEstablished}; +use libp2p::swarm::{ + ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent, + THandlerOutEvent, ToSwarm, dummy, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Refusal { + /// The peer is serving a local ban. + Banned, + /// Every inbound slot is taken. + InboundFull, + /// Only slots reserved for registered peers are left. + ReservedForRegistered, +} + +impl Refusal { + pub const fn as_str(self) -> &'static str { + match self { + Self::Banned => "banned", + Self::InboundFull => "inbound_full", + Self::ReservedForRegistered => "reserved_for_registered", + } + } +} + +impl std::fmt::Display for Refusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "connection refused: {}", self.as_str()) + } +} + +impl std::error::Error for Refusal {} + +/// Inbound slot accounting. +#[derive(Debug)] +pub struct InboundSlots { + max_inbound: usize, + /// Slots of `max_inbound` only registered peers may take. + registered_reserve: usize, + /// Established inbound connections; `true` when admitted as registered. + established: HashMap, +} + +impl InboundSlots { + pub fn new(max_inbound: usize, registered_reserve: usize) -> Self { + Self { + max_inbound, + registered_reserve: registered_reserve.min(max_inbound), + established: HashMap::new(), + } + } + + pub fn admit(&self, registered: bool) -> Result<(), Refusal> { + if self.established.len() >= self.max_inbound { + return Err(Refusal::InboundFull); + } + let unregistered = self.established.values().filter(|registered| !**registered).count(); + if !registered && unregistered >= self.max_inbound - self.registered_reserve { + return Err(Refusal::ReservedForRegistered); + } + Ok(()) + } + + fn opened(&mut self, connection: ConnectionId, registered: bool) { + self.established.insert(connection, registered); + } + + fn closed(&mut self, connection: &ConnectionId) { + self.established.remove(connection); + } +} + +pub struct ConnectionGate { + slots: InboundSlots, + /// Peer ids the deployment guarantees a reserved slot to. + reserved_peers: std::collections::HashSet, + /// Class decided when the connection was admitted, until it is established. + admitting: HashMap, +} + +impl ConnectionGate { + pub fn new( + max_inbound: usize, + registered_reserve: usize, + reserved_peers: std::collections::HashSet, + ) -> Self { + Self { + slots: InboundSlots::new(max_inbound, registered_reserve), + reserved_peers, + admitting: HashMap::new(), + } + } + + /// Whether `peer` may use the reserved slots. + fn is_privileged(&self, peer: &PeerId) -> bool { + self.reserved_peers.contains(peer) + || crate::p2p_admission::peer_registry() + .sender_class(peer, Instant::now()) + .is_registered() + } + + fn refuse(peer: &PeerId, refusal: Refusal) -> ConnectionDenied { + if refusal == Refusal::ReservedForRegistered { + crate::p2p_admission::note_refused_peer(*peer); + } + ConnectionDenied::new(refusal) + } + + fn check_ban(peer: &PeerId) -> Result<(), ConnectionDenied> { + if crate::p2p_admission::direct_peer_strikes().is_banned(peer, Instant::now()) { + return Err(Self::refuse(peer, Refusal::Banned)); + } + Ok(()) + } +} + +impl NetworkBehaviour for ConnectionGate { + type ConnectionHandler = dummy::ConnectionHandler; + type ToSwarm = Infallible; + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + _: &Multiaddr, + _: &Multiaddr, + ) -> Result, ConnectionDenied> { + Self::check_ban(&peer)?; + let registered = self.is_privileged(&peer); + self.slots.admit(registered).map_err(|refusal| Self::refuse(&peer, refusal))?; + self.admitting.insert(connection_id, registered); + Ok(dummy::ConnectionHandler) + } + + fn handle_established_outbound_connection( + &mut self, + _: ConnectionId, + peer: PeerId, + _: &Multiaddr, + _: Endpoint, + _: PortUse, + ) -> Result, ConnectionDenied> { + // Outbound dials are this node's own choice and are not limited, but a + // ban holds in both directions. + Self::check_ban(&peer)?; + Ok(dummy::ConnectionHandler) + } + + fn on_swarm_event(&mut self, event: FromSwarm) { + match event { + FromSwarm::ConnectionEstablished(ConnectionEstablished { + connection_id, + endpoint: ConnectedPoint::Listener { .. }, + .. + }) => { + let registered = self.admitting.remove(&connection_id).unwrap_or(false); + self.slots.opened(connection_id, registered); + } + FromSwarm::ConnectionClosed(ConnectionClosed { connection_id, .. }) => { + self.slots.closed(&connection_id); + } + FromSwarm::ListenFailure(failure) => { + self.admitting.remove(&failure.connection_id); + } + _ => {} + } + } + + fn on_connection_handler_event( + &mut self, + _: PeerId, + _: ConnectionId, + event: THandlerOutEvent, + ) { + match event {} + } + + fn poll(&mut self, _: &mut Context<'_>) -> Poll>> { + Poll::Pending + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Verify configured peers receive connection reservation only. + #[test] + fn listed_peers_may_use_the_reserved_slots() { + let (watchtower, stranger) = (PeerId::random(), PeerId::random()); + let listed = crate::env::parse_reserved_peers(&format!(" {watchtower} ,not-a-peer-id,,")); + assert_eq!(listed, std::collections::HashSet::from([watchtower])); + + let gate = ConnectionGate::new(4, 2, listed); + assert!(gate.is_privileged(&watchtower)); + assert!(!gate.is_privileged(&stranger)); + assert!( + !crate::p2p_admission::peer_registry() + .sender_class(&watchtower, Instant::now()) + .is_registered(), + "a reserved slot is not a sender class" + ); + } + + /// Unregistered peers can fill only what is not reserved; registered peers + /// still get in after that, up to the ceiling. + #[test] + fn reserved_inbound_slots_are_kept_for_registered_peers() { + let mut slots = InboundSlots::new(4, 2); + for _ in 0..2 { + assert_eq!(slots.admit(false), Ok(())); + slots.opened(ConnectionId::new_unchecked(slots.established.len()), false); + } + assert_eq!(slots.admit(false), Err(Refusal::ReservedForRegistered)); + for _ in 0..2 { + assert_eq!(slots.admit(true), Ok(())); + slots.opened(ConnectionId::new_unchecked(slots.established.len()), true); + } + assert_eq!(slots.admit(true), Err(Refusal::InboundFull)); + + // A closed unregistered connection frees an open slot again. + slots.closed(&ConnectionId::new_unchecked(0)); + assert_eq!(slots.admit(false), Ok(())); + + // Registered peers are not confined to the reserve. + let mut slots = InboundSlots::new(3, 1); + for index in 0..3 { + assert_eq!(slots.admit(true), Ok(())); + slots.opened(ConnectionId::new_unchecked(index), true); + } + assert_eq!(slots.admit(true), Err(Refusal::InboundFull)); + } +} diff --git a/node/src/middleware/mod.rs b/node/src/middleware/mod.rs index 5c0ab494..7c7ec948 100644 --- a/node/src/middleware/mod.rs +++ b/node/src/middleware/mod.rs @@ -1,4 +1,6 @@ pub mod behaviour; +pub mod connection_gate; +pub mod publisher; pub mod swarm; pub use behaviour::{AllBehaviours, get_topic_name, split_topic_name}; diff --git a/node/src/middleware/publisher.rs b/node/src/middleware/publisher.rs new file mode 100644 index 00000000..b4b454bf --- /dev/null +++ b/node/src/middleware/publisher.rs @@ -0,0 +1,114 @@ +//! Bounded, acknowledged access to the swarm from the business worker. +use futures::future::BoxFuture; +use libp2p::{ + Swarm, + gossipsub::{IdentTopic, MessageId}, +}; +use tokio::sync::{mpsc, oneshot}; + +use super::AllBehaviours; + +pub trait MessagePublisher: Send { + fn publish_message( + &mut self, + topic: IdentTopic, + data: Vec, + ) -> BoxFuture<'_, anyhow::Result>; +} + +impl MessagePublisher for Swarm { + fn publish_message( + &mut self, + topic: IdentTopic, + data: Vec, + ) -> BoxFuture<'_, anyhow::Result> { + Box::pin( + async move { self.behaviour_mut().gossipsub.publish(topic, data).map_err(Into::into) }, + ) + } +} + +impl MessagePublisher for super::swarm::BitvmSwarmWrapper { + fn publish_message( + &mut self, + topic: IdentTopic, + data: Vec, + ) -> BoxFuture<'_, anyhow::Result> { + self.0.publish_message(topic, data) + } +} + +pub struct PublishCommand { + topic: IdentTopic, + data: Vec, + result: oneshot::Sender>, +} + +impl PublishCommand { + pub fn execute(self, swarm: &mut Swarm) { + // A timed-out/cancelled caller must not leave a delayed broadcast behind. + if !self.result.is_closed() { + let result = swarm.behaviour_mut().gossipsub.publish(self.topic, self.data); + let _ = self.result.send(result.map_err(Into::into)); + } + } +} + +/// Maximum publish wait; cancellation closes the reply and skips queued commands. +const PUBLISH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[derive(Clone)] +pub struct NetworkPublisher(mpsc::Sender); + +pub fn channel() -> (NetworkPublisher, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(8); + (NetworkPublisher(tx), rx) +} + +impl MessagePublisher for NetworkPublisher { + fn publish_message( + &mut self, + topic: IdentTopic, + data: Vec, + ) -> BoxFuture<'_, anyhow::Result> { + Box::pin(async move { + let (tx, rx) = oneshot::channel(); + let publish = async { + self.0 + .send(PublishCommand { topic, data, result: tx }) + .await + .map_err(|_| anyhow::anyhow!("network publisher stopped"))?; + rx.await + .map_err(|_| anyhow::anyhow!("network publisher stopped before publishing"))? + }; + tokio::time::timeout(PUBLISH_TIMEOUT, publish) + .await + .map_err(|_| anyhow::anyhow!("network task did not take the publish in time"))? + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn publish_waits_for_actual_network_result_and_cancellation_closes_reply() { + let (mut publisher, mut commands) = channel(); + let mut send = publisher.publish_message(IdentTopic::new("test"), vec![1]); + let command = tokio::select! { + result = &mut send => panic!("must wait for network: {result:?}"), + command = commands.recv() => command.unwrap(), + }; + command.result.send(Err(anyhow::anyhow!("not published"))).unwrap(); + assert!(send.await.unwrap_err().to_string().contains("not published")); + + let mut send = publisher.publish_message(IdentTopic::new("test"), vec![2]); + let command = tokio::select! { + _ = &mut send => panic!("must wait for network"), + command = commands.recv() => command.unwrap(), + }; + drop(send); + assert!(command.result.is_closed()); + } +} diff --git a/node/src/middleware/swarm.rs b/node/src/middleware/swarm.rs index bf7b5e50..9d325dec 100644 --- a/node/src/middleware/swarm.rs +++ b/node/src/middleware/swarm.rs @@ -21,7 +21,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; use zeroize::Zeroizing; -pub struct BitvmSwarmWrapper(pub Swarm); +pub struct BitvmSwarmWrapper(pub Swarm, Option); impl std::fmt::Debug for BitvmSwarmWrapper { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -31,7 +31,11 @@ impl std::fmt::Debug for BitvmSwarmWrapper { impl BitvmSwarmWrapper { pub fn new(swarm: Swarm) -> Self { - Self(swarm) + Self(swarm, None) + } + + pub fn publisher(&self) -> super::publisher::NetworkPublisher { + self.1.clone().expect("network manager owns the publish command receiver") } pub fn inner(&self) -> &Swarm { @@ -67,11 +71,16 @@ pub enum TickMessageType { #[allow(async_fn_in_trait)] pub trait P2pMessageHandler { + /// Handle received gossip and report one verdict for `id` and `propagation_source`. + /// `sequence_number` is signed by the author. + #[allow(clippy::too_many_arguments)] async fn recv_and_dispatch( &self, swarm: &mut BitvmSwarmWrapper, actor: Actor, from_peer_id: PeerId, + propagation_source: PeerId, + sequence_number: Option, id: MessageId, message: &[u8], ) -> anyhow::Result<()>; @@ -112,6 +121,7 @@ pub struct BitvmNetworkManager { swarm: BitvmSwarmWrapper, connected_peers: Gauge, required_topics_healthy: Gauge, + publish_commands: tokio::sync::mpsc::Receiver, } impl BitvmNetworkManager { pub fn new( @@ -149,12 +159,14 @@ impl BitvmNetworkManager { let (peer_id, multi_addr) = parse_boot_node_str(peer)?; swarm.behaviour_mut().kademlia.add_address(&peer_id, multi_addr); } + let (publisher, publish_commands) = super::publisher::channel(); Ok(BitvmNetworkManager { config, - swarm: BitvmSwarmWrapper::new(swarm), + swarm: BitvmSwarmWrapper(swarm, Some(publisher)), peer_id: key_pair.public().to_peer_id(), connected_peers, required_topics_healthy, + publish_commands, }) } @@ -212,15 +224,20 @@ impl BitvmNetworkManager { info!("multi_addr: {}/p2p/{}", address.to_string(), self.peer_id.to_string()); let mut heart_beat_interval = interval(Duration::from_secs(self.config.heartbeat_interval)); let mut interval = interval(Duration::from_secs(self.config.regular_task_interval)); + // Use Delay ticks and coalesced worker notifications. + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { select! { + Some(command) = self.publish_commands.recv() => { + command.execute(&mut self.swarm); + } _ = cancellation_token.cancelled() => { info!("Swarm received shutdown signal"); msg_handler.graceful_shutdown().await?; return Ok("swarm_shutdown".to_string()); } - _ticker = interval.tick() => { + _ = interval.tick() => { match msg_handler.handle_tick_message(&mut self.swarm, self.peer_id, actor.clone(), TickMessageType::RegularlyAction).await { Ok(_) => {} Err(e) => { @@ -255,7 +272,7 @@ impl BitvmNetworkManager { let data_prefix = hex::encode(&message.data[..message.data.len().min(16)]); let data_starts_with_goatbin = message.data.starts_with(b"GOATBIN1"); match msg_handler.recv_and_dispatch(&mut self.swarm, actor.clone(), - source, id.clone(), &message.data).await { + source, propagation_source, message.sequence_number, id.clone(), &message.data).await { Ok(_) => {},Err(e) => { tracing::error!( error = ?e, diff --git a/node/src/p2p_admission.rs b/node/src/p2p_admission.rs new file mode 100644 index 00000000..1e34b3c3 --- /dev/null +++ b/node/src/p2p_admission.rs @@ -0,0 +1,4087 @@ +//! Inbound gossip admission: envelope, identity, replay, rate and storage checks. +//! Durable messages are admitted before persistence and forwarding. +//! Rejected messages use gossipsub `Ignore`. + +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use client::goat_chain::GOATClient; +use libp2p::PeerId; +use sha2::{Digest, Sha256}; +use store::localdb::LocalDB; +use store::{P2pInboxAdmissionClass, P2pInboxClassUsage}; + +use crate::action::{GOATMessage, GOATMessageContent, P2PMessageDelivery}; +use crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE; + +/// Pending retention for unregistered senders; registered senders use the normal retention window. +pub const UNREGISTERED_PENDING_TTL_SECS: i64 = 2 * 60 * 60; + +const VERIFIER_SET_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); +const VERIFIER_SET_RETRY_INTERVAL: Duration = Duration::from_secs(30); +const COMMITTEE_POSITIVE_TTL: Duration = Duration::from_secs(10 * 60); +const COMMITTEE_NEGATIVE_TTL: Duration = Duration::from_secs(2 * 60); +/// Maximum stale age for retaining a registered classification during refresh. +const COMMITTEE_STALE_LIMIT: Duration = Duration::from_secs(30 * 60); +const COMMITTEE_CACHE_MAX_ENTRIES: usize = 4096; +/// First-time lookup rate; known-member refreshes are exempt. +const COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE: u32 = 60; +const COMMITTEE_MAX_IN_FLIGHT_LOOKUPS: usize = 16; +/// Maximum unknown-identity lookups in flight. +const COMMITTEE_MAX_UNKNOWN_IN_FLIGHT: usize = 12; +const REGISTRY_RPC_TIMEOUT: Duration = Duration::from_secs(10); + +/// Separate discovery budget for operator stake lookups. +const OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE: u32 = 60; +const OPERATOR_MAX_IN_FLIGHT_LOOKUPS: usize = 16; +const OPERATOR_MAX_UNKNOWN_IN_FLIGHT: usize = 12; +const REFUSED_PEER_LOOKUPS_PER_MINUTE: u32 = 30; + +/// A confirmed operator binding keeps its class this long before re-validation; +/// a shorter window applies while the stake is still unknown. +const OPERATOR_STAKE_TTL: Duration = Duration::from_secs(10 * 60); +const OPERATOR_STAKE_STALE_LIMIT: Duration = Duration::from_secs(30 * 60); +const OPERATOR_CACHE_MAX_ENTRIES: usize = 4096; +/// Maximum persisted operator bindings loaded at startup. +pub const OPERATOR_BINDING_LOAD_LIMIT: i64 = OPERATOR_CACHE_MAX_ENTRIES as i64; + +const IMMEDIATE_BURST: f64 = 240.0; +const IMMEDIATE_REFILL_PER_SEC: f64 = 2.0; +const IMMEDIATE_MAX_TRACKED_PEERS: usize = 4096; + +/// NodeInfo broadcast response cooldown. +const NODE_INFO_RESPONSE_COOLDOWN: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Quota { + pub rows: i64, + pub bytes: i64, +} + +impl Quota { + fn admits(&self, rows: i64, bytes: i64) -> bool { + rows <= self.rows && bytes <= self.bytes + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InboundLimits { + pub max_json_bytes: usize, + pub max_binary_bytes: usize, + /// Every queued row, whatever its class. + pub global: Quota, + /// Global queue capacity reserved for committee. + pub committee_reserve: Quota, + /// Every queued row from unregistered senders. + pub unregistered_class: Quota, + pub registered_peer: Quota, + pub unregistered_peer: Quota, +} + +impl InboundLimits { + /// Derive byte quotas from the global ceiling; row limits remain fixed. + pub fn with_queued_bytes(max_json_bytes: usize, max_queued_bytes: i64) -> Self { + Self { + max_json_bytes, + max_binary_bytes: MAX_GOSSIPSUB_TRANSMIT_SIZE, + global: Quota { rows: 16_384, bytes: max_queued_bytes }, + committee_reserve: Quota { rows: 4_096, bytes: max_queued_bytes / 4 }, + unregistered_class: Quota { rows: 4_096, bytes: max_queued_bytes / 4 }, + registered_peer: Quota { rows: 1_024, bytes: max_queued_bytes / 8 }, + unregistered_peer: Quota { rows: 512, bytes: max_queued_bytes / 32 }, + } + } + + pub fn from_env() -> Self { + Self::with_queued_bytes( + crate::env::get_p2p_max_json_message_bytes(), + crate::env::get_p2p_inbox_max_queued_bytes(), + ) + } +} + +static INBOUND_LIMITS: LazyLock = LazyLock::new(InboundLimits::from_env); + +pub fn inbound_limits() -> &'static InboundLimits { + &INBOUND_LIMITS +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum DropReason { + OversizedJson, + OversizedBinary, + BinaryFromNonVerifier, + /// A registered author's message whose sequence number was already seen, or + /// is older than the replay window: a verbatim re-publish by someone else. + ReplayedSequence, + /// Sequence number exceeds the accepted clock lead. + FutureSequence, + /// The chain has said the author does not hold the role this kind of + /// message can only come from. + SenderRoleDenied, + /// A message addressed to another role, from an unregistered author, beyond + /// what this node relays for such authors. + ForwardRate, + DirectPeerRate, + AuthorRate, + UnregisteredRate, + GlobalRate, + Undecodable, + UnexpectedBinaryKind, + ImmediateRateLimited, + /// The bounded queue in front of the control worker (NodeInfo, ACKs) had no + /// room, by rows or by bytes. Distinct from `ImmediateRateLimited`: that one + /// is a sender exceeding its rate, this one is the node falling behind. + ControlQueueFull, + SenderQuota, + UnregisteredClassQuota, + GlobalQuota, + /// Not a message drop: a `NodeInfo` from a sender that is not registered was + /// not stored because the `node` table is at its ceiling for such senders. + NodeTableFull, + /// NodeInfo rejected for invalid identity, fields or binding. + NodeInfoRejected, + /// Not a drop: a message from an unregistered sender that was queued. Only + /// counted for the log summary, never exported as a drop metric. + AdmittedUnregistered, + /// Duplicate queued payload, recorded in the periodic summary. + DuplicatePayload, +} + +impl DropReason { + pub const fn as_str(self) -> &'static str { + match self { + Self::OversizedJson => "oversized_json", + Self::OversizedBinary => "oversized_binary", + Self::BinaryFromNonVerifier => "binary_from_non_verifier", + Self::ReplayedSequence => "replayed_sequence", + Self::FutureSequence => "future_sequence", + Self::SenderRoleDenied => "sender_role_denied", + Self::ForwardRate => "forward_rate", + Self::DirectPeerRate => "direct_peer_rate", + Self::AuthorRate => "author_rate", + Self::UnregisteredRate => "unregistered_rate", + Self::GlobalRate => "global_rate", + Self::Undecodable => "undecodable", + Self::UnexpectedBinaryKind => "unexpected_binary_kind", + Self::ImmediateRateLimited => "immediate_rate_limited", + Self::ControlQueueFull => "control_queue_full", + Self::SenderQuota => "sender_quota", + Self::UnregisteredClassQuota => "unregistered_class_quota", + Self::GlobalQuota => "global_quota", + Self::NodeTableFull => "node_table_full", + Self::NodeInfoRejected => "node_info_rejected", + Self::AdmittedUnregistered => "admitted_unregistered", + Self::DuplicatePayload => "duplicate_payload", + } + } + + /// Whether a drop is attributable to the direct peer, independent of local state and configuration. + pub const fn blames_direct_peer(self) -> bool { + matches!(self, Self::OversizedBinary | Self::Undecodable | Self::UnexpectedBinaryKind) + } + + /// Whether the payload was decoded before it was dropped. + pub const fn after_decode(self) -> bool { + matches!( + self, + Self::UnexpectedBinaryKind + | Self::SenderRoleDenied + | Self::ForwardRate + | Self::ImmediateRateLimited + | Self::ControlQueueFull + | Self::SenderQuota + | Self::UnregisteredClassQuota + | Self::GlobalQuota + ) + } +} + +/// Checks that need nothing but the raw bytes and the sender's identity. +pub fn check_envelope( + data: &[u8], + sender_is_verifier: bool, + limits: &InboundLimits, +) -> Result<(), DropReason> { + if GOATMessage::is_binary_envelope(data) { + if !sender_is_verifier { + return Err(DropReason::BinaryFromNonVerifier); + } + if data.len() > limits.max_binary_bytes { + return Err(DropReason::OversizedBinary); + } + } else if data.len() > limits.max_json_bytes { + return Err(DropReason::OversizedJson); + } + Ok(()) +} + +/// Whether one more queued row of `content_len` bytes from a sender of `class` +/// fits. `usage` is the queued state *before* the row is added. +pub fn check_inbox_quota( + class: P2pInboxAdmissionClass, + content_len: usize, + usage: &[P2pInboxClassUsage], + limits: &InboundLimits, +) -> Result<(), DropReason> { + let content_len = content_len as i64; + let total_rows: i64 = usage.iter().map(|usage| usage.rows).sum(); + let total_bytes: i64 = usage.iter().map(|usage| usage.bytes).sum(); + if !limits.global.admits(total_rows + 1, total_bytes + content_len) { + return Err(DropReason::GlobalQuota); + } + if class != P2pInboxAdmissionClass::Committee { + let committee = P2pInboxAdmissionClass::Committee.to_string(); + let (rows, bytes) = usage + .iter() + .filter(|usage| usage.admission_class != committee) + .fold((0, 0), |(rows, bytes), usage| (rows + usage.rows, bytes + usage.bytes)); + let open = Quota { + rows: limits.global.rows - limits.committee_reserve.rows, + bytes: limits.global.bytes - limits.committee_reserve.bytes, + }; + if !open.admits(rows + 1, bytes + content_len) { + return Err(DropReason::GlobalQuota); + } + } + + // A sender can hold rows in more than one class: it is classified when each + // row arrives, and a registration lookup may complete in between. + let peer_rows: i64 = usage.iter().map(|usage| usage.peer_rows).sum(); + let peer_bytes: i64 = usage.iter().map(|usage| usage.peer_bytes).sum(); + let peer_quota = + if class.is_registered() { limits.registered_peer } else { limits.unregistered_peer }; + if !peer_quota.admits(peer_rows + 1, peer_bytes + content_len) { + return Err(DropReason::SenderQuota); + } + + if !class.is_registered() { + let class_name = P2pInboxAdmissionClass::Unregistered.to_string(); + let (class_rows, class_bytes) = usage + .iter() + .find(|usage| usage.admission_class == class_name) + .map_or((0, 0), |usage| (usage.rows, usage.bytes)); + if !limits.unregistered_class.admits(class_rows + 1, class_bytes + content_len) { + return Err(DropReason::UnregisteredClassQuota); + } + } + Ok(()) +} + +/// Persistent C/R/C/U rotation; empty classes yield their turn. +#[derive(Debug, Default)] +pub struct InboxSchedule { + position: usize, +} + +/// Indices into the per-class queues of [`InboxSchedule::next`]. +pub const SCHEDULE_COMMITTEE: usize = 0; +pub const SCHEDULE_REGISTERED: usize = 1; +pub const SCHEDULE_UNREGISTERED: usize = 2; + +const SCHEDULE_ROTA: [usize; 4] = + [SCHEDULE_COMMITTEE, SCHEDULE_REGISTERED, SCHEDULE_COMMITTEE, SCHEDULE_UNREGISTERED]; + +impl InboxSchedule { + /// The queue a row of `admission_class` belongs to. + pub fn queue_of(admission_class: &str) -> usize { + if admission_class == P2pInboxAdmissionClass::Committee.to_string() { + SCHEDULE_COMMITTEE + } else if admission_class == P2pInboxAdmissionClass::Registered.to_string() { + SCHEDULE_REGISTERED + } else { + SCHEDULE_UNREGISTERED + } + } + + /// Advance the rotation only for a row about to be dispatched. + pub fn next(&mut self, queues: &mut [std::collections::VecDeque; 3]) -> Option { + for step in 0..SCHEDULE_ROTA.len() { + let position = (self.position + step) % SCHEDULE_ROTA.len(); + if let Some(row) = queues[SCHEDULE_ROTA[position]].pop_front() { + self.position = (position + 1) % SCHEDULE_ROTA.len(); + return Some(row); + } + } + None + } +} + +static INBOX_SCHEDULE: LazyLock> = + LazyLock::new(|| Mutex::new(InboxSchedule::default())); + +/// The process-wide rota. Only the inbox worker, on the swarm loop, touches it. +pub fn inbox_schedule() -> std::sync::MutexGuard<'static, InboxSchedule> { + INBOX_SCHEDULE.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +pub fn content_hash(data: &[u8]) -> [u8; 32] { + Sha256::digest(data).into() +} + +#[derive(Clone, Copy, Debug)] +struct CommitteeEntry { + registered: bool, + fetched_at: Instant, +} + +impl CommitteeEntry { + fn ttl(&self) -> Duration { + if self.registered { COMMITTEE_POSITIVE_TTL } else { COMMITTEE_NEGATIVE_TTL } + } + + fn needs_refresh(&self, now: Instant) -> bool { + now.saturating_duration_since(self.fetched_at) >= self.ttl() + } + + fn is_registered(&self, now: Instant) -> bool { + self.registered && now.saturating_duration_since(self.fetched_at) < COMMITTEE_STALE_LIMIT + } +} + +/// Canonical operator identity: the x-only master public key. +pub type OperatorKey = [u8; 32]; + +pub fn operator_key(pubkey: &bitcoin::XOnlyPublicKey) -> OperatorKey { + pubkey.serialize() +} + +/// A peer that proved (via a NodeInfo binding) it owns a master key, plus what +/// the chain says about that key's operator stake. +#[derive(Clone, Debug)] +struct OperatorBinding { + pubkey: OperatorKey, + issued_at: i64, + /// `Some(true)` once the key is confirmed to be a sufficiently staked + /// operator on chain; `None` until the lookup completes. + staked: Option, + fetched_at: Option, + /// Configured binding; bypasses discovery budget but still requires a positive stake verdict. + trusted: bool, +} + +impl OperatorBinding { + fn needs_refresh(&self, now: Instant) -> bool { + match self.fetched_at { + None => true, + Some(fetched_at) => now.saturating_duration_since(fetched_at) >= OPERATOR_STAKE_TTL, + } + } + + fn is_registered(&self, now: Instant) -> bool { + self.staked == Some(true) + && self + .fetched_at + .is_some_and(|at| now.saturating_duration_since(at) < OPERATOR_STAKE_STALE_LIMIT) + } +} + +#[derive(Debug, Default)] +struct RegistryState { + verifiers: HashSet>, + verifiers_fetched_at: Option, + verifiers_attempted_at: Option, + verifiers_in_flight: bool, + committee: HashMap, + committee_in_flight: HashSet, + /// Operator bindings keyed by the peer that proved them. + operators: HashMap, + /// Reverse index from master key to its current peer ID. + operator_owner: HashMap, + operators_in_flight: HashSet, + committee_discovery: DiscoveryBudget, + /// Separate lookup budget for connected neighbours. + neighbour_discovery: DiscoveryBudget, + /// For peers the connection gate refused; see [`LookupBudget::Refused`]. + refused_discovery: DiscoveryBudget, + operator_discovery: DiscoveryBudget, +} + +/// Per-minute budget for first-time (unknown-identity) registry lookups. +#[derive(Debug, Default)] +struct DiscoveryBudget { + window_started_at: Option, + lookups_in_window: u32, +} + +impl DiscoveryBudget { + /// Charge one lookup. Returns false when the budget is exhausted. + fn take(&mut self, per_minute: u32, now: Instant) -> bool { + let window_open = self.window_started_at.is_some_and(|started_at| { + now.saturating_duration_since(started_at) < Duration::from_secs(60) + }); + if !window_open { + self.window_started_at = Some(now); + self.lookups_in_window = 0; + } + if self.lookups_in_window >= per_minute { + return false; + } + self.lookups_in_window += 1; + true + } +} + +/// Whose budget a first-time committee lookup is charged to. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LookupBudget { + /// Discovery budget for signed message authors. + Author, + /// A peer this node holds a connection with; bounded by the connection + /// ceiling. Local bans depend on these lookups coming back. + Neighbour, + /// A peer whose connection was refused for lack of an open slot. + Refused, +} + +impl RegistryState { + /// Claim the verifier set refresh if it is due. The caller must fetch it and + /// call `record_verifier_set`, or it stays marked as in flight. + fn claim_verifier_set_refresh(&mut self, now: Instant) -> bool { + let due = match (self.verifiers_fetched_at, self.verifiers_attempted_at) { + (_, Some(attempted_at)) + if now.saturating_duration_since(attempted_at) < VERIFIER_SET_RETRY_INTERVAL => + { + false + } + (Some(fetched_at), _) => { + now.saturating_duration_since(fetched_at) >= VERIFIER_SET_REFRESH_INTERVAL + } + (None, _) => true, + }; + if !due || self.verifiers_in_flight { + return false; + } + self.verifiers_in_flight = true; + self.verifiers_attempted_at = Some(now); + true + } + + /// Claim a committee lookup for `peer` if one is due and a slot is free. The + /// caller must resolve it with `record_committee_peer`. + fn claim_committee_lookup( + &mut self, + peer: &PeerId, + budget: LookupBudget, + now: Instant, + ) -> bool { + if self.verifiers.contains(&peer.to_bytes()) || self.committee_in_flight.contains(peer) { + return false; + } + let known_registered = match self.committee.get(peer) { + Some(entry) if !entry.needs_refresh(now) => return false, + Some(entry) => entry.registered, + None => false, + }; + // Known-member refreshes may use every slot without discovery charges. + let in_flight = self.committee_in_flight.len(); + let admitted = if known_registered { + in_flight < COMMITTEE_MAX_IN_FLIGHT_LOOKUPS + } else { + match budget { + LookupBudget::Author => { + in_flight < COMMITTEE_MAX_UNKNOWN_IN_FLIGHT + && self + .committee_discovery + .take(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, now) + } + LookupBudget::Neighbour => { + in_flight < COMMITTEE_MAX_IN_FLIGHT_LOOKUPS + && self + .neighbour_discovery + .take(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, now) + } + LookupBudget::Refused => { + in_flight < COMMITTEE_MAX_UNKNOWN_IN_FLIGHT + && self.refused_discovery.take(REFUSED_PEER_LOOKUPS_PER_MINUTE, now) + } + } + }; + if admitted { + self.committee_in_flight.insert(*peer); + } + admitted + } + + /// Claim a due operator lookup; confirmed operators bypass discovery limits. + fn claim_operator_refresh( + &mut self, + peer: &PeerId, + pubkey: &OperatorKey, + now: Instant, + ) -> bool { + if self.operators_in_flight.contains(peer) { + return false; + } + let known_registered = match self.operators.get(peer) { + // Only the current owner of the key is worth validating. + Some(binding) if binding.pubkey != *pubkey => return false, + Some(binding) if !binding.needs_refresh(now) => return false, + Some(binding) => binding.staked == Some(true) || binding.trusted, + None => return false, + }; + let in_flight = self.operators_in_flight.len(); + if known_registered { + if in_flight >= OPERATOR_MAX_IN_FLIGHT_LOOKUPS { + return false; + } + } else if in_flight >= OPERATOR_MAX_UNKNOWN_IN_FLIGHT + || !self.operator_discovery.take(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE, now) + { + return false; + } + self.operators_in_flight.insert(*peer); + true + } +} + +/// Trust window for restored registrations awaiting chain refresh. +const SEEDED_TRUST: Duration = Duration::from_secs(5 * 60); + +/// How far back a seeded entry is dated: due for re-validation right away, and +/// `SEEDED_TRUST` short of the stale limit at which it stops being trusted. +fn seeded_fetched_at(now: Instant, stale_limit: Duration) -> Instant { + now.checked_sub(stale_limit.saturating_sub(SEEDED_TRUST)).unwrap_or(now) +} + +/// Why a configured operator binding was not applied. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TrustedBindingConflict { + /// The peer already holds a binding, signed by another key. + PeerBoundToAnotherKey(OperatorKey), + /// The key is already bound, by its own signature, to another peer id. + KeyBoundToAnotherPeer(PeerId), +} + +/// The role a kind of message can only legitimately come from. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SenderRole { + /// Chain observations and requests any node may publish. + Any, + Committee, + Verifier, + Operator, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RoleVerdict { + Confirmed, + Denied, + Unknown, +} + +/// What the caller should fetch from the chain, decided under the cache lock so +/// concurrent messages from one peer start a single lookup. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RefreshPlan { + pub verifier_set: bool, + pub committee_peer: bool, +} + +/// Cached chain registration; misses are Unregistered and trigger background lookups. +#[derive(Debug, Default)] +pub struct PeerRegistry { + state: Mutex, +} + +impl PeerRegistry { + fn lock(&self) -> std::sync::MutexGuard<'_, RegistryState> { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub fn is_verifier(&self, peer: &PeerId) -> bool { + self.lock().verifiers.contains(&peer.to_bytes()) + } + + pub fn sender_class(&self, peer: &PeerId, now: Instant) -> P2pInboxAdmissionClass { + let state = self.lock(); + if state.committee.get(peer).is_some_and(|entry| entry.is_registered(now)) { + P2pInboxAdmissionClass::Committee + } else if state.verifiers.contains(&peer.to_bytes()) + || state.operators.get(peer).is_some_and(|binding| binding.is_registered(now)) + { + P2pInboxAdmissionClass::Registered + } else { + P2pInboxAdmissionClass::Unregistered + } + } + + /// Whether `peer` is a committee member, from the cache. For handlers whose + /// authorisation is the committee specifically, not any registered sender. + pub fn is_committee(&self, peer: &PeerId, now: Instant) -> bool { + self.sender_class(peer, now) == P2pInboxAdmissionClass::Committee + } + + /// Cached role verdict. Only fresh negative answers are Denied; missing or stale answers are Unknown. + pub fn role_verdict(&self, peer: &PeerId, role: SenderRole, now: Instant) -> RoleVerdict { + let state = self.lock(); + match role { + SenderRole::Any => RoleVerdict::Confirmed, + SenderRole::Committee => match state.committee.get(peer) { + Some(entry) if entry.is_registered(now) => RoleVerdict::Confirmed, + Some(entry) if !entry.registered && !entry.needs_refresh(now) => { + RoleVerdict::Denied + } + _ => RoleVerdict::Unknown, + }, + SenderRole::Verifier => { + if state.verifiers.contains(&peer.to_bytes()) { + RoleVerdict::Confirmed + } else if state.verifiers_fetched_at.is_some_and(|at| { + now.saturating_duration_since(at) < VERIFIER_SET_REFRESH_INTERVAL * 2 + }) { + RoleVerdict::Denied + } else { + RoleVerdict::Unknown + } + } + SenderRole::Operator => match state.operators.get(peer) { + Some(binding) if binding.is_registered(now) => RoleVerdict::Confirmed, + Some(binding) if binding.staked == Some(false) && !binding.needs_refresh(now) => { + RoleVerdict::Denied + } + _ => RoleVerdict::Unknown, + }, + } + } + + /// Whether fresh chain results identify the peer as unregistered. + pub fn is_known_unregistered(&self, peer: &PeerId, now: Instant) -> bool { + // A stale "no" does not count: the peer may have registered since. + let looked_up = self + .lock() + .committee + .get(peer) + .is_some_and(|entry| !entry.registered && !entry.needs_refresh(now)); + looked_up && !self.sender_class(peer, now).is_registered() + } + + /// Registered verifier peer IDs; `None` means not yet fetched. + pub fn verifier_peer_ids(&self) -> Option> { + let state = self.lock(); + state.verifiers_fetched_at?; + Some( + state + .verifiers + .iter() + .filter_map(|bytes| PeerId::from_bytes(bytes).ok()) + .map(|peer| peer.to_string()) + .collect(), + ) + } + + /// Record a verified binding; newer issuance wins and each key maps to one peer. + /// Returns whether the peer's previous persisted registration must be revoked. + pub fn observe_operator_binding( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + issued_at: i64, + ) -> bool { + let mut state = self.lock(); + let mut released_confirmed_key = false; + // Move the key to the peer with the newer binding. + if let Some(current_owner) = state.operator_owner.get(pubkey).copied() + && current_owner != *peer + { + let newer = state + .operators + .get(¤t_owner) + // Require a strictly newer issuance time for ownership transfer. + .is_none_or(|existing| issued_at > existing.issued_at); + if !newer { + return false; + } + state.operators.remove(¤t_owner); + } + if let Some(previous) = state.operators.get_mut(peer) { + if previous.pubkey == *pubkey { + // Preserve the stake verdict when the peer re-announces the same key. + previous.issued_at = previous.issued_at.max(issued_at); + state.operator_owner.insert(*pubkey, *peer); + return false; + } + // A peer that rebinds to a different key releases its old key's + // ownership, and whatever was confirmed for that key with it. + released_confirmed_key = previous.staked == Some(true); + let previous_pubkey = previous.pubkey; + if state.operator_owner.get(&previous_pubkey) == Some(peer) { + state.operator_owner.remove(&previous_pubkey); + } + } + if state.operators.len() >= OPERATOR_CACHE_MAX_ENTRIES + && !state.operators.contains_key(peer) + { + let now = Instant::now(); + state.operators.retain(|_, binding| binding.is_registered(now) || binding.trusted); + let live: HashSet = state.operators.keys().copied().collect(); + state.operator_owner.retain(|_, owner| live.contains(owner)); + } + state.operator_owner.insert(*pubkey, *peer); + state.operators.insert( + *peer, + OperatorBinding { + pubkey: *pubkey, + issued_at, + staked: None, + fetched_at: None, + trusted: false, + }, + ); + released_confirmed_key + } + + /// Record a configured binding without overriding a signed binding. + /// See `P2P_TRUSTED_OPERATOR_BINDINGS`. + pub fn trust_operator_binding( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + ) -> Result<(), TrustedBindingConflict> { + let mut state = self.lock(); + if let Some(existing) = state.operators.get(peer) + && existing.pubkey != *pubkey + { + return Err(TrustedBindingConflict::PeerBoundToAnotherKey(existing.pubkey)); + } + if let Some(owner) = state.operator_owner.get(pubkey) + && owner != peer + { + return Err(TrustedBindingConflict::KeyBoundToAnotherPeer(*owner)); + } + state.operator_owner.insert(*pubkey, *peer); + state.operators.entry(*peer).and_modify(|binding| binding.trusted = true).or_insert( + OperatorBinding { + pubkey: *pubkey, + issued_at: 0, + staked: None, + fetched_at: None, + trusted: true, + }, + ); + Ok(()) + } + + /// Whether an operator stake lookup for `peer`/`pubkey` should start now. + /// Claimed lookups must be resolved with [`Self::record_operator_stake`]. + pub fn plan_operator_refresh(&self, peer: &PeerId, pubkey: &OperatorKey, now: Instant) -> bool { + self.lock().claim_operator_refresh(peer, pubkey, now) + } + + /// Claim due operator refreshes, confirmed operators first. + pub fn plan_due_operator_refreshes(&self, now: Instant) -> Vec<(PeerId, OperatorKey)> { + let mut state = self.lock(); + let mut due: Vec<(PeerId, OperatorKey, bool)> = state + .operators + .iter() + // Retry a negative stake verdict only after another announcement. + .filter(|(peer, binding)| { + binding.needs_refresh(now) + && (binding.staked != Some(false) || binding.trusted) + && !state.operators_in_flight.contains(*peer) + }) + .map(|(peer, binding)| { + (*peer, binding.pubkey, binding.staked == Some(true) || binding.trusted) + }) + .collect(); + due.sort_by_key(|(_, _, known)| !*known); + due.into_iter() + .filter(|(peer, pubkey, _)| state.claim_operator_refresh(peer, pubkey, now)) + .map(|(peer, pubkey, _)| (peer, pubkey)) + .collect() + } + + /// Record the verdict for the peer's current binding and return the persistence update. + pub fn record_operator_stake( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + staked: Option, + now: Instant, + ) -> Option { + let mut state = self.lock(); + state.operators_in_flight.remove(peer); + let staked = staked?; + // Ignore a result for a binding that was superseded while in flight. + let binding = state.operators.get_mut(peer).filter(|binding| binding.pubkey == *pubkey)?; + binding.staked = Some(staked); + binding.fetched_at = Some(now); + Some(staked) + } + + /// Restore a confirmed operator only when its current binding matches the stored key. + /// Schedule immediate revalidation. + pub fn seed_verified_operator( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + issued_at: i64, + now: Instant, + ) { + self.observe_operator_binding(peer, pubkey, issued_at); + let mut state = self.lock(); + if let Some(binding) = state.operators.get_mut(peer) + && binding.pubkey == *pubkey + && binding.staked.is_none() + { + binding.staked = Some(true); + binding.fetched_at = Some(seeded_fetched_at(now, OPERATOR_STAKE_STALE_LIMIT)); + } + } + + /// Re-instate a committee member confirmed in an earlier session; see + /// [`Self::seed_verified_operator`]. + pub fn seed_verified_committee_peer(&self, peer: &PeerId, now: Instant) { + self.lock().committee.entry(*peer).or_insert(CommitteeEntry { + registered: true, + fetched_at: seeded_fetched_at(now, COMMITTEE_STALE_LIMIT), + }); + } + + /// Claim due committee-member refreshes. + pub fn plan_due_committee_refreshes(&self, now: Instant) -> Vec { + let mut state = self.lock(); + let due: Vec = state + .committee + .iter() + .filter(|(peer, entry)| { + entry.registered + && entry.needs_refresh(now) + && !state.committee_in_flight.contains(*peer) + }) + .map(|(peer, _)| *peer) + .collect(); + let free = COMMITTEE_MAX_IN_FLIGHT_LOOKUPS.saturating_sub(state.committee_in_flight.len()); + let claimed: Vec = due.into_iter().take(free).collect(); + state.committee_in_flight.extend(claimed.iter().copied()); + claimed + } + + /// Claim the lookups that are due. Whatever is claimed must be resolved with + /// the matching `record_*` call, or it stays marked as in flight. + pub fn plan_refresh(&self, peer: Option<&PeerId>, now: Instant) -> RefreshPlan { + let mut state = self.lock(); + RefreshPlan { + verifier_set: state.claim_verifier_set_refresh(now), + committee_peer: peer + .is_some_and(|peer| state.claim_committee_lookup(peer, LookupBudget::Author, now)), + } + } + + /// Claim only the neighbour's committee lookup, using the neighbour budget. + pub fn plan_neighbour_lookup(&self, peer: &PeerId, now: Instant) -> bool { + self.lock().claim_committee_lookup(peer, LookupBudget::Neighbour, now) + } + + /// Claim refused-peer lookups using their separate budget. + pub fn plan_refused_peer_lookup(&self, peer: &PeerId, now: Instant) -> bool { + self.lock().claim_committee_lookup(peer, LookupBudget::Refused, now) + } + + pub fn record_verifier_set(&self, verifiers: Option>>, now: Instant) { + let mut state = self.lock(); + state.verifiers_in_flight = false; + // Retain the previous verifier set on fetch failure. + if let Some(verifiers) = verifiers { + state.verifiers = verifiers.into_iter().collect(); + state.verifiers_fetched_at = Some(now); + } + } + + /// Record a committee lookup. Returns the definite verdict, for the caller + /// to persist; see [`Self::record_operator_stake`]. + pub fn record_committee_peer( + &self, + peer: &PeerId, + registered: Option, + now: Instant, + ) -> Option { + let mut state = self.lock(); + state.committee_in_flight.remove(peer); + let registered = registered?; + let changed = Some(registered); + if !state.committee.contains_key(peer) + && state.committee.len() >= COMMITTEE_CACHE_MAX_ENTRIES + { + state.committee.retain(|_, entry| !entry.needs_refresh(now)); + if state.committee.len() >= COMMITTEE_CACHE_MAX_ENTRIES { + if !registered { + return changed; + } + let evict = state + .committee + .iter() + .find(|(_, entry)| !entry.registered) + .map(|(peer, _)| *peer); + if let Some(evict) = evict { + state.committee.remove(&evict); + } + } + } + state.committee.insert(*peer, CommitteeEntry { registered, fetched_at: now }); + changed + } +} + +static PEER_REGISTRY: LazyLock = LazyLock::new(PeerRegistry::default); + +pub fn peer_registry() -> &'static PeerRegistry { + &PEER_REGISTRY +} + +/// Persisted registration kinds. +const REGISTERED_KIND_COMMITTEE: &str = "Committee"; +const REGISTERED_KIND_OPERATOR: &str = "Operator"; + +/// Persist a change in a peer's confirmed registration. Best effort: the cache +/// stays authoritative for this session, the table only warms the next one. +async fn persist_registration_change( + local_db: &LocalDB, + peer: &PeerId, + kind: &str, + pubkey: Option<&OperatorKey>, + change: Option, +) { + let Some(registered) = change else { + return; + }; + let peer_id = peer.to_string(); + let pubkey = pubkey.map(hex::encode).unwrap_or_default(); + let result = async { + let mut storage = local_db.acquire().await?; + if registered { + storage.upsert_p2p_registered_peer(&peer_id, kind, &pubkey).await + } else { + storage.delete_p2p_registered_peer(&peer_id, kind).await + } + } + .await; + if let Err(error) = result { + tracing::debug!( + event = "p2p_admission", + outcome = "registration_persist_failed", + peer_id = %peer_id, + kind, + error = %error, + "failed to persist a confirmed registration; it will be rediscovered after a restart" + ); + } +} + +/// Revoke persisted operator registration after a key change. +pub async fn revoke_persisted_operator(local_db: &LocalDB, peer: &PeerId) { + persist_registration_change(local_db, peer, REGISTERED_KIND_OPERATOR, None, Some(false)).await; +} + +/// Restore confirmed peers; operator bindings must be signature-checked and match stored keys. +pub async fn seed_registry_from_store( + local_db: &LocalDB, + registry: &PeerRegistry, + operator_bindings: &HashMap, + now: Instant, +) -> Result { + let persisted = local_db.acquire().await?.load_p2p_registered_peers().await?; + let mut seeded = 0; + for (peer_id, kind, confirmed_pubkey) in persisted { + let Ok(peer) = PeerId::from_str(&peer_id) else { + continue; + }; + match kind.as_str() { + REGISTERED_KIND_COMMITTEE => { + registry.seed_verified_committee_peer(&peer, now); + seeded += 1; + } + REGISTERED_KIND_OPERATOR => { + if let Some((pubkey, issued_at)) = operator_bindings.get(&peer) + && hex::encode(pubkey) == confirmed_pubkey + { + registry.seed_verified_operator(&peer, pubkey, *issued_at, now); + seeded += 1; + } + } + _ => {} + } + } + Ok(seeded) +} + +fn spawn_committee_lookup(local_db: &LocalDB, goat_client: &Arc, peer: PeerId) { + let registry = peer_registry(); + let (local_db, goat_client) = (local_db.clone(), goat_client.clone()); + tokio::spawn(async move { + let registered = match tokio::time::timeout( + REGISTRY_RPC_TIMEOUT, + goat_client.committee_mana_is_validate_peer_id(&peer.to_bytes()), + ) + .await + { + Ok(Ok(registered)) => Some(registered), + Ok(Err(error)) => { + tracing::debug!( + event = "p2p_admission", + outcome = "committee_lookup_failed", + peer_id = %peer, + error = %error, + "failed to look up a peer in the committee registry" + ); + None + } + Err(_) => None, + }; + let change = registry.record_committee_peer(&peer, registered, Instant::now()); + persist_registration_change(&local_db, &peer, REGISTERED_KIND_COMMITTEE, None, change) + .await; + }); +} + +fn spawn_operator_stake_lookup( + local_db: &LocalDB, + goat_client: &Arc, + peer: PeerId, + operator_key: OperatorKey, +) { + use crate::utils::OperatorStakeStatus; + + let registry = peer_registry(); + // The stake is registered under the x-only key; the parity chosen here is + // dropped again by the lookup. + let Ok(xonly) = bitcoin::XOnlyPublicKey::from_slice(&operator_key) else { + registry.record_operator_stake(&peer, &operator_key, Some(false), Instant::now()); + return; + }; + let pubkey = bitcoin::PublicKey::new(xonly.public_key(bitcoin::secp256k1::Parity::Even)); + let (local_db, goat_client) = (local_db.clone(), goat_client.clone()); + tokio::spawn(async move { + let staked = match tokio::time::timeout( + REGISTRY_RPC_TIMEOUT, + crate::utils::operator_stake_status(&goat_client, &pubkey), + ) + .await + { + Ok(Ok(OperatorStakeStatus::Staked)) => Some(true), + Ok(Ok( + OperatorStakeStatus::NotRegistered | OperatorStakeStatus::Insufficient { .. }, + )) => Some(false), + // Preserve the cached stake verdict on RPC failure. + Ok(Err(error)) => { + tracing::debug!( + event = "p2p_admission", + outcome = "operator_stake_lookup_failed", + peer_id = %peer, + error = %error, + "failed to confirm operator stake; leaving the binding unverified" + ); + None + } + Err(_) => None, + }; + let change = registry.record_operator_stake(&peer, &operator_key, staked, Instant::now()); + persist_registration_change( + &local_db, + &peer, + REGISTERED_KIND_OPERATOR, + Some(&operator_key), + change, + ) + .await; + }); +} + +/// Look up a peer this node is connected to; see +/// [`PeerRegistry::plan_neighbour_lookup`]. +pub fn refresh_neighbour_in_background( + local_db: &LocalDB, + goat_client: &Arc, + peer: PeerId, +) { + if peer_registry().plan_neighbour_lookup(&peer, Instant::now()) { + spawn_committee_lookup(local_db, goat_client, peer); + } +} + +/// Start whatever registry lookups are due for `peer` without waiting for them. +pub fn refresh_registry_in_background( + local_db: &LocalDB, + goat_client: &Arc, + peer: Option, +) { + let registry = peer_registry(); + let plan = registry.plan_refresh(peer.as_ref(), Instant::now()); + if plan.verifier_set { + let goat_client = goat_client.clone(); + tokio::spawn(async move { + let fetched = match tokio::time::timeout( + REGISTRY_RPC_TIMEOUT, + goat_client.committee_mana_get_verifiers(), + ) + .await + { + Ok(Ok(verifiers)) => Some(verifiers), + Ok(Err(error)) => { + tracing::warn!( + event = "p2p_admission", + outcome = "verifier_set_refresh_failed", + error = %error, + "failed to refresh the registered verifier set; keeping the previous one" + ); + None + } + Err(_) => { + tracing::warn!( + event = "p2p_admission", + outcome = "verifier_set_refresh_timeout", + "timed out refreshing the registered verifier set; keeping the previous one" + ); + None + } + }; + registry.record_verifier_set(fetched, Instant::now()); + }); + } + if let Some(peer) = peer.filter(|_| plan.committee_peer) { + spawn_committee_lookup(local_db, goat_client, peer); + } +} + +/// Schedule refreshes for known registrations that are due. +pub fn refresh_due_registrations_in_background(local_db: &LocalDB, goat_client: &Arc) { + let registry = peer_registry(); + let now = Instant::now(); + refresh_registry_in_background(local_db, goat_client, None); + for peer in registry.plan_due_committee_refreshes(now) { + spawn_committee_lookup(local_db, goat_client, peer); + } + for peer in take_refused_peers() { + if registry.plan_refused_peer_lookup(&peer, now) { + spawn_committee_lookup(local_db, goat_client, peer); + } + } + for (peer, operator_key) in registry.plan_due_operator_refreshes(now) { + spawn_operator_stake_lookup(local_db, goat_client, peer, operator_key); + } +} + +/// Confirm, off the event loop, that a proven operator binding belongs to a +/// sufficiently staked operator, and cache the verdict for classification. +pub fn refresh_operator_binding_in_background( + local_db: &LocalDB, + goat_client: &Arc, + peer: PeerId, + operator_key: OperatorKey, +) { + if !peer_registry().plan_operator_refresh(&peer, &operator_key, Instant::now()) { + return; + } + spawn_operator_stake_lookup(local_db, goat_client, peer, operator_key); +} + +/// Per-peer token bucket for messages that are dispatched without being queued. +#[derive(Debug)] +pub struct PeerRateLimiter { + buckets: Mutex>, + burst: f64, + refill_per_sec: f64, + max_peers: usize, +} + +impl PeerRateLimiter { + pub fn new(burst: f64, refill_per_sec: f64, max_peers: usize) -> Self { + Self { buckets: Mutex::new(HashMap::new()), burst, refill_per_sec, max_peers } + } + + pub fn allow(&self, peer: &PeerId, now: Instant) -> bool { + let mut buckets = self.buckets.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !buckets.contains_key(peer) && buckets.len() >= self.max_peers { + // A bucket that has refilled completely belongs to an idle peer and + // carries no state worth keeping. + let (burst, refill_per_sec) = (self.burst, self.refill_per_sec); + buckets.retain(|_, bucket| { + let elapsed = now.saturating_duration_since(bucket.refilled_at).as_secs_f64(); + bucket.tokens + elapsed * refill_per_sec < burst + }); + if buckets.len() >= self.max_peers { + return false; + } + } + let bucket = buckets.entry(*peer).or_insert_with(|| RateBucket::full(self.burst, now)); + bucket.refill(self.refill_per_sec, self.burst, now); + if bucket.tokens < 1.0 { + return false; + } + bucket.tokens -= 1.0; + true + } +} + +static IMMEDIATE_LIMITER: LazyLock = LazyLock::new(|| { + PeerRateLimiter::new(IMMEDIATE_BURST, IMMEDIATE_REFILL_PER_SEC, IMMEDIATE_MAX_TRACKED_PEERS) +}); + +pub fn immediate_limiter() -> &'static PeerRateLimiter { + &IMMEDIATE_LIMITER +} + +/// A token bucket carrying both a rate and a burst, refilled lazily on access. +#[derive(Clone, Copy, Debug)] +struct RateBucket { + tokens: f64, + refilled_at: Instant, +} + +impl RateBucket { + fn full(burst: f64, now: Instant) -> Self { + Self { tokens: burst, refilled_at: now } + } + + fn refill(&mut self, rate: f64, burst: f64, now: Instant) { + let elapsed = now.saturating_duration_since(self.refilled_at).as_secs_f64(); + self.tokens = (self.tokens + elapsed * rate).min(burst); + self.refilled_at = now; + } +} + +/// One message costs a whole message token plus `bytes` byte tokens. +#[derive(Clone, Copy, Debug)] +struct DualBucket { + msgs: RateBucket, + bytes: RateBucket, +} + +/// Rate + burst for one tier, in messages and bytes per second. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TierRate { + pub msg_burst: u32, + pub msg_per_sec: u32, + pub byte_burst: i64, + pub byte_per_sec: i64, +} + +impl TierRate { + fn new_bucket(&self, now: Instant) -> DualBucket { + DualBucket { + msgs: RateBucket::full(self.msg_burst as f64, now), + bytes: RateBucket::full(self.byte_burst as f64, now), + } + } + + /// Refill then test whether one message of `bytes` fits, without spending. + fn peek(&self, bucket: &mut DualBucket, bytes: i64, now: Instant) -> bool { + self.peek_above(bucket, bytes, 0.0, now) + } + + /// Check capacity while retaining the specified fraction of burst tokens. + fn peek_above(&self, bucket: &mut DualBucket, bytes: i64, reserve: f64, now: Instant) -> bool { + bucket.msgs.refill(self.msg_per_sec as f64, self.msg_burst as f64, now); + bucket.bytes.refill(self.byte_per_sec as f64, self.byte_burst as f64, now); + bucket.msgs.tokens - 1.0 >= self.msg_burst as f64 * reserve + && bucket.bytes.tokens - bytes as f64 >= self.byte_burst as f64 * reserve + } + + fn spend(&self, bucket: &mut DualBucket, bytes: i64) { + bucket.msgs.tokens -= 1.0; + bucket.bytes.tokens -= bytes as f64; + } +} + +/// Pre-decode message and byte limits: direct peer, author, unregistered class and global. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct RateLimits { + pub direct_peer: TierRate, + pub registered_author: TierRate, + pub unregistered_author: TierRate, + pub unregistered_total: TierRate, + /// Forward-only budget for unregistered authors. + pub unregistered_forward: TierRate, + pub global: TierRate, + /// Committee-only reserve in the global and direct-peer budgets. + pub committee_reserve: f64, + pub max_tracked_peers: usize, +} + +impl RateLimits { + pub const fn defaults() -> Self { + const MIB: i64 = 1024 * 1024; + Self { + // Direct-peer rate ceiling. + direct_peer: TierRate { + msg_burst: 16_384, + msg_per_sec: 4_096, + byte_burst: 512 * MIB, + byte_per_sec: 32 * MIB, + }, + // Generous: one GenCircuits is ~11.5 MiB, and an operator may send a + // few graph messages back to back. + registered_author: TierRate { + msg_burst: 8_192, + msg_per_sec: 512, + byte_burst: 256 * MIB, + byte_per_sec: 8 * MIB, + }, + // Room for one maximum-size JSON message and a node's heartbeats, + // not for a stream. + unregistered_author: TierRate { + msg_burst: 32, + msg_per_sec: 2, + byte_burst: 4 * MIB, + byte_per_sec: 128 * 1024, + }, + // Shared budget for all unregistered authors. + unregistered_total: TierRate { + msg_burst: 1_024, + msg_per_sec: 32, + byte_burst: 32 * MIB, + byte_per_sec: MIB, + }, + // Sized for the chain observations honest unregistered nodes + // (watchtowers, challengers) publish, not for bulk. + unregistered_forward: TierRate { + msg_burst: 256, + msg_per_sec: 16, + byte_burst: 8 * MIB, + byte_per_sec: 512 * 1024, + }, + global: TierRate { + msg_burst: 16_384, + msg_per_sec: 4_096, + byte_burst: 512 * MIB, + byte_per_sec: 32 * MIB, + }, + committee_reserve: 0.25, + max_tracked_peers: 8_192, + } + } +} + +/// Idle duration before forgetting a tracked bucket. +const RATE_BUCKET_IDLE_FORGET: Duration = Duration::from_secs(60); +/// Minimum interval between full-table sweeps. +const RATE_TABLE_SWEEP_INTERVAL: Duration = Duration::from_secs(5); + +/// Bounded per-key buckets; excess keys share one overflow bucket. +#[derive(Debug)] +struct BucketTable { + buckets: HashMap, + overflow: DualBucket, + swept_at: Option, +} + +/// Which bucket of a [`BucketTable`] a message is charged to. +#[derive(Clone, Copy, Debug)] +enum BucketSlot { + Tracked(PeerId), + Overflow, +} + +impl BucketTable { + fn new(overflow_tier: TierRate, now: Instant) -> Self { + Self { buckets: HashMap::new(), overflow: overflow_tier.new_bucket(now), swept_at: None } + } + + /// The slot `key` is charged to. `pinned` keys are always tracked: they are + /// bounded by an on-chain registry, not by whoever generates key pairs. + fn slot( + &mut self, + key: &PeerId, + tier: TierRate, + cap: usize, + pinned: bool, + now: Instant, + ) -> BucketSlot { + if !self.buckets.contains_key(key) { + if self.buckets.len() >= cap + && self + .swept_at + .is_none_or(|at| now.saturating_duration_since(at) >= RATE_TABLE_SWEEP_INTERVAL) + { + self.swept_at = Some(now); + self.buckets.retain(|_, bucket| { + now.saturating_duration_since(bucket.msgs.refilled_at) < RATE_BUCKET_IDLE_FORGET + }); + } + if self.buckets.len() >= cap && !pinned { + return BucketSlot::Overflow; + } + self.buckets.insert(*key, tier.new_bucket(now)); + } + BucketSlot::Tracked(*key) + } + + fn get(&self, slot: BucketSlot) -> DualBucket { + match slot { + BucketSlot::Tracked(key) => self.buckets[&key], + BucketSlot::Overflow => self.overflow, + } + } + + fn set(&mut self, slot: BucketSlot, bucket: DualBucket) { + match slot { + BucketSlot::Tracked(key) => { + self.buckets.insert(key, bucket); + } + BucketSlot::Overflow => self.overflow = bucket, + } + } +} + +#[derive(Debug)] +struct RateState { + direct: BucketTable, + authors: BucketTable, + unregistered_total: DualBucket, + unregistered_forward: DualBucket, + global: DualBucket, +} + +/// Multi-tier pre-decode rate budget. +#[derive(Debug)] +pub struct InboundRateLimiter { + state: Mutex, + limits: RateLimits, +} + +impl InboundRateLimiter { + pub fn new(limits: RateLimits) -> Self { + let now = Instant::now(); + Self { + state: Mutex::new(RateState { + direct: BucketTable::new(limits.direct_peer, now), + authors: BucketTable::new(limits.unregistered_author, now), + unregistered_total: limits.unregistered_total.new_bucket(now), + unregistered_forward: limits.unregistered_forward.new_bucket(now), + global: limits.global.new_bucket(now), + }), + limits, + } + } + + /// Charge the relay budget for a message from an unregistered author that + /// this node forwards without storing. + pub fn charge_forward(&self, bytes: i64, now: Instant) -> Result<(), DropReason> { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let tier = self.limits.unregistered_forward; + let mut bucket = state.unregistered_forward; + let admitted = tier.peek(&mut bucket, bytes, now); + if admitted { + tier.spend(&mut bucket, bytes); + } + state.unregistered_forward = bucket; + admitted.then_some(()).ok_or(DropReason::ForwardRate) + } + + #[cfg(test)] + fn tracked_authors(&self) -> usize { + self.state.lock().unwrap().authors.buckets.len() + } + + /// Check all applicable rate tiers before charging any of them. + pub fn charge( + &self, + direct_peer: &PeerId, + author: &PeerId, + class: P2pInboxAdmissionClass, + bytes: i64, + now: Instant, + ) -> Result<(), DropReason> { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let author_tier = if class.is_registered() { + self.limits.registered_author + } else { + self.limits.unregistered_author + }; + let cap = self.limits.max_tracked_peers; + let direct_tier = self.limits.direct_peer; + let registered = class.is_registered(); + let direct_slot = state.direct.slot(direct_peer, direct_tier, cap, false, now); + let author_slot = state.authors.slot(author, author_tier, cap, registered, now); + let mut direct = state.direct.get(direct_slot); + let mut author_bucket = state.authors.get(author_slot); + let mut unregistered = state.unregistered_total; + let mut global = state.global; + + let charge_unregistered = !class.is_registered(); + // The shared tiers keep a reserve only the committee may draw on. + let reserve = if class == P2pInboxAdmissionClass::Committee { + 0.0 + } else { + self.limits.committee_reserve + }; + let direct_ok = direct_tier.peek_above(&mut direct, bytes, reserve, now); + let author_ok = author_tier.peek(&mut author_bucket, bytes, now); + let unregistered_ok = !charge_unregistered + || self.limits.unregistered_total.peek(&mut unregistered, bytes, now); + let global_ok = self.limits.global.peek_above(&mut global, bytes, reserve, now); + + // Persist refilled buckets even when admission fails. + let outcome = if !global_ok { + Err(DropReason::GlobalRate) + } else if !unregistered_ok { + Err(DropReason::UnregisteredRate) + } else if !author_ok { + Err(DropReason::AuthorRate) + } else if !direct_ok { + Err(DropReason::DirectPeerRate) + } else { + direct_tier.spend(&mut direct, bytes); + author_tier.spend(&mut author_bucket, bytes); + if charge_unregistered { + self.limits.unregistered_total.spend(&mut unregistered, bytes); + } + self.limits.global.spend(&mut global, bytes); + Ok(()) + }; + state.direct.set(direct_slot, direct); + state.authors.set(author_slot, author_bucket); + state.unregistered_total = unregistered; + state.global = global; + outcome + } +} + +static INBOUND_RATE_LIMITER: LazyLock = + LazyLock::new(|| InboundRateLimiter::new(RateLimits::defaults())); + +pub fn inbound_rate_limiter() -> &'static InboundRateLimiter { + &INBOUND_RATE_LIMITER +} + +#[derive(Debug, Default)] +struct ResponseGateState { + sent_at: Option, + pending: bool, +} + +/// Coalesce NodeInfo requests into one response per cooldown. +#[derive(Debug)] +pub struct ResponseGate { + state: Mutex, + cooldown: Duration, +} + +impl ResponseGate { + pub fn new(cooldown: Duration) -> Self { + Self { state: Mutex::new(ResponseGateState::default()), cooldown } + } + + fn cooled_down(&self, state: &ResponseGateState, now: Instant) -> bool { + state.sent_at.is_none_or(|sent_at| now.saturating_duration_since(sent_at) >= self.cooldown) + } + + /// `true` when the caller should respond now; otherwise the request is + /// remembered for [`Self::take_pending`]. + pub fn try_respond(&self, now: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if self.cooled_down(&state, now) { + state.sent_at = Some(now); + state.pending = false; + true + } else { + state.pending = true; + false + } + } + + /// `true` when a deferred request is due; the caller then responds. + pub fn take_pending(&self, now: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.pending && self.cooled_down(&state, now) { + state.sent_at = Some(now); + state.pending = false; + true + } else { + false + } + } +} + +static NODE_INFO_RESPONSE_GATE: LazyLock = + LazyLock::new(|| ResponseGate::new(NODE_INFO_RESPONSE_COOLDOWN)); + +pub fn node_info_response_gate() -> &'static ResponseGate { + &NODE_INFO_RESPONSE_GATE +} + +/// Cooldown for duplicate graph-setup ACKs. +const DEDUP_ACK_COOLDOWN: Duration = Duration::from_secs(8); +/// Cooldown for peer liveness updates. +const PEER_TIMESTAMP_COOLDOWN: Duration = Duration::from_secs(30); +const COOLDOWN_GATE_MAX_ENTRIES: usize = 8_192; + +/// Minimum interval between full gate/set sweeps. +const GATE_SWEEP_INTERVAL: Duration = Duration::from_secs(5); + +/// Bounded keyed timestamps; reject new keys when all entries are live. +#[derive(Debug, Default)] +struct BoundedStamps { + stamps: HashMap, + swept_at: Option, +} + +impl BoundedStamps { + fn is_live(&self, key: &str, ttl: Duration, now: Instant) -> bool { + self.stamps.get(key).is_some_and(|at| now.saturating_duration_since(*at) < ttl) + } + + /// Stamp `key` with `now`. Returns false when the table is full of live + /// entries and `key` is not among them. + fn stamp(&mut self, key: &str, ttl: Duration, max_entries: usize, now: Instant) -> bool { + if !self.stamps.contains_key(key) && self.stamps.len() >= max_entries { + if self + .swept_at + .is_none_or(|at| now.saturating_duration_since(at) >= GATE_SWEEP_INTERVAL) + { + self.swept_at = Some(now); + self.stamps.retain(|_, at| now.saturating_duration_since(*at) < ttl); + } + if self.stamps.len() >= max_entries { + return false; + } + } + self.stamps.insert(key.to_string(), now); + true + } +} + +/// Allows an action for a key at most once per cooldown, within a hard bound on +/// its own memory. A key it has no room to track is denied: every action gated +/// here is one the peer retries, and none is worth an unbounded table. +#[derive(Debug)] +pub struct CooldownGate { + seen: Mutex, + cooldown: Duration, + max_entries: usize, +} + +impl CooldownGate { + fn new(cooldown: Duration, max_entries: usize) -> Self { + Self { seen: Mutex::new(BoundedStamps::default()), cooldown, max_entries } + } + + /// Whether `key` is inside its cooldown. Records nothing. + pub fn is_cooling(&self, key: &str, now: Instant) -> bool { + let seen = self.seen.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + seen.is_live(key, self.cooldown, now) + } + + /// `true` when the action for `key` may proceed, recording it as just done. + pub fn allow(&self, key: &str, now: Instant) -> bool { + let mut seen = self.seen.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + !seen.is_live(key, self.cooldown, now) + && seen.stamp(key, self.cooldown, self.max_entries, now) + } + + #[cfg(test)] + fn len(&self) -> usize { + self.seen.lock().unwrap().stamps.len() + } +} + +static DEDUP_ACK_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(DEDUP_ACK_COOLDOWN, COOLDOWN_GATE_MAX_ENTRIES)); + +pub fn dedup_ack_gate() -> &'static CooldownGate { + &DEDUP_ACK_GATE +} + +static PEER_TIMESTAMP_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(PEER_TIMESTAMP_COOLDOWN, COOLDOWN_GATE_MAX_ENTRIES)); + +pub fn peer_timestamp_gate() -> &'static CooldownGate { + &PEER_TIMESTAMP_GATE +} + +static NODE_TABLE_PURGED_AT: Mutex> = Mutex::new(None); + +/// `true` at most once per `interval`: when the `node` table purge should run. +pub fn node_table_purge_due(interval: Duration, now: Instant) -> bool { + let mut purged_at = + NODE_TABLE_PURGED_AT.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if purged_at.is_some_and(|at| now.saturating_duration_since(at) < interval) { + return false; + } + *purged_at = Some(now); + true +} + +const STRIKE_WINDOW: Duration = Duration::from_secs(60); +const STRIKE_LIMIT: u32 = 256; +const STRIKE_BAN: Duration = Duration::from_secs(10 * 60); +const STRIKE_MAX_TRACKED_PEERS: usize = 4_096; + +#[derive(Debug, Default)] +struct StrikeState { + /// Window start and strikes in it, per direct peer. + strikes: HashMap, + banned_until: HashMap, +} + +/// Temporary local bans based on attributable relay strikes; registered peers are exempt. +#[derive(Debug, Default)] +pub struct DirectPeerStrikes { + state: Mutex, +} + +impl DirectPeerStrikes { + /// Count one strike against `peer`. Returns `true` when it just reached the + /// limit and should be banned now. + pub fn strike(&self, peer: &PeerId, now: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.banned_until.contains_key(peer) { + return false; + } + if !state.strikes.contains_key(peer) && state.strikes.len() >= STRIKE_MAX_TRACKED_PEERS { + state.strikes.retain(|_, (started_at, _)| { + now.saturating_duration_since(*started_at) < STRIKE_WINDOW + }); + if state.strikes.len() >= STRIKE_MAX_TRACKED_PEERS { + return false; + } + } + let (started_at, count) = state.strikes.entry(*peer).or_insert((now, 0)); + if now.saturating_duration_since(*started_at) >= STRIKE_WINDOW { + (*started_at, *count) = (now, 0); + } + *count += 1; + if *count < STRIKE_LIMIT { + return false; + } + state.strikes.remove(peer); + state.banned_until.insert(*peer, now + STRIKE_BAN); + true + } + + /// Check the peer's current ban for each connection. + pub fn is_banned(&self, peer: &PeerId, now: Instant) -> bool { + let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.banned_until.get(peer).is_some_and(|until| now < *until) + } + + /// Peers whose ban has run out, removed from the ban list. + pub fn take_expired_bans(&self, now: Instant) -> Vec { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let expired: Vec = state + .banned_until + .iter() + .filter(|(_, until)| now >= **until) + .map(|(peer, _)| *peer) + .collect(); + for peer in &expired { + state.banned_until.remove(peer); + } + expired + } +} + +static DIRECT_PEER_STRIKES: LazyLock = LazyLock::new(DirectPeerStrikes::default); + +pub fn direct_peer_strikes() -> &'static DirectPeerStrikes { + &DIRECT_PEER_STRIKES +} + +const REFUSED_PEERS_MAX: usize = 256; + +static REFUSED_PEERS: Mutex> = Mutex::new(Vec::new()); + +/// Queue refused peers for committee lookup on the next maintenance pass. +pub fn note_refused_peer(peer: PeerId) { + let mut refused = REFUSED_PEERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if refused.len() < REFUSED_PEERS_MAX && !refused.contains(&peer) { + refused.push(peer); + } +} + +fn take_refused_peers() -> Vec { + std::mem::take(&mut *REFUSED_PEERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner)) +} + +/// How often a protocol message this node has already stored is published again. +const PROTOCOL_REPUBLISH_COOLDOWN: Duration = Duration::from_secs(30); + +static PROTOCOL_REPUBLISH_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(PROTOCOL_REPUBLISH_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); + +/// Throttle stored-value recovery; stamp only on success. +/// A full gate leaves recovery open; live outbox rows retain their own retry schedule. +pub fn protocol_republish_gate() -> &'static CooldownGate { + &PROTOCOL_REPUBLISH_GATE +} + +/// How long a graph a relay just answered a sync request for stays on cooldown. +const SYNC_GRAPH_RESPONSE_COOLDOWN: Duration = Duration::from_secs(60); +/// TTL for locally requested graph responses. +const SYNC_GRAPH_REQUEST_TTL: Duration = Duration::from_secs(15 * 60); +const GRAPH_SET_MAX_ENTRIES: usize = 16_384; + +/// A global token bucket, for a resource no per-peer key bounds. +#[derive(Debug)] +pub struct GlobalRate { + bucket: Mutex, + rate: f64, + burst: f64, +} + +impl GlobalRate { + fn new(burst: f64, rate: f64) -> Self { + Self { bucket: Mutex::new(RateBucket::full(burst, Instant::now())), rate, burst } + } + + pub fn allow(&self, now: Instant) -> bool { + let mut bucket = self.bucket.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + bucket.refill(self.rate, self.burst, now); + if bucket.tokens < 1.0 { + return false; + } + bucket.tokens -= 1.0; + true + } +} + +/// A bounded set of keys, each valid until its TTL lapses. +#[derive(Debug)] +pub struct TtlSet { + entries: Mutex, + ttl: Duration, + max_entries: usize, +} + +impl TtlSet { + fn new(ttl: Duration, max_entries: usize) -> Self { + Self { entries: Mutex::new(BoundedStamps::default()), ttl, max_entries } + } + + /// Returns false when the set is full of unexpired keys and has no room. + pub fn insert(&self, key: &str, now: Instant) -> bool { + let mut entries = self.entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + entries.stamp(key, self.ttl, self.max_entries, now) + } + + /// Whether `key` is still present and unexpired. + pub fn contains(&self, key: &str, now: Instant) -> bool { + let entries = self.entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + entries.is_live(key, self.ttl, now) + } + + pub fn remove(&self, key: &str) { + let mut entries = self.entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + entries.stamps.remove(key); + } +} + +static SYNC_GRAPH_RESPONSE_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(SYNC_GRAPH_RESPONSE_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); + +/// Per-graph cooldown for answering `SyncGraphRequest` on the relay side. +pub fn sync_graph_response_gate() -> &'static CooldownGate { + &SYNC_GRAPH_RESPONSE_GATE +} + +// Separate aggregate response budgets for registered and unregistered requesters. +static SYNC_GRAPH_RESPONSE_BUDGET_REGISTERED: LazyLock = + LazyLock::new(|| GlobalRate::new(32.0, 2.0)); +static SYNC_GRAPH_RESPONSE_BUDGET_UNREGISTERED: LazyLock = + LazyLock::new(|| GlobalRate::new(8.0, 0.2)); + +/// Aggregate SyncGraph response budgets by registration class. +pub fn sync_graph_response_budget(requester_registered: bool) -> &'static GlobalRate { + if requester_registered { + &SYNC_GRAPH_RESPONSE_BUDGET_REGISTERED + } else { + &SYNC_GRAPH_RESPONSE_BUDGET_UNREGISTERED + } +} + +static REQUESTED_GRAPHS: LazyLock = + LazyLock::new(|| TtlSet::new(SYNC_GRAPH_REQUEST_TTL, GRAPH_SET_MAX_ENTRIES)); + +/// Outstanding local graph requests. +pub fn requested_graphs() -> &'static TtlSet { + &REQUESTED_GRAPHS +} + +/// How often one requested graph may be put through validation. A `SyncGraph` +/// costs chain queries, a graph rebuild and two signature sweeps; a sender that +/// keeps answering the same request with a bad graph gets one try per window. +const SYNC_GRAPH_VALIDATION_COOLDOWN: Duration = Duration::from_secs(30); + +static SYNC_GRAPH_VALIDATION_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(SYNC_GRAPH_VALIDATION_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); + +pub fn sync_graph_validation_gate() -> &'static CooldownGate { + &SYNC_GRAPH_VALIDATION_GATE +} + +/// Aggregate drop counters for periodic logging. +#[derive(Debug, Default)] +pub struct DropStats { + counts: Mutex>, +} + +impl DropStats { + pub fn record(&self, reason: DropReason) { + let mut counts = self.counts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + *counts.entry(reason).or_default() += 1; + } + + pub fn take(&self) -> Vec<(DropReason, u64)> { + let mut counts = self.counts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut taken: Vec<_> = counts.drain().collect(); + taken.sort_by_key(|(reason, _)| reason.as_str()); + taken + } +} + +static DROP_STATS: LazyLock = LazyLock::new(DropStats::default); + +pub fn record_drop(reason: DropReason) { + DROP_STATS.record(reason); +} + +/// Count an admitted message from an unregistered sender for the summary line. +/// Those are not logged one by one: whoever floods decides how many there are. +pub fn record_admitted_unregistered() { + DROP_STATS.record(DropReason::AdmittedUnregistered); +} + +/// Emit one line for everything dropped since the previous call. +pub fn log_drop_summary() { + let dropped = DROP_STATS.take(); + if dropped.is_empty() { + return; + } + let total: u64 = dropped.iter().map(|(_, count)| count).sum(); + let breakdown = dropped + .iter() + .map(|(reason, count)| format!("{}={count}", reason.as_str())) + .collect::>() + .join(" "); + tracing::warn!( + event = "p2p_admission", + outcome = "summary", + total, + breakdown, + "inbound P2P messages dropped, de-duplicated or admitted unlogged by admission control" + ); +} + +/// Number of sequence positions retained below the high-water mark. +const REPLAY_WINDOW: u64 = 1_024; +const REPLAY_MAX_TRACKED_AUTHORS: usize = 16_384; + +/// Maximum accepted sequence-number lead over local wall time. +const REPLAY_MAX_FUTURE: Duration = Duration::from_secs(10 * 60); +/// Below-floor rejections of one author per tick that are worth an operator's +/// attention; see [`ReplayGuard::take_lockout_suspects`]. +const REPLAY_LOCKOUT_SUSPECT_REJECTIONS: u32 = 8; + +#[derive(Debug, Default)] +struct AuthorSequence { + highest: u64, + /// Sequence numbers seen within `REPLAY_WINDOW` of `highest`. + recent: std::collections::BTreeSet, + /// Persisted rejection floor restored at startup. + floor: Option, + /// The mark the store is known to hold. `highest` above it is still owed. + persisted: u64, + /// Rejection count since the last report; diagnostic only. + rejected: u32, +} + +/// Per-registered-author sequence window with periodically persisted high-water marks. +/// Accept unseen numbers within the window; restored marks are rejection floors. +/// Authors must keep numbering monotonic across restarts. +/// `P2P_REPLAY_MARK_RESET_PEERS` clears selected persisted marks at startup. +#[derive(Debug, Default)] +pub struct ReplayGuard { + authors: Mutex>, + /// How far each registered author's numbering ran ahead of the local clock + /// since the last report; see [`Self::take_clock_leads`]. + clock_leads: Mutex>, +} + +/// Authors remembered per tick for the clock report. It is a diagnostic, and the +/// table must stay small whatever arrives. +const CLOCK_LEADS_MAX_AUTHORS: usize = 64; + +impl ReplayGuard { + /// Record `sequence_number` for `author`, or refuse it. `unix_nanos` is the + /// local wall clock. + pub fn admit( + &self, + author: &PeerId, + sequence_number: Option, + unix_nanos: u64, + ) -> Result<(), DropReason> { + // Require a signed sequence number. + let sequence_number = sequence_number.ok_or(DropReason::ReplayedSequence)?; + if sequence_number > unix_nanos { + self.note_clock_lead(author, sequence_number - unix_nanos); + } + if sequence_number > unix_nanos.saturating_add(REPLAY_MAX_FUTURE.as_nanos() as u64) { + return Err(DropReason::FutureSequence); + } + let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !authors.contains_key(author) && authors.len() >= REPLAY_MAX_TRACKED_AUTHORS { + // Only reachable if the registries outgrow the table; forgetting an + // author merely re-opens its window once. + if let Some(evicted) = authors.keys().next().copied() { + authors.remove(&evicted); + } + } + let state = authors.entry(*author).or_default(); + if state.floor.is_some_and(|floor| sequence_number <= floor) { + state.rejected = state.rejected.saturating_add(1); + return Err(DropReason::ReplayedSequence); + } + if sequence_number > state.highest { + state.highest = sequence_number; + let floor = sequence_number.saturating_sub(REPLAY_WINDOW); + state.recent = state.recent.split_off(&floor); + } else if state.highest - sequence_number >= REPLAY_WINDOW + || state.recent.contains(&sequence_number) + { + state.rejected = state.rejected.saturating_add(1); + return Err(DropReason::ReplayedSequence); + } + state.recent.insert(sequence_number); + Ok(()) + } + + fn note_clock_lead(&self, author: &PeerId, lead_nanos: u64) { + let mut leads = self.clock_leads.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if leads.len() < CLOCK_LEADS_MAX_AUTHORS || leads.contains_key(author) { + let lead = leads.entry(*author).or_default(); + *lead = (*lead).max(lead_nanos); + } + } + + /// Return and clear each registered author's maximum clock lead since the previous call. + pub fn take_clock_leads(&self) -> Vec<(PeerId, Duration)> { + let mut leads = self.clock_leads.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + leads.drain().map(|(author, lead)| (author, Duration::from_nanos(lead))).collect() + } + + /// Restore an author's persisted mark. Call before any message is admitted. + pub fn restore_mark(&self, author: &PeerId, highest: u64) { + let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if authors.len() >= REPLAY_MAX_TRACKED_AUTHORS { + return; + } + let state = authors.entry(*author).or_default(); + state.highest = state.highest.max(highest); + state.persisted = state.persisted.max(highest); + state.floor = Some(state.floor.map_or(highest, |floor| floor.max(highest))); + } + + /// Marks awaiting successful persistence confirmation. + pub fn pending_marks(&self) -> Vec<(PeerId, u64)> { + let authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + authors + .iter() + .filter(|(_, state)| state.highest > state.persisted) + .map(|(author, state)| (*author, state.highest)) + .collect() + } + + /// Acknowledge marks that were written. A mark that has moved on since it + /// was read stays owed for the difference. + pub fn confirm_persisted(&self, marks: &[(PeerId, u64)]) { + let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for (author, written) in marks { + if let Some(state) = authors.get_mut(author) { + state.persisted = state.persisted.max(*written); + } + } + } + + /// Return authors exceeding the rejection reporting threshold. + pub fn take_lockout_suspects(&self) -> Vec<(PeerId, u32)> { + let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + authors + .iter_mut() + .filter_map(|(author, state)| { + let rejected = std::mem::take(&mut state.rejected); + (rejected >= REPLAY_LOCKOUT_SUSPECT_REJECTIONS).then_some((*author, rejected)) + }) + .collect() + } +} + +/// The local wall clock in the unit gossipsub numbers its messages in. +pub fn unix_nanos_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos().min(u128::from(u64::MAX)) as u64) +} + +pub fn replay_guard() -> &'static ReplayGuard { + &REPLAY_GUARD +} + +static REPLAY_GUARD: LazyLock = LazyLock::new(ReplayGuard::default); + +/// How long the per-class inbox totals are served from memory. +const INBOX_CLASS_TOTALS_TTL: Duration = Duration::from_secs(2); + +/// Cache per-class queue totals and increment them on admission. +/// Worker removals are reflected at the next cache refresh. +#[derive(Debug, Default)] +pub struct InboxUsageCache { + totals: Mutex)>>, +} + +impl InboxUsageCache { + fn fresh(&self, now: Instant) -> Option> { + let totals = self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + totals + .as_ref() + .filter(|(at, _)| now.saturating_duration_since(*at) < INBOX_CLASS_TOTALS_TTL) + .map(|(_, totals)| totals.clone()) + } + + fn store(&self, totals: Vec, now: Instant) { + *self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some((now, totals)); + } + + /// Count a row that is about to be queued. + fn note_admitted(&self, class: P2pInboxAdmissionClass, bytes: i64) { + let mut totals = self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let Some((_, totals)) = totals.as_mut() else { + return; + }; + let class = class.to_string(); + match totals.iter_mut().find(|usage| usage.admission_class == class) { + Some(usage) => { + usage.rows += 1; + usage.bytes += bytes; + } + None => totals.push(P2pInboxClassUsage { + admission_class: class, + rows: 1, + bytes, + ..Default::default() + }), + } + } + + /// Combine cached class totals with fresh per-peer usage. + async fn usage( + &self, + storage: &mut store::localdb::StorageProcessor<'_>, + from_peer: &str, + now: Instant, + ) -> Result> { + let mut usage = match self.fresh(now) { + Some(totals) => totals, + None => { + let totals = storage.p2p_inbox_class_totals().await?; + self.store(totals.clone(), now); + totals + } + }; + for peer in storage.p2p_inbox_peer_usage(from_peer).await? { + match usage.iter_mut().find(|total| total.admission_class == peer.admission_class) { + Some(total) => { + total.peer_rows = peer.peer_rows; + total.peer_bytes = peer.peer_bytes; + } + None => usage.push(peer), + } + } + Ok(usage) + } +} + +static INBOX_USAGE_CACHE: LazyLock = LazyLock::new(InboxUsageCache::default); + +/// Injectable admission gates. +#[derive(Clone, Copy)] +pub struct AdmissionGates<'a> { + pub registry: &'a PeerRegistry, + pub replay_guard: &'a ReplayGuard, + pub rate_limiter: &'a InboundRateLimiter, + pub immediate_limiter: &'a PeerRateLimiter, + pub usage_cache: &'a InboxUsageCache, + pub limits: &'a InboundLimits, + /// This node's role. A durable message no handler of this role acts on is + /// relayed but not stored. + pub local_actor: &'a bitvm_lib::actors::Actor, + /// Graphs this node asked a relayer for. A `SyncGraph` for anything else is + /// somebody else's answer. + pub requested_graphs: &'a TtlSet, +} + +impl AdmissionGates<'_> { + /// Whether this node stores an admitted durable message of `kind`, or only + /// relays it. `synced_graph` is the graph a `SyncGraph` carries. + pub fn keeps( + &self, + kind: crate::action::MessageKind, + synced_graph: Option, + now: Instant, + ) -> bool { + kind.handled_by(self.local_actor) + && synced_graph + .is_none_or(|graph_id| self.requested_graphs.contains(&graph_id.to_string(), now)) + } +} + +impl<'a> AdmissionGates<'a> { + /// The process-wide gates the node runs with. + pub fn global(local_actor: &'a bitvm_lib::actors::Actor) -> Self { + Self { + local_actor, + requested_graphs: requested_graphs(), + registry: peer_registry(), + replay_guard: &REPLAY_GUARD, + rate_limiter: inbound_rate_limiter(), + immediate_limiter: immediate_limiter(), + usage_cache: &INBOX_USAGE_CACHE, + limits: inbound_limits(), + } + } +} + +/// One received gossipsub message, as admission sees it. +#[derive(Clone, Copy, Debug)] +pub struct InboundGossip<'a> { + /// The neighbour that relayed the message to this node. + pub direct_peer: &'a PeerId, + /// The signed author. + pub source: &'a PeerId, + /// The author's gossipsub sequence number, covered by its signature. + pub sequence_number: Option, + pub data: &'a [u8], +} + +/// What to do with one inbound gossipsub message. +pub enum InboundVerdict { + /// Neither processed nor forwarded. + Drop(DropReason), + /// Dispatch now; nothing is persisted. + Immediate(GOATMessage), + /// Forward without persistence, dispatch or ACK. + Forward, + /// Persist for the inbox worker. + Enqueue { message: GOATMessage, class: P2pInboxAdmissionClass, content_hash: [u8; 32] }, + /// The same sender already has this exact payload queued. + Duplicate(GOATMessage), +} + +/// Evaluate admission without swarm or chain access; propagate database errors. +pub async fn evaluate_inbound_message( + local_db: &LocalDB, + gates: &AdmissionGates<'_>, + inbound: &InboundGossip<'_>, + now: Instant, +) -> Result { + let InboundGossip { direct_peer, source, sequence_number, data } = *inbound; + if let Err(reason) = check_envelope(data, gates.registry.is_verifier(source), gates.limits) { + return Ok(InboundVerdict::Drop(reason)); + } + // Classification reads only the cache and never blocks. + let class = gates.registry.sender_class(source, now); + // Check replay before charging the registered author's rate budget. + if class.is_registered() + && let Err(reason) = gates.replay_guard.admit(source, sequence_number, unix_nanos_now()) + { + return Ok(InboundVerdict::Drop(reason)); + } + // Apply rate limits before decoding. + if let Err(reason) = + gates.rate_limiter.charge(direct_peer, source, class, data.len() as i64, now) + { + return Ok(InboundVerdict::Drop(reason)); + } + let Ok(message) = GOATMessage::deserialize_message(data).await else { + return Ok(InboundVerdict::Drop(DropReason::Undecodable)); + }; + // Require GenCircuits inside the binary envelope. + if GOATMessage::is_binary_envelope(data) + && !matches!(message.content, GOATMessageContent::GenCircuits(_)) + { + return Ok(InboundVerdict::Drop(DropReason::UnexpectedBinaryKind)); + } + + // Reject only definite sender-role denials; unknown identities remain subject to unregistered limits. + let sender_role = message.content.kind().sender_role(); + if gates.registry.role_verdict(source, sender_role, now) == RoleVerdict::Denied { + return Ok(InboundVerdict::Drop(DropReason::SenderRoleDenied)); + } + + if message.content.p2p_delivery() == P2PMessageDelivery::Immediate { + if !gates.immediate_limiter.allow(source, now) { + return Ok(InboundVerdict::Drop(DropReason::ImmediateRateLimited)); + } + return Ok(InboundVerdict::Immediate(message)); + } + + // Forward unsolicited SyncGraph responses without storing them locally. + let synced_graph = match &message.content { + GOATMessageContent::SyncGraph(sync) => Some(sync.graph_id), + _ => None, + }; + if !gates.keeps(message.content.kind(), synced_graph, now) { + // Apply the unregistered forwarding budget. + if !class.is_registered() + && let Err(reason) = gates.rate_limiter.charge_forward(data.len() as i64, now) + { + return Ok(InboundVerdict::Drop(reason)); + } + return Ok(InboundVerdict::Forward); + } + + let content_hash = content_hash(data); + let from_peer = source.to_string(); + let mut storage = local_db.acquire().await?; + if storage.has_queued_p2p_inbox_payload(&from_peer, &content_hash).await? { + return Ok(InboundVerdict::Duplicate(message)); + } + let usage = gates.usage_cache.usage(&mut storage, &from_peer, now).await?; + if let Err(reason) = check_inbox_quota(class, data.len(), &usage, gates.limits) { + return Ok(InboundVerdict::Drop(reason)); + } + gates.usage_cache.note_admitted(class, data.len() as i64); + Ok(InboundVerdict::Enqueue { message, class, content_hash }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::action::KickoffSent; + use bitvm_lib::actors::Actor; + use uuid::Uuid; + + const SMALL: InboundLimits = InboundLimits { + max_json_bytes: 1024, + max_binary_bytes: 4096, + global: Quota { rows: 6, bytes: 6000 }, + committee_reserve: Quota { rows: 0, bytes: 0 }, + unregistered_class: Quota { rows: 4, bytes: 3000 }, + registered_peer: Quota { rows: 3, bytes: 5000 }, + unregistered_peer: Quota { rows: 2, bytes: 2000 }, + }; + + fn usage( + class: P2pInboxAdmissionClass, + rows: i64, + bytes: i64, + peer: (i64, i64), + ) -> P2pInboxClassUsage { + P2pInboxClassUsage { + admission_class: class.to_string(), + rows, + bytes, + peer_rows: peer.0, + peer_bytes: peer.1, + } + } + + /// Use isolated gates in tests. + struct TestGates { + registry: PeerRegistry, + replay_guard: ReplayGuard, + rate_limiter: InboundRateLimiter, + immediate_limiter: PeerRateLimiter, + usage_cache: InboxUsageCache, + local_actor: Actor, + requested_graphs: TtlSet, + } + + /// A wall clock comfortably ahead of every sequence number the tests use. + const WALL_CLOCK: u64 = 1_800_000_000 * 1_000_000_000; + + /// A distinct operator key per `index`. The registry never parses it. + fn key(index: u32) -> OperatorKey { + let mut key = [0x11; 32]; + key[..4].copy_from_slice(&index.to_be_bytes()); + key + } + + impl TestGates { + fn new(immediate_limiter: PeerRateLimiter) -> Self { + Self { + requested_graphs: TtlSet::new(SYNC_GRAPH_REQUEST_TTL, 16), + local_actor: Actor::Committee, + registry: PeerRegistry::default(), + replay_guard: ReplayGuard::default(), + rate_limiter: InboundRateLimiter::new(RateLimits::defaults()), + immediate_limiter, + usage_cache: InboxUsageCache::default(), + } + } + + /// Evaluate `data` as authored and relayed by `source`. + async fn evaluate( + &self, + local_db: &LocalDB, + limits: &InboundLimits, + source: &PeerId, + sequence_number: u64, + data: &[u8], + now: Instant, + ) -> InboundVerdict { + let gates = AdmissionGates { + registry: &self.registry, + replay_guard: &self.replay_guard, + rate_limiter: &self.rate_limiter, + immediate_limiter: &self.immediate_limiter, + usage_cache: &self.usage_cache, + limits, + local_actor: &self.local_actor, + requested_graphs: &self.requested_graphs, + }; + let inbound = InboundGossip { + direct_peer: source, + source, + sequence_number: Some(sequence_number), + data, + }; + evaluate_inbound_message(local_db, &gates, &inbound, now).await.unwrap() + } + } + + async fn kickoff_sent_bytes() -> Vec { + GOATMessage::new( + Actor::Committee, + GOATMessageContent::KickoffSent(KickoffSent { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + }), + ) + .serialize_message() + .await + .unwrap() + } + + #[test] + fn envelope_limits_depend_on_the_encoding_and_the_sender() { + let json = vec![b'{'; 1025]; + assert_eq!(check_envelope(&json, true, &SMALL), Err(DropReason::OversizedJson)); + assert_eq!(check_envelope(&json[..1024], false, &SMALL), Ok(())); + + let mut binary = b"GOATBIN1".to_vec(); + binary.resize(2048, 0); + assert_eq!(check_envelope(&binary, false, &SMALL), Err(DropReason::BinaryFromNonVerifier)); + assert_eq!(check_envelope(&binary, true, &SMALL), Ok(()), "above the JSON ceiling"); + binary.resize(4097, 0); + assert_eq!(check_envelope(&binary, true, &SMALL), Err(DropReason::OversizedBinary)); + } + + /// Verify GenCircuits fits the binary transport ceiling. + #[tokio::test] + async fn gen_circuits_fits_the_binary_envelope() { + use crate::action::GenCircuits; + use bitvm_lib::babe_adapter::{BABE_N_CC, build_setup_package}; + + let secp = bitcoin::secp256k1::Secp256k1::new(); + let secret = bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(); + let message = GOATMessage::new( + Actor::Operator, + GOATMessageContent::GenCircuits(GenCircuits { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + verifier_pubkey: bitcoin::PublicKey::new(secret.public_key(&secp)), + setup_package: build_setup_package(BABE_N_CC).unwrap(), + }), + ); + let bytes = message.serialize_message().await.unwrap(); + assert!(GOATMessage::is_binary_envelope(&bytes)); + let limits = InboundLimits::with_queued_bytes(2 << 20, 2 << 30); + assert!(bytes.len() > limits.max_json_bytes, "GenCircuits is {} bytes", bytes.len()); + assert!( + bytes.len() * 5 <= limits.max_binary_bytes * 4, + "GenCircuits is {} bytes, within 20% of the {} byte ceiling", + bytes.len(), + limits.max_binary_bytes + ); + assert_eq!(check_envelope(&bytes, true, &limits), Ok(())); + assert!( + (bytes.len() as i64) * 8 <= limits.registered_peer.bytes, + "a verifier must be able to have several setups queued" + ); + } + + /// Verify graph-bearing JSON fits the configured verifier-slot capacity. + #[test] + fn default_json_ceiling_fits_graphs_with_several_verifier_slots() { + use bitvm_lib::types::BitvmGcCircuitData; + use goat::assert_scripts::WireHash; + + let secp = bitcoin::secp256k1::Secp256k1::new(); + let secret = bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(); + // 0xff is the widest byte in JSON's decimal array encoding. + let slot = BitvmGcCircuitData { + verifier_pubkey: bitcoin::PublicKey::new(secret.public_key(&secp)), + final_msg_hashlocks: vec![[0xff; 20]; bitvm_lib::babe_adapter::BABE_M_CC], + wire_hashes: std::array::from_fn(|_| WireHash { + true_label_hash: [0xff; 20], + false_label_hash: [0xff; 20], + }), + }; + let slot_len = serde_json::to_vec(&slot).unwrap().len(); + assert!( + slot_len * 8 <= crate::env::DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES, + "one GC slot is {slot_len} bytes of JSON" + ); + } + + #[test] + fn quota_is_charged_per_sender_per_class_and_globally() { + use P2pInboxAdmissionClass::{Registered, Unregistered}; + + assert_eq!(check_inbox_quota(Unregistered, 100, &[], &SMALL), Ok(())); + + // The sender's own rows, in whichever class they were admitted. + let own = [usage(Unregistered, 1, 100, (1, 100)), usage(Registered, 1, 100, (1, 100))]; + assert_eq!( + check_inbox_quota(Unregistered, 100, &own, &SMALL), + Err(DropReason::SenderQuota) + ); + assert_eq!(check_inbox_quota(Registered, 100, &own, &SMALL), Ok(())); + let own_bytes = [usage(Unregistered, 1, 1950, (1, 1950))]; + assert_eq!( + check_inbox_quota(Unregistered, 100, &own_bytes, &SMALL), + Err(DropReason::SenderQuota) + ); + + // Other unregistered senders exhaust the shared pool; a registered + // sender is not charged against it. + let crowded = [usage(Unregistered, 4, 400, (0, 0))]; + assert_eq!( + check_inbox_quota(Unregistered, 100, &crowded, &SMALL), + Err(DropReason::UnregisteredClassQuota) + ); + assert_eq!(check_inbox_quota(Registered, 100, &crowded, &SMALL), Ok(())); + + // The global ceiling counts every class, including pre-migration rows. + let full = [usage(Unregistered, 3, 300, (0, 0)), usage(Registered, 3, 300, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &full, &SMALL), Err(DropReason::GlobalQuota)); + let heavy = [usage(Registered, 1, 5950, (0, 0))]; + assert_eq!( + check_inbox_quota(Registered, 100, &heavy, &SMALL), + Err(DropReason::GlobalQuota) + ); + } + + #[test] + fn derived_quotas_nest_inside_the_global_ceiling() { + let limits = InboundLimits::with_queued_bytes(2 << 20, 2 << 30); + assert!(limits.unregistered_peer.bytes < limits.unregistered_class.bytes); + assert!(limits.unregistered_class.bytes < limits.global.bytes); + assert!(limits.registered_peer.bytes < limits.global.bytes); + assert!(limits.unregistered_peer.rows < limits.unregistered_class.rows); + assert!(limits.unregistered_class.rows < limits.global.rows); + assert!(limits.max_json_bytes as i64 <= limits.unregistered_peer.bytes); + } + + #[test] + fn registry_classifies_from_cache_and_never_blocks() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (verifier, member, stranger) = (PeerId::random(), PeerId::random(), PeerId::random()); + + // Nothing is known yet: everyone is unregistered, and lookups are due. + assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Unregistered); + assert_eq!( + registry.plan_refresh(Some(&member), now), + RefreshPlan { verifier_set: true, committee_peer: true } + ); + assert_eq!( + registry.plan_refresh(Some(&member), now), + RefreshPlan::default(), + "a lookup already in flight is not started twice" + ); + + registry.record_verifier_set(Some(vec![verifier.to_bytes()]), now); + registry.record_committee_peer(&member, Some(true), now); + assert!(registry.is_verifier(&verifier)); + assert_eq!(registry.sender_class(&verifier, now), P2pInboxAdmissionClass::Registered); + assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Committee); + assert!(registry.is_committee(&member, now)); + assert!(!registry.is_committee(&verifier, now), "registered, but not the committee"); + assert_eq!(registry.sender_class(&stranger, now), P2pInboxAdmissionClass::Unregistered); + assert_eq!( + registry.plan_refresh(Some(&verifier), now), + RefreshPlan::default(), + "a verifier needs no committee lookup" + ); + + // Past its TTL the member is re-validated but keeps its class meanwhile. + // The verifier set has aged out by then as well. + let later = now + COMMITTEE_POSITIVE_TTL; + assert_eq!(registry.sender_class(&member, later), P2pInboxAdmissionClass::Committee); + assert_eq!( + registry.plan_refresh(Some(&member), later), + RefreshPlan { verifier_set: true, committee_peer: true } + ); + registry.record_committee_peer(&member, None, later); + assert_eq!( + registry.sender_class(&member, later), + P2pInboxAdmissionClass::Committee, + "a failed lookup leaves the cached answer in place" + ); + assert_eq!( + registry.sender_class(&member, now + COMMITTEE_STALE_LIMIT), + P2pInboxAdmissionClass::Unregistered, + "but it is not trusted forever" + ); + + // A failed verifier refresh keeps the previous set and is retried soon, + // but not on every message. + registry.record_verifier_set(None, later); + assert!(registry.is_verifier(&verifier)); + assert!(!registry.plan_refresh(None, later + Duration::from_secs(1)).verifier_set); + assert!(registry.plan_refresh(None, later + VERIFIER_SET_RETRY_INTERVAL).verifier_set); + } + + #[test] + fn unknown_identities_cannot_starve_revalidation_of_known_members() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + let member = PeerId::random(); + registry.record_committee_peer(&member, Some(true), now); + + let later = now + COMMITTEE_POSITIVE_TTL; + let mut started = 0; + for _ in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let stranger = PeerId::random(); + if registry.plan_refresh(Some(&stranger), later).committee_peer { + started += 1; + registry.record_committee_peer(&stranger, Some(false), later); + } + } + assert_eq!(started, COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE); + assert!( + registry.plan_refresh(Some(&member), later).committee_peer, + "re-validating a registered peer is not charged to the discovery budget" + ); + } + + /// Item 5: even with the unknown-lookup slots saturated and in flight, a + /// known member's re-check must still find a reserved slot. + #[test] + fn known_member_recheck_survives_saturated_unknown_lookups() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + let member = PeerId::random(); + registry.record_committee_peer(&member, Some(true), now); + + // Fill and hold the unknown-lookup slots (do not resolve them). + let mut unknown_in_flight = 0; + for _ in 0..COMMITTEE_MAX_IN_FLIGHT_LOOKUPS * 2 { + if registry.plan_refresh(Some(&PeerId::random()), now).committee_peer { + unknown_in_flight += 1; + } + } + assert_eq!( + unknown_in_flight, COMMITTEE_MAX_UNKNOWN_IN_FLIGHT, + "unknown lookups are capped below the total, leaving slots reserved" + ); + let later = now + COMMITTEE_POSITIVE_TTL; + assert!( + registry.plan_refresh(Some(&member), later).committee_peer, + "a known member re-check uses a reserved slot despite the unknown flood" + ); + } + + /// An operator becomes registered only once its proven binding is confirmed + /// to be a staked operator, and a key maps to one peer id at a time. + #[test] + fn operator_binding_grants_registered_only_after_stake_confirmation() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let peer = PeerId::random(); + let pubkey = &key(1); + + registry.observe_operator_binding(&peer, pubkey, 1); + assert_eq!( + registry.sender_class(&peer, now), + P2pInboxAdmissionClass::Unregistered, + "a proven binding alone is not enough; stake is unconfirmed" + ); + assert!(registry.plan_operator_refresh(&peer, pubkey, now)); + assert!( + !registry.plan_operator_refresh(&peer, pubkey, now), + "a lookup already in flight is not started twice" + ); + registry.record_operator_stake(&peer, pubkey, Some(true), now); + assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); + + // Verify one key registers only one peer. + let new_peer = PeerId::random(); + registry.observe_operator_binding(&new_peer, pubkey, 2); + registry.record_operator_stake(&new_peer, pubkey, Some(true), now); + assert_eq!(registry.sender_class(&new_peer, now), P2pInboxAdmissionClass::Registered); + assert_eq!( + registry.sender_class(&peer, now), + P2pInboxAdmissionClass::Unregistered, + "the superseded peer id loses the class" + ); + + // An older binding for a key already owned by another peer is ignored. + registry.observe_operator_binding(&peer, pubkey, 1); + assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Unregistered); + } + + /// The rate budget rejects a flood before decode, charges every tier, and + /// only debits when all tiers pass. + #[test] + fn inbound_rate_budget_bounds_bytes_and_messages_per_tier() { + const MIB: i64 = 1024 * 1024; + let limits = RateLimits { + direct_peer: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + registered_author: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + unregistered_author: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 2 * MIB, + byte_per_sec: 0, + }, + unregistered_total: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 3 * MIB, + byte_per_sec: 0, + }, + unregistered_forward: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + global: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + committee_reserve: 0.0, + max_tracked_peers: 16, + }; + let limiter = InboundRateLimiter::new(limits); + let now = Instant::now(); + let relay = PeerId::random(); + use P2pInboxAdmissionClass::{Registered, Unregistered}; + + // One unregistered author is capped at its own 2 MiB byte burst. + let a = PeerId::random(); + assert!(limiter.charge(&relay, &a, Unregistered, MIB, now).is_ok()); + assert_eq!( + limiter.charge(&relay, &a, Unregistered, 2 * MIB, now), + Err(DropReason::AuthorRate), + "the author's own bucket has only 1 MiB left" + ); + + // Verify distinct unregistered authors share the class byte budget. + let (b, c, d) = (PeerId::random(), PeerId::random(), PeerId::random()); + assert!(limiter.charge(&relay, &b, Unregistered, MIB, now).is_ok()); + assert!(limiter.charge(&relay, &c, Unregistered, MIB, now).is_ok()); + assert_eq!( + limiter.charge(&relay, &d, Unregistered, MIB, now), + Err(DropReason::UnregisteredRate) + ); + + // A registered author is not charged against the unregistered pool, and + // the messages rejected above did not drain the global tier: a fresh + // registered author is still admitted up to the global burst. + let op = PeerId::random(); + assert!(limiter.charge(&relay, &op, Registered, 50 * MIB, now).is_ok()); + } + + #[test] + fn rate_limiter_refills_and_bounds_its_own_memory() { + let limiter = PeerRateLimiter::new(2.0, 1.0, 2); + let now = Instant::now(); + let (a, b, c) = (PeerId::random(), PeerId::random(), PeerId::random()); + assert!(limiter.allow(&a, now)); + assert!(limiter.allow(&a, now)); + assert!(!limiter.allow(&a, now), "the burst is spent"); + assert!(limiter.allow(&a, now + Duration::from_secs(1)), "one token per second"); + + assert!(limiter.allow(&b, now + Duration::from_secs(1))); + assert!( + !limiter.allow(&c, now + Duration::from_secs(1)), + "no room to track a new peer while the tracked ones are active" + ); + assert!( + limiter.allow(&c, now + Duration::from_secs(60)), + "idle peers are forgotten to make room" + ); + } + + #[test] + fn response_gate_coalesces_requests_inside_the_cooldown() { + let gate = ResponseGate::new(Duration::from_secs(10)); + let now = Instant::now(); + assert!(gate.try_respond(now)); + assert!(!gate.take_pending(now + Duration::from_secs(20)), "nothing was deferred"); + + assert!(!gate.try_respond(now + Duration::from_secs(1))); + assert!(!gate.try_respond(now + Duration::from_secs(2))); + assert!(!gate.take_pending(now + Duration::from_secs(5)), "still cooling down"); + assert!(gate.take_pending(now + Duration::from_secs(10)), "one response for both"); + assert!(!gate.take_pending(now + Duration::from_secs(30))); + assert!(gate.try_respond(now + Duration::from_secs(30))); + } + + #[tokio::test] + async fn verdicts_follow_the_queue_state() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + let limits = InboundLimits { unregistered_peer: Quota { rows: 1, bytes: 4096 }, ..SMALL }; + + let verdict = gates.evaluate(&local_db, &limits, &source, 1, b"not a message", now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::Undecodable))); + + let first = kickoff_sent_bytes().await; + let verdict = gates.evaluate(&local_db, &limits, &source, 2, &first, now).await; + let InboundVerdict::Enqueue { class, content_hash, .. } = verdict else { + panic!("an empty inbox admits the first message"); + }; + assert_eq!(class, P2pInboxAdmissionClass::Unregistered); + let mut storage = local_db.acquire().await.unwrap(); + storage + .insert_p2p_inbox_message(&store::P2pInboxMessage { + message_id: "first".to_string(), + actor: "Committee".to_string(), + from_peer: source.to_string(), + msg_type: "KickoffSent".to_string(), + content: first.clone(), + content_size: first.len() as i64, + admission_class: class.to_string(), + content_hash: Some(content_hash.to_vec()), + ..Default::default() + }) + .await + .unwrap(); + drop(storage); + + // Verify duplicate detection precedes quota enforcement. + let verdict = gates.evaluate(&local_db, &limits, &source, 3, &first, now).await; + assert!(matches!(verdict, InboundVerdict::Duplicate(_))); + + // A different payload from the same sender is over its row quota ... + let second = kickoff_sent_bytes().await; + let verdict = gates.evaluate(&local_db, &limits, &source, 4, &second, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::SenderQuota))); + + // ... unless the sender turns out to be registered. + gates.registry.record_committee_peer(&source, Some(true), now); + let verdict = gates.evaluate(&local_db, &limits, &source, 5, &second, now).await; + assert!(matches!( + verdict, + InboundVerdict::Enqueue { class: P2pInboxAdmissionClass::Committee, .. } + )); + } + + /// Verify replay neither passes admission nor consumes rate budget. + #[tokio::test] + async fn replayed_registered_messages_are_dropped_before_the_rate_charge() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let member = PeerId::random(); + gates.registry.record_committee_peer(&member, Some(true), now); + + let message = kickoff_sent_bytes().await; + let verdict = gates.evaluate(&local_db, &SMALL, &member, 100, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + for _ in 0..3 { + let verdict = gates.evaluate(&local_db, &SMALL, &member, 100, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ReplayedSequence))); + } + + // Unregistered authors use rate limits rather than replay tracking. + let stranger = PeerId::random(); + let other = kickoff_sent_bytes().await; + let verdict = gates.evaluate(&local_db, &SMALL, &stranger, 7, &other, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + } + + #[test] + fn replay_window_tolerates_reordering_but_not_reuse() { + let guard = ReplayGuard::default(); + let author = PeerId::random(); + let base = 1_000_000; + assert_eq!(guard.admit(&author, Some(base), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&author, Some(base + 5), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&author, Some(base + 3), WALL_CLOCK), Ok(()), "reordered, but new"); + assert_eq!( + guard.admit(&author, Some(base + 3), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + assert_eq!( + guard.admit(&author, Some(base + 5), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + assert_eq!(guard.admit(&author, None, WALL_CLOCK), Err(DropReason::ReplayedSequence)); + + // The author restarts: numbering jumps to the new start-up time, and + // everything from the previous session falls out of the window. + let restarted = base + 10 * REPLAY_WINDOW; + assert_eq!(guard.admit(&author, Some(restarted), WALL_CLOCK), Ok(())); + assert_eq!( + guard.admit(&author, Some(base + 4), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + assert_eq!(guard.admit(&author, Some(restarted - REPLAY_WINDOW + 1), WALL_CLOCK), Ok(())); + assert_eq!( + guard.admit(&author, Some(restarted - REPLAY_WINDOW), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + + // Windows are per author. + assert_eq!(guard.admit(&PeerId::random(), Some(base), WALL_CLOCK), Ok(())); + } + + /// Verify re-announcement preserves stake classification. + #[test] + fn reannounced_operator_binding_keeps_its_stake_verdict() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let peer = PeerId::random(); + let pubkey = &key(1); + + registry.observe_operator_binding(&peer, pubkey, 100); + assert!(registry.plan_operator_refresh(&peer, pubkey, now)); + assert_eq!(registry.record_operator_stake(&peer, pubkey, Some(true), now), Some(true)); + assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); + + registry.observe_operator_binding(&peer, pubkey, 400); + assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); + assert!( + !registry.plan_operator_refresh(&peer, pubkey, now), + "the verdict is still fresh, so no lookup is due" + ); + + // Keep the newest binding timestamp. + let usurper = PeerId::random(); + registry.observe_operator_binding(&usurper, pubkey, 300); + assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); + assert!(!registry.plan_operator_refresh(&usurper, pubkey, now)); + + // Every definite verdict is handed back for persisting, changed or not. + let later = now + OPERATOR_STAKE_TTL; + assert!(registry.plan_operator_refresh(&peer, pubkey, later)); + assert_eq!(registry.record_operator_stake(&peer, pubkey, Some(true), later), Some(true)); + assert!(registry.plan_operator_refresh(&peer, pubkey, later + OPERATOR_STAKE_TTL)); + assert_eq!( + registry.record_operator_stake(&peer, pubkey, Some(false), later + OPERATOR_STAKE_TTL), + Some(false) + ); + } + + /// Self-signed operator bindings are free to mint. They may exhaust the + /// operator discovery budget, but not the committee's, and not the + /// re-validation of operators already confirmed. + #[test] + fn operator_binding_flood_spends_only_the_operator_discovery_budget() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + let operator = PeerId::random(); + let real = key(0); + registry.observe_operator_binding(&operator, &real, 1); + assert!(registry.plan_operator_refresh(&operator, &real, now)); + registry.record_operator_stake(&operator, &real, Some(true), now); + + let mut started = 0; + for index in 0..(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let (sybil, sybil_key) = (PeerId::random(), key(index + 1)); + registry.observe_operator_binding(&sybil, &sybil_key, 1); + if registry.plan_operator_refresh(&sybil, &sybil_key, now) { + started += 1; + registry.record_operator_stake(&sybil, &sybil_key, Some(false), now); + } + } + // The real operator's own first lookup took one token of the same budget. + assert_eq!(started + 1, OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE); + + assert!( + registry.plan_refresh(Some(&PeerId::random()), now).committee_peer, + "committee discovery has its own budget" + ); + let due = registry.plan_due_operator_refreshes(now + OPERATOR_STAKE_TTL); + assert!( + due.first().is_some_and(|(peer, _)| *peer == operator), + "the confirmed operator is re-validated first, outside the spent budget" + ); + } + + /// Registrations confirmed in an earlier session are trusted at once and are + /// re-validated as known members, not rediscovered through the budget. + #[test] + fn seeded_registrations_are_trusted_and_revalidated_as_known() { + let registry = PeerRegistry::default(); + // `Instant` cannot be dated before the machine booted; keep clear of it. + let now = Instant::now() + COMMITTEE_STALE_LIMIT; + let (member, operator) = (PeerId::random(), PeerId::random()); + registry.seed_verified_committee_peer(&member, now); + registry.seed_verified_operator(&operator, &key(0), 1, now); + assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Committee); + assert_eq!(registry.sender_class(&operator, now), P2pInboxAdmissionClass::Registered); + + // Spend both discovery budgets on strangers. + registry.record_verifier_set(Some(vec![]), now); + for index in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let stranger = PeerId::random(); + if registry.plan_refresh(Some(&stranger), now).committee_peer { + registry.record_committee_peer(&stranger, Some(false), now); + } + let sybil_key = key(index + 1); + registry.observe_operator_binding(&stranger, &sybil_key, 1); + if registry.plan_operator_refresh(&stranger, &sybil_key, now) { + registry.record_operator_stake(&stranger, &sybil_key, Some(false), now); + } + } + + assert_eq!(registry.plan_due_committee_refreshes(now), vec![member]); + assert!( + registry.plan_due_operator_refreshes(now).iter().any(|(peer, _)| *peer == operator) + ); + // Revoke restored registrations after a negative chain answer. + assert_eq!(registry.record_committee_peer(&member, Some(false), now), Some(false)); + assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Unregistered); + } + + /// A flood of fresh identities must neither grow the bucket table nor evict + /// the peers it already tracks: past the cap they share one bucket. + #[test] + fn rate_table_is_bounded_and_overflow_identities_share_one_bucket() { + let tier = TierRate { msg_burst: 2, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; + let roomy = + TierRate { msg_burst: 1_000, msg_per_sec: 0, byte_burst: 1 << 30, byte_per_sec: 0 }; + let limiter = InboundRateLimiter::new(RateLimits { + direct_peer: roomy, + registered_author: roomy, + unregistered_author: tier, + unregistered_total: roomy, + unregistered_forward: roomy, + global: roomy, + committee_reserve: 0.0, + max_tracked_peers: 4, + }); + let now = Instant::now(); + let relay = PeerId::random(); + use P2pInboxAdmissionClass::{Registered, Unregistered}; + + for _ in 0..4 { + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); + } + // New authors share overflow capacity when the table is full. + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); + for _ in 0..64 { + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now), + Err(DropReason::AuthorRate) + ); + } + assert_eq!(limiter.tracked_authors(), 4, "fresh identities do not grow the table"); + + // A registered author is bounded by the chain registry and always tracked. + let member = PeerId::random(); + assert!(limiter.charge(&relay, &member, Registered, 1, now).is_ok()); + assert_eq!(limiter.tracked_authors(), 5); + + // Idle buckets are forgotten, making room for new authors again. + let later = now + RATE_BUCKET_IDLE_FORGET; + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, later).is_ok()); + assert_eq!(limiter.tracked_authors(), 1); + } + + /// Forgetting an idle bucket must not hand its owner tokens it had spent. + #[test] + fn default_tiers_refill_within_the_idle_forget_window() { + let limits = RateLimits::defaults(); + for tier in [ + limits.direct_peer, + limits.registered_author, + limits.unregistered_author, + limits.unregistered_total, + limits.unregistered_forward, + limits.global, + ] { + let window = RATE_BUCKET_IDLE_FORGET.as_secs() as i64; + assert!(i64::from(tier.msg_burst) <= i64::from(tier.msg_per_sec) * window); + assert!(tier.byte_burst <= tier.byte_per_sec * window); + } + } + + /// Verify admissions immediately update cached class totals. + #[tokio::test] + async fn cached_class_totals_count_admissions_immediately() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let limits = InboundLimits { + unregistered_class: Quota { rows: 2, bytes: 3000 }, + unregistered_peer: Quota { rows: 2, bytes: 3000 }, + ..SMALL + }; + + // Nothing is inserted between evaluations: only the cache can know that + // the class already has two rows on their way into the inbox. + for sequence_number in 0..2 { + let message = kickoff_sent_bytes().await; + let verdict = gates + .evaluate(&local_db, &limits, &PeerId::random(), sequence_number, &message, now) + .await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + } + let message = kickoff_sent_bytes().await; + let verdict = gates.evaluate(&local_db, &limits, &PeerId::random(), 9, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::UnregisteredClassQuota))); + + // Reload totals after cache expiry. + let later = now + INBOX_CLASS_TOTALS_TTL; + let verdict = + gates.evaluate(&local_db, &limits, &PeerId::random(), 10, &message, later).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + } + + #[tokio::test] + async fn binary_envelope_is_reserved_for_gen_circuits_from_verifiers() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + + // A small message wrapped in the binary envelope: well-formed, but it + // is not the payload the envelope exists for. + let message = GOATMessage::new( + Actor::Committee, + GOATMessageContent::KickoffSent(KickoffSent { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + }), + ); + let mut wrapped = b"GOATBIN1".to_vec(); + wrapped.extend(bincode::serialize(&message).unwrap()); + + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &wrapped, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::BinaryFromNonVerifier))); + + gates.registry.record_verifier_set(Some(vec![source.to_bytes()]), now); + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &wrapped, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::UnexpectedBinaryKind))); + } + + #[tokio::test] + async fn immediate_messages_are_rate_limited_per_sender() { + use crate::action::NodeInfo; + + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(1.0, 0.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + let request = GOATMessage::new( + Actor::All, + GOATMessageContent::RequestNodeInfo(NodeInfo { + peer_id: source.to_string(), + actor: "Operator".to_string(), + goat_addr: String::new(), + btc_pub_key: String::new(), + socket_addr: String::new(), + node_name: String::new(), + service_fee_rate: 0.0, + available_peg_btc: "0".to_string(), + ..Default::default() + }), + ) + .serialize_message() + .await + .unwrap(); + + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Immediate(_))); + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ImmediateRateLimited))); + let other = PeerId::random(); + let verdict = gates.evaluate(&local_db, &SMALL, &other, 1, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Immediate(_)), "limits are per sender"); + } + + /// Verify equivalent key encodings share one operator identity. + #[test] + fn operator_identity_ignores_how_the_key_is_spelled() { + use crate::action::{NodeInfo, sign_node_info_binding, verify_node_info_binding}; + use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; + + let secp = Secp256k1::new(); + let keypair = Keypair::from_secret_key(&secp, &SecretKey::from_slice(&[9; 32]).unwrap()); + let compressed = bitcoin::PublicKey::new(keypair.public_key()); + let mut other_parity = compressed.to_string(); + other_parity.replace_range(..2, if other_parity.starts_with("02") { "03" } else { "02" }); + let mut uncompressed = compressed; + uncompressed.compressed = false; + let spellings = [ + compressed.to_string(), + compressed.to_string().to_uppercase(), + other_parity, + uncompressed.to_string(), + ]; + + let registry = PeerRegistry::default(); + let now = Instant::now(); + let mut keys = HashSet::new(); + let mut peers = Vec::new(); + for (index, spelling) in spellings.iter().enumerate() { + let peer = PeerId::random(); + let issued_at = 100 + index as i64; + let node_info = NodeInfo { + peer_id: peer.to_string(), + btc_pub_key: spelling.clone(), + binding_sig: sign_node_info_binding( + &peer.to_string(), + spelling, + issued_at, + &keypair, + ), + binding_issued_at: issued_at, + ..Default::default() + }; + let verified = verify_node_info_binding(&node_info) + .unwrap_or_else(|| panic!("{spelling} is a valid spelling of the key")); + let operator = operator_key(&verified); + keys.insert(operator); + registry.observe_operator_binding(&peer, &operator, issued_at); + if registry.plan_operator_refresh(&peer, &operator, now) { + registry.record_operator_stake(&peer, &operator, Some(true), now); + } + peers.push(peer); + } + assert_eq!(keys.len(), 1, "every spelling is the same operator"); + let registered = + peers.iter().filter(|peer| registry.sender_class(peer, now).is_registered()).count(); + assert_eq!(registered, 1, "one stake backs one peer id, however the key is written"); + assert!(registry.sender_class(peers.last().unwrap(), now).is_registered()); + } + + /// Verify equal timestamps cannot transfer ownership and key changes revoke prior registration. + #[test] + fn operator_key_changes_hands_only_forward_and_rebinding_revokes() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (first, second) = (PeerId::random(), PeerId::random()); + let (staked, unstaked) = (key(1), key(2)); + + assert!(!registry.observe_operator_binding(&first, &staked, 100)); + assert!(registry.plan_operator_refresh(&first, &staked, now)); + assert_eq!(registry.record_operator_stake(&first, &staked, Some(true), now), Some(true)); + + registry.observe_operator_binding(&second, &staked, 100); + assert!(registry.sender_class(&first, now).is_registered(), "same issue time: no change"); + assert!(!registry.plan_operator_refresh(&second, &staked, now)); + + // Moving to an unstaked key gives up the confirmed one ... + assert!(registry.observe_operator_binding(&first, &unstaked, 200)); + assert!(!registry.sender_class(&first, now).is_registered()); + // ... and the negative verdict for the new key is still handed back, so + // the store is cleaned even though nothing "changed" for this binding. + assert!(registry.plan_operator_refresh(&first, &unstaked, now)); + assert_eq!( + registry.record_operator_stake(&first, &unstaked, Some(false), now), + Some(false) + ); + // A verdict for the key the peer no longer holds is ignored. + assert_eq!(registry.record_operator_stake(&first, &staked, Some(true), now), None); + } + + /// A stored confirmation is restored only for the key it was made for. + #[tokio::test] + async fn stored_operator_confirmation_is_bound_to_its_key() { + let local_db = store::create_local_db("sqlite::memory:").await; + let now = Instant::now() + OPERATOR_STAKE_STALE_LIMIT; + let (kept, rebound, member) = (PeerId::random(), PeerId::random(), PeerId::random()); + let (staked, unstaked) = (key(1), key(2)); + { + let mut storage = local_db.acquire().await.unwrap(); + for peer in [&kept, &rebound] { + storage + .upsert_p2p_registered_peer( + &peer.to_string(), + REGISTERED_KIND_OPERATOR, + &hex::encode(if peer == &kept { staked } else { key(3) }), + ) + .await + .unwrap(); + } + storage + .upsert_p2p_registered_peer(&member.to_string(), REGISTERED_KIND_COMMITTEE, "") + .await + .unwrap(); + } + // `rebound` was confirmed for key 3 but now presents a (validly signed) + // binding for an unstaked key. + let bindings = HashMap::from([(kept, (staked, 10)), (rebound, (unstaked, 20))]); + let registry = PeerRegistry::default(); + for (peer, (operator, issued_at)) in &bindings { + registry.observe_operator_binding(peer, operator, *issued_at); + } + let seeded = seed_registry_from_store(&local_db, ®istry, &bindings, now).await.unwrap(); + assert_eq!(seeded, 2, "the kept operator and the committee member"); + assert_eq!(registry.sender_class(&kept, now), P2pInboxAdmissionClass::Registered); + assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Committee); + assert_eq!(registry.sender_class(&rebound, now), P2pInboxAdmissionClass::Unregistered); + + // The revocation a re-binding triggers removes the stored row. + revoke_persisted_operator(&local_db, &rebound).await; + let stored = local_db.acquire().await.unwrap().load_p2p_registered_peers().await.unwrap(); + assert!(!stored.iter().any(|(peer_id, _, _)| *peer_id == rebound.to_string())); + } + + #[test] + fn committee_reserve_is_closed_to_other_senders() { + use P2pInboxAdmissionClass::{Committee, Registered, Unregistered}; + let limits = InboundLimits { committee_reserve: Quota { rows: 2, bytes: 2000 }, ..SMALL }; + + // Four rows from staked operators fill everything outside the reserve. + let crowded = [usage(Registered, 4, 400, (0, 0))]; + assert_eq!( + check_inbox_quota(Registered, 100, &crowded, &limits), + Err(DropReason::GlobalQuota) + ); + assert_eq!( + check_inbox_quota(Unregistered, 100, &crowded, &limits), + Err(DropReason::GlobalQuota) + ); + assert_eq!(check_inbox_quota(Committee, 100, &crowded, &limits), Ok(())); + // Committee rows do not count against the others' share. + let committee_heavy = [usage(Committee, 3, 300, (0, 0)), usage(Registered, 2, 200, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &committee_heavy, &limits), Ok(())); + // The committee is still bounded by the global ceiling. + let full = [usage(Committee, 6, 600, (0, 0))]; + assert_eq!(check_inbox_quota(Committee, 100, &full, &limits), Err(DropReason::GlobalQuota)); + } + + /// Verify a full gate rejects new live keys. + #[test] + fn gates_and_sets_refuse_new_keys_when_full_of_live_ones() { + let now = Instant::now(); + let gate = CooldownGate::new(Duration::from_secs(60), 4); + for index in 0..4 { + assert!(gate.allow(&format!("key-{index}"), now)); + } + for index in 4..64 { + assert!(!gate.allow(&format!("key-{index}"), now + Duration::from_secs(1))); + } + assert_eq!(gate.len(), 4); + assert!(gate.is_cooling("key-0", now + Duration::from_secs(1))); + assert!(!gate.is_cooling("key-9", now + Duration::from_secs(1)), "never recorded"); + // Expired keys make room again, once the sweep interval has passed. + assert!(gate.allow("late", now + Duration::from_secs(61))); + assert_eq!(gate.len(), 1); + + let set = TtlSet::new(Duration::from_secs(60), 2); + assert!(set.insert("a", now)); + assert!(set.insert("b", now)); + assert!(!set.insert("c", now), "full of unexpired keys"); + assert!(set.insert("a", now), "refreshing a member needs no room"); + set.remove("a"); + assert!(!set.contains("a", now)); + assert!(set.insert("c", now)); + } + + /// Verify restored replay marks reject older sequence numbers. + #[test] + fn restored_replay_mark_is_a_floor() { + let guard = ReplayGuard::default(); + let author = PeerId::random(); + assert_eq!(guard.admit(&author, Some(500), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&author, Some(498), WALL_CLOCK), Ok(())); + assert_eq!(guard.pending_marks(), vec![(author, 500)]); + + let restarted = ReplayGuard::default(); + restarted.restore_mark(&author, 500); + assert!(restarted.pending_marks().is_empty(), "a restored mark is already stored"); + for replayed in [497, 498, 499, 500] { + assert_eq!( + restarted.admit(&author, Some(replayed), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + } + assert_eq!(restarted.admit(&author, Some(501), WALL_CLOCK), Ok(())); + assert_eq!(restarted.pending_marks(), vec![(author, 501)]); + } + + /// Persisted confirmation must retain failed writes and newer pending marks. + #[test] + fn replay_marks_stay_owed_until_the_write_is_confirmed() { + let guard = ReplayGuard::default(); + let author = PeerId::random(); + assert_eq!(guard.admit(&author, Some(10), WALL_CLOCK), Ok(())); + + // Tick 1: the marks are read, the write fails, nothing is confirmed. + let owed = guard.pending_marks(); + assert_eq!(owed, vec![(author, 10)]); + assert_eq!(guard.pending_marks(), owed, "still owed on the next tick"); + + // Tick 2: the write is under way when a newer message is admitted. + let written = guard.pending_marks(); + assert_eq!(guard.admit(&author, Some(11), WALL_CLOCK), Ok(())); + guard.confirm_persisted(&written); + assert_eq!(guard.pending_marks(), vec![(author, 11)], "only 10 reached the store"); + + guard.confirm_persisted(&[(author, 11)]); + assert!(guard.pending_marks().is_empty()); + // A late confirmation of an older write never moves the record back. + guard.confirm_persisted(&[(author, 10)]); + assert!(guard.pending_marks().is_empty()); + } + + /// Verify future sequence numbers do not raise replay marks. + #[test] + fn future_sequence_numbers_cannot_raise_the_mark() { + let guard = ReplayGuard::default(); + let author = PeerId::random(); + let ahead = WALL_CLOCK + REPLAY_MAX_FUTURE.as_nanos() as u64; + assert_eq!( + guard.admit(&author, Some(ahead + 1), WALL_CLOCK), + Err(DropReason::FutureSequence) + ); + assert!(guard.pending_marks().is_empty(), "nothing was recorded"); + assert_eq!(guard.admit(&author, Some(ahead), WALL_CLOCK), Ok(()), "inside the tolerance"); + + // The corrected clock numbers lower than the mark: refused, and surfaced + // for the operator once it keeps happening. + assert!(guard.take_lockout_suspects().is_empty()); + for offset in 0..REPLAY_LOCKOUT_SUSPECT_REJECTIONS as u64 { + assert_eq!( + guard.admit(&author, Some(WALL_CLOCK - REPLAY_WINDOW * 2 + offset), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + } + assert_eq!( + guard.take_lockout_suspects(), + vec![(author, REPLAY_LOCKOUT_SUSPECT_REJECTIONS)] + ); + assert!(guard.take_lockout_suspects().is_empty(), "reported once per tick"); + } + + #[test] + fn direct_peer_is_banned_at_the_strike_limit_and_released_later() { + let strikes = DirectPeerStrikes::default(); + let now = Instant::now(); + let (flooder, bystander) = (PeerId::random(), PeerId::random()); + for _ in 0..(STRIKE_LIMIT - 1) { + assert!(!strikes.strike(&flooder, now)); + } + assert!(!strikes.strike(&bystander, now)); + assert!(strikes.strike(&flooder, now), "the limit is reached exactly once"); + assert!(!strikes.strike(&flooder, now), "already banned"); + assert!(strikes.take_expired_bans(now).is_empty()); + assert_eq!(strikes.take_expired_bans(now + STRIKE_BAN), vec![flooder]); + + // Strikes spread thinner than the window never add up. + let slow = PeerId::random(); + for round in 0..(STRIKE_LIMIT * 2) { + assert!(!strikes.strike(&slow, now + STRIKE_WINDOW * round)); + } + + // Only verdicts that do not depend on this node's state are blamed on + // the neighbour; and an unanswered lookup is not "known unregistered". + assert!(DropReason::Undecodable.blames_direct_peer()); + assert!(DropReason::UnexpectedBinaryKind.blames_direct_peer()); + for local in [ + DropReason::SenderQuota, + DropReason::ReplayedSequence, + // An honest relay earns this one just by being the only neighbour. + DropReason::DirectPeerRate, + // The JSON ceiling is configurable, so nodes may disagree on it. + DropReason::OversizedJson, + DropReason::BinaryFromNonVerifier, + ] { + assert!(!local.blames_direct_peer(), "{local:?} depends on this node"); + } + assert!(!strikes.is_banned(&bystander, now)); + let banned = PeerId::random(); + for _ in 0..STRIKE_LIMIT { + strikes.strike(&banned, now); + } + assert!(strikes.is_banned(&banned, now), "a ban holds against reconnecting"); + assert!(!strikes.is_banned(&banned, now + STRIKE_BAN)); + let registry = PeerRegistry::default(); + assert!(!registry.is_known_unregistered(&flooder, now)); + registry.record_committee_peer(&flooder, Some(false), now); + assert!(registry.is_known_unregistered(&flooder, now)); + registry.record_committee_peer(&bystander, Some(true), now); + assert!(!registry.is_known_unregistered(&bystander, now)); + } + + /// A durable message no handler of this node's role acts on is relayed, but + /// takes no inbox row and no quota. + #[tokio::test] + async fn messages_for_another_role_are_forwarded_without_being_stored() { + use crate::action::NackReady; + + let local_db = store::create_local_db("sqlite::memory:").await; + let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + let nack_ready = GOATMessage::new( + Actor::Verifier, + GOATMessageContent::NackReady(NackReady { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + }), + ) + .serialize_message() + .await + .unwrap(); + + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &nack_ready, now).await; + assert!(matches!(verdict, InboundVerdict::Forward), "a committee node has no handler"); + gates.local_actor = Actor::Verifier; + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &nack_ready, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + } + + /// Verify every dispatch role arm is covered by `MessageKind::handled_by`. + #[test] + fn dispatch_arms_are_covered_by_the_role_table() { + use crate::action::MessageKind; + use std::str::FromStr; + + let source = include_str!("handle.rs"); + let dispatcher = { + let start = source.find("pub(crate) fn heavy_task_from_content(").unwrap(); + let end = source.find("fn make_message(").unwrap(); + &source[start..end] + }; + let roles = [Actor::Committee, Actor::Operator, Actor::Verifier, Actor::Watchtower]; + let mut arms = 0; + for (at, _) in dispatcher.match_indices("GOATMessageContent::") { + // Match dispatch patterns ending in `=>`, excluding expressions in arm bodies. + let rest = &dispatcher[at + "GOATMessageContent::".len()..]; + let name: String = rest.chars().take_while(|c| c.is_alphanumeric()).collect(); + let Ok(kind) = MessageKind::from_str(&name) else { + continue; + }; + let Some(arrow) = rest.find("=>") else { + continue; + }; + let pattern = &rest[..arrow]; + if pattern.contains(';') || pattern.contains(".await") || pattern.contains("Some(") { + continue; + } + arms += 1; + let named: Vec<&Actor> = + roles.iter().filter(|role| pattern.contains(&format!("Actor::{role}"))).collect(); + // No role named: the `_` arm, which every role reaches. + let reached: Vec<&Actor> = + if named.is_empty() { roles.iter().collect() } else { named }; + for role in reached { + assert!( + kind.handled_by(role), + "dispatch has an arm for ({name}, {role}) but handled_by says it is not \ + handled: the message would be relayed without ever being processed" + ); + } + } + assert!(arms >= 40, "the scan found only {arms} arms; has dispatch moved?"); + } + + /// Verify committee-only reserves in shared rate tiers. + #[test] + fn shared_rate_tiers_keep_a_reserve_for_the_committee() { + let tier = TierRate { msg_burst: 8, msg_per_sec: 0, byte_burst: 8_000, byte_per_sec: 0 }; + let roomy = + TierRate { msg_burst: 1_000, msg_per_sec: 0, byte_burst: 1 << 30, byte_per_sec: 0 }; + let limiter = InboundRateLimiter::new(RateLimits { + direct_peer: roomy, + registered_author: roomy, + unregistered_author: roomy, + unregistered_total: roomy, + unregistered_forward: roomy, + global: tier, + committee_reserve: 0.25, + max_tracked_peers: 16, + }); + let now = Instant::now(); + let relay = PeerId::random(); + use P2pInboxAdmissionClass::{Committee, Registered, Unregistered}; + + // Six of eight messages are open to everyone ... + for _ in 0..6 { + assert!(limiter.charge(&relay, &PeerId::random(), Registered, 100, now).is_ok()); + } + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Registered, 100, now), + Err(DropReason::GlobalRate) + ); + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Unregistered, 100, now), + Err(DropReason::GlobalRate) + ); + // ... and the last two only to the committee. + for _ in 0..2 { + assert!(limiter.charge(&relay, &PeerId::random(), Committee, 100, now).is_ok()); + } + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Committee, 100, now), + Err(DropReason::GlobalRate) + ); + } + + /// Verify unregistered forward-only traffic is rate-limited. + #[tokio::test] + async fn forwarding_for_unregistered_authors_is_budgeted() { + use crate::action::NackReady; + + let local_db = store::create_local_db("sqlite::memory:").await; + let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let tier = TierRate { msg_burst: 2, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; + gates.rate_limiter = InboundRateLimiter::new(RateLimits { + unregistered_forward: tier, + ..RateLimits::defaults() + }); + let now = Instant::now(); + let nack_ready = |graph_id| { + GOATMessage::new( + Actor::Verifier, + GOATMessageContent::NackReady(NackReady { instance_id: Uuid::new_v4(), graph_id }), + ) + }; + + for sequence_number in 0..2 { + let data = nack_ready(Uuid::new_v4()).serialize_message().await.unwrap(); + let verdict = gates + .evaluate(&local_db, &SMALL, &PeerId::random(), sequence_number, &data, now) + .await; + assert!(matches!(verdict, InboundVerdict::Forward)); + } + let data = nack_ready(Uuid::new_v4()).serialize_message().await.unwrap(); + let verdict = gates.evaluate(&local_db, &SMALL, &PeerId::random(), 9, &data, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ForwardRate))); + + let member = PeerId::random(); + gates.registry.record_committee_peer(&member, Some(true), now); + let verdict = gates.evaluate(&local_db, &SMALL, &member, 1, &data, now).await; + assert!(matches!(verdict, InboundVerdict::Forward), "registered authors are not charged"); + } + + /// A kind only one role can send is refused once the chain has said the + /// author does not hold that role — and only then. + #[tokio::test] + async fn sender_role_is_refused_only_on_a_definite_answer() { + use crate::action::{AggNonceConsensus, MessageKind}; + + assert_eq!(MessageKind::NonceGeneration.sender_role(), SenderRole::Committee); + assert_eq!(MessageKind::GenCircuits.sender_role(), SenderRole::Verifier); + assert_eq!(MessageKind::InitGraph.sender_role(), SenderRole::Operator); + assert_eq!(MessageKind::KickoffSent.sender_role(), SenderRole::Any); + // Keep the prefilter no stricter than handler authorization. + assert_eq!(MessageKind::SolderingProofReady.sender_role(), SenderRole::Any); + + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (unknown, outsider, member) = (PeerId::random(), PeerId::random(), PeerId::random()); + registry.record_committee_peer(&outsider, Some(false), now); + registry.record_committee_peer(&member, Some(true), now); + let verdict = |peer, role, at| registry.role_verdict(peer, role, at); + assert_eq!(verdict(&unknown, SenderRole::Committee, now), RoleVerdict::Unknown); + assert_eq!(verdict(&outsider, SenderRole::Committee, now), RoleVerdict::Denied); + assert_eq!(verdict(&member, SenderRole::Committee, now), RoleVerdict::Confirmed); + assert_eq!(verdict(&outsider, SenderRole::Any, now), RoleVerdict::Confirmed); + assert_eq!( + verdict(&outsider, SenderRole::Committee, now + COMMITTEE_NEGATIVE_TTL), + RoleVerdict::Unknown, + "a stale \"no\" is not a \"no\": the peer may have registered since" + ); + // The verifier set is definite once fetched; before that nobody is denied. + assert_eq!(verdict(&unknown, SenderRole::Verifier, now), RoleVerdict::Unknown); + registry.record_verifier_set(Some(vec![member.to_bytes()]), now); + assert_eq!(verdict(&unknown, SenderRole::Verifier, now), RoleVerdict::Denied); + assert_eq!(verdict(&member, SenderRole::Verifier, now), RoleVerdict::Confirmed); + // An operator is denied only by a negative stake verdict for its binding. + assert_eq!(verdict(&unknown, SenderRole::Operator, now), RoleVerdict::Unknown); + registry.observe_operator_binding(&outsider, &key(1), 1); + assert_eq!(verdict(&outsider, SenderRole::Operator, now), RoleVerdict::Unknown); + assert!(registry.plan_operator_refresh(&outsider, &key(1), now)); + registry.record_operator_stake(&outsider, &key(1), Some(false), now); + assert_eq!(verdict(&outsider, SenderRole::Operator, now), RoleVerdict::Denied); + + // End to end: the same committee-only message, from an unknown author + // and from one the chain has turned down. + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + gates.registry.record_committee_peer(&outsider, Some(false), now); + let secp = bitcoin::secp256k1::Secp256k1::new(); + let keypair = bitcoin::secp256k1::Keypair::from_secret_key( + &secp, + &bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), + ); + let digest = bitcoin::secp256k1::Message::from_digest([3; 32]); + let data = GOATMessage::new( + Actor::Committee, + GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + committee_pubkey: bitcoin::PublicKey::new(keypair.public_key()), + consensus_hash: [3; 32], + signature: secp.sign_schnorr(&digest, &keypair), + }), + ) + .serialize_message() + .await + .unwrap(); + let verdict = gates.evaluate(&local_db, &SMALL, &unknown, 1, &data, now).await; + assert!( + matches!(verdict, InboundVerdict::Enqueue { .. }), + "unknown is not denied: the handler stays the authority" + ); + let verdict = gates.evaluate(&local_db, &SMALL, &outsider, 1, &data, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::SenderRoleDenied))); + } + + /// Verify neighbour discovery is independent of author discovery. + #[test] + fn neighbour_lookups_have_their_own_budget() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + for _ in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let author = PeerId::random(); + if registry.plan_refresh(Some(&author), now).committee_peer { + registry.record_committee_peer(&author, Some(false), now); + } + } + let neighbour = PeerId::random(); + assert!(!registry.plan_refresh(Some(&neighbour), now).committee_peer, "budget spent"); + assert!(registry.plan_neighbour_lookup(&neighbour, now)); + + // Verify refused peers cannot consume neighbour discovery budget. + let mut refused = 0; + for _ in 0..(REFUSED_PEER_LOOKUPS_PER_MINUTE * 2) { + let peer = PeerId::random(); + if registry.plan_refused_peer_lookup(&peer, now) { + refused += 1; + registry.record_committee_peer(&peer, Some(false), now); + } + } + assert_eq!(refused, REFUSED_PEER_LOOKUPS_PER_MINUTE); + assert!(registry.plan_neighbour_lookup(&PeerId::random(), now)); + } + + /// Verify committee-only lookup does not claim verifier-set refresh. + #[test] + fn neighbour_lookup_does_not_claim_the_verifier_set() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + // The verifier set has never been fetched, so it is due. + assert!(registry.plan_neighbour_lookup(&PeerId::random(), now)); + assert!(registry.plan_refused_peer_lookup(&PeerId::random(), now)); + assert!( + registry.plan_refresh(None, now).verifier_set, + "the verifier set is still there to be claimed by a caller that fetches it" + ); + } + + /// Verify weighted rotation with persistent backlog and one dispatch per tick. + #[test] + fn rota_serves_every_class_across_ticks_when_each_tick_runs_one_message() { + use std::collections::VecDeque; + + /// One tick as the worker runs it: list a batch by shares, then serve it + /// through the rota until the budget (in messages) is spent. The first + /// message always runs. + fn tick(schedule: &mut InboxSchedule, backlog: [usize; 3], budget: usize) -> Vec { + let shares = [8, 4, 4]; + let mut queues: [VecDeque; 3] = Default::default(); + for class in 0..3 { + queues[class].extend(std::iter::repeat_n(class, backlog[class].min(shares[class]))); + } + let mut served = Vec::new(); + loop { + if !served.is_empty() && served.len() >= budget { + break; + } + let Some(class) = schedule.next(&mut queues) else { + break; + }; + served.push(class); + } + served + } + + let mut schedule = InboxSchedule::default(); + let mut served = [0usize; 3]; + let mut last_served = [0usize; 3]; + let mut longest_wait = [0usize; 3]; + for tick_number in 1..=400 { + for class in tick(&mut schedule, [100, 100, 100], 1) { + served[class] += 1; + longest_wait[class] = longest_wait[class].max(tick_number - last_served[class]); + last_served[class] = tick_number; + } + } + assert_eq!(served, [200, 100, 100], "committee twice as often, nobody starved"); + assert!(longest_wait.iter().all(|wait| *wait <= 4), "{longest_wait:?}"); + + // Compare against restarting rotation at each tick. + let mut served = [0usize; 3]; + for _ in 0..400 { + for class in tick(&mut InboxSchedule::default(), [100, 100, 100], 1) { + served[class] += 1; + } + } + assert_eq!(served, [400, 0, 0]); + + // A class with nothing to claim passes its turn on instead of wasting it. + let mut schedule = InboxSchedule::default(); + let mut served = [0usize; 3]; + for _ in 0..300 { + for class in tick(&mut schedule, [100, 0, 100], 1) { + served[class] += 1; + } + } + assert_eq!(served, [200, 0, 100]); + + // With budget to spare a tick serves the batch interleaved, and the next + // tick picks up where this one stopped. + let mut schedule = InboxSchedule::default(); + assert_eq!(tick(&mut schedule, [100, 100, 100], 6), [0, 1, 0, 2, 0, 1]); + assert_eq!(tick(&mut schedule, [100, 100, 100], 2), [0, 2]); + assert_eq!(InboxSchedule::queue_of("Committee"), SCHEDULE_COMMITTEE); + assert_eq!(InboxSchedule::queue_of("Registered"), SCHEDULE_REGISTERED); + assert_eq!(InboxSchedule::queue_of("Unregistered"), SCHEDULE_UNREGISTERED); + assert_eq!(InboxSchedule::queue_of(""), SCHEDULE_UNREGISTERED, "unclassified is untrusted"); + } + + /// Verify configured bindings still require confirmed stake. + #[test] + fn trusted_operator_binding_needs_the_chain_but_not_the_budget() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + // Spend the operator discovery budget on self-signed bindings. + for index in 0..(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let (sybil, sybil_key) = (PeerId::random(), key(index + 10)); + registry.observe_operator_binding(&sybil, &sybil_key, 1); + if registry.plan_operator_refresh(&sybil, &sybil_key, now) { + registry.record_operator_stake(&sybil, &sybil_key, Some(false), now); + } + } + let (newcomer, newcomer_key) = (PeerId::random(), key(1)); + registry.observe_operator_binding(&newcomer, &newcomer_key, 1); + assert!(!registry.plan_operator_refresh(&newcomer, &newcomer_key, now), "budget spent"); + + let (trusted, trusted_key) = (PeerId::random(), key(2)); + assert_eq!(registry.trust_operator_binding(&trusted, &trusted_key), Ok(())); + assert!(!registry.sender_class(&trusted, now).is_registered(), "configured is not staked"); + let due = registry.plan_due_operator_refreshes(now); + assert!(due.contains(&(trusted, trusted_key)), "looked up despite the spent budget"); + assert!( + due.iter().all(|(_, operator)| *operator == trusted_key), + "keys the chain said are not staked are not swept again, and the newcomer still \ + has no budget: {} due", + due.len() + ); + registry.record_operator_stake(&trusted, &trusted_key, Some(true), now); + assert_eq!(registry.sender_class(&trusted, now), P2pInboxAdmissionClass::Registered); + + // The operator's own signed announcement later keeps the verdict ... + registry.observe_operator_binding(&trusted, &trusted_key, 500); + assert!(registry.sender_class(&trusted, now).is_registered()); + // ... and if it moves to another peer id, its signature outranks the + // static configuration. + let moved = PeerId::random(); + registry.observe_operator_binding(&moved, &trusted_key, 600); + assert!(!registry.sender_class(&trusted, now).is_registered()); + } + + /// Verify signed bindings take precedence over configuration. + #[test] + fn configured_binding_does_not_override_a_signed_one() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (operator, other_peer) = (PeerId::random(), PeerId::random()); + let (signed_key, configured_key) = (key(1), key(2)); + + // Restored from the store: the operator's own signed binding, confirmed. + registry.seed_verified_operator(&operator, &signed_key, 100, now); + let now = now + Duration::from_secs(1); + assert!(registry.sender_class(&operator, now).is_registered()); + + // The config still names an older key for that peer ... + assert_eq!( + registry.trust_operator_binding(&operator, &configured_key), + Err(TrustedBindingConflict::PeerBoundToAnotherKey(signed_key)) + ); + // ... or names the operator's key under a peer id it has moved away from. + assert_eq!( + registry.trust_operator_binding(&other_peer, &signed_key), + Err(TrustedBindingConflict::KeyBoundToAnotherPeer(operator)) + ); + assert!(registry.sender_class(&operator, now).is_registered(), "the signed binding stands"); + assert!(!registry.sender_class(&other_peer, now).is_registered()); + + // Agreeing with the signed binding keeps its verdict and adds the trust. + assert_eq!(registry.trust_operator_binding(&operator, &signed_key), Ok(())); + assert!(registry.sender_class(&operator, now).is_registered()); + + // The same peer configured twice with different keys: first one stands. + let fresh = PeerId::random(); + assert_eq!(registry.trust_operator_binding(&fresh, &key(3)), Ok(())); + assert_eq!( + registry.trust_operator_binding(&fresh, &key(4)), + Err(TrustedBindingConflict::PeerBoundToAnotherKey(key(3))) + ); + } + + /// Verify clock leads are recorded within and beyond tolerance. + #[test] + fn clock_leads_are_reported_per_author_and_then_forgotten() { + let guard = ReplayGuard::default(); + let (slightly_ahead, far_ahead, on_time) = + (PeerId::random(), PeerId::random(), PeerId::random()); + let minute = Duration::from_secs(60).as_nanos() as u64; + + assert_eq!(guard.admit(&on_time, Some(WALL_CLOCK - minute), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&slightly_ahead, Some(WALL_CLOCK + 2 * minute), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&slightly_ahead, Some(WALL_CLOCK + 3 * minute), WALL_CLOCK), Ok(())); + assert_eq!( + guard.admit(&far_ahead, Some(WALL_CLOCK + 30 * minute), WALL_CLOCK), + Err(DropReason::FutureSequence) + ); + + let mut leads = guard.take_clock_leads(); + leads.sort_by_key(|(_, lead)| *lead); + assert_eq!( + leads, + vec![ + (slightly_ahead, Duration::from_secs(3 * 60)), + (far_ahead, Duration::from_secs(30 * 60)) + ], + "the largest lead per author; an author on time is not listed" + ); + assert!(guard.take_clock_leads().is_empty()); + } + + /// Verify only successful recovery closes the gate. + #[test] + fn republish_gate_stays_open_until_a_publish_succeeds() { + let gate = CooldownGate::new(PROTOCOL_REPUBLISH_COOLDOWN, 16); + let now = Instant::now(); + let key = "committee-presign:graph"; + + // First run: the value is stored, the publish fails, nothing is stamped. + assert!(!gate.is_cooling(key, now)); + // The retry, seconds later, therefore publishes. + let retry = now + Duration::from_secs(10); + assert!(!gate.is_cooling(key, retry)); + assert!(gate.allow(key, retry), "stamped after the publish went through"); + // Later messages of the round do not publish it again ... + assert!(gate.is_cooling(key, retry + Duration::from_secs(1))); + // ... until the cooldown has passed, and other rounds are unaffected. + assert!(!gate.is_cooling(key, retry + PROTOCOL_REPUBLISH_COOLDOWN)); + assert!(!gate.is_cooling("committee-presign:another-graph", retry)); + } + + /// A "no" that has gone stale is not grounds for acting against a neighbour. + #[test] + fn a_stale_negative_is_not_known_unregistered() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let peer = PeerId::random(); + registry.record_committee_peer(&peer, Some(false), now); + assert!(registry.is_known_unregistered(&peer, now)); + assert!(!registry.is_known_unregistered(&peer, now + COMMITTEE_NEGATIVE_TTL)); + } + + #[test] + fn trusted_operator_bindings_are_parsed_strictly() { + use crate::env::parse_trusted_operator_bindings; + use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; + + let secp = Secp256k1::new(); + let keypair = Keypair::from_secret_key(&secp, &SecretKey::from_slice(&[9; 32]).unwrap()); + let pubkey = bitcoin::PublicKey::new(keypair.public_key()); + let xonly = bitcoin::XOnlyPublicKey::from(pubkey).serialize(); + let (first, second) = (PeerId::random(), PeerId::random()); + let value = format!( + " {first}={pubkey} , not-a-binding, {second} = {} ,{first}=zz,", + pubkey.to_string().to_uppercase() + ); + assert_eq!( + parse_trusted_operator_bindings(&value), + vec![(first, xonly), (second, xonly)], + "malformed entries are skipped, and every spelling is the same key" + ); + assert!(parse_trusted_operator_bindings("").is_empty()); + } + + /// Verify only requested SyncGraph responses are kept locally. + #[tokio::test] + async fn unsolicited_sync_graph_is_relayed_but_not_stored() { + use crate::action::{MessageKind, SyncGraphRequest}; + + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let relayer = PeerId::random(); + gates.registry.record_committee_peer(&relayer, Some(true), now); + + // Exercise SyncGraph admission by message kind and graph ID. + for actor in [Actor::Committee, Actor::Operator, Actor::Verifier, Actor::Watchtower] { + assert!(MessageKind::SyncGraph.handled_by(&actor)); + } + // ... and the request for one is stored only where it can be answered. + let request = GOATMessage::new( + Actor::All, + GOATMessageContent::SyncGraphRequest(SyncGraphRequest { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + }), + ) + .serialize_message() + .await + .unwrap(); + let verdict = gates.evaluate(&local_db, &SMALL, &relayer, 1, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. }), "a committee node answers"); + + // The decision itself: an answer is kept only by the node that asked, + // and stops being expected once the request has been served. + let (asked, unasked) = (Uuid::new_v4(), Uuid::new_v4()); + assert!(gates.requested_graphs.insert(&asked.to_string(), now)); + let admission = AdmissionGates { + registry: &gates.registry, + replay_guard: &gates.replay_guard, + rate_limiter: &gates.rate_limiter, + immediate_limiter: &gates.immediate_limiter, + usage_cache: &gates.usage_cache, + limits: &SMALL, + local_actor: &gates.local_actor, + requested_graphs: &gates.requested_graphs, + }; + assert!(admission.keeps(MessageKind::SyncGraph, Some(asked), now)); + assert!(!admission.keeps(MessageKind::SyncGraph, Some(unasked), now), "relayed only"); + assert!( + !admission.keeps(MessageKind::SyncGraph, Some(asked), now + SYNC_GRAPH_REQUEST_TTL), + "a request that has expired no longer expects an answer" + ); + gates.requested_graphs.remove(&asked.to_string()); + assert!(!admission.keeps(MessageKind::SyncGraph, Some(asked), now), "already served"); + // Other kinds are unaffected by the set. + assert!(admission.keeps(MessageKind::KickoffSent, None, now)); + assert!(!admission.keeps(MessageKind::NackReady, None, now), "not a committee message"); + assert_eq!(DropReason::ControlQueueFull.as_str(), "control_queue_full"); + } +} diff --git a/node/src/p2p_msg_handler.rs b/node/src/p2p_msg_handler.rs index 55bca8fa..5ac2c6ad 100644 --- a/node/src/p2p_msg_handler.rs +++ b/node/src/p2p_msg_handler.rs @@ -3,26 +3,236 @@ use crate::action::{ }; use crate::env::get_local_node_info; use crate::metrics_service::MetricsState; +use crate::middleware::publisher::NetworkPublisher; use crate::middleware::swarm::{BitvmSwarmWrapper, P2pMessageHandler, TickMessageType}; use crate::utils::detect_heart_beat; use bitvm_lib::actors::Actor; use bitvm_lib::babe_adapter::BabeBundleBuilder; use client::http_client::async_client::HttpAsyncClient; use client::{btc_chain::BTCClient, goat_chain::GOATClient}; +use futures::FutureExt; use libp2p::PeerId; use libp2p::gossipsub::MessageId; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use store::localdb::LocalDB; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc}; use tokio_util::sync::CancellationToken; +#[derive(Clone)] pub struct BitvmNodeProcessor { pub local_db: LocalDB, pub btc_client: Arc, pub goat_client: Arc, - pub http_client: HttpAsyncClient, + pub http_client: Arc, pub soldering_builder: Option>, pub metrics_state: MetricsState, pub shutdown_token: CancellationToken, + pub worker: Arc>, +} + +pub struct ImmediateMessage { + pub source: PeerId, + pub id: MessageId, + pub message: GOATMessage, + _bytes: OwnedSemaphorePermit, +} + +pub struct WorkerControl { + immediate: mpsc::Sender, + bytes: Arc, + tick: Arc, + joins: tokio::sync::Mutex>>, +} + +fn spawn_isolated_worker( + name: &'static str, + shutdown: CancellationToken, + work: F, +) -> tokio::task::JoinHandle<()> +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + 'static, +{ + let runtime = tokio::runtime::Handle::current(); + let (done, finished) = tokio::sync::oneshot::channel(); + // Run named workers with 32 MiB stacks. + let thread = std::thread::Builder::new() + .name(name.into()) + .stack_size(32 * 1024 * 1024) + .spawn(move || { + runtime.block_on(async move { + tokio::select! { + biased; + _ = shutdown.cancelled() => {}, + result = std::panic::AssertUnwindSafe(async move { work().await }).catch_unwind() => { + if result.is_err() { + tracing::error!(event = "p2p_worker_panic", "worker panicked; stopping node"); + shutdown.cancel(); + } + } + } + }); + let _ = done.send(()); + }) + .expect("start isolated protocol worker"); + tokio::spawn(async move { + finished.await.expect("isolated worker stopped unexpectedly"); + // Done is sent after dispatch futures have been dropped; joining only + // waits for the thread's epilogue, never for business work. + thread.join().expect("isolated worker panicked after shutdown"); + }) +} + +impl WorkerControl { + pub fn enqueue( + &self, + source: PeerId, + id: MessageId, + message: GOATMessage, + bytes: usize, + ) -> bool { + let Ok(bytes) = u32::try_from(bytes.max(1)) else { return false }; + let Ok(permit) = self.bytes.clone().try_acquire_many_owned(bytes) else { return false }; + self.immediate.try_send(ImmediateMessage { source, id, message, _bytes: permit }).is_ok() + } +} + +impl BitvmNodeProcessor { + fn worker(&self, swarm: &BitvmSwarmWrapper, actor: Actor) -> &WorkerControl { + self.worker.get_or_init(|| { + // Run sequential business dispatch on a separate thread. + let (tx, rx) = mpsc::channel(16); + let tick = Arc::new(Notify::new()); + let mut processor = self.clone(); + processor.worker = Default::default(); + let publisher = swarm.publisher(); + let worker_tick = tick.clone(); + let control_processor = processor.clone(); + let control_publisher = publisher.clone(); + let control_actor = actor.clone(); + let control_join = spawn_isolated_worker( + "bitvm-control", + self.shutdown_token.clone(), + move || async move { + control_processor + .run_immediate_worker(control_publisher, control_actor, rx) + .await; + }, + ); + let maintenance_join = + tokio::spawn(processor.clone().run_admission_maintenance(publisher.clone())); + let join = spawn_isolated_worker( + "bitvm-business", + self.shutdown_token.clone(), + move || async move { + processor.run_worker(publisher, actor, worker_tick).await; + }, + ); + WorkerControl { + immediate: tx, + bytes: Arc::new(Semaphore::new(16 * 1024 * 1024)), + tick, + joins: tokio::sync::Mutex::new(vec![join, control_join, maintenance_join]), + } + }) + } + + async fn run_worker(&self, mut publisher: NetworkPublisher, actor: Actor, tick: Arc) { + let mut backlog = false; + loop { + // Coalesce tick notifications and continue immediately while backlogged. + // Each pass serves local messages and outbox before inbox. + if backlog { + tokio::task::yield_now().await; + } else { + tick.notified().await; + } + let data = + GOATMessage::new(actor.clone(), GOATMessageContent::Tick).serialize_message().await; + let result = match data { + Ok(data) => { + handle_self_p2p_msg( + &mut publisher, + &self.local_db, + &self.btc_client, + &self.goat_client, + &self.http_client, + &self.soldering_builder, + actor.clone(), + crate::env::get_peer_id().parse().expect("configured peer id"), + GOATMessage::default_message_id(), + &data, + &self.metrics_state, + &self.shutdown_token, + ) + .await + } + Err(error) => Err(error), + }; + backlog = match result { + Ok(backlog) => backlog, + Err(error) => { + tracing::error!(event = "p2p_worker", error = %error, "business dispatch failed"); + false + } + }; + } + } + + async fn run_immediate_worker( + &self, + mut publisher: NetworkPublisher, + actor: Actor, + mut immediate: mpsc::Receiver, + ) { + // Control queue: NodeInfo and ACK only; graph sync uses the durable inbox. + while let Some(message) = immediate.recv().await { + if let Err(error) = crate::action::dispatch_immediate_message( + &mut publisher, + &self.local_db, + &self.btc_client, + &self.goat_client, + &self.http_client, + &self.soldering_builder, + actor.clone(), + message.source, + message.id, + message.message, + &self.metrics_state, + ) + .await + { + tracing::debug!(event = "p2p_control_worker", error = %error, "control message failed"); + } + } + } + + /// Independently refresh registrations, persist replay marks and report clock skew. + /// This task is the sole caller of admission maintenance. + async fn run_admission_maintenance(self, mut publisher: NetworkPublisher) { + let mut clock = tokio::time::interval(std::time::Duration::from_secs( + crate::env::REGULAR_TASK_INTERVAL_SECOND, + )); + clock.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = self.shutdown_token.cancelled() => return, + _ = clock.tick() => {} + } + // A pass is cut short by shutdown too: `graceful_shutdown` joins + // this task before it releases the queue claims. + tokio::select! { + biased; + _ = self.shutdown_token.cancelled() => return, + _ = async { + crate::action::run_p2p_admission_maintenance(&self.local_db, &self.goat_client) + .await; + crate::handle::flush_deferred_node_info_response(&mut publisher).await; + } => {} + } + } + } } impl P2pMessageHandler for BitvmNodeProcessor { async fn recv_and_dispatch( @@ -30,9 +240,12 @@ impl P2pMessageHandler for BitvmNodeProcessor { swarm: &mut BitvmSwarmWrapper, actor: Actor, from_peer_id: PeerId, + propagation_source: PeerId, + sequence_number: Option, id: MessageId, message: &[u8], ) -> anyhow::Result<()> { + let worker = self.worker(swarm, actor.clone()); handle_inbound_p2p_message( swarm, &self.local_db, @@ -42,9 +255,12 @@ impl P2pMessageHandler for BitvmNodeProcessor { &self.soldering_builder, actor, from_peer_id, + propagation_source, + sequence_number, id, message, &self.metrics_state, + worker, ) .await } @@ -52,7 +268,7 @@ impl P2pMessageHandler for BitvmNodeProcessor { async fn handle_tick_message( &self, swarm: &mut BitvmSwarmWrapper, - peer_id: PeerId, + _peer_id: PeerId, actor: Actor, msg_type: TickMessageType, ) -> anyhow::Result<()> { @@ -68,27 +284,8 @@ impl P2pMessageHandler for BitvmNodeProcessor { Ok(()) } TickMessageType::RegularlyAction => { - tracing::debug!("Handling regular action tick message"); - let tick_data = - GOATMessage { actor: actor.clone(), content: GOATMessageContent::Tick } - .serialize_message() - .await?; - - handle_self_p2p_msg( - swarm, - &self.local_db, - &self.btc_client, - &self.goat_client, - &self.http_client, - &self.soldering_builder, - actor, - peer_id, - GOATMessage::default_message_id(), - &tick_data, - &self.metrics_state, - &self.shutdown_token, - ) - .await + self.worker(swarm, actor).tick.notify_one(); + Ok(()) } } } @@ -112,6 +309,12 @@ impl P2pMessageHandler for BitvmNodeProcessor { } async fn graceful_shutdown(&self) -> anyhow::Result<()> { + self.shutdown_token.cancel(); + if let Some(worker) = self.worker.get() { + for join in std::mem::take(&mut *worker.joins.lock().await) { + join.await?; + } + } let mut storage = self.local_db.start_immediate_transaction().await?; let local_released = storage.release_processing_local_messages().await?; let inbox_released = storage.release_processing_p2p_inbox_messages().await?; @@ -129,6 +332,56 @@ impl P2pMessageHandler for BitvmNodeProcessor { #[cfg(test)] mod tests { + #[tokio::test] + async fn blocking_business_work_does_not_block_the_network_executor() { + let shutdown = tokio_util::sync::CancellationToken::new(); + let (entered, ready) = tokio::sync::oneshot::channel(); + let (release, wait) = std::sync::mpsc::channel(); + let worker = + super::spawn_isolated_worker("bitvm-test", shutdown.clone(), move || async move { + let _ = entered.send(()); + // Deliberately no await: model synchronous graph reconstruction. + let _ = wait.recv(); + }); + ready.await.unwrap(); + let network_progress = tokio::time::timeout(std::time::Duration::from_secs(1), async { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + 1 + }) + .await; + let _ = release.send(()); + worker.await.unwrap(); + assert_eq!(network_progress.unwrap(), 1); + + let worker = super::spawn_isolated_worker("bitvm-test", shutdown.clone(), || async { + std::future::pending::<()>().await; + }); + shutdown.cancel(); + tokio::time::timeout(std::time::Duration::from_secs(1), worker).await.unwrap().unwrap(); + } + + #[tokio::test] + async fn ephemeral_queue_is_bounded_by_rows_and_bytes() { + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let control = super::WorkerControl { + immediate: tx, + bytes: std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + tick: Default::default(), + joins: Default::default(), + }; + let source = libp2p::PeerId::random(); + let message = || { + super::GOATMessage::new(bitvm_lib::actors::Actor::All, super::GOATMessageContent::Tick) + }; + let id = || super::GOATMessage::default_message_id(); + assert!(!control.enqueue(source, id(), message(), 5)); + assert!(control.enqueue(source, id(), message(), 4)); + assert!(!control.enqueue(source, id(), message(), 1)); + drop(rx.recv().await.unwrap()); + assert!(control.enqueue(source, id(), message(), 1)); + assert!(!control.enqueue(source, id(), message(), 1)); + assert_eq!(control.bytes.available_permits(), 3, "failed enqueue releases its permits"); + } use crate::action::{GOATMessage, GOATMessageContent, NodeInfo, send_to_peer}; use crate::env::get_rpc_support_actors; use crate::middleware::swarm::{ @@ -227,12 +480,19 @@ mod tests { #[tracing::instrument(level = Level::INFO)] async fn recv_and_dispatch( &self, - _swarm: &mut BitvmSwarmWrapper, + swarm: &mut BitvmSwarmWrapper, actor: Actor, from_peer_id: PeerId, + propagation_source: PeerId, + _sequence_number: Option, id: MessageId, message: &[u8], ) -> anyhow::Result<()> { + swarm.behaviour_mut().gossipsub.report_message_validation_result( + &id, + &propagation_source, + libp2p::gossipsub::MessageAcceptance::Accept, + ); if id == GOATMessage::default_message_id() { tracing::info!("recv_and_dispatch receive local message"); return Ok(()); @@ -351,4 +611,274 @@ mod tests { assert!(success); Ok(()) } + + /// Publishes whatever the test queued, bytes as given, on the next tick. It + /// stands in for a peer that is not running this implementation. + struct RawPublisher { + queue: std::sync::Arc>>>, + } + + impl P2pMessageHandler for RawPublisher { + async fn recv_and_dispatch( + &self, + swarm: &mut BitvmSwarmWrapper, + _actor: Actor, + _from_peer_id: PeerId, + propagation_source: PeerId, + _sequence_number: Option, + id: MessageId, + _message: &[u8], + ) -> anyhow::Result<()> { + swarm.behaviour_mut().gossipsub.report_message_validation_result( + &id, + &propagation_source, + libp2p::gossipsub::MessageAcceptance::Accept, + ); + Ok(()) + } + + async fn handle_tick_message( + &self, + swarm: &mut BitvmSwarmWrapper, + _peer_id: PeerId, + _actor: Actor, + msg_type: TickMessageType, + ) -> anyhow::Result<()> { + if !matches!(msg_type, TickMessageType::RegularlyAction) { + return Ok(()); + } + let topic = libp2p::gossipsub::IdentTopic::new(crate::middleware::get_topic_name( + &Actor::Committee.to_string(), + )); + let mut queue = self.queue.lock().unwrap(); + // Publishing fails until the mesh has formed; keep the payload queued. + queue.retain(|payload| { + swarm.behaviour_mut().gossipsub.publish(topic.clone(), payload.clone()).is_err() + }); + Ok(()) + } + + async fn finish_subscribe_topic( + &self, + _swarm: &mut BitvmSwarmWrapper, + _actor: Actor, + _topic: &str, + ) -> anyhow::Result<()> { + Ok(()) + } + } + + /// Accepts and records every payload that reaches it. + struct Recorder { + received: std::sync::Arc>>>, + } + + impl P2pMessageHandler for Recorder { + async fn recv_and_dispatch( + &self, + swarm: &mut BitvmSwarmWrapper, + _actor: Actor, + _from_peer_id: PeerId, + propagation_source: PeerId, + _sequence_number: Option, + id: MessageId, + message: &[u8], + ) -> anyhow::Result<()> { + swarm.behaviour_mut().gossipsub.report_message_validation_result( + &id, + &propagation_source, + libp2p::gossipsub::MessageAcceptance::Accept, + ); + self.received.lock().unwrap().push(message.to_vec()); + Ok(()) + } + + async fn handle_tick_message( + &self, + _swarm: &mut BitvmSwarmWrapper, + _peer_id: PeerId, + _actor: Actor, + _msg_type: TickMessageType, + ) -> anyhow::Result<()> { + Ok(()) + } + + async fn finish_subscribe_topic( + &self, + _swarm: &mut BitvmSwarmWrapper, + _actor: Actor, + _topic: &str, + ) -> anyhow::Result<()> { + Ok(()) + } + } + + fn free_tcp_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() + } + + fn swarm_config(local_key: String, p2p_port: u16, bootnodes: Vec) -> BitvmSwarmConfig { + BitvmSwarmConfig { + local_key, + p2p_port, + bootnodes, + topic_names: vec![Actor::Committee.to_string(), Actor::All.to_string()], + heartbeat_interval: 3600, + regular_task_interval: 1, + } + } + + async fn wait_until(what: &str, mut done: impl FnMut() -> bool) { + for _ in 0..600 { + if done() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("timed out waiting until {what}"); + } + + /// Verify the relay forwards admitted messages and withholds rejected ones. + #[test] + fn relay_forwards_admitted_messages_and_withholds_dropped_ones() { + // The node processor's dispatch future is deep: `main` drives it on the + // 8 MiB main thread, and it overflows the 2 MiB stack of a test thread. + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(relay_scenario()); + }) + .unwrap() + .join() + .unwrap(); + } + + async fn relay_scenario() { + use crate::action::KickoffSent; + use crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE; + + init(); + let cancel = CancellationToken::new(); + let relay_db = store::create_local_db(&temp_sqlite_db_path()).await; + let (relay_key, relay_peer_id) = gen_local_key().unwrap(); + let relay_port = free_tcp_port(); + // The real processor announces itself on its heartbeat tick, which reads + // the node identity from the environment. + unsafe { + std::env::set_var(crate::env::ENV_PEER_KEY, &relay_key); + if std::env::var(crate::env::ENV_BITVM_SECRET).is_err() { + std::env::set_var(crate::env::ENV_BITVM_SECRET, "seed:relay-admission-test"); + } + } + let bootnode = generate_bootnode_url(&relay_peer_id, relay_port); + + let relay = super::BitvmNodeProcessor { + local_db: relay_db.clone(), + btc_client: std::sync::Arc::new(client::btc_chain::BTCClient::new_mock_client().0), + goat_client: std::sync::Arc::new(client::goat_chain::GOATClient::new_mock_client().0), + http_client: std::sync::Arc::new( + client::http_client::async_client::HttpAsyncClient::new(None), + ), + soldering_builder: None, + metrics_state: crate::metrics_service::MetricsState::new(std::sync::Arc::new( + std::sync::Mutex::new(Registry::default()), + )), + shutdown_token: cancel.clone(), + worker: Default::default(), + }; + let mut relay_manager = BitvmNetworkManager::new( + swarm_config(relay_key, relay_port, vec![]), + &mut Registry::default(), + ) + .unwrap(); + // The node processor's future is not `Send`; it runs on this task, next + // to the scenario below, the same way `main` drives it. + let relay_run = relay_manager.run(Actor::Committee, relay, cancel.clone()); + + let queue = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let received = std::sync::Arc::new(std::sync::Mutex::new(Vec::>::new())); + let mut publisher_manager = BitvmNetworkManager::new( + swarm_config(gen_local_key().unwrap().0, 0, vec![bootnode.clone()]), + &mut Registry::default(), + ) + .unwrap(); + let mut recorder_manager = BitvmNetworkManager::new( + swarm_config(gen_local_key().unwrap().0, 0, vec![bootnode]), + &mut Registry::default(), + ) + .unwrap(); + let (publisher, publisher_cancel) = (RawPublisher { queue: queue.clone() }, cancel.clone()); + tokio::spawn(async move { + publisher_manager.run(Actor::Operator, publisher, publisher_cancel).await.unwrap(); + }); + let (recorder, recorder_cancel) = (Recorder { received: received.clone() }, cancel.clone()); + tokio::spawn(async move { + recorder_manager.run(Actor::Verifier, recorder, recorder_cancel).await.unwrap(); + }); + + let scenario = async { + let protocol_message = |graph_id: uuid::Uuid| { + GOATMessage::new( + Actor::Committee, + GOATMessageContent::KickoffSent(KickoffSent { + instance_id: uuid::Uuid::new_v4(), + graph_id, + }), + ) + }; + let (first, sentinel) = (uuid::Uuid::new_v4(), uuid::Uuid::new_v4()); + let first = protocol_message(first).serialize_message().await.unwrap(); + let sentinel = protocol_message(sentinel).serialize_message().await.unwrap(); + + // The recorder is not connected to the publisher: whatever it sees came + // through the relay, after the relay admitted it. + queue.lock().unwrap().push(first.clone()); + wait_until("the relay forwarded the first protocol message", || { + received.lock().unwrap().contains(&first) + }) + .await; + + let oversized_json = vec![b'{'; crate::env::DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES + 1]; + let undecodable = b"not a protocol message".to_vec(); + let mut binary_from_non_verifier = b"GOATBIN1".to_vec(); + binary_from_non_verifier.resize(MAX_GOSSIPSUB_TRANSMIT_SIZE / 4, 0); + let junk = [oversized_json, undecodable, binary_from_non_verifier]; + // Wait for the sentinel after all preceding messages have been processed. + queue.lock().unwrap().extend(junk.iter().cloned().chain([sentinel.clone()])); + wait_until("the relay forwarded the sentinel", || { + received.lock().unwrap().contains(&sentinel) + }) + .await; + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + let received = received.lock().unwrap().clone(); + for payload in &junk { + assert!( + !received.contains(payload), + "the relay forwarded a {}-byte payload its own admission dropped", + payload.len() + ); + } + + let mut storage = relay_db.acquire().await.unwrap(); + let queued = storage.p2p_inbox_class_totals().await.unwrap(); + let (rows, bytes): (i64, i64) = queued + .iter() + .fold((0, 0), |(rows, bytes), usage| (rows + usage.rows, bytes + usage.bytes)); + assert!(rows <= 2, "only the two protocol messages may be queued, found {rows}"); + assert!( + bytes <= (first.len() + sentinel.len()) as i64, + "the relay persisted {bytes} bytes; junk must never reach the inbox" + ); + }; + tokio::select! { + result = relay_run => panic!("the relay stopped before the scenario finished: {result:?}"), + () = scenario => {} + } + cancel.cancel(); + } } diff --git a/node/src/utils.rs b/node/src/utils.rs index 3827ace6..9598c86b 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -445,20 +445,25 @@ pub async fn validate_init_graph_base( Ok(()) } -/// Verify the stake conditions required for an operator to create a graph. -/// This mirrors the Gateway graph-posting requirement and is shared by graph -/// validation and the early InitGraph admission check. -pub async fn validate_operator_stake( +/// Operator stake verdict; RPC failures are returned as errors. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperatorStakeStatus { + Staked, + NotRegistered, + Insufficient { locked: u64, min: u64 }, +} + +pub async fn operator_stake_status( goat_client: &GOATClient, operator_pubkey: &PublicKey, -) -> Result<()> { +) -> Result { let operator_xonly_pubkey = XOnlyPublicKey::from(*operator_pubkey).serialize(); let operator_addr = goat_client .stake_mana_pubkey_to_address(&operator_xonly_pubkey) .await .context("query operator address")?; if operator_addr == [0; 20] { - bail!("operator not registered"); + return Ok(OperatorStakeStatus::NotRegistered); } let min_stake_amount = goat_client.gateway_get_min_stake_amount().await.context("query minimum operator stake")?; @@ -467,9 +472,25 @@ pub async fn validate_operator_stake( .await .context("query operator locked stake")?; if locked_stake < min_stake_amount { - bail!("insufficient operator stake: locked={locked_stake}, min={min_stake_amount}"); + return Ok(OperatorStakeStatus::Insufficient { + locked: locked_stake, + min: min_stake_amount, + }); + } + Ok(OperatorStakeStatus::Staked) +} + +pub async fn validate_operator_stake( + goat_client: &GOATClient, + operator_pubkey: &PublicKey, +) -> Result<()> { + match operator_stake_status(goat_client, operator_pubkey).await? { + OperatorStakeStatus::Staked => Ok(()), + OperatorStakeStatus::NotRegistered => bail!("operator not registered"), + OperatorStakeStatus::Insufficient { locked, min } => { + bail!("insufficient operator stake: locked={locked}, min={min}") + } } - Ok(()) } pub fn validate_verifier_graph_params_endorsements( From 88482cb3ddf076e99303dcad26e76483e3a023a4 Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 21 Sep 2026 22:57:28 +0800 Subject: [PATCH 12/17] fix: recover graph setup and signing-round delivery --- node/src/action.rs | 187 ++-- node/src/handle.rs | 908 ++++++++++++++---- .../instance_maintenance_tasks.rs | 8 +- 3 files changed, 831 insertions(+), 272 deletions(-) diff --git a/node/src/action.rs b/node/src/action.rs index f6061bf6..6700cf56 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -3613,31 +3613,18 @@ pub async fn send_to_peer( } } -pub async fn push_local_unhandled_messages_with_reason( +/// Defer the message currently being dispatched, or enqueue a new local task. +pub async fn defer_or_enqueue_message( local_db: &LocalDB, message: &GOATMessage, delay_secs: usize, reason: MessageDeferReason, reason_detail: &str, ) -> Result<()> { - // Keep self-deferral in the original sender-specific inbox row. - let deferred_in_inbox = ACTIVE_INBOX_DISPATCH - .try_with(|active| { - if DispatchFingerprint::of(message.content()) != active.fingerprint { - return false; - } - *active.retry.borrow_mut() = Some(retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(delay_secs.max(1) as i64), - reason_detail.to_owned(), - )); - true - }) - .unwrap_or(false); - if deferred_in_inbox { + if defer_current_inbox(message, delay_secs, reason_detail) { return Ok(()); } - let mut storage_processor = local_db.start_immediate_transaction().await?; + let actor = message.actor.clone(); let content: GOATMessageContent = message.content().clone(); let key = LocalMessageKey::from_content(actor.clone(), &content)?; @@ -3647,35 +3634,30 @@ pub async fn push_local_unhandled_messages_with_reason( let active_claim = ACTIVE_LOCAL_MESSAGE_CLAIM .try_with(|claim| (claim.message_id.clone(), claim.message_version)) .ok(); - let claimed_message = if let Some((message_id, _)) = active_claim.as_ref() { - storage_processor.find_messages_by_id(message_id).await? - } else { - None - }; - let owns_requeued_message = claimed_message.as_ref().is_some_and(|existing| { - active_claim.as_ref().is_some_and(|(message_id, message_version)| { - existing.message_id == message_id.as_str() - && existing.message_version == *message_version - && existing.business_id == business_id - && existing.msg_type == message_type - }) - }); - let self_deferred = if let Some(existing) = claimed_message.as_ref() - && existing.state == MessageState::Processing.to_string() - && owns_requeued_message + let mut storage_processor = local_db.start_immediate_transaction().await?; + + let queued_message_id = if let Some((message_id, message_version)) = + active_claim.filter(|(message_id, _)| message_id == &target_message_id) { - storage_processor + let deferred = storage_processor .self_defer_local_message( - &existing.message_id, - existing.message_version, + &message_id, + message_version, current_time_secs() + delay_secs as i64, reason_detail, ) - .await? + .await?; + if !deferred { + return Err(retryable_dispatch_error( + RetryableDispatchReason::ResourceLocked, + Some(delay_secs.max(1) as i64), + format!( + "local message {business_id}:{message_type} is no longer owned by this claim" + ), + )); + } + message_id } else { - false - }; - if !self_deferred { let upserted = upsert_message( &mut storage_processor, true, @@ -3701,20 +3683,11 @@ pub async fn push_local_unhandled_messages_with_reason( )); } } - } - let queued_message_id = if self_deferred { - claimed_message.as_ref().map(|message| message.message_id.as_str()) - } else { - Some(target_message_id.as_str()) - }; - let persist_result = match queued_message_id { - Some(message_id) => { - storage_processor - .upsert_message_debug_reason(message_id, reason.code(), reason_detail) - .await - } - None => Ok(()), + target_message_id }; + let persist_result = storage_processor + .upsert_message_debug_reason(&queued_message_id, reason.code(), reason_detail) + .await; if let Err(error) = persist_result { tracing::warn!( event = "local_message_queue", @@ -3732,6 +3705,22 @@ pub async fn push_local_unhandled_messages_with_reason( Ok(()) } +fn defer_current_inbox(message: &GOATMessage, delay_secs: usize, reason_detail: &str) -> bool { + ACTIVE_INBOX_DISPATCH + .try_with(|active| { + if DispatchFingerprint::of(message.content()) != active.fingerprint { + return false; + } + *active.retry.borrow_mut() = Some(retryable_dispatch_error( + RetryableDispatchReason::DependencyPending, + Some(delay_secs.max(1) as i64), + reason_detail.to_owned(), + )); + true + }) + .unwrap_or(false) +} + /// Helper: try to get graph. If missing, send SyncGraphRequest and defer current handling. pub(crate) async fn get_graph_or_defer( swarm: &mut dyn MessagePublisher, @@ -3763,7 +3752,7 @@ pub(crate) async fn get_graph_or_defer( "submitted" }; let delay_secs: usize = 60; // 1 min default retry - if let Err(error) = push_local_unhandled_messages_with_reason( + if let Err(error) = defer_or_enqueue_message( local_db, message, delay_secs, @@ -4238,7 +4227,7 @@ mod tests { ); let mut compensated = false; let result = track_inbox_retry(DispatchFingerprint::of(message.content()), async { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( &local_db, &message, 30, @@ -4284,7 +4273,7 @@ mod tests { ); let result = track_inbox_retry(DispatchFingerprint::of(dispatched.content()), async { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( &local_db, &other_event, 30, @@ -4292,7 +4281,7 @@ mod tests { "a different event", ) .await?; - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( &local_db, &own_copy, 45, @@ -4430,7 +4419,7 @@ mod tests { Actor::Operator, GOATMessageContent::PostReady(PostReady { instance_id }), ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( &local_db, &message, 0, @@ -4455,7 +4444,7 @@ mod tests { }; assert_eq!(claimed.len(), 1); - let error = push_local_unhandled_messages_with_reason( + let error = defer_or_enqueue_message( &local_db, &message, 30, @@ -4563,7 +4552,7 @@ mod tests { message_id: claimed.message_id.clone(), message_version: claimed.message_version, }, - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( &local_db, &message, 30, @@ -4591,4 +4580,82 @@ mod tests { 0 ); } + + #[tokio::test] + async fn local_claim_is_not_deferred_by_another_actor_message() { + let local_db = store::create_local_db("sqlite::memory:").await; + let instance_id = Uuid::new_v4(); + let claimed_message = GOATMessage::new( + Actor::Operator, + GOATMessageContent::PostReady(PostReady { instance_id }), + ); + let other_actor_message = + GOATMessage::new(Actor::Committee, claimed_message.content.clone()); + let claimed_id = + LocalMessageKey::from_content(claimed_message.actor.clone(), claimed_message.content()) + .unwrap() + .message_id(); + let other_id = LocalMessageKey::from_content( + other_actor_message.actor.clone(), + other_actor_message.content(), + ) + .unwrap() + .message_id(); + { + let mut storage = local_db.acquire().await.unwrap(); + upsert_message( + &mut storage, + false, + SELF_SENDER.to_owned(), + claimed_message.actor.clone(), + claimed_message.content.clone(), + 0, + 0, + ) + .await + .unwrap(); + } + let claimed = local_db + .acquire() + .await + .unwrap() + .claim_local_messages( + current_time_secs() + 1, + current_time_secs() + 300, + 0, + 1, + QUEUE_MAX_ABANDONS, + ) + .await + .unwrap() + .pop() + .unwrap(); + + ACTIVE_LOCAL_MESSAGE_CLAIM + .scope( + LocalMessageClaim { + message_id: claimed.message_id.clone(), + message_version: claimed.message_version, + }, + defer_or_enqueue_message( + &local_db, + &other_actor_message, + 30, + MessageDeferReason::HandlerError, + "other actor", + ), + ) + .await + .unwrap(); + + let mut storage = local_db.acquire().await.unwrap(); + assert_eq!( + storage.find_messages_by_id(&claimed_id).await.unwrap().unwrap().state, + MessageState::Processing.to_string() + ); + assert_eq!( + storage.find_messages_by_id(&other_id).await.unwrap().unwrap().state, + MessageState::Pending.to_string() + ); + } } diff --git a/node/src/handle.rs b/node/src/handle.rs index 33e3d124..10b13f4a 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -794,6 +794,73 @@ pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent } } +/// Apply the business-key cooldown to stored-value recovery. +async fn publish_stored( + ctx: &mut HandlerContext<'_>, + gate_key: &str, + target: Actor, + content: GOATMessageContent, + origin: StoredValue, +) -> Result<()> { + let now = std::time::Instant::now(); + let gate = crate::p2p_admission::protocol_republish_gate(); + let message = GOATMessage::new(target, content); + if origin == StoredValue::Found && gate.is_cooling(gate_key, now) { + return Ok(()); + } + send_protocol_message(ctx.swarm, ctx.local_db, message, origin).await?; + gate.allow(gate_key, now); + Ok(()) +} + +/// Check local graph-signing completion; missing, foreign and Obsoleted rows are not complete. +async fn graph_signing_round_is_over( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, +) -> Result { + let Some(row) = local_db.acquire().await?.find_graph(&graph_id).await? else { + return Ok(false); + }; + if row.instance_id != instance_id { + return Ok(false); + } + Ok(GraphStatus::from_str(&row.status).is_ok_and(|status| match status { + GraphStatus::OperatorPresigned | GraphStatus::Obsoleted => false, + GraphStatus::CommitteePresigned + | GraphStatus::OperatorDataPushed + | GraphStatus::PreKickoff + | GraphStatus::OperatorKickOff + | GraphStatus::Challenge + | GraphStatus::Disprove + | GraphStatus::Skipped + | GraphStatus::OperatorTake1 + | GraphStatus::OperatorTake2 => true, + })) +} + +/// The committee key this node signs with for `instance_id`. +fn local_committee_pubkey(instance_id: Uuid) -> Result { + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + Ok(load_committee_instance_keypair(&committee_master_key, instance_id)?.public_key().into()) +} + +/// Recover this node's own PeginConfirm value; leave live outbox rows on their retry schedule. +async fn redeliver_own_pegin_confirm_value( + ctx: &mut HandlerContext<'_>, + instance_id: Uuid, + gate_key: &str, + sender: &PublicKey, + content: &GOATMessageContent, +) -> Result<()> { + if ctx.id != GOATMessage::default_message_id() + || *sender != local_committee_pubkey(instance_id)? + { + return Ok(()); + } + publish_stored(ctx, gate_key, Actor::Committee, content.clone(), StoredValue::Found).await +} + fn make_message(ctx: &HandlerContext<'_>, content: &GOATMessageContent) -> GOATMessage { GOATMessage::new(ctx.actor.clone(), content.clone()) } @@ -1524,7 +1591,7 @@ async fn defer_confirm_instance_until_previous_graph_presigned( let Some((previous_instance_id, previous_graph_id)) = get_graph_id_by_nonce(ctx.local_db, previous_nonce, operator_pubkey).await? else { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &retry_message, 60, @@ -1552,7 +1619,7 @@ async fn defer_confirm_instance_until_previous_graph_presigned( "Failed to send SyncGraphRequest for previous graph {previous_instance_id}:{previous_graph_id}: {error}" ); } - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &retry_message, 60, @@ -1569,7 +1636,7 @@ async fn defer_confirm_instance_until_previous_graph_presigned( return Ok(false); } - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &retry_message, 60, @@ -1583,7 +1650,13 @@ async fn defer_confirm_instance_until_previous_graph_presigned( graph_nonce: previous_graph.parameters.graph_nonce, graph: previous_graph, }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::All, message_content)).await?; + send_protocol_message( + ctx.swarm, + ctx.local_db, + GOATMessage::new(Actor::All, message_content), + StoredValue::Fresh, + ) + .await?; tracing::info!( "Defer ConfirmInstance for {instance_id}: re-broadcast previous CreateGraph {previous_instance_id}:{previous_graph_id} until it is committee pre-signed" ); @@ -1625,7 +1698,7 @@ async fn handle_confirm_instance_operator( graph, }); let msg = GOATMessage::new(Actor::All, message_content); - send_to_peer(ctx.swarm, msg).await?; + send_protocol_message(ctx.swarm, ctx.local_db, msg, StoredValue::Fresh).await?; return Ok(()); } @@ -1744,12 +1817,20 @@ async fn handle_init_graph_verifier(context: &HeavyTaskContext, message: InitGra ); return Ok(()); } - if let Err(error) = validate_operator_stake(&context.goat_client, operator_pubkey).await { + if !matches!( + operator_stake_status(&context.goat_client, operator_pubkey).await.map_err(|error| { + retryable_dispatch_error( + RetryableDispatchReason::ExternalRpcUnavailable, + None, + format!("InitGraph operator stake lookup failed: {error:#}"), + ) + })?, + OperatorStakeStatus::Staked + ) { tracing::warn!( instance_id = %instance_id, graph_id = %graph_id, operator_pubkey = %operator_pubkey, - error = %error, "Ignore InitGraph from an operator that does not meet stake requirements" ); return Ok(()); @@ -1961,10 +2042,44 @@ async fn handle_gen_circuits_operator( if operator_state.candidate_verifier_pubkeys.is_none() { freeze_operator_candidates(operator_state)?; } - let cut_candidates = if was_frozen { Vec::new() } else { operator_state.candidates.clone() }; + // On every retry, enqueue any CutCircuits still owed by the frozen candidate set. + let owed = cut_circuits_owed(operator_state, current_time_secs()); save_babe_setup_state(ctx.local_db, instance_id, graph_id, &state)?; - for candidate in cut_candidates { + ensure_cut_circuits_outbox(ctx.local_db, instance_id, graph_id, &owed).await?; + Ok(()) +} + +/// Candidates still owed CutCircuits, excluding delivered proofs and closed selections. +fn cut_circuits_owed(state: &OperatorBabeSetupState, now: i64) -> Vec { + if state.candidate_verifier_pubkeys.is_none() || state.selected_verifier_pubkeys.is_some() { + return Vec::new(); + } + let delivered = state.candidates.iter().filter(|candidate| candidate.gc_data.is_some()).count(); + let window_over = state.proof_collection_started_at.is_some_and(|started_at| { + now - started_at >= get_verifier_candidate_collection_window_secs() + }); + if window_over && delivered >= min_required_verifier() { + return Vec::new(); + } + state + .candidates + .iter() + .filter(|candidate| { + candidate.gc_data.is_none() && candidate.soldering_proof_ready.is_none() + }) + .cloned() + .collect() +} + +/// Create missing CutCircuits outbox entries; leave all existing entries unchanged. +async fn ensure_cut_circuits_outbox( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + owed: &[OperatorVerifierCandidate], +) -> Result<()> { + for candidate in owed { let message = GOATMessage::new( Actor::Verifier, GOATMessageContent::CutCircuits(CutCircuits { @@ -1973,13 +2088,13 @@ async fn handle_gen_circuits_operator( verifier_pubkey: candidate.verifier_pubkey, candidate_index: candidate .candidate_index - .expect("candidate slot assigned before CutCircuits"), - selected_circuit_indexes: candidate.selected_circuit_indexes, + .context("frozen candidate has no slot assigned")?, + selected_circuit_indexes: candidate.selected_circuit_indexes.clone(), }), ); let ack_peer_id = PeerId::from_bytes(&candidate.verifier_peer_id).map(|peer_id| peer_id.to_string()).ok(); - enqueue_graph_setup_outbox_message(ctx.local_db, message, ack_peer_id.as_deref()).await?; + enqueue_graph_setup_outbox_message(local_db, message, ack_peer_id.as_deref()).await?; } Ok(()) } @@ -2567,92 +2682,126 @@ async fn handle_compact_soldering_proof_operator( "all verifier soldering proofs are ready to build the graph" ); - let instance_params = get_instance_parameters(&context.local_db, instance_id) - .await? - .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; - - let (graph_nonce, cur_prekickoff_txn) = - match get_current_prekickoff_tx(&context.local_db, &local_operator_pubkey).await? { - Some((graph_nonce, prekickoff_tx)) => (graph_nonce, prekickoff_tx), - None => { - (0, build_genesis_prekickoff_tx(&context.btc_client, &context.goat_client).await?) - } - }; - let prekickoff_params = - build_prekickoff_params(&context.btc_client, graph_nonce, cur_prekickoff_txn).await?; - - let graph_build_started_at = Instant::now(); - tracing::info!( - event = "operator_soldering_proof", - stage = "graph_build", - outcome = "started", - verifier_slots = bitvm_gc_circuit_datas.len(), - graph_nonce, - "building graph parameters from verified soldering proofs" - ); - let mut graph_params = build_graph_params( + // Reuse an existing graph for this ID and resume CreateGraph delivery. + let (graph_nonce, graph, definition_hash, already_finalized) = match get_graph( &context.local_db, - &context.goat_client, - instance_params, - prekickoff_params, - bitvm_gc_circuit_datas, - graph_nonce, + instance_id, graph_id, ) - .await?; + .await? + { + Some(stored) => { + let graph_nonce = stored.parameters.graph_nonce; + let definition_hash = hex::encode(stored.parameters_hash()?); + let already_finalized = BitvmGcGraph::from_simplified(&stored)?.committee_pre_signed(); + tracing::info!( + event = "operator_graph_creation", + outcome = "resumed", + stage = "definition_store", + graph_nonce, + definition_hash = %definition_hash, + already_finalized, + "graph was stored by an earlier run; resuming at the CreateGraph outbox" + ); + (graph_nonce, stored, definition_hash, already_finalized) + } + None => { + let instance_params = get_instance_parameters(&context.local_db, instance_id) + .await? + .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; + + let (graph_nonce, cur_prekickoff_txn) = + match get_current_prekickoff_tx(&context.local_db, &local_operator_pubkey).await? { + Some((graph_nonce, prekickoff_tx)) => (graph_nonce, prekickoff_tx), + None => ( + 0, + build_genesis_prekickoff_tx(&context.btc_client, &context.goat_client) + .await?, + ), + }; + let prekickoff_params = + build_prekickoff_params(&context.btc_client, graph_nonce, cur_prekickoff_txn) + .await?; - let challenge_init_txid = generate_bitvm_graph(graph_params.clone())? - .watchtower_challenge_init - .tx() - .compute_txid() - .to_byte_array(); - graph_params.pubin_disprove_constant = - get_guest_constant_value(graph_id, challenge_init_txid, &graph_params.watchtower_pubkeys)?; - let mut graph = generate_bitvm_graph(graph_params)?; - anyhow::ensure!( - graph.watchtower_challenge_init.tx().compute_txid().to_byte_array() == challenge_init_txid, - "Operator constant unexpectedly changes the watchtower challenge init transaction" - ); - operator_pre_sign(operator_master_key.master_keypair(), &mut graph)?; + let graph_build_started_at = Instant::now(); + tracing::info!( + event = "operator_soldering_proof", + stage = "graph_build", + outcome = "started", + verifier_slots = bitvm_gc_circuit_datas.len(), + graph_nonce, + "building graph parameters from verified soldering proofs" + ); + let mut graph_params = build_graph_params( + &context.local_db, + &context.goat_client, + instance_params, + prekickoff_params, + bitvm_gc_circuit_datas, + graph_nonce, + graph_id, + ) + .await?; - let graph = graph.to_simplified()?; - tracing::info!( - event = "operator_soldering_proof", - stage = "graph_build", - outcome = "completed", - graph_nonce, - elapsed_ms = graph_build_started_at.elapsed().as_millis(), - "built operator-pre-signed graph" - ); - let definition_hash = hex::encode(graph.parameters_hash()?); - tracing::info!( - event = "operator_graph_creation", - outcome = "started", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - "storing operator-pre-signed graph definition" - ); - if let Err(error) = store_operator_presigned_graph(&context.local_db, &graph).await { - tracing::error!( - event = "operator_graph_creation", - outcome = "failed", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - error = %error, - "failed to store operator-pre-signed graph definition" - ); - return Err(error).context("store operator-pre-signed graph definition"); - } - tracing::info!( - event = "operator_graph_creation", - outcome = "committed", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - "stored operator-pre-signed graph definition" - ); + let challenge_init_txid = generate_bitvm_graph(graph_params.clone())? + .watchtower_challenge_init + .tx() + .compute_txid() + .to_byte_array(); + graph_params.pubin_disprove_constant = get_guest_constant_value( + graph_id, + challenge_init_txid, + &graph_params.watchtower_pubkeys, + )?; + let mut graph = generate_bitvm_graph(graph_params)?; + anyhow::ensure!( + graph.watchtower_challenge_init.tx().compute_txid().to_byte_array() + == challenge_init_txid, + "Operator constant unexpectedly changes the watchtower challenge init transaction" + ); + operator_pre_sign(operator_master_key.master_keypair(), &mut graph)?; + + let graph = graph.to_simplified()?; + tracing::info!( + event = "operator_soldering_proof", + stage = "graph_build", + outcome = "completed", + graph_nonce, + elapsed_ms = graph_build_started_at.elapsed().as_millis(), + "built operator-pre-signed graph" + ); + let definition_hash = hex::encode(graph.parameters_hash()?); + tracing::info!( + event = "operator_graph_creation", + outcome = "started", + stage = "definition_store", + graph_nonce, + definition_hash = %definition_hash, + "storing operator-pre-signed graph definition" + ); + if let Err(error) = store_operator_presigned_graph(&context.local_db, &graph).await { + tracing::error!( + event = "operator_graph_creation", + outcome = "failed", + stage = "definition_store", + graph_nonce, + definition_hash = %definition_hash, + error = %error, + "failed to store operator-pre-signed graph definition" + ); + return Err(error).context("store operator-pre-signed graph definition"); + } + tracing::info!( + event = "operator_graph_creation", + outcome = "committed", + stage = "definition_store", + graph_nonce, + definition_hash = %definition_hash, + "stored operator-pre-signed graph definition" + ); + (graph_nonce, graph, definition_hash, false) + } + }; let message = GOATMessage::new( Actor::All, @@ -2661,9 +2810,13 @@ async fn handle_compact_soldering_proof_operator( let serialized = message.serialize_message().await?; let outbox_id = format!("create-graph:{graph_id}"); let mut storage = context.local_db.acquire().await?; - storage - .insert_p2p_outbox_message(&outbox_id, message.content.event_type(), &serialized) - .await?; + // Idempotent: an entry that exists is left as it is. A graph the committee + // has already signed was evidently announced; only the clean-up is left. + if !already_finalized { + storage + .insert_p2p_outbox_message(&outbox_id, message.content.event_type(), &serialized) + .await?; + } let mut cancelled_setup_messages = 0; for setup_outbox_id in obsolete_setup_outbox_ids { cancelled_setup_messages += @@ -2923,7 +3076,13 @@ async fn try_start_graph_committee_setup( pub_nonces: pub_nonces.clone(), nonce_sigs, }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::All, message_content)).await?; + send_protocol_message( + ctx.swarm, + ctx.local_db, + GOATMessage::new(Actor::All, message_content), + StoredValue::Fresh, + ) + .await?; store_committee_pub_nonces_for_graph( ctx.local_db, instance_id, @@ -2984,15 +3143,29 @@ async fn maybe_vote_and_presign_graph( let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; let local_committee_pubkey = instance_keypair.public_key().into(); - if get_committee_partial_sigs_for_graph_member( + if let Some(stored_partial_sigs) = get_committee_partial_sigs_for_graph_member( ctx.local_db, instance_id, graph_id, &local_committee_pubkey, ) .await? - .is_some() { + // Publish stored partial signatures without signing again. + publish_stored( + ctx, + &format!("committee-presign:{graph_id}"), + Actor::All, + GOATMessageContent::CommitteePresign(CommitteePresign { + instance_id, + graph_id, + committee_pubkey: local_committee_pubkey, + committee_partial_sigs: stored_partial_sigs, + agg_nonces, + }), + StoredValue::Found, + ) + .await?; return endorse_graph_if_presigned(ctx, instance_id, graph_id, &full_graph).await; } @@ -3007,30 +3180,42 @@ async fn maybe_vote_and_presign_graph( "local committee agg nonce consensus differs for graph {graph_id}" ))); } - if !consensus_votes + // Recover delivery of the stored vote. + let local_vote = consensus_votes .iter() - .any(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) - { - let signature = - SECP256K1.sign_schnorr(&SecpMessage::from_digest(consensus_hash), &instance_keypair); - store_committee_agg_nonce_consensus_for_graph( - ctx.local_db, - instance_id, - graph_id, - local_committee_pubkey, - consensus_hash, - signature, - ) - .await?; - let message_content = GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + .find(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) + .map(|(_, _, signature)| *signature); + let (signature, origin) = match local_vote { + Some(signature) => (signature, StoredValue::Found), + None => { + let signature = SECP256K1 + .sign_schnorr(&SecpMessage::from_digest(consensus_hash), &instance_keypair); + store_committee_agg_nonce_consensus_for_graph( + ctx.local_db, + instance_id, + graph_id, + local_committee_pubkey, + consensus_hash, + signature, + ) + .await?; + (signature, StoredValue::Fresh) + } + }; + publish_stored( + ctx, + &format!("agg-nonce-consensus:{graph_id}"), + Actor::Committee, + GOATMessageContent::AggNonceConsensus(AggNonceConsensus { instance_id, graph_id, committee_pubkey: local_committee_pubkey, consensus_hash, signature, - }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::Committee, message_content)).await?; - } + }), + origin, + ) + .await?; let consensus_votes = get_committee_agg_nonce_consensus_for_graph(ctx.local_db, instance_id, graph_id).await?; @@ -3085,14 +3270,20 @@ async fn maybe_vote_and_presign_graph( committee_partial_sigs.clone(), ) .await?; - let message_content = GOATMessageContent::CommitteePresign(CommitteePresign { - instance_id, - graph_id, - committee_pubkey: local_committee_pubkey, - committee_partial_sigs, - agg_nonces, - }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::All, message_content)).await?; + publish_stored( + ctx, + &format!("committee-presign:{graph_id}"), + Actor::All, + GOATMessageContent::CommitteePresign(CommitteePresign { + instance_id, + graph_id, + committee_pubkey: local_committee_pubkey, + committee_partial_sigs, + agg_nonces, + }), + StoredValue::Fresh, + ) + .await?; endorse_graph_if_presigned(ctx, instance_id, graph_id, &full_graph).await } @@ -3108,6 +3299,13 @@ async fn endorse_graph_if_presigned( if committee_partial_sigs.len() != committee_pubkeys.len() { return Ok(()); } + // Check the recovery gate before signing the endorsement. + let gate_key = format!("endorse-graph:{graph_id}"); + if crate::p2p_admission::protocol_republish_gate() + .is_cooling(&gate_key, std::time::Instant::now()) + { + return Ok(()); + } let committee_sig_for_graph = endorse_graph(ctx.goat_client, graph).await?; let committee_sig_for_params = endorse_graph_params(graph).await?; @@ -3124,8 +3322,9 @@ async fn endorse_graph_if_presigned( committee_sig_for_params: committee_sig_for_params.as_bytes().to_vec(), committee_evm_address, }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::All, message_content)).await?; - Ok(()) + // `Found`: whether this is the first endorsement or a repeat is exactly what + // the outbox row records, and a repeat must not be published past it. + publish_stored(ctx, &gate_key, Actor::All, message_content, StoredValue::Found).await } #[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] @@ -3172,7 +3371,7 @@ async fn handle_create_graph_committee( ); } let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 60, @@ -3187,7 +3386,7 @@ async fn handle_create_graph_committee( }; if !previous_graph.committee_pre_signed() { let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 60, @@ -3203,7 +3402,7 @@ async fn handle_create_graph_committee( } None => { let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 60, @@ -3319,6 +3518,12 @@ async fn handle_nonce_generation_committee( nonce_sigs: &CommitteeNonceSignatures, content: &GOATMessageContent, ) -> Result<()> { + if graph_signing_round_is_over(ctx.local_db, instance_id, graph_id).await? { + tracing::debug!( + "Ignore NonceGeneration for {instance_id}:{graph_id}: signing round is over" + ); + return Ok(()); + } // received from Committee members if !ensure_self_or_valid_committee( ctx, @@ -3384,6 +3589,12 @@ async fn handle_agg_nonce_consensus_committee( signature: &secp256k1::schnorr::Signature, content: &GOATMessageContent, ) -> Result<()> { + if graph_signing_round_is_over(ctx.local_db, instance_id, graph_id).await? { + tracing::debug!( + "Ignore AggNonceConsensus for {instance_id}:{graph_id}: signing round is over" + ); + return Ok(()); + } if !ensure_self_or_valid_committee( ctx, instance_id, @@ -3413,7 +3624,7 @@ async fn handle_agg_nonce_consensus_committee( let Some((_, _, _, expected_consensus_hash)) = graph_nonce_consensus_context(ctx, instance_id, graph_id, &graph).await? else { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 30, @@ -3570,7 +3781,7 @@ async fn validate_committee_presign_for_graph( let pub_nonces_unchecked = get_committee_pub_nonces_for_graph(ctx.local_db, instance_id, graph_id).await?; if pub_nonces_unchecked.len() != committee_pubkeys.len() { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 30, @@ -3646,6 +3857,12 @@ async fn handle_committee_presign_committee( _agg_nonces: &CommitteeAggNonces, content: &GOATMessageContent, ) -> Result<()> { + if graph_signing_round_is_over(ctx.local_db, instance_id, graph_id).await? { + tracing::debug!( + "Ignore CommitteePresign for {instance_id}:{graph_id}: signing round is over" + ); + return Ok(()); + } // received from Committee members if !ensure_self_or_valid_committee( ctx, @@ -3952,10 +4169,39 @@ async fn handle_graph_finalize_committee( pub_nonce, ) .await?; - send_to_peer(ctx.swarm, GOATMessage::new(Actor::Committee, message_content)) - .await?; + send_protocol_message( + ctx.swarm, + ctx.local_db, + GOATMessage::new(Actor::Committee, message_content), + StoredValue::Fresh, + ) + .await?; } } + if let Some(pub_nonce) = stored_pub_nonce { + // Recover the stored nonce proof after Presigned without recomputing MuSig signatures. + let (_, expected_nonce, nonce_sig) = committee_master_key + .nonce_for_instance_job_with_keypair( + &graph.parameters.instance_parameters, + instance_keypair, + )?; + if expected_nonce != pub_nonce { + bail!("stored pegin public nonce differs from deterministic nonce"); + } + publish_stored( + ctx, + &format!("pegin-nonce:{instance_id}"), + Actor::Committee, + GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { + instance_id, + committee_pubkey: local_committee_pubkey, + pub_nonce, + nonce_sig, + }), + StoredValue::Found, + ) + .await?; + } maybe_vote_and_sign_pegin_confirm(ctx, instance_id).await?; } // 4. (Relayer) try to call Gateway.postGraphData @@ -4170,10 +4416,47 @@ async fn maybe_vote_and_sign_pegin_confirm( let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; let local_committee_pubkey = instance_keypair.public_key().into(); - if get_committee_partial_sig_for_instance(ctx.local_db, instance_id, &local_committee_pubkey) - .await? - .is_some() + if let Some(stored_partial_sig) = + get_committee_partial_sig_for_instance(ctx.local_db, instance_id, &local_committee_pubkey) + .await? { + // Recreate a missing endorsement while preserving the stored partial signature. + let stored_endorse_sig = get_committee_endorse_sigs_for_pegin(ctx.local_db, instance_id) + .await? + .into_iter() + .find(|(committee_pubkey, _)| *committee_pubkey == local_committee_pubkey) + .map(|(_, endorse_sig)| endorse_sig); + let endorse_sig = match stored_endorse_sig { + Some(endorse_sig) => endorse_sig, + None => { + let pegin_txid = instance_parameters.build_pegin_tx()?.1.tx().compute_txid(); + let endorse_sig = endorse_pegin(ctx.goat_client, instance_id, &pegin_txid) + .await? + .as_bytes() + .to_vec(); + store_committee_endorse_sig_for_pegin( + ctx.local_db, + instance_id, + local_committee_pubkey, + endorse_sig.clone(), + ) + .await?; + endorse_sig + } + }; + publish_stored( + ctx, + &format!("pegin-confirm-partial-sig:{instance_id}"), + Actor::Committee, + GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { + instance_id, + committee_pubkey: local_committee_pubkey, + partial_sig: stored_partial_sig, + endorse_sig, + }), + StoredValue::Found, + ) + .await?; return broadcast_pegin_confirm_if_presigned( ctx, instance_id, @@ -4194,29 +4477,41 @@ async fn maybe_vote_and_sign_pegin_confirm( "local PeginConfirm nonce consensus differs for instance {instance_id}" ))); } - if !consensus_votes + // Stored, then published: see the same step for graphs in + // `maybe_vote_and_presign_graph`. + let local_vote = consensus_votes .iter() - .any(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) - { - let signature = - SECP256K1.sign_schnorr(&SecpMessage::from_digest(consensus_hash), &instance_keypair); - store_committee_agg_nonce_consensus_for_instance( - ctx.local_db, - instance_id, - local_committee_pubkey, - consensus_hash, - signature, - ) - .await?; - let message_content = - GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { + .find(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) + .map(|(_, _, signature)| *signature); + let (signature, origin) = match local_vote { + Some(signature) => (signature, StoredValue::Found), + None => { + let signature = SECP256K1 + .sign_schnorr(&SecpMessage::from_digest(consensus_hash), &instance_keypair); + store_committee_agg_nonce_consensus_for_instance( + ctx.local_db, instance_id, - committee_pubkey: local_committee_pubkey, + local_committee_pubkey, consensus_hash, signature, - }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::Committee, message_content)).await?; - } + ) + .await?; + (signature, StoredValue::Fresh) + } + }; + publish_stored( + ctx, + &format!("pegin-confirm-nonce-consensus:{instance_id}"), + Actor::Committee, + GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { + instance_id, + committee_pubkey: local_committee_pubkey, + consensus_hash, + signature, + }), + origin, + ) + .await?; if !has_complete_pegin_confirm_nonce_consensus( ctx, instance_id, @@ -4254,13 +4549,19 @@ async fn maybe_vote_and_sign_pegin_confirm( endorse_sig.as_bytes().to_vec(), ) .await?; - let message_content = GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { - instance_id, - committee_pubkey: local_committee_pubkey, - partial_sig, - endorse_sig: endorse_sig.as_bytes().to_vec(), - }); - send_to_peer(ctx.swarm, GOATMessage::new(Actor::Committee, message_content)).await?; + publish_stored( + ctx, + &format!("pegin-confirm-partial-sig:{instance_id}"), + Actor::Committee, + GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { + instance_id, + committee_pubkey: local_committee_pubkey, + partial_sig, + endorse_sig: endorse_sig.as_bytes().to_vec(), + }), + StoredValue::Fresh, + ) + .await?; broadcast_pegin_confirm_if_presigned(ctx, instance_id, &instance_parameters, &committee_pubkeys) .await } @@ -4303,9 +4604,14 @@ async fn handle_pegin_confirm_nonce_committee( pub_nonce.clone(), ) .await?; - if ctx.id == GOATMessage::default_message_id() { - send_to_peer(ctx.swarm, GOATMessage::new(Actor::Committee, content.clone())).await?; - } + redeliver_own_pegin_confirm_value( + ctx, + instance_id, + &format!("pegin-nonce:{instance_id}"), + received_committee_pubkey, + content, + ) + .await?; // 3. Agree on the full nonce transcript before generating a partial signature. maybe_vote_and_sign_pegin_confirm(ctx, instance_id).await } @@ -4334,7 +4640,7 @@ async fn handle_pegin_confirm_nonce_consensus_committee( let Some((_, _, _, _, expected_consensus_hash)) = pegin_confirm_nonce_consensus_context(ctx, instance_id).await? else { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 30, @@ -4376,6 +4682,16 @@ async fn handle_pegin_confirm_nonce_consensus_committee( *signature, ) .await?; + // A locally queued copy is the recovery monitor re-delivering this node's own + // stored vote, as the nonce and partial-signature handlers do for theirs. + redeliver_own_pegin_confirm_value( + ctx, + instance_id, + &format!("pegin-confirm-nonce-consensus:{instance_id}"), + received_committee_pubkey, + content, + ) + .await?; maybe_vote_and_sign_pegin_confirm(ctx, instance_id).await } @@ -4405,7 +4721,7 @@ async fn handle_pegin_confirm_partial_sig_committee( let pub_nonces_unchecked = get_committee_pub_nonces_for_instance(ctx.local_db, instance_id).await?; if pub_nonces_unchecked.len() != committee_pubkeys.len() { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 30, @@ -4460,7 +4776,7 @@ async fn handle_pegin_confirm_partial_sig_committee( ) .await? { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 30, @@ -4507,7 +4823,7 @@ async fn handle_pegin_confirm_partial_sig_committee( return Ok(()); } Err(e) => { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, 30, @@ -4537,9 +4853,14 @@ async fn handle_pegin_confirm_partial_sig_committee( endorse_sig.to_owned(), ) .await?; - if ctx.id == GOATMessage::default_message_id() { - send_to_peer(ctx.swarm, GOATMessage::new(Actor::Committee, content.clone())).await?; - } + redeliver_own_pegin_confirm_value( + ctx, + instance_id, + &format!("pegin-confirm-partial-sig:{instance_id}"), + received_committee_pubkey, + content, + ) + .await?; broadcast_pegin_confirm_if_presigned(ctx, instance_id, &instance_params, &committee_pubkeys) .await } @@ -4570,7 +4891,7 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ctx.actor.clone(), GOATMessageContent::PostReady(PostReady { instance_id }), ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4595,7 +4916,7 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ctx.actor.clone(), GOATMessageContent::PostReady(PostReady { instance_id }), ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4617,7 +4938,7 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ctx.actor.clone(), GOATMessageContent::PostReady(PostReady { instance_id }), ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4639,7 +4960,7 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ctx.actor.clone(), GOATMessageContent::PostReady(PostReady { instance_id }), ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4699,7 +5020,7 @@ async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> R ctx.actor.clone(), GOATMessageContent::PostReady(PostReady { instance_id }), ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4829,7 +5150,7 @@ async fn handle_kickoff_ready_operator( ) as u64 * avg_block_time_secs(ctx.btc_client.network()); let delay_secs = min_pegout_time_secs * nonce_interval; - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4844,7 +5165,7 @@ async fn handle_kickoff_ready_operator( "Operator {operator_pubkey} skipped obsoleted graph {current_instance_id}:{current_graph_id}" ); let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 blocks - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4868,7 +5189,7 @@ async fn handle_kickoff_ready_operator( ) as u64 * avg_block_time_secs(ctx.btc_client.network()); let delay_secs = min_pegout_time_secs * nonce_interval; - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4883,7 +5204,7 @@ async fn handle_kickoff_ready_operator( "Operator {operator_pubkey} skipped non-posted graph {current_instance_id}:{current_graph_id}" ); let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 blocks - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4942,7 +5263,7 @@ async fn handle_kickoff_sent_committee( None => { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -4961,7 +5282,7 @@ async fn handle_kickoff_sent_committee( let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * (kickoff_height - goat_confirmed_btc_height); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -5004,8 +5325,18 @@ async fn handle_kickoff_sent_verifier( let kickoff_height = match ctx.btc_client.get_tx_status(&kickoff_txid).await?.block_height { Some(height) => height, None => { - tracing::warn!( - "Ignore KickoffSent for {instance_id}:{graph_id}: kickoff tx not confirmed yet" + // Retry after Bitcoin confirmation. + let delay_secs = avg_block_time_secs(ctx.btc_client.network()); + defer_or_enqueue_message( + ctx.local_db, + &message, + delay_secs as usize, + MessageDeferReason::BitcoinConfirmationPending, + "kickoff transaction is not confirmed on Bitcoin", + ) + .await?; + tracing::info!( + "Retry KickoffSent later for {instance_id}:{graph_id}: kickoff tx not confirmed yet" ); return Ok(()); } @@ -5034,7 +5365,7 @@ async fn handle_kickoff_sent_verifier( if kickoff_height >= goat_confirmed_btc_height { let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * (kickoff_height - goat_confirmed_btc_height) as u64; - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -5358,7 +5689,7 @@ async fn handle_watchtower_challenge_init_sent_watchtower( tracing::warn!( "Retry WatchtowerChallengeInitSent for {instance_id}:{graph_id} later: watchtower proof not ready, retry after {wait_secs} seconds" ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, wait_secs, @@ -5379,9 +5710,18 @@ async fn handle_watchtower_challenge_init_sent_watchtower( { Ok(txid) => txid, Err(e) => { + // Retry broadcast failures until the connector is spent. tracing::warn!( - "Ignore WatchtowerChallengeInitSent for {instance_id}:{graph_id}: failed to send watchtower challenge tx: {e}" + "Retry WatchtowerChallengeInitSent for {instance_id}:{graph_id} later: failed to send watchtower challenge tx: {e}" ); + defer_or_enqueue_message( + ctx.local_db, + &message, + 30, + MessageDeferReason::BitcoinTransactionPending, + "watchtower challenge transaction could not be broadcast", + ) + .await?; return Ok(()); } }; @@ -5676,7 +6016,7 @@ async fn handle_operator_commit_pubin_ready_operator( tracing::info!( "Retry OperatorCommitPubinReady later for {instance_id}:{graph_id}: challenge info is not ready: {e}" ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, wait_secs, @@ -5700,7 +6040,7 @@ async fn handle_operator_commit_pubin_ready_operator( tracing::info!( "Retry OperatorCommitPubinReady later for {instance_id}:{graph_id}: operator pubin inputs are not ready: {e}" ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, wait_secs, @@ -5830,7 +6170,7 @@ async fn handle_assert_ready_operator( tracing::info!( "Retry AssertReady later for {instance_id}:{graph_id}: operator proof is not ready" ); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, wait_secs, @@ -5994,7 +6334,7 @@ async fn handle_assert_sent_verifier( else { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6013,7 +6353,7 @@ async fn handle_assert_sent_verifier( if !ctx.btc_client.get_tx_status(&connector_e_spent_txid).await?.confirmed { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6026,7 +6366,7 @@ async fn handle_assert_sent_verifier( let Some(commit_pubin_tx) = ctx.btc_client.get_tx(&connector_e_spent_txid).await? else { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6056,7 +6396,7 @@ async fn handle_assert_sent_verifier( Err(error) => { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6266,7 +6606,7 @@ async fn handle_challenge_assert_sent_operator( let Some(challenge_assert_tx) = ctx.btc_client.get_tx(&challenge_assert_txid).await? else { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6433,7 +6773,7 @@ async fn handle_wrongly_challenge_timeout_verifier( let delay_secs = avg_block_time_secs(ctx.btc_client.network()); let message = make_message(ctx, content); if ctx.btc_client.get_tx(&challenge_assert_txid).await?.is_none() { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6455,7 +6795,7 @@ async fn handle_wrongly_challenge_timeout_verifier( { Some(height) => height as u64, None => { - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6479,7 +6819,7 @@ async fn handle_wrongly_challenge_timeout_verifier( if bitcoin_height < disprove_height { let retry_secs = avg_block_time_secs(ctx.btc_client.network()) * (disprove_height - bitcoin_height); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, retry_secs as usize, @@ -6579,7 +6919,7 @@ async fn handle_disprove_sent_committee( Some(height) => height as u64, None => { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6597,7 +6937,7 @@ async fn handle_disprove_sent_committee( if goat_confirmed_height < challenge_finish_height { let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * (challenge_finish_height - goat_confirmed_height); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6744,7 +7084,7 @@ async fn handle_take1_sent_committee( if withdraw_status == WithdrawStatus::Initialized { // Kickoff not posted yet, wait for it let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * 6; // wait for 6 blocks - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6767,7 +7107,7 @@ async fn handle_take1_sent_committee( Some(height) => height as u64, None => { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 block - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6785,7 +7125,7 @@ async fn handle_take1_sent_committee( if goat_confirmed_height < take1_height { let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * (take1_height - goat_confirmed_height); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6956,7 +7296,7 @@ async fn handle_take2_sent_committee( if withdraw_status == WithdrawStatus::Initialized { // Kickoff not posted yet, wait for it let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * 6; // wait for 6 blocks - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6979,7 +7319,7 @@ async fn handle_take2_sent_committee( Some(height) => height as u64, None => { let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 block - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -6997,7 +7337,7 @@ async fn handle_take2_sent_committee( if goat_confirmed_height < take2_height { let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * (take2_height - goat_confirmed_height); - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( ctx.local_db, &message, delay_secs as usize, @@ -7412,6 +7752,98 @@ mod tests { } } + /// Verify retries fill missing CutCircuits entries without changing existing entries. + #[tokio::test] + async fn retry_after_a_cut_off_freeze_completes_the_cut_circuits() { + use bitcoin::secp256k1::{Secp256k1, SecretKey}; + + let local_db = store::create_local_db("sqlite::memory:").await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let package = build_setup_package(BABE_M_CC + 1).unwrap(); + let secp = Secp256k1::new(); + let mut state = OperatorBabeSetupState { + candidate_verifier_pubkeys: None, + candidates: (1..=3u8) + .map(|index| OperatorVerifierCandidate { + verifier_peer_id: PeerId::random().to_bytes(), + verifier_pubkey: PublicKey::new( + SecretKey::from_slice(&[index; 32]).unwrap().public_key(&secp), + ), + setup_package: package.clone(), + candidate_index: None, + selected_circuit_indexes: vec![], + gc_data: None, + soldering_proof_ready: None, + }) + .collect(), + candidate_collection_started_at: None, + proof_collection_started_at: None, + selected_verifier_pubkeys: None, + asserted_operator_proof: None, + }; + let now = current_time_secs(); + assert!(cut_circuits_owed(&state, now).is_empty(), "nothing is owed before the freeze"); + freeze_operator_candidates(&mut state).unwrap(); + let outbox_id = |candidate: &OperatorVerifierCandidate| { + format!("cut-circuits:{instance_id}:{graph_id}:{}", candidate.verifier_pubkey) + }; + let outbox_state = async |message_id: String| -> Option<(String, i64)> { + local_db.acquire().await.unwrap().p2p_outbox_entry_state(&message_id).await.unwrap() + }; + + // First run: the set is frozen and saved, one CutCircuits is written, and + // then the handler is cut off. + let owed = cut_circuits_owed(&state, now); + assert_eq!(owed.len(), 3); + ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed[..1]).await.unwrap(); + // That verifier acknowledges it before the retry comes round. + let first_peer = PeerId::from_bytes(&owed[0].verifier_peer_id).unwrap().to_string(); + assert!( + local_db + .acquire() + .await + .unwrap() + .acknowledge_p2p_outbox_message(&outbox_id(&owed[0]), &first_peer) + .await + .unwrap() + ); + assert!(outbox_state(outbox_id(&owed[1])).await.is_none(), "never written"); + + // The retry: the set is frozen already, and everything is still owed. + let owed = cut_circuits_owed(&state, now); + assert_eq!(owed.len(), 3); + ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed).await.unwrap(); + assert_eq!( + outbox_state(outbox_id(&owed[0])).await, + Some(("Processed".to_string(), 0)), + "the acknowledged entry is left as it is, not re-opened" + ); + for candidate in &owed[1..] { + let (entry_state, len) = outbox_state(outbox_id(candidate)).await.unwrap(); + assert_eq!(entry_state, "Pending"); + assert!(len > 0, "the missing CutCircuits is now queued for delivery"); + } + // And again: nothing changes. + ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed).await.unwrap(); + assert_eq!(outbox_state(outbox_id(&owed[0])).await, Some(("Processed".to_string(), 0))); + + // A candidate that has delivered its proof is owed nothing more ... + state.candidates[0].soldering_proof_ready = Some(soldering_proof_ready(1)); + assert_eq!(cut_circuits_owed(&state, now).len(), 2); + // ... nobody is once the proof window has closed with enough proofs in + // hand, so a late retry does not set verifiers garbling for nothing ... + let (_, gc_data, _) = gc_submission(); + state.candidates[0].gc_data = Some(gc_data); + state.proof_collection_started_at = Some(now); + assert_eq!(cut_circuits_owed(&state, now).len(), 2, "the window is still open"); + let after_window = now + get_verifier_candidate_collection_window_secs(); + assert!(cut_circuits_owed(&state, after_window).is_empty()); + // ... and nobody is once the selection is sealed. + state.proof_collection_started_at = None; + state.selected_verifier_pubkeys = Some(vec![state.candidates[0].verifier_pubkey]); + assert!(cut_circuits_owed(&state, now).is_empty()); + } + #[test] fn freeze_operator_candidate_selects_protocol_finalized_count() { let package = build_setup_package(BABE_M_CC + 1).unwrap(); @@ -7696,4 +8128,64 @@ mod tests { tx.base_size() ); } + #[tokio::test] + async fn late_round_messages_are_dropped_only_once_the_round_is_provably_over() { + let local_db = store::create_local_db("sqlite::memory:").await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + // No row yet: the full path must run, it is what asks for the graph. + assert!(!graph_signing_round_is_over(&local_db, instance_id, graph_id).await.unwrap()); + + let mut storage = local_db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&store::Graph { + graph_id, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_owned(), + ..Default::default() + }) + .await + .unwrap(); + drop(storage); + assert!(!graph_signing_round_is_over(&local_db, instance_id, graph_id).await.unwrap()); + + // Obsoleted straight from OperatorPresigned: the round never finished. + let obsoleted = Uuid::new_v4(); + let mut storage = local_db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&store::Graph { + graph_id: obsoleted, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_owned(), + ..Default::default() + }) + .await + .unwrap(); + storage + .transition_graph_status( + instance_id, + obsoleted, + GraphStatus::Obsoleted, + store::GraphStatusSource::ChainReconcile, + None, + ) + .await + .unwrap(); + storage + .transition_graph_status( + instance_id, + graph_id, + GraphStatus::CommitteePresigned, + store::GraphStatusSource::Definition, + None, + ) + .await + .unwrap(); + drop(storage); + assert!(!graph_signing_round_is_over(&local_db, instance_id, obsoleted).await.unwrap()); + assert!(graph_signing_round_is_over(&local_db, instance_id, graph_id).await.unwrap()); + // The row answers for its own instance only. + assert!(!graph_signing_round_is_over(&local_db, Uuid::new_v4(), graph_id).await.unwrap()); + } } diff --git a/node/src/scheduled_tasks/instance_maintenance_tasks.rs b/node/src/scheduled_tasks/instance_maintenance_tasks.rs index 6f94bc06..5faa781a 100644 --- a/node/src/scheduled_tasks/instance_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/instance_maintenance_tasks.rs @@ -1,7 +1,7 @@ use crate::action::{ ConfirmInstance, GOATMessage, GOATMessageContent, MessageDeferReason, PeginConfirmNonce, PeginConfirmNonceConsensus, PeginConfirmPartialSig, PeginRequest, PostReady, - RetryableDispatchError, RetryableDispatchReason, push_local_unhandled_messages_with_reason, + RetryableDispatchError, RetryableDispatchReason, defer_or_enqueue_message, }; use crate::env::{ COMMITTEE_INSTANCE_KEYS_DIR, get_bitvm_key, get_committee_instance_key_delete_timelock_blocks, @@ -625,7 +625,7 @@ pub async fn pegin_confirm_recovery_monitor( }), ); if !finish_recovery_enqueue( - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( local_db, &message, 0, @@ -685,7 +685,7 @@ pub async fn pegin_confirm_recovery_monitor( }), ); if !finish_recovery_enqueue( - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( local_db, &message, 0, @@ -720,7 +720,7 @@ pub async fn pegin_confirm_recovery_monitor( }), ); if !finish_recovery_enqueue( - push_local_unhandled_messages_with_reason( + defer_or_enqueue_message( local_db, &message, 0, From a5ee3ad6947876bed7571880ec50504f63be64de Mon Sep 17 00:00:00 2001 From: ethan Date: Wed, 23 Sep 2026 00:28:54 +0800 Subject: [PATCH 13/17] fix: recover exhausted graph setup delivery --- crates/store/src/localdb.rs | 59 ++++++++++++++++++++++++--- node/src/action.rs | 81 ++++++++++++++++++++++++++++++++++++- node/src/env.rs | 2 +- 3 files changed, 135 insertions(+), 7 deletions(-) diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index 3b6c2dfe..de4f4e54 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -3806,8 +3806,12 @@ impl<'a> StorageProcessor<'a> { /// End a bounded-retry message once its delivery window is exhausted. pub async fn expire_p2p_outbox_retry_messages(&mut self, now: i64) -> anyhow::Result { let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'RetryExhausted', content = X'', lease_until = 0, next_retry_at = 0, \ - retry_until = 0, retry_interval_secs = 0, ack_peer_id = '', \ + "UPDATE p2p_outbox SET state = 'RetryExhausted', \ + content = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN content ELSE X'' END, \ + lease_until = 0, next_retry_at = 0, \ + retry_until = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN retry_until ELSE 0 END, \ + retry_interval_secs = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN retry_interval_secs ELSE 0 END, \ + ack_peer_id = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN ack_peer_id ELSE '' END, \ last_error = 'retry window expired without expected ACK', updated_at = ? \ WHERE retry_until > 0 AND retry_until <= ? AND state IN ('Pending', 'Processing')", ) @@ -3818,6 +3822,45 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } + /// At most one retained setup payload per pass, oldest recovery first. + pub async fn next_exhausted_setup( + &mut self, + now: i64, + ) -> anyhow::Result, i64)>> { + Ok(sqlx::query_as( + "SELECT message_id, content, created_at FROM p2p_outbox \ + WHERE state = 'RetryExhausted' AND length(content) > 0 \ + AND msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') \ + AND updated_at <= ? ORDER BY updated_at, message_id LIMIT 1", + ) + .bind(now - 60) + .fetch_optional(self.conn()) + .await?) + } + + /// Called inside an immediate transaction: transfer bytes, never reopen a terminal row. + pub async fn recover_setup_delivery( + &mut self, + id: &str, + canonical_id: &str, + now: i64, + window: i64, + resume: bool, + ) -> anyhow::Result<()> { + if resume { + let next_id = format!("{canonical_id}:recovery:{now}"); + sqlx::query( + "INSERT INTO p2p_outbox (message_id,msg_type,content,state,attempt_count,next_retry_at,lease_until, \ + retry_until,retry_interval_secs,ack_peer_id,publish_count,created_at,updated_at) \ + SELECT ?,msg_type,content,'Pending',0,?,0,?,retry_interval_secs,ack_peer_id,publish_count,created_at,? \ + FROM p2p_outbox WHERE message_id = ? AND state = 'RetryExhausted' AND length(content) > 0", + ).bind(next_id).bind(now).bind(now + window).bind(now).bind(id).execute(self.conn()).await?; + } + sqlx::query("UPDATE p2p_outbox SET content = X'', updated_at = ? WHERE message_id = ? AND state = 'RetryExhausted'") + .bind(now).bind(id).execute(self.conn()).await?; + Ok(()) + } + /// Record a publish that went through and set when the row is next due. pub async fn schedule_p2p_outbox_retry( &mut self, @@ -3873,11 +3916,14 @@ impl<'a> StorageProcessor<'a> { ) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_outbox SET state = 'Processed', content = X'', lease_until = 0, next_retry_at = 0, updated_at = ? \ - WHERE message_id = ? AND retry_until > 0 AND ack_peer_id = ? \ - AND state IN ('Pending', 'Processing')", + WHERE (message_id = ? OR substr(message_id, 1, length(?) + 10) = ? || ':recovery:') \ + AND ack_peer_id = ? AND ack_peer_id != '' \ + AND state IN ('Pending', 'Processing', 'RetryExhausted')", ) .bind(get_current_timestamp_secs()) .bind(message_id) + .bind(message_id) + .bind(message_id) .bind(peer_id) .execute(self.conn()) .await?; @@ -3890,10 +3936,13 @@ impl<'a> StorageProcessor<'a> { let result = sqlx::query( "UPDATE p2p_outbox SET state = 'Cancelled', content = X'', lease_until = 0, next_retry_at = 0, \ updated_at = ? \ - WHERE message_id = ? AND state IN ('Pending', 'Processing')", + WHERE (message_id = ? OR substr(message_id, 1, length(?) + 10) = ? || ':recovery:') \ + AND state IN ('Pending', 'Processing', 'RetryExhausted')", ) .bind(get_current_timestamp_secs()) .bind(message_id) + .bind(message_id) + .bind(message_id) .execute(self.conn()) .await?; Ok(result.rows_affected() > 0) diff --git a/node/src/action.rs b/node/src/action.rs index 6700cf56..ab5c468f 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -2621,9 +2621,66 @@ async fn handle_p2p_outbox_messages( "outbound messages reached the end of their retry window: graph-setup messages without the expected ACK, signing-round messages without their round closing" ); } + recover_exhausted_setup(local_db).await?; drain_p2p_outbox(swarm, local_db, deadline).await } +/// Recovery is driven by durable delivery state even when no new gossip arrives. +async fn recover_exhausted_setup(local_db: &LocalDB) -> Result<()> { + let now = current_time_secs(); + let Some((id, bytes, created_at)) = local_db.acquire().await?.next_exhausted_setup(now).await? + else { + return Ok(()); + }; + let decoded = GOATMessage::deserialize_message(&bytes).await; + let mut storage = local_db.start_immediate_transaction().await?; + let mut resume = false; + let mut canonical = id.clone(); + if let Ok(message) = decoded + && let Some(key) = graph_setup_outbox_id(&message.content) + && let BusinessRef::Graph { instance_id, graph_id } = message.content.business_ref() + { + canonical = key; + let instance = storage.find_instance(&instance_id).await?; + let graph = storage.find_graph(&graph_id).await?; + resume = now - created_at < MESSAGE_EXPIRE_TIME + && graph.is_none() + && instance.is_some_and(|row| { + matches!( + InstanceBridgeInStatus::from_str(&row.status), + Ok(InstanceBridgeInStatus::UserBroadcastPeginPrepare + | InstanceBridgeInStatus::Presigned + | InstanceBridgeInStatus::RelayerL1Broadcasted + | InstanceBridgeInStatus::RelayerL2Minted) + ) + }); + } + storage + .recover_setup_delivery( + &id, + &canonical, + now, + get_p2p_graph_setup_retry_window_secs(), + resume, + ) + .await?; + storage.commit().await?; + tracing::info!(event = "setup_delivery_recovery", message_id = %id, resume, + "settled exhausted setup delivery"); + Ok(()) +} + +fn setup_retry_interval_secs(publish_count: i64, base: i64, id: &str) -> i64 { + let base = base.max(1); + let delay = base + .saturating_mul(1_i64 << publish_count.saturating_sub(1).clamp(0, 20)) + .min(base.max(60)); + // Stable jitter keeps retries reproducible across restarts without synchronizing peers. + let jitter = + id.bytes().fold(publish_count as u64, |n, b| n.wrapping_mul(31).wrapping_add(b as u64)) % 7; + delay + jitter as i64 +} + async fn drain_p2p_outbox( swarm: &mut dyn MessagePublisher, local_db: &LocalDB, @@ -2698,6 +2755,24 @@ async fn publish_p2p_outbox_row( return Ok(OutboxRowOutcome::Closed); } }; + if let Some(canonical_id) = graph_setup_outbox_id(&outbound.content) + && let BusinessRef::Graph { instance_id, graph_id } = outbound.content.business_ref() + { + let mut storage = local_db.acquire().await?; + let instance = storage.find_instance(&instance_id).await?; + let failed = instance.as_ref().is_some_and(|row| { + InstanceBridgeInStatus::from_str(&row.status).is_ok_and(|status| { + SigningRound::GraphSigning.delivery_finished(None, Some(&status)) + }) + }); + if failed + || storage.find_graph(&graph_id).await?.is_some() + || current_time_secs() - message.created_at >= MESSAGE_EXPIRE_TIME + { + storage.cancel_p2p_outbox_message(&canonical_id).await?; + return Ok(OutboxRowOutcome::Closed); + } + } let signing_round = if message.message_id.starts_with(PROTOCOL_OUTBOX_PREFIX) { let round = SigningRound::of(&outbound.content) .filter(|_| outbound.content.business_ref() != BusinessRef::Unscoped); @@ -2738,7 +2813,11 @@ async fn publish_p2p_outbox_row( message.publish_count + 1, round.retry_ceiling_secs(crate::env::get_p2p_protocol_retry_max_secs()), ), - None => message.retry_interval_secs, + None => setup_retry_interval_secs( + message.publish_count + 1, + message.retry_interval_secs, + &message.message_id, + ), }; let next_retry_at = current_time_secs() + interval; storage.schedule_p2p_outbox_retry(&message.message_id, next_retry_at).await?; diff --git a/node/src/env.rs b/node/src/env.rs index 66318658..6b209fb0 100644 --- a/node/src/env.rs +++ b/node/src/env.rs @@ -80,7 +80,7 @@ pub const DEFAULT_MIN_REQUIRED_VERIFIER: usize = 1; pub const DEFAULT_VERIFIER_CANDIDATE_BACKUP_COUNT: usize = 2; pub const DEFAULT_VERIFIER_CANDIDATE_COLLECTION_WINDOW_SECS: i64 = 15; pub const DEFAULT_P2P_GRAPH_SETUP_RETRY_INTERVAL_SECS: i64 = 10; -pub const DEFAULT_P2P_GRAPH_SETUP_RETRY_WINDOW_SECS: i64 = 180; +pub const DEFAULT_P2P_GRAPH_SETUP_RETRY_WINDOW_SECS: i64 = 600; pub const ENV_BITCOIN_NETWORK: &str = "BITCOIN_NETWORK"; pub const ENV_GOAT_NETWORK: &str = "GOAT_NETWORK"; From eb13ff54335911e461e376f99788020b6a729ff5 Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 28 Sep 2026 14:00:08 +0800 Subject: [PATCH 14/17] fix: bind graph setup ACKs to payload and peers --- crates/store/src/localdb.rs | 29 ++++++---- node/src/action.rs | 105 +++++++++++++++++++++++------------- node/src/handle.rs | 33 +++++------- 3 files changed, 99 insertions(+), 68 deletions(-) diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index de4f4e54..df80f10a 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -3729,6 +3729,7 @@ impl<'a> StorageProcessor<'a> { pub async fn enqueue_p2p_outbox_retry_message( &mut self, message_id: &str, + delivery_id: Option<&str>, msg_type: &str, content: &[u8], retry_until: i64, @@ -3739,11 +3740,12 @@ impl<'a> StorageProcessor<'a> { let now = get_current_timestamp_secs(); let result = sqlx::query( "INSERT INTO p2p_outbox \ - (message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, retry_until, retry_interval_secs, ack_peer_id, created_at, updated_at) \ - VALUES (?, ?, ?, 'Pending', 0, ?, 0, ?, ?, ?, ?, ?) \ + (message_id, delivery_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, retry_until, retry_interval_secs, ack_peer_id, created_at, updated_at) \ + VALUES (?, ?, ?, ?, 'Pending', 0, ?, 0, ?, ?, ?, ?, ?) \ ON CONFLICT(message_id) DO NOTHING", ) .bind(message_id) + .bind(delivery_id.unwrap_or_default()) .bind(msg_type) .bind(content) .bind(first_publish_at) @@ -3850,9 +3852,9 @@ impl<'a> StorageProcessor<'a> { if resume { let next_id = format!("{canonical_id}:recovery:{now}"); sqlx::query( - "INSERT INTO p2p_outbox (message_id,msg_type,content,state,attempt_count,next_retry_at,lease_until, \ + "INSERT INTO p2p_outbox (message_id,delivery_id,msg_type,content,state,attempt_count,next_retry_at,lease_until, \ retry_until,retry_interval_secs,ack_peer_id,publish_count,created_at,updated_at) \ - SELECT ?,msg_type,content,'Pending',0,?,0,?,retry_interval_secs,ack_peer_id,publish_count,created_at,? \ + SELECT ?,delivery_id,msg_type,content,'Pending',0,?,0,?,retry_interval_secs,ack_peer_id,publish_count,created_at,? \ FROM p2p_outbox WHERE message_id = ? AND state = 'RetryExhausted' AND length(content) > 0", ).bind(next_id).bind(now).bind(now + window).bind(now).bind(id).execute(self.conn()).await?; } @@ -3909,21 +3911,28 @@ impl<'a> StorageProcessor<'a> { row.map(|row| Ok((row.try_get("state")?, row.try_get("content_len")?))).transpose() } - pub async fn acknowledge_p2p_outbox_message( + pub async fn p2p_outbox_delivery_id( &mut self, message_id: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_scalar("SELECT delivery_id FROM p2p_outbox WHERE message_id = ?") + .bind(message_id) + .fetch_optional(self.conn()) + .await?) + } + + pub async fn acknowledge_p2p_outbox_message( + &mut self, + delivery_id: &str, peer_id: &str, ) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_outbox SET state = 'Processed', content = X'', lease_until = 0, next_retry_at = 0, updated_at = ? \ - WHERE (message_id = ? OR substr(message_id, 1, length(?) + 10) = ? || ':recovery:') \ - AND ack_peer_id = ? AND ack_peer_id != '' \ + WHERE delivery_id = ? AND delivery_id != '' AND ack_peer_id = ? AND ack_peer_id != '' \ AND state IN ('Pending', 'Processing', 'RetryExhausted')", ) .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(message_id) - .bind(message_id) + .bind(delivery_id) .bind(peer_id) .execute(self.conn()) .await?; diff --git a/node/src/action.rs b/node/src/action.rs index ab5c468f..fe34f26d 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -554,7 +554,7 @@ pub enum GOATMessageContent { CutCircuits(CutCircuits), #[business_ref(graph)] SolderingProofReady(SolderingProofReady), - #[business_ref(graph)] + #[business_ref(unscoped)] GraphSetupAck(GraphSetupAck), #[business_ref(graph)] VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement), @@ -867,20 +867,37 @@ pub struct SolderingProofReady { pub total_len: usize, } -#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] -pub enum GraphSetupStage { - GenCircuits, - CutCircuits, - SolderingProofReady, -} - #[derive(Debug, Serialize, Deserialize, Clone)] pub struct GraphSetupAck { - pub outbox_id: String, - pub instance_id: Uuid, - pub graph_id: Uuid, - pub stage: GraphSetupStage, - pub acknowledger_peer_id: String, + pub delivery_id: String, +} + +fn graph_setup_payload_hash(payload: &[u8]) -> [u8; 32] { + Sha256::digest(payload).into() +} + +fn graph_setup_delivery_id( + sender_peer_id: &str, + recipient_peer_id: &str, + payload_hash: &[u8; 32], +) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"goat/graph-setup-delivery/v2"); + for field in [sender_peer_id.as_bytes(), recipient_peer_id.as_bytes(), payload_hash.as_slice()] + { + hasher.update((field.len() as u64).to_be_bytes()); + hasher.update(field); + } + format!("graph-setup-v2:{}", hex::encode(hasher.finalize())) +} + +fn graph_setup_ack_required(content: &GOATMessageContent) -> bool { + matches!( + content, + GOATMessageContent::GenCircuits(_) + | GOATMessageContent::CutCircuits(_) + | GOATMessageContent::SolderingProofReady(_) + ) } pub fn graph_setup_outbox_id(content: &GOATMessageContent) -> Option { @@ -904,25 +921,20 @@ pub fn graph_setup_outbox_id(content: &GOATMessageContent) -> Option { } } -fn graph_setup_ack(content: &GOATMessageContent) -> Option { - let (instance_id, graph_id, stage) = match content { - GOATMessageContent::GenCircuits(message) => { - (message.instance_id, message.graph_id, GraphSetupStage::GenCircuits) - } - GOATMessageContent::CutCircuits(message) => { - (message.instance_id, message.graph_id, GraphSetupStage::CutCircuits) - } - GOATMessageContent::SolderingProofReady(message) => { - (message.instance_id, message.graph_id, GraphSetupStage::SolderingProofReady) - } - _ => return None, - }; +fn graph_setup_ack( + content: &GOATMessageContent, + sender_peer_id: &PeerId, + payload: &[u8], +) -> Option { + if !graph_setup_ack_required(content) { + return None; + } + // Bind to the authenticated gossip author, never an identity from the payload. + let sender_peer_id = sender_peer_id.to_string(); + let recipient_peer_id = crate::env::get_peer_id(); + let payload_hash = graph_setup_payload_hash(payload); Some(GraphSetupAck { - outbox_id: graph_setup_outbox_id(content)?, - instance_id, - graph_id, - stage, - acknowledger_peer_id: crate::env::get_peer_id(), + delivery_id: graph_setup_delivery_id(&sender_peer_id, &recipient_peer_id, &payload_hash), }) } @@ -934,17 +946,32 @@ pub async fn enqueue_graph_setup_outbox_message( let outbox_id = graph_setup_outbox_id(&message.content) .ok_or_else(|| anyhow!("not a graph setup message"))?; let serialized = message.serialize_message().await?; + let sender = crate::env::get_peer_id(); + let recipient = if graph_setup_ack_required(&message.content) { + let ack_peer_id = ack_peer_id.context("graph setup ACK requires a recipient peer")?; + Some( + PeerId::from_str(ack_peer_id) + .context("invalid graph setup ACK recipient peer id")? + .to_string(), + ) + } else { + None + }; + let delivery_id = recipient.as_ref().map(|recipient| { + graph_setup_delivery_id(&sender, recipient, &graph_setup_payload_hash(&serialized)) + }); let now = current_time_secs(); local_db .acquire() .await? .enqueue_p2p_outbox_retry_message( &outbox_id, + delivery_id.as_deref(), message.content.event_type(), &serialized, now + get_p2p_graph_setup_retry_window_secs(), get_p2p_graph_setup_retry_interval_secs(), - ack_peer_id, + recipient.as_deref(), // Only the outbox ever sends these, so the row is due at once. 0, ) @@ -1392,7 +1419,7 @@ pub async fn handle_inbound_p2p_message( report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); metrics_state.record_p2p_receive(true); refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; - maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, now).await; + maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, message, now).await; Ok(()) } // Forward duplicates with rate-limited ACKs and logs. @@ -1401,7 +1428,7 @@ pub async fn handle_inbound_p2p_message( metrics_state.record_p2p_receive(true); p2p_admission::record_drop(p2p_admission::DropReason::DuplicatePayload); refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; - maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, now).await; + maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, message, now).await; Ok(()) } InboundVerdict::Immediate(decoded) => { @@ -1514,18 +1541,18 @@ async fn refresh_peer_timestamp_throttled(local_db: &LocalDB, from_peer_id: Peer } } -/// Rate-limit graph-setup ACKs by sender and outbox slot. +/// Rate-limit graph-setup ACKs by delivery identity (including both peers). async fn maybe_send_graph_setup_ack( swarm: &mut dyn MessagePublisher, decoded: &GOATMessage, from_peer_id: PeerId, + payload: &[u8], now: Instant, ) { - let Some(ack) = graph_setup_ack(&decoded.content) else { + let Some(ack) = graph_setup_ack(&decoded.content, &from_peer_id, payload) else { return; }; - let key = format!("{from_peer_id}:{}", ack.outbox_id); - if !crate::p2p_admission::dedup_ack_gate().allow(&key, now) { + if !crate::p2p_admission::dedup_ack_gate().allow(&ack.delivery_id, now) { return; } if let Err(error) = @@ -3569,6 +3596,7 @@ async fn store_protocol_message(local_db: &LocalDB, message: &GOATMessage) -> Re .await? .enqueue_p2p_outbox_retry_message( &protocol_outbox_id(&message.content), + None, message.content.event_type(), &bytes, now + MESSAGE_EXPIRE_TIME, @@ -4255,6 +4283,7 @@ mod tests { .unwrap() .enqueue_p2p_outbox_retry_message( "protocol:AggNonceConsensus:graph:corrupt", + None, "AggNonceConsensus", b"not a message", now + 600, diff --git a/node/src/handle.rs b/node/src/handle.rs index 10b13f4a..c42fbf1d 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -227,31 +227,16 @@ fn load_or_create_committee_instance_keypair( pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent) -> Result<()> { match (content, &ctx.actor) { - ( - GOATMessageContent::GraphSetupAck(GraphSetupAck { - outbox_id, - acknowledger_peer_id, - .. - }), - _, - ) => { - if acknowledger_peer_id != &ctx.from_peer_id.to_string() { - tracing::warn!( - outbox_id, - from_peer_id = %ctx.from_peer_id, - acknowledger_peer_id, - "Ignore GraphSetupAck with mismatched source peer" - ); - return Ok(()); - } + (GOATMessageContent::GraphSetupAck(GraphSetupAck { delivery_id }), _) => { + let source_peer_id = ctx.from_peer_id.to_string(); let acknowledged = ctx .local_db .acquire() .await? - .acknowledge_p2p_outbox_message(outbox_id, acknowledger_peer_id) + .acknowledge_p2p_outbox_message(delivery_id, &source_peer_id) .await?; tracing::debug!( - outbox_id, + delivery_id, from_peer_id = %ctx.from_peer_id, acknowledged, "processed GraphSetupAck" @@ -7798,12 +7783,20 @@ mod tests { ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed[..1]).await.unwrap(); // That verifier acknowledges it before the retry comes round. let first_peer = PeerId::from_bytes(&owed[0].verifier_peer_id).unwrap().to_string(); + let first_delivery_id = local_db + .acquire() + .await + .unwrap() + .p2p_outbox_delivery_id(&outbox_id(&owed[0])) + .await + .unwrap() + .unwrap(); assert!( local_db .acquire() .await .unwrap() - .acknowledge_p2p_outbox_message(&outbox_id(&owed[0]), &first_peer) + .acknowledge_p2p_outbox_message(&first_delivery_id, &first_peer) .await .unwrap() ); From 7434821e6e54f344b301c7f09b2c3ddc17078477 Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 28 Sep 2026 19:20:37 +0800 Subject: [PATCH 15/17] fix: reject duplicate verifier peers in graph endorsements --- ...000000_add_p2p_setup_delivery_metadata.sql | 4 ++ node/src/handle.rs | 19 +++--- node/src/utils.rs | 62 ++++++++++++++----- 3 files changed, 61 insertions(+), 24 deletions(-) create mode 100644 crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql diff --git a/crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql b/crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql new file mode 100644 index 00000000..45aa50d8 --- /dev/null +++ b/crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql @@ -0,0 +1,4 @@ +ALTER TABLE p2p_outbox ADD COLUMN delivery_id TEXT NOT NULL DEFAULT ''; + +CREATE INDEX IF NOT EXISTS idx_p2p_outbox_delivery_id + ON p2p_outbox (delivery_id, state); diff --git a/node/src/handle.rs b/node/src/handle.rs index c42fbf1d..8d68a6b1 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -1963,11 +1963,9 @@ async fn handle_gen_circuits_operator( if existing.setup_package != *setup_package { bail!("conflicting GenCircuits setup package for verifier {verifier_pubkey}"); } - } else if operator_state - .candidates - .iter() - .any(|candidate| candidate.verifier_pubkey == *verifier_pubkey) - { + } else if operator_state.candidates.iter().any(|candidate| { + XOnlyPublicKey::from(candidate.verifier_pubkey) == XOnlyPublicKey::from(*verifier_pubkey) + }) { tracing::warn!( "Ignore GenCircuits for {instance_id}:{graph_id}: verifier public key {verifier_pubkey} is already bound to another peer" ); @@ -3243,6 +3241,12 @@ async fn maybe_vote_and_presign_graph( })?; } + // Recheck the persisted bindings at the actual signing boundary, including recovery. + let verifier_endorsements = + get_verifier_graph_params_endorsements_for_graph(ctx.local_db, instance_id, graph_id) + .await?; + validate_verifier_graph_params_endorsements(ctx.goat_client, graph, &verifier_endorsements) + .await?; let (_, sec_nonces, _) = committee_master_key.nonces_for_graph_job_with_keypair(&full_graph, instance_keypair)?; let committee_partial_sigs = @@ -3431,9 +3435,7 @@ async fn handle_verifier_graph_params_endorsement_committee( signature: &secp256k1::schnorr::Signature, content: &GOATMessageContent, ) -> Result<()> { - if !ctx.is_self_peer - && !ctx.goat_client.committee_mana_is_verifier(&ctx.from_peer_id.to_bytes()).await? - { + if !ctx.goat_client.committee_mana_is_verifier(&ctx.from_peer_id.to_bytes()).await? { tracing::warn!( "Ignore VerifierGraphParamsEndorsement for {instance_id}:{graph_id}: sender {} is not a registered verifier", ctx.from_peer_id @@ -3487,6 +3489,7 @@ async fn handle_verifier_graph_params_endorsement_committee( graph_id, *verifier_pubkey, verifier_index, + &ctx.from_peer_id.to_string(), *signature, ) .await?; diff --git a/node/src/utils.rs b/node/src/utils.rs index 9598c86b..91329f62 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -300,7 +300,7 @@ fn validate_graph_policy(parameters: &BitvmGcGraphParameters) -> Result<()> { let mut verifier_pubkeys = std::collections::HashSet::new(); for gc_data in ¶meters.gc_data { - if !verifier_pubkeys.insert(gc_data.verifier_pubkey) { + if !verifier_pubkeys.insert(XOnlyPublicKey::from(gc_data.verifier_pubkey)) { bail!(SpecialError::InvalidGraph(format!( "duplicate verifier pubkey in GC data: {}", gc_data.verifier_pubkey @@ -493,9 +493,10 @@ pub async fn validate_operator_stake( } } -pub fn validate_verifier_graph_params_endorsements( +pub async fn validate_verifier_graph_params_endorsements( + goat_client: &GOATClient, graph: &SimplifiedBitvmGcGraph, - verifier_endorsements: &[(PublicKey, usize, SchnorrSignature)], + verifier_endorsements: &[(PublicKey, usize, String, SchnorrSignature)], ) -> Result<()> { let expected_verifier_num = graph.parameters.gc_data.len(); if verifier_endorsements.len() != expected_verifier_num { @@ -508,7 +509,13 @@ pub fn validate_verifier_graph_params_endorsements( let mut seen_pubkeys = std::collections::HashSet::new(); let mut seen_indices = std::collections::HashSet::new(); - for (verifier_pubkey, verifier_index, signature) in verifier_endorsements { + let mut seen_peers = std::collections::HashSet::new(); + for (verifier_pubkey, verifier_index, verifier_peer_id, signature) in verifier_endorsements { + let peer = + PeerId::from_str(verifier_peer_id).context("invalid verifier endorsement peer")?; + if !seen_peers.insert(peer) { + bail!(SpecialError::InvalidGraph("duplicate verifier endorsement peer".into())); + } if *verifier_index >= expected_verifier_num { bail!(SpecialError::InvalidGraph(format!( "verifier params endorsement index {verifier_index} out of range" @@ -521,7 +528,7 @@ pub fn validate_verifier_graph_params_endorsements( verifier_pubkey ))); } - if !seen_pubkeys.insert(*verifier_pubkey) { + if !seen_pubkeys.insert(XOnlyPublicKey::from(*verifier_pubkey)) { bail!(SpecialError::InvalidGraph(format!( "duplicate verifier params endorsement pubkey: {verifier_pubkey}" ))); @@ -539,6 +546,18 @@ pub fn validate_verifier_graph_params_endorsements( } } + for (_, _, peer, _) in verifier_endorsements { + let peer = PeerId::from_str(peer)?; + if !goat_client + .committee_mana_is_verifier(&peer.to_bytes()) + .await + .context("recheck verifier registration before committee setup")? + { + bail!(SpecialError::InvalidGraph( + "endorsement peer is not a registered verifier".into() + )); + } + } Ok(()) } @@ -547,11 +566,10 @@ pub async fn validate_init_graph( btc_client: &BTCClient, goat_client: &GOATClient, graph: &SimplifiedBitvmGcGraph, - verifier_endorsements: &[(PublicKey, usize, SchnorrSignature)], + verifier_endorsements: &[(PublicKey, usize, String, SchnorrSignature)], ) -> Result<()> { validate_init_graph_base(local_db, btc_client, goat_client, graph).await?; - validate_verifier_graph_params_endorsements(graph, verifier_endorsements)?; - Ok(()) + validate_verifier_graph_params_endorsements(goat_client, graph, verifier_endorsements).await } pub async fn validate_finalized_graph( btc_client: &BTCClient, @@ -4304,6 +4322,7 @@ pub struct GraphProcessDataItem { pub endorse_signature: Vec, pub params_endorse_signature: Vec, pub verifier_index: Option, + pub verifier_peer_id: Option, pub verifier_params_signature: Option, } pub type GraphProcessDataMap = IndexMap; @@ -5136,12 +5155,9 @@ pub async fn find_pegin_graph_process_data( storage_processor: &mut StorageProcessor<'_>, graph_id: Uuid, ) -> Result<(bool, GraphProcessDataMap)> { - if let Ok(Some(data)) = storage_processor.find_pegin_graph_process_data(&graph_id).await - && let Ok(process_data) = serde_json::from_str(data.process_data.as_str()) - { - Ok((data.is_endorsed, process_data)) - } else { - Ok((false, IndexMap::new())) + match storage_processor.find_pegin_graph_process_data(&graph_id).await? { + Some(data) => Ok((data.is_endorsed, serde_json::from_str(&data.process_data)?)), + None => Ok((false, IndexMap::new())), } } @@ -5454,11 +5470,22 @@ pub async fn store_verifier_graph_params_endorsement( graph_id: Uuid, verifier_pubkey: PublicKey, verifier_index: usize, + verifier_peer_id: &str, signature: SchnorrSignature, ) -> Result<()> { let mut storage_processor = local_db.acquire().await?; let (is_endorsed, mut process_data) = find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + for (key, item) in &process_data { + if let Some(peer) = &item.verifier_peer_id + && ((*key == verifier_pubkey && peer != verifier_peer_id) + || (*key != verifier_pubkey + && (peer == verifier_peer_id + || XOnlyPublicKey::from(*key) == XOnlyPublicKey::from(verifier_pubkey)))) + { + bail!(SpecialError::InvalidGraph("conflicting verifier peer/key binding".into())); + } + } if let Some(existing_index) = process_data.get(&verifier_pubkey).and_then(|v| v.verifier_index) && existing_index != verifier_index { @@ -5478,10 +5505,12 @@ pub async fn store_verifier_graph_params_endorsement( .entry(verifier_pubkey) .and_modify(|v| { v.verifier_index = Some(verifier_index); + v.verifier_peer_id = Some(verifier_peer_id.to_owned()); v.verifier_params_signature = Some(signature); }) .or_insert_with(|| GraphProcessDataItem { verifier_index: Some(verifier_index), + verifier_peer_id: Some(verifier_peer_id.to_owned()), verifier_params_signature: Some(signature), ..Default::default() }); @@ -5500,7 +5529,7 @@ pub async fn get_verifier_graph_params_endorsements_for_graph( local_db: &LocalDB, _instance_id: Uuid, graph_id: Uuid, -) -> Result> { +) -> Result> { let mut storage_processor = local_db.acquire().await?; let (_is_endorsed, process_data) = find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; @@ -5509,7 +5538,8 @@ pub async fn get_verifier_graph_params_endorsements_for_graph( .filter_map(|(k, v)| { v.verifier_index .zip(v.verifier_params_signature.as_ref()) - .map(|(index, signature)| (*k, index, *signature)) + .zip(v.verifier_peer_id.as_ref()) + .map(|((index, signature), peer)| (*k, index, peer.clone(), *signature)) }) .collect()) } From c6a8743371d76775fb716dc8b4fccc72d63944cf Mon Sep 17 00:00:00 2001 From: ethan Date: Mon, 28 Sep 2026 23:51:29 +0800 Subject: [PATCH 16/17] fix ci --- node/src/action.rs | 14 ++++++- node/src/handle.rs | 26 +++++++++--- node/src/rpc_service/mod.rs | 84 ++++++++++++++++++++++--------------- 3 files changed, 83 insertions(+), 41 deletions(-) diff --git a/node/src/action.rs b/node/src/action.rs index fe34f26d..bfde134c 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -942,11 +942,21 @@ pub async fn enqueue_graph_setup_outbox_message( local_db: &LocalDB, message: GOATMessage, ack_peer_id: Option<&str>, +) -> Result { + let sender_peer_id = crate::env::get_peer_id(); + enqueue_graph_setup_outbox_message_from_peer(local_db, message, ack_peer_id, &sender_peer_id) + .await +} + +pub(crate) async fn enqueue_graph_setup_outbox_message_from_peer( + local_db: &LocalDB, + message: GOATMessage, + ack_peer_id: Option<&str>, + sender: &str, ) -> Result { let outbox_id = graph_setup_outbox_id(&message.content) .ok_or_else(|| anyhow!("not a graph setup message"))?; let serialized = message.serialize_message().await?; - let sender = crate::env::get_peer_id(); let recipient = if graph_setup_ack_required(&message.content) { let ack_peer_id = ack_peer_id.context("graph setup ACK requires a recipient peer")?; Some( @@ -958,7 +968,7 @@ pub async fn enqueue_graph_setup_outbox_message( None }; let delivery_id = recipient.as_ref().map(|recipient| { - graph_setup_delivery_id(&sender, recipient, &graph_setup_payload_hash(&serialized)) + graph_setup_delivery_id(sender, recipient, &graph_setup_payload_hash(&serialized)) }); let now = current_time_secs(); local_db diff --git a/node/src/handle.rs b/node/src/handle.rs index 8d68a6b1..d36521bf 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -2027,9 +2027,10 @@ async fn handle_gen_circuits_operator( } // On every retry, enqueue any CutCircuits still owed by the frozen candidate set. let owed = cut_circuits_owed(operator_state, current_time_secs()); + let sender_peer_id = get_peer_id(); save_babe_setup_state(ctx.local_db, instance_id, graph_id, &state)?; - ensure_cut_circuits_outbox(ctx.local_db, instance_id, graph_id, &owed).await?; + ensure_cut_circuits_outbox(ctx.local_db, instance_id, graph_id, &owed, &sender_peer_id).await?; Ok(()) } @@ -2061,6 +2062,7 @@ async fn ensure_cut_circuits_outbox( instance_id: Uuid, graph_id: Uuid, owed: &[OperatorVerifierCandidate], + sender_peer_id: &str, ) -> Result<()> { for candidate in owed { let message = GOATMessage::new( @@ -2077,7 +2079,13 @@ async fn ensure_cut_circuits_outbox( ); let ack_peer_id = PeerId::from_bytes(&candidate.verifier_peer_id).map(|peer_id| peer_id.to_string()).ok(); - enqueue_graph_setup_outbox_message(local_db, message, ack_peer_id.as_deref()).await?; + enqueue_graph_setup_outbox_message_from_peer( + local_db, + message, + ack_peer_id.as_deref(), + sender_peer_id, + ) + .await?; } Ok(()) } @@ -7745,6 +7753,8 @@ mod tests { async fn retry_after_a_cut_off_freeze_completes_the_cut_circuits() { use bitcoin::secp256k1::{Secp256k1, SecretKey}; + let sender_peer_id = generate_local_key().public().to_peer_id().to_string(); + let local_db = store::create_local_db("sqlite::memory:").await; let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); let package = build_setup_package(BABE_M_CC + 1).unwrap(); @@ -7783,7 +7793,9 @@ mod tests { // then the handler is cut off. let owed = cut_circuits_owed(&state, now); assert_eq!(owed.len(), 3); - ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed[..1]).await.unwrap(); + ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed[..1], &sender_peer_id) + .await + .unwrap(); // That verifier acknowledges it before the retry comes round. let first_peer = PeerId::from_bytes(&owed[0].verifier_peer_id).unwrap().to_string(); let first_delivery_id = local_db @@ -7808,7 +7820,9 @@ mod tests { // The retry: the set is frozen already, and everything is still owed. let owed = cut_circuits_owed(&state, now); assert_eq!(owed.len(), 3); - ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed).await.unwrap(); + ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed, &sender_peer_id) + .await + .unwrap(); assert_eq!( outbox_state(outbox_id(&owed[0])).await, Some(("Processed".to_string(), 0)), @@ -7820,7 +7834,9 @@ mod tests { assert!(len > 0, "the missing CutCircuits is now queued for delivery"); } // And again: nothing changes. - ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed).await.unwrap(); + ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed, &sender_peer_id) + .await + .unwrap(); assert_eq!(outbox_state(outbox_id(&owed[0])).await, Some(("Processed".to_string(), 0))); // A candidate that has delivered its proof is owed nothing more ... diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index 621f1799..39b94769 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -189,6 +189,17 @@ pub async fn serve_with_app_state( addr: String, app_state: Arc, cancellation_token: CancellationToken, +) -> anyhow::Result { + let listener = TcpListener::bind(&addr) + .await + .with_context(|| format!("failed to bind RPC listener to {addr}"))?; + serve_with_listener(listener, app_state, cancellation_token).await +} + +async fn serve_with_listener( + listener: TcpListener, + app_state: Arc, + cancellation_token: CancellationToken, ) -> anyhow::Result { let node_span = tracing::Span::current(); let server = build_business_router(app_state.clone()) @@ -235,9 +246,6 @@ pub async fn serve_with_app_state( ) .layer(middleware::from_fn_with_state(app_state, metrics_middleware)); - let listener = TcpListener::bind(&addr) - .await - .with_context(|| format!("failed to bind RPC listener to {addr}"))?; let listening_addr = listener.local_addr().context("failed to determine RPC listener address")?; tracing::info!( @@ -360,7 +368,7 @@ mod tests { InstanceListResponse, InstanceOverviewResponse, InstanceSettingResponse, }; use crate::rpc_service::node::{NodeListResponse, NodeOverViewResponse}; - use crate::rpc_service::{self, Actor, current_time_secs, routes}; + use crate::rpc_service::{self, Actor, AppState, current_time_secs, routes}; use crate::utils::{ generate_local_key, generate_random_bytes, get_rand_btc_address_p2wpkh, get_rand_goat_address, temp_sqlite_db_path, @@ -380,7 +388,7 @@ mod tests { Graph, GraphStatus, GraphStatusSource, Instance, InstanceBridgeInStatus, Node, create_local_db, }; - use tokio::time::sleep; + use tokio::{net::TcpListener, time::sleep}; use tokio_util::sync::CancellationToken; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -471,11 +479,6 @@ mod tests { let _ = tracing_subscriber::fmt().with_env_filter(EnvFilter::from_default_env()).try_init(); } - fn available_addr() -> String { - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - listener.local_addr().unwrap().to_string() - } - async fn spawn_metrics_listener( app_state: Arc, metrics_path: &str, @@ -610,11 +613,12 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn metrics_are_served_only_by_the_dedicated_listener() -> Result<(), Box> { - let addr = available_addr(); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); let app_state = mock_app_state().await?; let cancellation_token = CancellationToken::new(); - let server = tokio::spawn(rpc_service::serve_with_app_state( - addr.clone(), + let server = tokio::spawn(rpc_service::serve_with_listener( + listener, app_state.clone(), cancellation_token.clone(), )); @@ -797,7 +801,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_nodes_api() -> Result<(), Box> { init(None); - let addr = available_addr(); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); let mut nodes = Vec::::new(); let (_, public_key) = Secp256k1::new().generate_keypair(&mut rand::thread_rng()); let pub_key = public_key.to_string(); @@ -840,12 +845,15 @@ mod tests { let local_db = create_local_db(&temp_sqlite_db_path()).await; init_nodes_data(&local_db, &nodes).await?; - tokio::spawn(rpc_service::serve( - addr.clone(), - local_db, - Actor::Verifier, - generate_local_key().public().to_peer_id().to_string(), - MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + tokio::spawn(rpc_service::serve_with_listener( + listener, + AppState::create_arc_app_state( + local_db.clone(), + Actor::Verifier, + generate_local_key().public().to_peer_id().to_string(), + MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + ) + .await?, CancellationToken::new(), )); sleep(Duration::from_secs(3)).await; @@ -885,7 +893,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_bitvm_api() -> Result<(), Box> { init(None); - let addr = available_addr(); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); let actor = Actor::Verifier; let local_key = generate_local_key(); let peer_id = local_key.public().to_peer_id().to_string(); @@ -1035,12 +1044,15 @@ mod tests { init_instance_graph_data(&local_db, &instances, &graphs).await?; - tokio::spawn(rpc_service::serve( - addr.clone(), - local_db.clone(), - actor.clone(), - peer_id.clone(), - MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + tokio::spawn(rpc_service::serve_with_listener( + listener, + AppState::create_arc_app_state( + local_db.clone(), + actor.clone(), + peer_id.clone(), + MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + ) + .await?, CancellationToken::new(), )); sleep(Duration::from_secs(3)).await; @@ -1155,17 +1167,21 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_proof_api() -> Result<(), Box> { init(None); - let addr = available_addr(); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); info!("Start api server"); let committee = Actor::Committee; let committee_peer_id = generate_local_key().public().to_peer_id().to_string(); let local_db = create_local_db(&temp_sqlite_db_path()).await; - tokio::spawn(rpc_service::serve( - addr.clone(), - local_db, - committee, - committee_peer_id, - MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + tokio::spawn(rpc_service::serve_with_listener( + listener, + AppState::create_arc_app_state( + local_db, + committee, + committee_peer_id, + MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + ) + .await?, CancellationToken::new(), )); sleep(Duration::from_secs(3)).await; From 3271ed0a44fd0c2641908e4a4ef4653d79887824 Mon Sep 17 00:00:00 2001 From: ethan Date: Tue, 29 Sep 2026 22:03:35 +0800 Subject: [PATCH 17/17] feat: Add CreateGraph delivery ACK retries --- crates/store/src/localdb.rs | 14 ++++++------- node/src/action.rs | 41 +++++++++++++++++++++++++++++-------- node/src/handle.rs | 39 ++++++++++++++++++++++++++--------- 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index df80f10a..69b9a090 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -3809,11 +3809,11 @@ impl<'a> StorageProcessor<'a> { pub async fn expire_p2p_outbox_retry_messages(&mut self, now: i64) -> anyhow::Result { let result = sqlx::query( "UPDATE p2p_outbox SET state = 'RetryExhausted', \ - content = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN content ELSE X'' END, \ + content = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN content ELSE X'' END, \ lease_until = 0, next_retry_at = 0, \ - retry_until = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN retry_until ELSE 0 END, \ - retry_interval_secs = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN retry_interval_secs ELSE 0 END, \ - ack_peer_id = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') THEN ack_peer_id ELSE '' END, \ + retry_until = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN retry_until ELSE 0 END, \ + retry_interval_secs = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN retry_interval_secs ELSE 0 END, \ + ack_peer_id = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN ack_peer_id ELSE '' END, \ last_error = 'retry window expired without expected ACK', updated_at = ? \ WHERE retry_until > 0 AND retry_until <= ? AND state IN ('Pending', 'Processing')", ) @@ -3828,11 +3828,11 @@ impl<'a> StorageProcessor<'a> { pub async fn next_exhausted_setup( &mut self, now: i64, - ) -> anyhow::Result, i64)>> { + ) -> anyhow::Result, i64, String)>> { Ok(sqlx::query_as( - "SELECT message_id, content, created_at FROM p2p_outbox \ + "SELECT message_id, content, created_at, ack_peer_id FROM p2p_outbox \ WHERE state = 'RetryExhausted' AND length(content) > 0 \ - AND msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady') \ + AND msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') \ AND updated_at <= ? ORDER BY updated_at, message_id LIMIT 1", ) .bind(now - 60) diff --git a/node/src/action.rs b/node/src/action.rs index bfde134c..53f86753 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -43,7 +43,7 @@ use std::str::FromStr; use std::sync::{Arc, LazyLock, Mutex}; use std::time::{Duration, Instant}; use store::localdb::LocalDB; -use store::{GraphStatus, InstanceBridgeInStatus, MessageState, P2pInboxMessage}; +use store::{Graph, GraphStatus, InstanceBridgeInStatus, MessageState, P2pInboxMessage}; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -897,10 +897,14 @@ fn graph_setup_ack_required(content: &GOATMessageContent) -> bool { GOATMessageContent::GenCircuits(_) | GOATMessageContent::CutCircuits(_) | GOATMessageContent::SolderingProofReady(_) + | GOATMessageContent::CreateGraph(_) ) } -pub fn graph_setup_outbox_id(content: &GOATMessageContent) -> Option { +pub fn graph_setup_outbox_id( + content: &GOATMessageContent, + ack_peer_id: Option<&str>, +) -> Option { match content { GOATMessageContent::InitGraph(message) => Some(format!("init-graph:{}", message.graph_id)), GOATMessageContent::GenCircuits(message) => Some(format!( @@ -917,10 +921,27 @@ pub fn graph_setup_outbox_id(content: &GOATMessageContent) -> Option { message.candidate_index, hex::encode(message.payload_hash), )), + GOATMessageContent::CreateGraph(message) => { + let id = format!("create-graph:{}", message.graph_id); + Some(match ack_peer_id.filter(|peer_id| !peer_id.is_empty()) { + Some(peer_id) => format!("{id}:{peer_id}"), + None => id, + }) + } _ => None, } } +fn graph_setup_delivery_finished(content: &GOATMessageContent, graph: Option<&Graph>) -> bool { + match content { + GOATMessageContent::CreateGraph(_) => graph.is_some_and(|graph| { + GraphStatus::from_str(&graph.status) + .is_ok_and(|status| status != GraphStatus::OperatorPresigned) + }), + _ => graph.is_some(), + } +} + fn graph_setup_ack( content: &GOATMessageContent, sender_peer_id: &PeerId, @@ -954,8 +975,6 @@ pub(crate) async fn enqueue_graph_setup_outbox_message_from_peer( ack_peer_id: Option<&str>, sender: &str, ) -> Result { - let outbox_id = graph_setup_outbox_id(&message.content) - .ok_or_else(|| anyhow!("not a graph setup message"))?; let serialized = message.serialize_message().await?; let recipient = if graph_setup_ack_required(&message.content) { let ack_peer_id = ack_peer_id.context("graph setup ACK requires a recipient peer")?; @@ -967,6 +986,8 @@ pub(crate) async fn enqueue_graph_setup_outbox_message_from_peer( } else { None }; + let outbox_id = graph_setup_outbox_id(&message.content, recipient.as_deref()) + .ok_or_else(|| anyhow!("not a graph setup message"))?; let delivery_id = recipient.as_ref().map(|recipient| { graph_setup_delivery_id(sender, recipient, &graph_setup_payload_hash(&serialized)) }); @@ -2665,7 +2686,8 @@ async fn handle_p2p_outbox_messages( /// Recovery is driven by durable delivery state even when no new gossip arrives. async fn recover_exhausted_setup(local_db: &LocalDB) -> Result<()> { let now = current_time_secs(); - let Some((id, bytes, created_at)) = local_db.acquire().await?.next_exhausted_setup(now).await? + let Some((id, bytes, created_at, ack_peer_id)) = + local_db.acquire().await?.next_exhausted_setup(now).await? else { return Ok(()); }; @@ -2674,14 +2696,14 @@ async fn recover_exhausted_setup(local_db: &LocalDB) -> Result<()> { let mut resume = false; let mut canonical = id.clone(); if let Ok(message) = decoded - && let Some(key) = graph_setup_outbox_id(&message.content) + && let Some(key) = graph_setup_outbox_id(&message.content, Some(&ack_peer_id)) && let BusinessRef::Graph { instance_id, graph_id } = message.content.business_ref() { canonical = key; let instance = storage.find_instance(&instance_id).await?; let graph = storage.find_graph(&graph_id).await?; resume = now - created_at < MESSAGE_EXPIRE_TIME - && graph.is_none() + && !graph_setup_delivery_finished(&message.content, graph.as_ref()) && instance.is_some_and(|row| { matches!( InstanceBridgeInStatus::from_str(&row.status), @@ -2792,18 +2814,19 @@ async fn publish_p2p_outbox_row( return Ok(OutboxRowOutcome::Closed); } }; - if let Some(canonical_id) = graph_setup_outbox_id(&outbound.content) + if let Some(canonical_id) = graph_setup_outbox_id(&outbound.content, Some(&message.ack_peer_id)) && let BusinessRef::Graph { instance_id, graph_id } = outbound.content.business_ref() { let mut storage = local_db.acquire().await?; let instance = storage.find_instance(&instance_id).await?; + let graph = storage.find_graph(&graph_id).await?; let failed = instance.as_ref().is_some_and(|row| { InstanceBridgeInStatus::from_str(&row.status).is_ok_and(|status| { SigningRound::GraphSigning.delivery_finished(None, Some(&status)) }) }); if failed - || storage.find_graph(&graph_id).await?.is_some() + || graph_setup_delivery_finished(&outbound.content, graph.as_ref()) || current_time_secs() - message.created_at >= MESSAGE_EXPIRE_TIME { storage.cancel_p2p_outbox_message(&canonical_id).await?; diff --git a/node/src/handle.rs b/node/src/handle.rs index d36521bf..442e92fd 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -2651,6 +2651,19 @@ async fn handle_compact_soldering_proof_operator( }; seal_selected_verifiers(operator_state, selected_verifier_pubkeys)?; let bitvm_gc_circuit_datas = selected_gc_data(operator_state)?; + let selected_verifier_count = operator_state + .selected_verifier_pubkeys + .as_ref() + .context("selected verifier set is missing after proof collection")? + .len(); + let selected_verifier_peer_ids = (0..selected_verifier_count) + .map(|verifier_index| { + let candidate = selected_candidate_for_graph_index(operator_state, verifier_index)?; + PeerId::from_bytes(&candidate.verifier_peer_id) + .context("decode selected verifier peer id for CreateGraph ACK") + .map(|peer_id| peer_id.to_string()) + }) + .collect::>>()?; let mut obsolete_setup_outbox_ids = vec![format!("init-graph:{graph_id}")]; obsolete_setup_outbox_ids.extend(operator_state.candidates.iter().map(|candidate| { format!("cut-circuits:{instance_id}:{graph_id}:{}", candidate.verifier_pubkey) @@ -2798,16 +2811,22 @@ async fn handle_compact_soldering_proof_operator( Actor::All, GOATMessageContent::CreateGraph(CreateGraph { instance_id, graph_id, graph_nonce, graph }), ); - let serialized = message.serialize_message().await?; - let outbox_id = format!("create-graph:{graph_id}"); - let mut storage = context.local_db.acquire().await?; - // Idempotent: an entry that exists is left as it is. A graph the committee - // has already signed was evidently announced; only the clean-up is left. + let mut outbox_ids = Vec::new(); if !already_finalized { - storage - .insert_p2p_outbox_message(&outbox_id, message.content.event_type(), &serialized) - .await?; + for ack_peer_id in &selected_verifier_peer_ids { + outbox_ids.push( + enqueue_graph_setup_outbox_message( + &context.local_db, + message.clone(), + Some(ack_peer_id), + ) + .await?, + ); + } } + let mut storage = context.local_db.acquire().await?; + // The per-verifier outbox rows retry until each selected verifier ACKs. + // A finalized graph was already announced; only the setup cleanup is left. let mut cancelled_setup_messages = 0; for setup_outbox_id in obsolete_setup_outbox_ids { cancelled_setup_messages += @@ -2820,9 +2839,9 @@ async fn handle_compact_soldering_proof_operator( stage = "create_graph_outbox", graph_nonce, definition_hash = %definition_hash, - outbox_id, + outbox_ids = ?outbox_ids, cancelled_setup_messages, - "enqueued CreateGraph for swarm publication" + "enqueued CreateGraph for selected verifier delivery" ); // The outbox is durable before this cleanup. A process crash before