diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp index c2726cda9..90a4dbc2f 100644 --- a/src/herder/HerderImpl.cpp +++ b/src/herder/HerderImpl.cpp @@ -198,6 +198,44 @@ selectPrewarmCandidates(std::vector&& envelopes, return selected; } +// The candidates a pre-validation works on: the first `windows[phase]` +// transactions of each phase in mempool order, leaving out those of the +// ledger being closed (`closingTxs`, still in the mempool) and those whose +// validity depends on the close time of the ledger they go into (their +// result could not be reused). +std::vector +selectPrevalidationCandidates(std::vector&& envelopes, + std::vector const& windows, + bool supportsSoroban, Hash const& networkID, + UnorderedSet const& closingTxs, + std::atomic const& stop) +{ + std::vector selected; + std::vector taken(windows.size(), 0); + for (auto const& env : envelopes) + { + if (stop.load(std::memory_order_relaxed)) + { + break; + } + size_t const phase = isSorobanEnvelope(env) ? 1 : 0; + if (phase >= windows.size() || taken[phase] >= windows[phase] || + (phase == 1 && !supportsSoroban) || + txValidityDependsOnCloseTime(env)) + { + continue; + } + auto tx = TransactionFrameBase::makeTransactionFromWire(networkID, env); + if (closingTxs.find(tx->getFullHash()) != closingTxs.end()) + { + continue; + } + ++taken[phase]; + selected.push_back(std::move(tx)); + } + return selected; +} + // The number of transactions in each phase of a generalized tx set. std::vector txCountPerPhase(TxSetXDRFrame const& txSet) @@ -481,6 +519,7 @@ HerderImpl::shutdown() { discardPreparedTxSet(); stopSignaturePrewarm(); + stopProposalPrevalidation(nullptr); mTrackingTimer.cancel(); mOutOfSyncTimer.cancel(); mTriggerTimer.cancel(); @@ -917,8 +956,329 @@ HerderImpl::maybeStartSpeculativeApply(SCPEnvelope const& envelope) stopSignaturePrewarm(); } mSpeculativeApplyValue = std::make_pair(st.slotIndex, *value); + // A pre-validation that is still running was for a build on top of the + // LCL that this node has not needed (it votes for another leader's + // value): keep what it found, and leave the cores to the execution. + validationCacheFor(lcl); + // A node that may lead the next ledger wants the state this execution + // leaves, to validate its candidates on while this ledger is in its + // ballot. A ledger with upgrades is left out: they are applied when the + // ledger commits, after that state. + LedgerManager::SpeculativeApplyExecuted onExecuted; + if (sv.upgrades.empty() && getSCP().isValidator() && + !buildsLegacyTxSet(lcl.header.ledgerVersion) && + mayLeadAfter(st.slotIndex, sv, /*appliedTransactions=*/true)) + { + onExecuted = [this](std::shared_ptr state) { + startProposalPrevalidation(std::move(state)); + }; + } mLedgerManager.startSpeculativeApply( - LedgerCloseData(static_cast(st.slotIndex), *txSet, sv)); + LedgerCloseData(static_cast(st.slotIndex), *txSet, sv), + std::move(onExecuted)); +} + +bool +HerderImpl::mayLeadAfter(uint64_t slotIndex, StellarValue const& value, + bool appliedTransactions) +{ + auto const schedule = mHerderSCPDriver.getExpectedLeaderSchedule( + slotIndex + 1, value, appliedTransactions); + auto const& self = getSCP().getLocalNodeID(); + return (schedule.size() > 0 && schedule[0] == self) || + (schedule.size() > 1 && schedule[1] == self); +} + +void +HerderImpl::startProposalPrevalidation( + std::shared_ptr state) +{ + ZoneScoped; + releaseAssert(threadIsMain()); + if (!state || mApp.isStopping()) + { + return; + } + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + // Only while the executed ledger is still the next one to close and this + // node is voting for the value that was executed. A ledger that applied + // nothing ends its leader's term, and a skipped set means the state is + // not the one the candidates would be applied on. + StellarValue votedValue; + if (!isTracking() || state->mLedgerSeq != lcl.header.ledgerSeq + 1 || + state->mTxSetSkipped || !state->mValue.upgrades.empty() || + !mSpeculativeApplyValue || + mSpeculativeApplyValue->first != state->mLedgerSeq || + !toStellarValue(mSpeculativeApplyValue->second, votedValue) || + !(votedValue == state->mValue) || + !mayLeadAfter(state->mLedgerSeq, state->mValue, state->mAppliedTxs > 0)) + { + return; + } + stopProposalPrevalidation(nullptr); + // The pre-validation verifies the same signatures, and more. + if (mSignaturePrewarm && mSignaturePrewarm->ledgerSeq > state->mLedgerSeq) + { + stopSignaturePrewarm(); + } + + auto& metrics = mApp.getMetrics(); + auto const windows = validTxTargets(lcl.header.ledgerVersion); + // The executed ledger's transactions are still in the mempool: look + // past that many more. + auto payload = [&]() { + auto timer = + metrics.NewTimer({"herder", "prevalidate", "pull"}).TimeScope(); + return std::make_shared const>( + pullTopTransactionsPayload(2 * maxTxSetCandidates() + + state->mTxHashes->size())); + }(); + uint32_t count = 0; + if (payload->size() >= 4) + { + std::memcpy(&count, payload->data(), 4); + } + if (count == 0) + { + return; + } + + size_t const workers = static_cast(mApp.getConfig().WORKER_THREADS); + size_t const tasks = std::max( + 1, std::min(mApp.getBatchExecutor().preferredTaskCount(), + workers > 2 ? workers - 2 : 1)); + auto job = std::make_shared(); + job->ledgerSeq = state->mLedgerSeq; + job->value = state->mValue; + job->start = std::chrono::steady_clock::now(); + job->pendingTasks = tasks; + mProposalPrevalidation = job; + metrics.NewMeter({"herder", "prevalidate", "started"}, "prevalidation") + .Mark(); + CLOG_DEBUG(Herder, + "Pre-validating candidates for ledger {} on the executed state " + "of ledger {}: {} pulled, {} tasks", + state->mLedgerSeq + 1, state->mLedgerSeq, count, tasks); + + bool const supportsSoroban = protocolVersionStartsFrom( + state->mHeader->ledgerVersion, SOROBAN_PROTOCOL_VERSION); + std::optional validationLedgerSeq; + if (protocolVersionStartsFrom(state->mHeader->ledgerVersion, + ProtocolVersion::V_19)) + { + validationLedgerSeq = state->mLedgerSeq + 1; + } + for (size_t task = 0; task < tasks; ++task) + { + mApp.postOnBackgroundThread( + [job, state, payload, windows, supportsSoroban, validationLedgerSeq, + networkID = mApp.getNetworkID(), + &appConnector = mApp.getAppConnector(), &metrics, task, tasks]() { + ZoneScopedN("proposal prevalidation"); + { + std::lock_guard lock(job->mutex); + --job->pendingTasks; + if (job->stop.load()) + { + return; + } + ++job->activeTasks; + } + // Results go to the job a few at a time; once it is stopped + // they are of no use to anyone. + std::vector> batch; + auto publish = [&]() { + std::lock_guard lock(job->mutex); + if (!job->stop.load()) + { + job->published.insert( + job->published.end(), + std::make_move_iterator(batch.begin()), + std::make_move_iterator(batch.end())); + } + batch.clear(); + }; + try + { + std::call_once(job->selected, [&]() { + // Must not throw out of call_once. + try + { + job->candidates = selectPrevalidationCandidates( + OverlayIPC::parseTopTransactions(*payload), + windows, supportsSoroban, networkID, + *state->mTxHashes, job->stop); + } + catch (std::exception const& e) + { + CLOG_WARNING(Herder, + "Proposal pre-validation could not " + "read its candidates: {}", + e.what()); + job->candidates.clear(); + std::lock_guard lock(job->mutex); + job->stop.store(true); + } + }); + auto const& candidates = job->candidates; + auto view = state->makeView(metrics); + auto const& header = *state->mHeader; + auto diagnostics = DiagnosticEventManager::createDisabled(); + for (size_t i = task; i < candidates.size(); i += tasks) + { + if (job->stop.load(std::memory_order_relaxed)) + { + break; + } + auto const& tx = candidates[i]; + TxValidationResult result; + result.mValid = + tx->checkValid(appConnector, *view, 0, 0, 0, + diagnostics, validationLedgerSeq) + ->isSuccess(); + if (result.mValid) + { + auto feeSource = + view->getAccount(tx->getFeeSourceID()); + if (feeSource) + { + result.mFeeSourceBalance = getAvailableBalance( + header, feeSource.current()); + } + } + batch.emplace_back(tx->getFullHash(), result); + if (batch.size() >= 32) + { + publish(); + } + } + } + catch (std::exception const& e) + { + // The build validates whatever has no result. What this + // task had not published yet is dropped with it. + CLOG_WARNING(Herder, "Proposal pre-validation failed: {}", + e.what()); + batch.clear(); + std::lock_guard lock(job->mutex); + job->stop.store(true); + } + publish(); + std::lock_guard lock(job->mutex); + // Complete only if every task ran to its end: a stopped job + // (by the build, or by a failure) is not. + if (--job->activeTasks == 0 && job->pendingTasks == 0 && + !job->stop.load()) + { + job->finishedAfterNs = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - job->start) + .count(); + } + }, + "Herder: proposal prevalidation"); + } +} + +void +HerderImpl::stopProposalPrevalidation(LedgerHeaderHistoryEntry const* lcl) +{ + if (!mProposalPrevalidation) + { + return; + } + auto job = std::move(mProposalPrevalidation); + mProposalPrevalidation.reset(); + std::vector> results; + int64_t finishedAfterNs = -1; + { + // Take what the tasks have handed over so far. Nothing is published + // after this, so there is nothing to wait for: tasks that have not + // started find `stop` set and leave, running ones after their + // current transaction. + std::lock_guard lock(job->mutex); + finishedAfterNs = job->finishedAfterNs.load(); + job->stop.store(true); + results.swap(job->published); + } + auto& metrics = mApp.getMetrics(); + metrics.NewTimer({"herder", "prevalidate", "duration"}) + .Update(finishedAfterNs >= 0 + ? std::chrono::nanoseconds(finishedAfterNs) + : std::chrono::duration_cast( + std::chrono::steady_clock::now() - job->start)); + metrics.NewMeter({"herder", "prevalidate", "txs"}, "transaction") + .Mark(results.size()); + if (finishedAfterNs < 0) + { + metrics + .NewMeter({"herder", "prevalidate", "incomplete"}, "prevalidation") + .Mark(); + } + // The results describe the ledger that was executed. They hold on top of + // the LCL only if that is the ledger, closed with the very same value: + // executing a ledger is deterministic, so the state it was validated on + // is then the state the LCL left (the value has no upgrades, see + // maybeStartSpeculativeApply). + if (!lcl || lcl->header.ledgerSeq != job->ledgerSeq || + !(lcl->header.scpValue == job->value) || + mValidationCache.lclHash != lcl->hash) + { + metrics + .NewMeter({"herder", "prevalidate", "discarded"}, "prevalidation") + .Mark(); + return; + } + metrics.NewMeter({"herder", "prevalidate", "used"}, "prevalidation").Mark(); + for (auto& r : results) + { + mValidationCache.results.emplace(std::move(r)); + } +} + +TxValidationCache* +HerderImpl::validationCacheFor(LedgerHeaderHistoryEntry const& lcl) +{ + if (mValidationCache.lclHash != lcl.hash) + { + mValidationCache.lclHash = lcl.hash; + TxValidationCache().swap(mValidationCache.results); + } + stopProposalPrevalidation(&lcl); + return &mValidationCache.results; +} + +std::chrono::milliseconds +HerderImpl::expectedTxSetBuildDuration() const +{ + // Until this node has built a set of some size: what a build costs per + // transaction on the slower of the machines measured (about 0.17 s for + // 4,000 transactions with most candidates validated ahead of time). + static std::chrono::nanoseconds const DEFAULT_COST_PER_TX{40'000}; + auto const perTx = mTxSetBuildCostPerTx > std::chrono::nanoseconds::zero() + ? mTxSetBuildCostPerTx + : DEFAULT_COST_PER_TX; + auto const bySize = std::chrono::duration_cast( + perTx * static_cast(mLastClosedTxSetSize)); + return std::max(mLastTxSetBuildDuration, bySize); +} + +std::chrono::milliseconds +HerderImpl::prepareTxSetLeadTime(std::chrono::milliseconds expectedCloseTime, + std::chrono::milliseconds lastBuild) +{ + using std::chrono::milliseconds; + // What a pushed set needs to reach the peers (measured: about 0.1 s + // across regions for a 4,000-transaction set). + milliseconds const pushAllowance{100}; + auto lead = + std::min(expectedCloseTime / 4, milliseconds(250)); + if (lastBuild > milliseconds::zero()) + { + lead = std::max(lead, lastBuild + lastBuild / 4 + + pushAllowance); + } + return std::min(lead, expectedCloseTime / 2); } TxSubmitStatus @@ -1496,6 +1856,27 @@ HerderImpl::lastClosedLedgerIncreased(bool latest, TxSetXDRFrameConstPtr txSet, { releaseAssert(threadIsMain()); discardPreparedTxSet(); + // What the next build has to be ready for (see + // expectedTxSetBuildDuration). + mLastClosedTxSetSize = txSet ? txSet->sizeTxTotal() : 0; + { + // A pre-validation is for the ledger that was executed + // speculatively; if another one closed, its results are of no use. + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + if (mProposalPrevalidation && + (mProposalPrevalidation->ledgerSeq != lcl.header.ledgerSeq || + !(mProposalPrevalidation->value == lcl.header.scpValue))) + { + stopProposalPrevalidation(nullptr); + } + // Results computed on the previous ledger are of no use either, + // whether or not this node builds on the new one. + if (mValidationCache.lclHash != lcl.hash) + { + mValidationCache.lclHash = lcl.hash; + TxValidationCache().swap(mValidationCache.results); + } + } // Ensure potential upgrades are handled in overlay maybeHandleUpgrade(); @@ -1788,9 +2169,15 @@ HerderImpl::setupTriggerNextLedger() VirtualClock::system_time_point::duration>( triggerTime - now), lcl.header.ledgerVersion)); - // Yield out of ledger close before touching its read view. - // This is main-thread work; only its scheduling changes. - mPrepareTxSetTimer.expires_at(now); + // Build shortly before the trigger rather than right away: + // the set then holds what arrived in the meantime, and is + // still built and on its way to the peers when consensus + // starts (see makeProposalValue). This also yields out of + // ledger close before touching its read view. + mPrepareTxSetTimer.expires_at(std::max( + now, triggerTime - + prepareTxSetLeadTime( + milliseconds, expectedTxSetBuildDuration()))); mPrepareTxSetTimer.async_wait( [this, nextIndex, closeTime]() { prepareTxSet(static_cast(nextIndex), @@ -1953,19 +2340,15 @@ HerderImpl::prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime) { return; } - mPreparedTxSet = buildTxSet(ledgerSeq, closeTime, /*pushToPeers=*/false); + // The prepared set is proposed unchanged at the trigger + // (makeProposalValue), so it goes to the peers as soon as its XDR + // exists: they can have it before the proposal does. + mPreparedTxSet = buildTxSet(ledgerSeq, closeTime, /*pushToPeers=*/true); CLOG_INFO(Herder, "Prepared TX set for ledger {} before trigger: {} transactions, " "close time {}", ledgerSeq, mPreparedTxSet->txSet->sizeTxTotal(), closeTime.toString()); - // A capacity-limited prepared set is proposed unchanged at the trigger - // (makeProposalValue), so peers can have it before the proposal exists. - // Any other prepared set is rebuilt at the trigger and is not pushed. - if (mPreparedTxSet->capacityLimited && mPreparedTxSet->applicableTxSet) - { - pushTxSetToPeers(mPreparedTxSet->txSet, ledgerSeq); - } } void @@ -2128,7 +2511,7 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, } } }, - targets); + targets, validationCacheFor(lcl)); markSignatureCacheLookups(mApp, "build", sigCacheHits, sigCacheMisses); CLOG_INFO(Herder, "Proposed TX set has {} transactions", proposedSet->sizeTxTotal()); @@ -2163,31 +2546,27 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime, } } - // If all valid candidates fit, refresh at the trigger so an early empty - // or small snapshot doesn't prevent collecting transactions during the - // timer wait. Reuse a snapshot when selection actually excluded valid - // candidates. Invalid transactions do not establish excess demand. - // Candidates past a phase's validated prefix count as valid here. There - // are such candidates only when the prefix already held more valid ones - // than the phase can include, so they don't change the result (except - // in a phase that can include nothing: it validates nothing, and counts - // as capacity limited whenever it has candidates). - size_t validCandidates = 0; - for (size_t i = 0; i < txPhases.size(); ++i) - { - validCandidates += txPhases[i].size() - invalidTxPhases[i].size(); - } - bool capacityLimited = validCandidates > proposedSet->sizeTxTotal(); + auto const buildDuration = + std::chrono::steady_clock::now() - txSetBuildStart; mSCPMetrics.mTxSetBuild.Update( - std::chrono::duration_cast( - std::chrono::steady_clock::now() - txSetBuildStart)); + std::chrono::duration_cast(buildDuration)); + // The next build is scheduled far enough ahead of its trigger to fit one + // like this (see prepareTxSetLeadTime). + mLastTxSetBuildDuration = + std::chrono::duration_cast(buildDuration); + if (proposedSet->sizeTxTotal() >= 100) + { + mTxSetBuildCostPerTx = + std::chrono::duration_cast( + buildDuration) / + static_cast(proposedSet->sizeTxTotal()); + } return PreparedTxSet{lcl.hash, ledgerSeq, closeTime, proposedSet, std::move(applicableProposedSet), - std::move(invalidTxHashes), - capacityLimited}; + std::move(invalidTxHashes)}; } // called to take a position during the next round @@ -2287,7 +2666,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger, } // There is no nomination: the leader of each round (a deterministic - // schedule seeded by the LCL hash) proposes straight into the ballot + // schedule derived from the LCL's header) proposes straight into the ballot // protocol. The value is only built if this node leads a round. mHerderSCPDriver.startSlot( ledgerSeqToTrigger, @@ -2586,8 +2965,7 @@ HerderImpl::makeProposalValue(uint32_t ledgerSeqToTrigger) auto prepared = std::move(mPreparedTxSet); mPreparedTxSet.reset(); mPrepareTxSetTimer.cancel(); - if (!prepared || !prepared->capacityLimited || - prepared->ledgerSeq != ledgerSeqToTrigger || + if (!prepared || prepared->ledgerSeq != ledgerSeqToTrigger || prepared->previousLedgerHash != lcl.hash || ctValidityOffset(prepared->closeTime) != std::chrono::milliseconds::zero()) diff --git a/src/herder/HerderImpl.h b/src/herder/HerderImpl.h index cd22b658d..6a3444d4e 100644 --- a/src/herder/HerderImpl.h +++ b/src/herder/HerderImpl.h @@ -9,6 +9,7 @@ #include "herder/LedgerCloseData.h" #include "herder/PendingEnvelopes.h" #include "herder/QuorumIntersectionChecker.h" +#include "herder/TxSetFrame.h" #include "herder/Upgrades.h" #include "ledger/LedgerHeaderUtils.h" #include "overlay/NetworkConstants.h" @@ -145,6 +146,7 @@ class HerderImpl : public Herder std::function(size_t)> mGetTopTransactionsForTesting; friend class LeaderBuildTestAccess; + friend class ProposalPrevalidationTestAccess; std::optional mMaxClassicTxSize; std::optional mMaxTxSizeOverride; @@ -273,8 +275,33 @@ class HerderImpl : public Herder TxSetXDRFrameConstPtr txSet; ApplicableTxSetFrameConstPtr applicableTxSet; std::vector invalidTxHashes; - bool capacityLimited; }; + // How long before the trigger the first-round leader builds its set, so + // that the set is built and on its way to the peers when consensus + // starts; what arrives in between waits for the next ledger. + // * At least a quarter of the target close time, but no more than + // 250 ms of a long one. + // * Long enough for the build this node expects (`expectedBuild`, see + // expectedTxSetBuildDuration; with a quarter to spare) plus the time + // the pushed set needs to reach the peers. + // * Never more than half of the target close time. + static std::chrono::milliseconds + prepareTxSetLeadTime(std::chrono::milliseconds expectedCloseTime, + std::chrono::milliseconds expectedBuild); + // How long this node's latest tx set build took (zero before the first), + // what it cost per transaction of the set it built (zero until it has + // built a set of some size), and how many transactions the last closed + // ledger's set held. + std::chrono::milliseconds mLastTxSetBuildDuration{0}; + std::chrono::nanoseconds mTxSetBuildCostPerTx{0}; + size_t mLastClosedTxSetSize{0}; + // What the next build is expected to take: a build like this node's last + // one, or what a set the size of the last closed ledger's costs at this + // node's rate, whichever is longer. The second matters for a node that + // has not built for a while: its last build may be from before the load + // rose (every node's first term under load starts that way), and a lead + // time sized for it would make the first ledger of the term late. + std::chrono::milliseconds expectedTxSetBuildDuration() const; std::optional mPreparedTxSet; // `pushToPeers`: this is the set the node is about to propose as a round // leader, so the overlay sends it to every peer as soon as its XDR exists @@ -328,6 +355,65 @@ class HerderImpl : public Herder std::vector const& pendingPerPhase); std::shared_ptr mSignaturePrewarm; + // Pre-validation of the next ledger's candidates. + // + // A node that expects to lead ledger N+1 (in round 1, or in round 2 if + // the first leader times out) starts validating its candidates as soon as + // it has executed ledger N speculatively, while N is still going through + // consensus: background tasks run `checkValid` on the mempool's top + // transactions against the state that execution left (see + // ExecutedLedgerState). When N has closed with the value that was + // executed, the build of N+1 takes those results instead of validating + // the same transactions on the main thread (see mValidationCache). + struct ProposalPrevalidation + { + // The executed ledger (N) and its value: the results hold on top of + // N if and only if N closes with this value. + uint32_t ledgerSeq{0}; + StellarValue value; + std::chrono::steady_clock::time_point start; + std::atomic stop{false}; + // Decoded and selected once, by whichever task gets there first. + std::once_flag selected; + std::vector candidates; + // Guards everything below. Tasks hand over their results in small + // batches, so whoever stops the job takes what is there and never + // waits for a task: nothing is published after `stop` is set. + std::mutex mutex; + std::vector> published; + size_t pendingTasks{0}; + size_t activeTasks{0}; + // Nanoseconds from `start` until the last task finished, or -1. + std::atomic finishedAfterNs{-1}; + }; + std::shared_ptr mProposalPrevalidation; + // Whether the local node would lead round 1 or 2 of the ledger after + // `slotIndex` if `slotIndex` closes with `value`. + bool mayLeadAfter(uint64_t slotIndex, StellarValue const& value, + bool appliedTransactions); + // Called on the main thread when the speculative execution of a ledger + // has finished (`state` is null if it was not executed). + void startProposalPrevalidation( + std::shared_ptr state); + // Stops the running pre-validation, if any. With `lcl`, its results are + // moved into mValidationCache if they hold on top of `lcl`. + void stopProposalPrevalidation(LedgerHeaderHistoryEntry const* lcl); + + // Validation results for candidates of the next ledger, computed against + // the last closed ledger `lclHash`: by the pre-validation above, and by + // every build on that ledger. A later build on the same ledger (the build + // at the trigger after an early one, a later round's) only validates what + // it has not seen. + struct ValidationCache + { + Hash lclHash; + TxValidationCache results; + }; + ValidationCache mValidationCache; + // The cache for a build on top of `lcl`, emptied if it was for another + // ledger and topped up with the pre-validation's results. + TxValidationCache* validationCacheFor(LedgerHeaderHistoryEntry const& lcl); + // Tx sets the overlay holds on stable storage: received from it, or // handed to it after building them. persistSCPState leaves those out of // the database; after a restart they come back from the overlay (see diff --git a/src/herder/HerderSCPDriver.cpp b/src/herder/HerderSCPDriver.cpp index e663c438c..96c1968d6 100644 --- a/src/herder/HerderSCPDriver.cpp +++ b/src/herder/HerderSCPDriver.cpp @@ -1080,15 +1080,18 @@ HerderSCPDriver::getLeaderSchedule(uint64_t slotIndex) auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); if (slotIndex != static_cast(lcl.header.ledgerSeq) + 1) { - // The seed is the hash of the previous ledger's header: unknown for - // any slot but the next one. + // The schedule is derived from the previous ledger's header (who + // closed it, and whether it applied): unknown for any slot but the + // next one. See getExpectedLeaderSchedule for the slot after that. return empty; } // Force a pending rebuild of the transitive quorum, if any mPendingEnvelopes.isNodeDefinitelyInQuorum(mSCP.getLocalNodeID()); auto const& quorum = mPendingEnvelopes.getCurrentlyTrackedQuorum(); auto& cache = mLeaderSchedule; + bool const haveResults = ledgersHaveResults(); if (cache.mSlotIndex != slotIndex || cache.mSeed != lcl.hash || + cache.mLedgersHaveResults != haveResults || cache.mQuorumSize != quorum.size() || cache.mQuorumRebuildCount != mPendingEnvelopes.getQuorumRebuildCount()) { @@ -1098,7 +1101,9 @@ HerderSCPDriver::getLeaderSchedule(uint64_t slotIndex) { nodes.emplace_back(n.first); } - cache.mSchedule = computeLeaderSchedule(lcl.hash, std::move(nodes)); + cache.mSchedule = computeNextLeaderSchedule( + lcl.header, std::move(nodes), haveResults); + cache.mLedgersHaveResults = haveResults; cache.mSlotIndex = slotIndex; cache.mSeed = lcl.hash; cache.mQuorumSize = quorum.size(); @@ -1107,6 +1112,46 @@ HerderSCPDriver::getLeaderSchedule(uint64_t slotIndex) return cache.mSchedule; } +bool +HerderSCPDriver::ledgersHaveResults() const +{ +#ifdef BUILD_TESTS + // Overlay-only mode closes ledgers without applying them. + return !mApp.getRunInOverlayOnlyMode(); +#else + return true; +#endif +} + +std::vector +HerderSCPDriver::getExpectedLeaderSchedule(uint64_t slotIndex, + StellarValue const& closingValue, + bool appliedTransactions) +{ + releaseAssert(threadIsMain()); + auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + if (slotIndex != static_cast(lcl.header.ledgerSeq) + 2) + { + return {}; + } + // The tracked quorum as it is: this is called while SCP emits an + // envelope, where a pending rebuild (database reads) does not belong, + // and a prediction made on a slightly older node set is still one. + auto const& quorum = mPendingEnvelopes.getCurrentlyTrackedQuorum(); + std::vector nodes; + nodes.reserve(quorum.size()); + for (auto const& n : quorum) + { + nodes.emplace_back(n.first); + } + // The ledger in between (LCL+1) will name the LCL's hash as its previous + // ledger hash: that is the seed of the slot after it. + return computeLeaderSchedule( + lcl.hash, std::move(nodes), + leaderTermOutcome(lcl.header.ledgerSeq + 1, closingValue, + appliedTransactions || !ledgersHaveResults())); +} + std::optional HerderSCPDriver::getRoundLeader(uint64 slotIndex, uint32 round) { @@ -1177,6 +1222,48 @@ HerderSCPDriver::measureAndRecordBallotBlockedOnTxSet(uint64_t slotIndex, mSCPMetrics.mBallotBlockedOnTxSet.Update(std::chrono::milliseconds(0)); } +ValueWrapperPtr +HerderSCPDriver::reproposeValue(uint64_t slotIndex, uint32 round, + Value const& value) +{ + // The leader of a later round keeps the value it already votes for (its + // transaction set is known to the nodes that voted for it), but puts it + // forward under its own signature: the node whose value closes a ledger + // leads the next one (see leaderTermOutcome), and a leader whose round + // timed out should not be that node. SCP only asks while the node is + // not tied to the value (it has not accepted it as prepared); after + // that the value, and with it the term, stays as it is. + StellarValue sv; + if (round <= 1 || !toStellarValue(value, sv) || + sv.ext.v() == STELLAR_VALUE_BASIC || isEmptyTxSetStellarValue(sv)) + { + return nullptr; + } + auto const& self = mSCP.getLocalNodeID(); + if (getLcValueSignature(sv).nodeID == self) + { + return nullptr; + } + // Only a set this node holds: its signature makes it answerable for the + // set (a ledger that applies nothing costs the signer its term). + auto txSet = mPendingEnvelopes.getTxSet(sv.txSetHash); + if (auto const* frame = std::get_if(&txSet); + !frame || !*frame) + { + return nullptr; + } + mApp.getMetrics().NewMeter({"scp", "leader", "resigned"}, "value").Mark(); + auto resigned = + mHerder.makeStellarValue(sv.txSetHash, getConsensusTime(sv), + sv.upgrades, mApp.getConfig().NODE_SEED); + CLOG_INFO(Herder, + "Slot {} round {}: proposing the value of {} under our own " + "signature", + slotIndex, round, + mApp.getConfig().toShortString(getLcValueSignature(sv).nodeID)); + return wrapStellarValue(resigned); +} + void HerderSCPDriver::proposedValue(uint64_t slotIndex, uint32 round, Value const& value) diff --git a/src/herder/HerderSCPDriver.h b/src/herder/HerderSCPDriver.h index 1067c0575..939bc1ec7 100644 --- a/src/herder/HerderSCPDriver.h +++ b/src/herder/HerderSCPDriver.h @@ -110,9 +110,9 @@ class HerderSCPDriver : public SCPDriver void valueExternalized(uint64_t slotIndex, Value const& value) override; // Deterministic leader schedule: the leader of `round` for `slotIndex`. - // The schedule is a permutation of the transitive quorum seeded by the - // hash of the last closed ledger header, so it is only defined for the - // next ledger to close (LCL+1). + // The schedule is derived from the header of the last closed ledger and + // the transitive quorum (see computeNextLeaderSchedule), so it is only + // defined for the next ledger to close (LCL+1). bool isProposalPending(uint64 slotIndex, NodeID const& leader) const override; std::optional getRoundLeader(uint64 slotIndex, @@ -122,6 +122,18 @@ class HerderSCPDriver : public SCPDriver std::vector const& getLeaderSchedule(uint64_t slotIndex); // True if the local node leads `round` of slotIndex. bool isLocalNodeRoundLeader(uint64_t slotIndex, uint32_t round); + // The schedule `slotIndex` = LCL+2 will have if LCL+1 closes with + // `closingValue` and applies transactions or not. Empty for any other + // slot. This is a prediction: nothing in consensus depends on it. + std::vector + getExpectedLeaderSchedule(uint64_t slotIndex, + StellarValue const& closingValue, + bool appliedTransactions); + // False when ledgers close without recording transaction results (the + // test-only overlay-only mode). + bool ledgersHaveResults() const; + ValueWrapperPtr reproposeValue(uint64_t slotIndex, uint32 round, + Value const& value) override; // Update metrics indicating that a value was replaced with an empty-tx-set // value for slotIndex. @@ -338,6 +350,7 @@ class HerderSCPDriver : public SCPDriver Hash mSeed; size_t mQuorumSize{0}; uint64_t mQuorumRebuildCount{0}; + bool mLedgersHaveResults{true}; std::vector mSchedule; }; LeaderScheduleCache mLeaderSchedule; diff --git a/src/herder/LeaderSchedule.cpp b/src/herder/LeaderSchedule.cpp index cc9599a71..8f3f4efc6 100644 --- a/src/herder/LeaderSchedule.cpp +++ b/src/herder/LeaderSchedule.cpp @@ -4,6 +4,7 @@ #include "herder/LeaderSchedule.h" #include "crypto/SHA.h" +#include "ledger/LedgerHeaderUtils.h" #include "util/XDROperators.h" #include "xdrpp/marshal.h" #include @@ -35,4 +36,69 @@ computeLeaderSchedule(Hash const& seed, std::vector nodes) } return res; } + +LeaderTermOutcome +leaderTermOutcome(uint32_t ledgerSeq, StellarValue const& value, + bool appliedTransactions) +{ + LeaderTermOutcome outcome; + if (value.ext.v() == STELLAR_VALUE_BASIC) + { + return outcome; + } + auto const& signer = getLcValueSignature(value).nodeID; + if (isEmptyTxSetStellarValue(value) || !appliedTransactions) + { + outcome.mForfeited = signer; + } + else if ((static_cast(ledgerSeq) + 1) % LEADER_TERM_LEDGERS != 0) + { + outcome.mIncumbent = signer; + } + return outcome; +} + +LeaderTermOutcome +leaderTermOutcome(LedgerHeader const& lcl, bool ledgersHaveResults) +{ + static Hash const emptyResults = xdrSha256(TransactionResultSet{}); + return leaderTermOutcome(lcl.ledgerSeq, lcl.scpValue, + !ledgersHaveResults || + lcl.txSetResultHash != emptyResults); +} + +std::vector +computeLeaderSchedule(Hash const& seed, std::vector nodes, + LeaderTermOutcome const& outcome) +{ + auto schedule = computeLeaderSchedule(seed, std::move(nodes)); + // Both keep the seeded order of everyone else. + if (outcome.mIncumbent) + { + auto it = + std::find(schedule.begin(), schedule.end(), *outcome.mIncumbent); + if (it != schedule.end()) + { + std::rotate(schedule.begin(), it, it + 1); + } + } + if (outcome.mForfeited) + { + auto it = + std::find(schedule.begin(), schedule.end(), *outcome.mForfeited); + if (it != schedule.end()) + { + std::rotate(it, it + 1, schedule.end()); + } + } + return schedule; +} + +std::vector +computeNextLeaderSchedule(LedgerHeader const& lcl, std::vector nodes, + bool ledgersHaveResults) +{ + return computeLeaderSchedule(lcl.previousLedgerHash, std::move(nodes), + leaderTermOutcome(lcl, ledgersHaveResults)); +} } diff --git a/src/herder/LeaderSchedule.h b/src/herder/LeaderSchedule.h index 9042d90ea..ddcba75d5 100644 --- a/src/herder/LeaderSchedule.h +++ b/src/herder/LeaderSchedule.h @@ -4,7 +4,9 @@ #pragma once +#include "xdr/Stellar-ledger.h" #include "xdr/Stellar-types.h" +#include #include namespace stellar @@ -18,9 +20,74 @@ namespace stellar // node leads twice. // // The order depends only on `seed` and the *set* of nodes, so every node that -// agrees on both computes the same schedule: with a symmetric quorum, seeding -// with the hash of the last closed ledger header gives the whole network the -// same leader for each round of the next ledger. +// agrees on both computes the same schedule. std::vector computeLeaderSchedule(Hash const& seed, std::vector nodes); + +// Leader terms: the node whose value closed a ledger leads the next one too, +// for up to LEADER_TERM_LEDGERS ledgers in a row. A stable leader can prepare +// its next transaction set while the previous ledger is still in its ballot +// (see HerderImpl::startProposalPrevalidation). +// +// Terms end on fixed ledger numbers (every multiple of LEADER_TERM_LEDGERS +// starts a new one), so that the schedule of a slot is a function of the +// previous ledger's header alone, as before. +constexpr uint32_t LEADER_TERM_LEDGERS = 5; + +// What the ledger `ledgerSeq`, closed with `value`, means for the leader of +// the next ledger. +// * `mIncumbent`: the node that keeps leading: the node that signed +// `value`. Set unless the term is over or was forfeited. +// * `mForfeited`: the node that lost its term early and goes to the end of +// the next ledger's order: the node that signed `value`, when +// - `value` is an empty-tx-set value (its leader's transaction set never +// arrived, so consensus dropped it), or +// - the ledger applied no transaction (`appliedTransactions` is false). +// That covers a ledger whose transaction set was found invalid at +// apply, and an honestly empty ledger: nothing is lost by rotating +// after one. +// There is no incumbent when the next ledger starts a new term (every +// multiple of LEADER_TERM_LEDGERS); a forfeit counts there too. Neither is +// set when `value` is unsigned (the genesis ledger). +// +// A leader whose round times out loses the term as well, without being named +// here: the leader of the round that closes the ledger signs the value it +// proposes, and so becomes the incumbent. (Exception: a later round's leader +// that has already accepted the previous leader's value as prepared keeps +// that value and its signature, see HerderSCPDriver::reproposeValue.) +struct LeaderTermOutcome +{ + std::optional mIncumbent; + std::optional mForfeited; +}; +LeaderTermOutcome leaderTermOutcome(uint32_t ledgerSeq, + StellarValue const& value, + bool appliedTransactions); + +// The same from the header of a closed ledger, where "applied no transaction" +// reads: `txSetResultHash` is the hash of an empty result set. Only checked +// with `ledgersHaveResults`; without it (the test-only overlay-only mode, +// where no ledger records results) every ledger counts as having applied. +LeaderTermOutcome leaderTermOutcome(LedgerHeader const& lcl, + bool ledgersHaveResults); + +// The leader schedule of the ledger after `lcl`. +// +// The order is seeded with `lcl.previousLedgerHash`, the hash of the ledger +// *before* `lcl`. `lcl`'s own hash would do as well for agreement, but it is +// only known once `lcl` has closed; with the older hash, the order for the +// next ledger is already known while `lcl` is going through consensus. +// +// An incumbent (see leaderTermOutcome) that is one of `nodes` leads round 1, +// and a node that forfeited its term comes last; the seeded order of the +// other nodes is kept. +std::vector computeNextLeaderSchedule(LedgerHeader const& lcl, + std::vector nodes, + bool ledgersHaveResults); + +// The same, for a ledger that has not closed yet: `seed` is the hash of the +// ledger before it and `outcome` what it is expected to mean for its leader. +std::vector computeLeaderSchedule(Hash const& seed, + std::vector nodes, + LeaderTermOutcome const& outcome); } diff --git a/src/herder/TxSetFrame.cpp b/src/herder/TxSetFrame.cpp index bf1602762..de090b70c 100644 --- a/src/herder/TxSetFrame.cpp +++ b/src/herder/TxSetFrame.cpp @@ -881,7 +881,8 @@ makeTxSetFromTransactions( #endif , TxSetReadyCallback const& onTxSetReady, - std::vector const& validTxTargets) + std::vector const& validTxTargets, + TxValidationCache* validationCache) { releaseAssert(threadIsMain()); releaseAssert(!app.getLedgerManager().isApplying()); @@ -926,7 +927,7 @@ makeTxSetFromTransactions( { validatedTxs = TxSetUtils::trimInvalid( phaseTxs, app, accountFeeMap, closeTimeOffset.seconds(), - closeTimeOffset.seconds(), invalid); + closeTimeOffset.seconds(), invalid, validationCache); } else { @@ -941,7 +942,7 @@ makeTxSetFromTransactions( SurgePricingPriorityQueue::GENERIC_LANE; }, app, accountFeeMap, closeTimeOffset.seconds(), - closeTimeOffset.seconds(), invalid); + closeTimeOffset.seconds(), invalid, validationCache); } #ifdef BUILD_TESTS // In tests we shouldn't be really trying to close ledgers with @@ -1925,6 +1926,40 @@ TxSetPhaseFrame::sortedForApply(Hash const& txSetHash) const } } +bool +txValidityDependsOnCloseTime(TransactionEnvelope const& env) +{ + auto timeBounded = [](TimeBounds const& tb) { + return tb.minTime != 0 || tb.maxTime != 0; + }; + auto conditional = [&](Preconditions const& cond) { + switch (cond.type()) + { + case PRECOND_NONE: + return false; + case PRECOND_TIME: + return timeBounded(cond.timeBounds()); + case PRECOND_V2: + return (cond.v2().timeBounds && + timeBounded(*cond.v2().timeBounds)) || + cond.v2().minSeqAge != 0; + } + // An unknown kind of precondition: assume the worst. + return true; + }; + switch (env.type()) + { + case ENVELOPE_TYPE_TX_V0: + return env.v0().tx.timeBounds && timeBounded(*env.v0().tx.timeBounds); + case ENVELOPE_TYPE_TX: + return conditional(env.v1().tx.cond); + case ENVELOPE_TYPE_TX_FEE_BUMP: + return conditional(env.feeBump().tx.innerTx.v1().tx.cond); + default: + return true; + } +} + TxSetValidationContext TxSetValidationContext::forLCL(Application& app) { diff --git a/src/herder/TxSetFrame.h b/src/herder/TxSetFrame.h index 1feb222ae..ef074dced 100644 --- a/src/herder/TxSetFrame.h +++ b/src/herder/TxSetFrame.h @@ -103,6 +103,28 @@ enum class TxSetValidationResult // Returns a string representation of the validation result for logging. std::string toString(TxSetValidationResult result); +// What validating one transaction against a ledger found: whether +// `checkValid` passed and, if it did, the available balance of the fee source +// (which the caller compares with the fees of all the transactions it pays +// for). +struct TxValidationResult +{ + bool mValid{false}; + std::optional mFeeSourceBalance; +}; + +// Validation results by full transaction hash, for transactions validated +// against one particular ledger. It only holds transactions whose validity +// does not depend on the close time of the ledger they go into (see +// `txValidityDependsOnCloseTime`), so a result is the same for every set +// built on that ledger. Whoever owns the cache makes sure it is only used +// with the ledger its results were computed on. +using TxValidationCache = UnorderedMap; + +// Whether the outcome of validating `env` can depend on the close time it is +// validated for: it has time bounds or a minimum sequence age. +bool txValidityDependsOnCloseTime(TransactionEnvelope const& env); + // The ledger state a transaction set is validated against: the ledger the // set would apply on top of. Validation only reads the state through this // context, so it can run wherever the context's views can be used: the main @@ -120,6 +142,11 @@ struct TxSetValidationContext // Upper bound on the number of tasks the per-transaction checks are // split into. 1 runs them on the calling thread. size_t mMaxTasks; + // Results of earlier validations against this ledger, if any: a + // transaction found in it is not validated again, and the results of the + // others are added to it. Only read while the validation tasks run, and + // only written by the calling thread. + TxValidationCache* mValidationCache{nullptr}; // The LCL as seen by the main thread. Main thread only. static TxSetValidationContext forLCL(Application& app); @@ -185,7 +212,10 @@ makeTxSetFromTransactions( #endif , TxSetReadyCallback const& onTxSetReady = {}, - std::vector const& validTxTargets = {}); + std::vector const& validTxTargets = {}, + // Results of earlier validations against the LCL (see + // `TxValidationCache`); extended with this build's. + TxValidationCache* validationCache = nullptr); #ifdef BUILD_TESTS std::pair @@ -449,7 +479,8 @@ class TxSetPhaseFrame #endif , TxSetReadyCallback const& onTxSetReady, - std::vector const& validTxTargets); + std::vector const& validTxTargets, + TxValidationCache* validationCache); #ifdef BUILD_TESTS friend std::pair makeTxSetFromTransactions( @@ -642,7 +673,8 @@ class ApplicableTxSetFrame #endif , TxSetReadyCallback const& onTxSetReady, - std::vector const& validTxTargets); + std::vector const& validTxTargets, + TxValidationCache* validationCache); #ifdef BUILD_TESTS friend std::pair makeTxSetFromTransactions( diff --git a/src/herder/TxSetUtils.cpp b/src/herder/TxSetUtils.cpp index 2405d40aa..00483cd04 100644 --- a/src/herder/TxSetUtils.cpp +++ b/src/herder/TxSetUtils.cpp @@ -16,11 +16,13 @@ #include "main/Application.h" #include "main/Config.h" #include "main/ErrorMessages.h" +#include "medida/meter.h" #include "transactions/MutableTransactionResult.h" #include "transactions/TransactionUtils.h" #include "util/BatchExecutor.h" #include "util/GlobalChecks.h" #include "util/Logging.h" +#include "util/MetricsRegistry.h" #include "util/ProtocolVersion.h" #include "util/UnorderedSet.h" #include "util/XDRCereal.h" @@ -222,11 +224,15 @@ TxSetUtils::getInvalidTxListWithErrors( std::vector>> txValidationResult( txs.size()); auto& appConnector = app.getAppConnector(); + // The tasks only read the cache; what they validated themselves is + // added to it below, by this thread. + TxValidationCache const* const cache = ctx.mValidationCache; + std::vector fromCache(cache ? txs.size() : 0, 0); app.getBatchExecutor().executeBatchOverRanges( txs.size(), taskCount, - [&appConnector, &txs, &ledgerViews, &txValidationResult, - lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, + [&appConnector, &txs, &ledgerViews, &txValidationResult, cache, + &fromCache, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, validationLedgerSeq](size_t begin, size_t end, size_t rangeIndex) { auto const& view = *ledgerViews.at(rangeIndex); auto const header = view.getLedgerHeader().current(); @@ -238,7 +244,18 @@ TxSetUtils::getInvalidTxListWithErrors( // cache it here, in parallel, rather than there. Each frame // is in one range only, and the batch completes before the // main thread reads it. - txs[i]->getFullHash(); + auto const& fullHash = txs[i]->getFullHash(); + if (cache) + { + auto it = cache->find(fullHash); + if (it != cache->end()) + { + txValidationResult[i] = {it->second.mValid, + it->second.mFeeSourceBalance}; + fromCache[i] = 1; + continue; + } + } auto res = txs[i]->checkValid(appConnector, view, 0, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, @@ -257,6 +274,30 @@ TxSetUtils::getInvalidTxListWithErrors( } }); + if (ctx.mValidationCache) + { + size_t hits = 0; + for (size_t i = 0; i < txs.size(); ++i) + { + if (fromCache[i]) + { + ++hits; + } + else if (!txValidityDependsOnCloseTime(txs[i]->getEnvelope())) + { + ctx.mValidationCache->emplace( + txs[i]->getFullHash(), + TxValidationResult{txValidationResult[i].first, + txValidationResult[i].second}); + } + } + auto& metrics = app.getMetrics(); + metrics.NewMeter({"herder", "txset", "validation-cache-hit"}, "tx") + .Mark(hits); + metrics.NewMeter({"herder", "txset", "validation-cache-miss"}, "tx") + .Mark(txs.size() - hits); + } + TxFrameListWithErrors invalidTxsWithError; auto& [invalidTxs, errorCode] = invalidTxsWithError; errorCode = TxSetValidationResult::VALID; @@ -336,9 +377,13 @@ TxSetUtils::trimInvalid(TxFrameList const& txs, Application& app, UnorderedMap& accountFeeMap, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, - TxFrameList& invalidTxs) + TxFrameList& invalidTxs, + TxValidationCache* validationCache) { - invalidTxs = getInvalidTxListWithErrors(txs, app, accountFeeMap, + releaseAssert(threadIsMain()); + auto ctx = TxSetValidationContext::forLCL(app); + ctx.mValidationCache = validationCache; + invalidTxs = getInvalidTxListWithErrors(txs, app, ctx, accountFeeMap, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset) .first; @@ -351,7 +396,7 @@ TxSetUtils::trimInvalidPrefix( std::function const& countsTowardTarget, Application& app, UnorderedMap& accountFeeMap, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, - TxFrameList& invalidTxs) + TxFrameList& invalidTxs, TxValidationCache* validationCache) { ZoneScoped; TxFrameList validTxs; @@ -374,9 +419,9 @@ TxSetUtils::trimInvalidPrefix( txs.begin() + validated + count); validated += count; TxFrameList batchInvalid; - for (auto const& tx : - trimInvalid(batch, app, accountFeeMap, lowerBoundCloseTimeOffset, - upperBoundCloseTimeOffset, batchInvalid)) + for (auto const& tx : trimInvalid( + batch, app, accountFeeMap, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, batchInvalid, validationCache)) { if (!countsTowardTarget || countsTowardTarget(*tx)) { diff --git a/src/herder/TxSetUtils.h b/src/herder/TxSetUtils.h index 4e459efe2..09ef4dc7c 100644 --- a/src/herder/TxSetUtils.h +++ b/src/herder/TxSetUtils.h @@ -71,7 +71,8 @@ class TxSetUtils trimInvalid(TxFrameList const& txs, Application& app, UnorderedMap& accountFeeMap, uint64_t lowerBoundCloseTimeOffset, - uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs); + uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, + TxValidationCache* validationCache = nullptr); // Like `trimInvalid`, for candidates in priority order when only the // first `targetValidTxs` valid ones that `countsTowardTarget` accepts are @@ -86,6 +87,6 @@ class TxSetUtils countsTowardTarget, Application& app, UnorderedMap& accountFeeMap, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, - TxFrameList& invalidTxs); + TxFrameList& invalidTxs, TxValidationCache* validationCache = nullptr); }; // class TxSetUtils } // namespace stellar diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index 983bf7b7c..012af8cb1 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -4,6 +4,7 @@ #include "bucket/BucketIndexUtils.h" #include "herder/HerderImpl.h" +#include "herder/LeaderSchedule.h" #include "herder/LedgerCloseData.h" #include "herder/test/TestTxSetUtils.h" #include "main/Application.h" @@ -5423,6 +5424,22 @@ class LeaderBuildTestAccess herder.setupTriggerNextLedger(); } + // When the first-round leader builds its set, and how long before the + // trigger that is. + static VirtualClock::time_point + prepareTime(HerderImpl& herder) + { + return herder.mPrepareTxSetTimer.expiry_time(); + } + + static std::chrono::milliseconds + prepareLeadTime(HerderImpl& herder) + { + return HerderImpl::prepareTxSetLeadTime( + herder.mLedgerManager.getExpectedLedgerCloseTime(), + herder.expectedTxSetBuildDuration()); + } + // The leader's full build (mempool pull through final checks). static TxSetXDRFrameConstPtr build(HerderImpl& herder, uint32_t ledgerSeq, ConsensusTime closeTime) @@ -5483,11 +5500,32 @@ class LeaderBuildTestAccess selectLocalFirstLeader(HerderImpl& herder) { auto& scp = herder.getSCP(); - auto const& lcl = - herder.mApp.getLedgerManager().getLastClosedLedgerHeader(); + auto& lm = herder.mApp.getLedgerManager(); + // A node whose own ledger applied nothing has forfeited its term and + // comes last in the next ledger's order, whatever the seed. Test + // setup reaches such ledgers (upgrade helpers close empty ledgers + // under the local node's signature): close one that applies a + // transaction, after which the node leads again or, at the start of + // a term, takes its place in the seeded order. + if (leaderTermOutcome(lm.getLastClosedLedgerHeader().header, true) + .mForfeited == scp.getLocalNodeID()) + { + auto root = herder.mApp.getRoot(); + txtest::closeLedger( + herder.mApp, + {root->tx({txtest::payment(root->getPublicKey(), 1)})}); + REQUIRE( + !leaderTermOutcome(lm.getLastClosedLedgerHeader().header, true) + .mForfeited); + } + auto const& lcl = lm.getLastClosedLedgerHeader(); auto qset = scp.getLocalQuorumSet(); REQUIRE(qset.validators.size() == 2); - for (int i = 0; i < 100; ++i) + // The local node's key is fixed (the instance number and the day's + // seed decide it) and may sort behind nearly every other key: allow + // enough candidates for the absent peer to find one that sorts + // behind even such a key. + for (int i = 0; i < 2000; ++i) { qset.validators = { scp.getLocalNodeID(), @@ -6107,7 +6145,7 @@ TEST_CASE("only round leaders construct local proposals", auto& scp = herder.getSCP(); auto const lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto const seq = lcl.header.ledgerSeq + 1; - // The schedule is seeded by the hash of the last closed ledger. + // The schedule is derived from the last closed ledger. if (!previous) { previous = lcl.hash; @@ -6147,8 +6185,30 @@ TEST_CASE("only round leaders construct local proposals", app, [&]() { return done; }, std::chrono::seconds(1)); REQUIRE(done); REQUIRE(clock.now() < herder.getTriggerTimer().expiry_time()); + // Nobody builds right away: the first-round leader does shortly + // before the trigger, so that its set holds what arrives until then. + REQUIRE(pulls == 0); + REQUIRE(!LeaderBuildTestAccess::prepared(herder)); + if (role == 0) + { + auto const lead = LeaderBuildTestAccess::prepareLeadTime(herder); + REQUIRE(lead > std::chrono::milliseconds(0)); + REQUIRE(lead <= std::chrono::milliseconds(250)); + auto const prepareAt = LeaderBuildTestAccess::prepareTime(herder); + REQUIRE(prepareAt == herder.getTriggerTimer().expiry_time() - lead); + REQUIRE(clock.now() < prepareAt); + clock.setCurrentVirtualTime(prepareAt); + testutil::crankUntil( + app, + [&]() { + return LeaderBuildTestAccess::prepared(herder) != nullptr; + }, + std::chrono::seconds(1)); + } + REQUIRE(clock.now() < herder.getTriggerTimer().expiry_time()); REQUIRE(pulls == (role == 0 ? 1 : 0)); REQUIRE(bool(LeaderBuildTestAccess::prepared(herder)) == (role == 0)); + auto const preparedSet = LeaderBuildTestAccess::prepared(herder); REQUIRE(scp.getLeaderRound(seq) == 0); REQUIRE(scp.getLatestMessagesSend(seq).empty()); auto const trigger = herder.getTriggerTimer().expiry_time(); @@ -6158,10 +6218,10 @@ TEST_CASE("only round leaders construct local proposals", app, [&]() { return scp.getLeaderRound(seq) != 0; }, std::chrono::seconds(1)); REQUIRE(scp.getLeaderRound(seq) == 1); - // The first leader refreshes its empty early snapshot. Followers - // pull the mempool once, only to pre-verify signatures; they do not - // build or publish a set. - REQUIRE(pulls == (role == 0 ? 2 : 1)); + // The first leader proposes the set it prepared, without pulling + // again. Followers pull the mempool once, only to pre-verify + // signatures; they do not build or publish a set. + REQUIRE(pulls == 1); REQUIRE(scp.getLatestMessagesSend(seq).empty() == (role != 0)); // With the first leader silent, real leader timers eventually @@ -6172,13 +6232,18 @@ TEST_CASE("only round leaders construct local proposals", std::chrono::seconds(30)); REQUIRE(scp.getLeaderRound(seq) == static_cast(role + 1)); REQUIRE(driver.isLocalNodeRoundLeader(seq, scp.getLeaderRound(seq))); - REQUIRE(pulls == 2); + REQUIRE(pulls == (role == 0 ? 1 : 2)); auto messages = scp.getLatestMessagesSend(seq); REQUIRE(messages.front().statement.pledges.type() == SCP_ST_PREPARE); auto const& proposed = messages.front().statement.pledges.prepare().ballot.value; StellarValue nominated; xdr::xdr_from_opaque(proposed, nominated); + if (role == 0) + { + REQUIRE(preparedSet); + REQUIRE(nominated.txSetHash == preparedSet->getContentsHash()); + } auto txSet = std::get( herder.getTxSet(nominated.txSetHash)); REQUIRE(txSet); @@ -6278,6 +6343,204 @@ TEST_CASE("a follower promoted after losing sync still proposes", REQUIRE(checkedFollower); } +// A leader whose round times out should not keep its term: the node whose +// value closes a ledger leads the next one. So when the leader of a later +// round puts forward the value it already votes for (the first leader's), it +// does so under its own signature, unless it is already tied to that value. +TEST_CASE("the leader of a later round signs the value it proposes again", + "[herder][early-proposal][leader-term]") +{ + // Three nodes. The first leader's proposal reaches the local node, which + // votes for it, and the round times out with the local node next in line. + // * not tied: all three nodes are needed. The third node votes for + // something else, so the first leader's value is not accepted as + // prepared: the local node proposes it again under its own signature. + // * tied: any two nodes are a quorum, so the first leader's and the + // local node's votes make the value accepted as prepared. The first + // leader may have confirmed it: the local node keeps it as it is. + auto const tied = GENERATE(false, true); + CAPTURE(tied); + std::vector keys; + SCPQuorumSet qset; + qset.threshold = tied ? 2 : 3; + for (int i = 0; i < 3; ++i) + { + keys.push_back(SecretKey::fromSeed( + sha256(fmt::format("early-proposal-node-{}", i)))); + qset.validators.push_back(keys.back().getPublicKey()); + } + + bool checkedSecondLeader = false; + for (int i = 0; i < 3 && !checkedSecondLeader; ++i) + { + VirtualClock clock; + clock.setCurrentVirtualTime(VirtualClock::from_time_t(1000)); + auto cfg = getTestConfig(i); + cfg.HTTP_PORT = 0; + cfg.MANUAL_CLOSE = false; + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = false; + cfg.FORCE_OLD_STYLE_PREPARE_START_TRIGGER_TIMER = true; + cfg.NODE_SEED = keys[i]; + cfg.QUORUM_SET = qset; + auto app = createTestApplication(clock, cfg); + auto& herder = static_cast(app->getHerder()); + auto& scp = herder.getSCP(); + auto const lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto const seq = lcl.header.ledgerSeq + 1; + auto& driver = herder.getHerderSCPDriver(); + if (!driver.isLocalNodeRoundLeader(seq, 2)) + { + continue; + } + checkedSecondLeader = true; + auto const firstLeader = driver.getRoundLeader(seq, 1); + REQUIRE(firstLeader); + REQUIRE(!(*firstLeader == scp.getLocalNodeID())); + auto keyOf = [&](NodeID const& id) { + auto it = + std::find_if(keys.begin(), keys.end(), [&](SecretKey const& k) { + return k.getPublicKey() == id; + }); + REQUIRE(it != keys.end()); + return *it; + }; + auto const firstKey = keyOf(*firstLeader); + auto const thirdKey = + *std::find_if(keys.begin(), keys.end(), [&](SecretKey const& k) { + return !(k.getPublicKey() == *firstLeader) && + !(k.getPublicKey() == scp.getLocalNodeID()); + }); + auto& resigned = + app->getMetrics().NewMeter({"scp", "leader", "resigned"}, "value"); + + herder.getHerderSCPDriver().recordBallotStart(lcl.header.ledgerSeq); + size_t pulls = 0; + herder.mGetTopTransactionsForTesting = [&](size_t) { + ++pulls; + return std::vector{}; + }; + LeaderBuildTestAccess::schedule(herder); + clock.setCurrentVirtualTime(herder.getTriggerTimer().expiry_time()); + testutil::crankUntil( + app, [&]() { return scp.getLeaderRound(seq) != 0; }, + std::chrono::seconds(1)); + REQUIRE(scp.getLeaderRound(seq) == 1); + REQUIRE(scp.getLatestMessagesSend(seq).empty()); + + // An empty set (the same on every node), proposed by `signer` at + // counter 1. + auto txSet = + makeTxSetFromTransactions(TxFrameList{}, *app, ApplyTimeOffset{}) + .first; + herder.getPendingEnvelopes().addTxSet(txSet->getContentsHash(), seq, + txSet); + auto const closeTime = + std::max(ConsensusTime::fromSystemTime(clock.system_now(), + lcl.header.ledgerVersion), + getConsensusTime(lcl.header.scpValue) + .next(lcl.header.ledgerVersion)); + auto prepareFrom = [&](SecretKey const& signer, ConsensusTime ct) { + auto const value = herder.makeStellarValue( + txSet->getContentsHash(), ct, emptyUpgradeSteps, signer); + SCPEnvelope env; + env.statement.slotIndex = seq; + env.statement.nodeID = signer.getPublicKey(); + env.statement.pledges.type(SCP_ST_PREPARE); + auto& prep = env.statement.pledges.prepare(); + prep.ballot.counter = 1; + prep.ballot.value = xdr::xdr_to_opaque(value); + // every node uses the same (normalized) quorum set + prep.quorumSetHash = scp.getLocalNode()->getQuorumSetHash(); + herder.signEnvelope(signer, env); + return std::make_pair(env, value); + }; + auto const [proposal, firstValue] = prepareFrom(firstKey, closeTime); + auto const firstBallotValue = + proposal.statement.pledges.prepare().ballot.value; + REQUIRE(herder.recvSCPEnvelope(proposal) == + Herder::ENVELOPE_STATUS_READY); + + auto ballotOf = [&]() -> std::optional { + auto messages = scp.getLatestMessagesSend(seq); + if (messages.empty() || + messages.front().statement.pledges.type() != SCP_ST_PREPARE) + { + return std::nullopt; + } + return messages.front().statement.pledges.prepare().ballot; + }; + // The local node votes for the first leader's value as it is. + testutil::crankUntil( + app, [&]() { return ballotOf().has_value(); }, + std::chrono::seconds(1)); + REQUIRE(ballotOf()); + REQUIRE(ballotOf()->counter == 1); + REQUIRE(ballotOf()->value == firstBallotValue); + if (!tied) + { + // The third node is at counter 1 with a value of its own: the + // local node has now heard from a quorum, and the first leader's + // value has two votes out of the three it needs. + auto const [other, otherValue] = + prepareFrom(thirdKey, closeTime.next(lcl.header.ledgerVersion)); + REQUIRE(herder.recvSCPEnvelope(other) == + Herder::ENVELOPE_STATUS_READY); + } + + // Nothing more arrives. The round times out and the local node + // leads round 2 without building anything. + testutil::crankUntil( + app, [&]() { return ballotOf() && ballotOf()->counter == 2; }, + std::chrono::seconds(60)); + REQUIRE(ballotOf()); + REQUIRE(ballotOf()->counter == 2); + REQUIRE(scp.getLeaderRound(seq) == 2); + StellarValue proposed; + xdr::xdr_from_opaque(ballotOf()->value, proposed); + // the same transaction set, close time and upgrades either way + REQUIRE(proposed.txSetHash == firstValue.txSetHash); + REQUIRE(getConsensusTime(proposed) == getConsensusTime(firstValue)); + REQUIRE(proposed.upgrades == firstValue.upgrades); + REQUIRE(getLcValueSignature(firstValue).nodeID == *firstLeader); + REQUIRE(driver.validateValue(seq, ballotOf()->value, true) == + SCPDriver::kFullyValidatedValue); + auto const outcome = leaderTermOutcome(seq, proposed, true); + bool const insideTerm = (seq + 1) % LEADER_TERM_LEDGERS != 0; + if (tied) + { + // kept as it is: if it closes the ledger, the first leader + // keeps its term + REQUIRE(ballotOf()->value == firstBallotValue); + REQUIRE(resigned.count() == 0); + if (insideTerm) + { + REQUIRE(outcome.mIncumbent == *firstLeader); + } + } + else + { + // under the local node's signature: if it closes the ledger, the + // local node leads the next one, and the first leader does not + REQUIRE(getLcValueSignature(proposed).nodeID == + scp.getLocalNodeID()); + REQUIRE(!(ballotOf()->value == firstBallotValue)); + REQUIRE(resigned.count() == 1); + if (insideTerm) + { + REQUIRE(outcome.mIncumbent == scp.getLocalNodeID()); + } + } + // The mempool was only pulled to pre-verify signatures: no set was + // built. + REQUIRE(pulls >= 1); + REQUIRE( + app->getMetrics().NewTimer({"herder", "txset", "build"}).count() == + 0); + herder.mGetTopTransactionsForTesting = nullptr; + } + REQUIRE(checkedSecondLeader); +} + TEST_CASE("a leader build that completes a fetch can externalize", "[herder][early-proposal]") { @@ -6658,7 +6921,10 @@ TEST_CASE("signature prewarm at the ballot covers the next ledger", herder.mGetTopTransactionsForTesting = nullptr; } -TEST_CASE("early underfilled proposals refresh at the trigger", +// The first-round leader builds once, shortly before the trigger, and +// proposes that set as it is: what arrives after the build waits for the +// next ledger, whether or not the set had room for it. +TEST_CASE("the set prepared before the trigger is proposed unchanged", "[herder][early-proposal]") { auto const invalidFirst = GENERATE(false, true); @@ -6686,7 +6952,7 @@ TEST_CASE("early underfilled proposals refresh at the trigger", auto extra = bob.tx({payment(alice, 100)}); auto bad = txtest::getGenesisAccount(*app, 2).tx({payment(alice, 100)}); getSignatures(bad).clear(); - std::vector mempool; + std::vector mempool{tx->getEnvelope()}; if (invalidFirst) { // More candidates than capacity, but they cannot fill the ledger. @@ -6703,24 +6969,31 @@ TEST_CASE("early underfilled proposals refresh at the trigger", [&]() { return LeaderBuildTestAccess::prepared(herder) != nullptr; }, std::chrono::seconds(1)); REQUIRE(pulls == 1); - REQUIRE(LeaderBuildTestAccess::prepared(herder)->sizeTxTotal() == 0); - // Transactions arrive while we are waiting for the normal trigger. + auto const prepared = LeaderBuildTestAccess::prepared(herder); + REQUIRE(prepared->sizeTxTotal() == (invalidFirst ? 0 : 1)); + // It was built a short lead time before the trigger, not earlier. + REQUIRE(clock.now() >= herder.getTriggerTimer().expiry_time() - + LeaderBuildTestAccess::prepareLeadTime(herder)); + REQUIRE(clock.now() < herder.getTriggerTimer().expiry_time()); + // Transactions arrive while we are waiting for the trigger. mempool = {tx->getEnvelope(), extra->getEnvelope()}; clock.setCurrentVirtualTime(herder.getTriggerTimer().expiry_time()); testutil::crankUntil( app, [&]() { return herder.getSCP().getLeaderRound(seq) != 0; }, std::chrono::seconds(1)); - REQUIRE(pulls == 2); - // Check the actual set cached for the proposal, independently of benchmark - // instrumentation. Both newly arrived transactions must be present. - auto expected = makeTxSetFromTransactions(TxFrameList{tx, extra}, *app, - ApplyTimeOffset{}) - .first; - REQUIRE(expected->sizeTxTotal() == 2); - auto nominated = std::get( - herder.getTxSet(expected->getContentsHash())); + // No second build: the proposal names the prepared set. + REQUIRE(pulls == 1); + auto messages = herder.getSCP().getLatestMessagesSend(seq); + REQUIRE(!messages.empty()); + REQUIRE(messages.front().statement.pledges.type() == SCP_ST_PREPARE); + StellarValue proposed; + xdr::xdr_from_opaque( + messages.front().statement.pledges.prepare().ballot.value, proposed); + REQUIRE(proposed.txSetHash == prepared->getContentsHash()); + auto nominated = + std::get(herder.getTxSet(proposed.txSetHash)); REQUIRE(nominated); - REQUIRE(nominated->sizeTxTotal() == 2); + REQUIRE(nominated->sizeTxTotal() == (invalidFirst ? 0 : 1)); REQUIRE(!LeaderBuildTestAccess::prepared(herder)); herder.mGetTopTransactionsForTesting = nullptr; } diff --git a/src/herder/test/LeaderRoundsTests.cpp b/src/herder/test/LeaderRoundsTests.cpp index ad1db120e..530d22a5a 100644 --- a/src/herder/test/LeaderRoundsTests.cpp +++ b/src/herder/test/LeaderRoundsTests.cpp @@ -3,8 +3,8 @@ // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 // Network-level tests of consensus without nomination: the leader of each -// round comes from a deterministic schedule seeded by the hash of the last -// closed ledger header (see herder/LeaderSchedule.h). +// round comes from a deterministic schedule derived from the last closed +// ledger header (see herder/LeaderSchedule.h). #include "crypto/SHA.h" #include "herder/Herder.h" @@ -38,13 +38,15 @@ acceleratedConfig(int configNum) struct ClosedLedger { - Hash mPreviousHash; + LedgerHeader mPrevious; NodeID mProposer; + Hash mResultHash; }; // Cranks until `app` closed `lastLedger`, recording for every ledger it -// closes on the way the hash of the ledger before it and the node that signed -// the externalized value (its proposer). +// closes on the way the header of the ledger before it (what its leader +// schedule is derived from) and the node that signed the externalized value +// (its proposer). std::map crankAndRecord(Simulation::pointer const& simulation, Application::pointer const& app, uint32_t lastLedger, @@ -63,7 +65,8 @@ crankAndRecord(Simulation::pointer const& simulation, { REQUIRE(cur.header.previousLedgerHash == prev.hash); res[cur.header.ledgerSeq] = ClosedLedger{ - prev.hash, getLcValueSignature(cur.header.scpValue).nodeID}; + prev.header, getLcValueSignature(cur.header.scpValue).nodeID, + cur.header.txSetResultHash}; } prev = cur; } @@ -107,15 +110,33 @@ TEST_CASE("leader schedule drives consensus on a healthy network", timeouts += leaderTimeouts(n) + ballotTimeouts(n); } - // Every node computes the same schedule from the previous ledger's hash, - // and the value that externalizes is the one its round 1 leader proposed - // (unless a round timed out somewhere, e.g. on a busy test machine). + // Every node computes the same schedule from the previous ledger's + // header, and the value that externalizes is the one its round 1 leader + // proposed (unless a round timed out somewhere, e.g. on a busy test + // machine). These ledgers are empty, so no leader keeps its term: the + // node that closed the previous ledger goes last, and the order seeded + // by the ledger before it is otherwise kept. std::set proposers; int64_t notFirstLeader = 0; REQUIRE(closed.size() >= 10); for (auto const& c : closed) { - auto schedule = computeLeaderSchedule(c.second.mPreviousHash, nodeIDs); + auto schedule = + computeNextLeaderSchedule(c.second.mPrevious, nodeIDs, true); + auto expected = computeLeaderSchedule( + c.second.mPrevious.previousLedgerHash, nodeIDs); + if (c.second.mPrevious.scpValue.ext.v() != STELLAR_VALUE_BASIC) + { + auto const previousProposer = + getLcValueSignature(c.second.mPrevious.scpValue).nodeID; + auto it = + std::find(expected.begin(), expected.end(), previousProposer); + REQUIRE(it != expected.end()); + std::rotate(it, it + 1, expected.end()); + // so two ledgers in a row are never led by the same node here + REQUIRE(schedule.front() != previousProposer); + } + REQUIRE(schedule == expected); REQUIRE(schedule.size() == 4); REQUIRE(std::find(schedule.begin(), schedule.end(), c.second.mProposer) != schedule.end()); @@ -200,7 +221,8 @@ TEST_CASE("leader schedule fails over when a validator is down", for (auto const& c : closed) { // the schedule covers the whole quorum set, including the dead node - auto schedule = computeLeaderSchedule(c.second.mPreviousHash, allNodes); + auto schedule = + computeNextLeaderSchedule(c.second.mPrevious, allNodes, true); REQUIRE(schedule.size() == 3); // every ledger closed anyway, with a live node's proposal REQUIRE(c.second.mProposer != dead); diff --git a/src/herder/test/LeaderScheduleTests.cpp b/src/herder/test/LeaderScheduleTests.cpp index c5bc3d149..abe4e92ee 100644 --- a/src/herder/test/LeaderScheduleTests.cpp +++ b/src/herder/test/LeaderScheduleTests.cpp @@ -10,6 +10,7 @@ #include "util/XDROperators.h" #include #include +#include #include #include @@ -121,3 +122,295 @@ TEST_CASE("leader schedule is fair across seeds", "[herder][leader-schedule]") REQUIRE(f.second < expectedPair * 1.4); } } + +namespace +{ +// A closed ledger's header as far as the schedule reads it: `proposer` signed +// its value, and it applied transactions or not. +LedgerHeader +closedLedger(uint32_t seq, NodeID const& proposer, bool appliedTransactions, + Hash const& previousLedgerHash) +{ + LedgerHeader header; + header.ledgerSeq = seq; + header.previousLedgerHash = previousLedgerHash; + header.scpValue.ext.v(STELLAR_VALUE_SIGNED); + header.scpValue.ext.lcValueSignature().nodeID = proposer; + header.txSetResultHash = appliedTransactions + ? sha256("some results") + : xdrSha256(TransactionResultSet{}); + return header; +} +} + +TEST_CASE("leader term outcome", "[herder][leader-schedule][leader-term]") +{ + auto nodes = makeNodes(7); + NodeID const proposer = nodes[2]; + Hash const prev = sha256("ledger 100"); + // 101 + 1 is not a multiple of the term length + REQUIRE(LEADER_TERM_LEDGERS == 5); + auto header = closedLedger(101, proposer, true, prev); + + SECTION("the node whose value closed the ledger keeps leading") + { + for (auto const& outcome : + {leaderTermOutcome(header, true), leaderTermOutcome(header, false), + leaderTermOutcome(101, header.scpValue, true)}) + { + REQUIRE(outcome.mIncumbent == proposer); + REQUIRE(!outcome.mForfeited); + } + } + SECTION("millisecond values name their proposer too") + { + header.scpValue.ext.v(STELLAR_VALUE_SIGNED_MS); + header.scpValue.ext.signedMsValue().lcValueSignature.nodeID = nodes[4]; + REQUIRE(leaderTermOutcome(header, true).mIncumbent == nodes[4]); + } + SECTION("a term ends on every multiple of the term length") + { + for (uint32_t seq = 100; seq < 100 + 3 * LEADER_TERM_LEDGERS; ++seq) + { + header.ledgerSeq = seq; + bool const nextStartsTerm = (seq + 1) % LEADER_TERM_LEDGERS == 0; + auto const outcome = leaderTermOutcome(header, true); + REQUIRE(outcome.mIncumbent.has_value() == !nextStartsTerm); + // the end of a term is not a forfeit + REQUIRE(!outcome.mForfeited); + } + } + SECTION("a ledger that applied nothing forfeits the term") + { + // an invalid ("skipped") transaction set, or an empty one + auto skipped = closedLedger(101, proposer, false, prev); + for (auto const& outcome : + {leaderTermOutcome(skipped, true), + leaderTermOutcome(101, skipped.scpValue, false)}) + { + REQUIRE(!outcome.mIncumbent); + REQUIRE(outcome.mForfeited == proposer); + } + // unless ledgers never record results (overlay-only mode) + REQUIRE(leaderTermOutcome(skipped, false).mIncumbent == proposer); + REQUIRE(!leaderTermOutcome(skipped, false).mForfeited); + // also at the end of a term: the new term's order is seeded, and + // must not start with the node whose ledger applied nothing + skipped.ledgerSeq = 104; + REQUIRE(!leaderTermOutcome(skipped, true).mIncumbent); + REQUIRE(leaderTermOutcome(skipped, true).mForfeited == proposer); + } + SECTION("a consensus-level empty-tx-set value forfeits the term") + { + header.scpValue.ext.v(STELLAR_VALUE_EMPTY_TX_SET); + header.scpValue.ext.proposedValue().lcValueSignature.nodeID = proposer; + for (bool haveResults : {true, false}) + { + auto const outcome = leaderTermOutcome(header, haveResults); + REQUIRE(!outcome.mIncumbent); + REQUIRE(outcome.mForfeited == proposer); + } + header.scpValue.ext.v(STELLAR_VALUE_EMPTY_TX_SET_MS); + header.scpValue.ext.proposedMsValue().lcValueSignature.nodeID = + proposer; + REQUIRE(leaderTermOutcome(header, false).mForfeited == proposer); + } + SECTION("an unsigned value names nobody") + { + header.scpValue.ext.v(STELLAR_VALUE_BASIC); + REQUIRE(!leaderTermOutcome(header, true).mIncumbent); + REQUIRE(!leaderTermOutcome(header, true).mForfeited); + } +} + +TEST_CASE("next leader schedule", "[herder][leader-schedule][leader-term]") +{ + auto nodes = makeNodes(9); + NodeID const proposer = nodes[5]; + Hash const prev = sha256("ledger 100"); + auto header = closedLedger(101, proposer, true, prev); + auto const seeded = computeLeaderSchedule(prev, nodes); + auto others = seeded; + others.erase(std::find(others.begin(), others.end(), proposer)); + + SECTION("the incumbent leads round 1, the seeded order follows") + { + auto schedule = computeNextLeaderSchedule(header, nodes, true); + REQUIRE(schedule.size() == nodes.size()); + REQUIRE(schedule[0] == proposer); + REQUIRE(std::vector(schedule.begin() + 1, schedule.end()) == + others); + } + SECTION("the order is seeded by the hash of the ledger before") + { + // Known one ledger early: nothing else of the closed ledger's + // header moves the order. + auto fresh = closedLedger(104, proposer, true, prev); + REQUIRE(computeNextLeaderSchedule(fresh, nodes, true) == seeded); + fresh.txSetResultHash = sha256("other results"); + fresh.scpValue.txSetHash = sha256("other set"); + fresh.bucketListHash = sha256("other state"); + REQUIRE(computeNextLeaderSchedule(fresh, nodes, true) == seeded); + fresh.previousLedgerHash = sha256("another ledger 103"); + REQUIRE(computeNextLeaderSchedule(fresh, nodes, true) == + computeLeaderSchedule(fresh.previousLedgerHash, nodes)); + } + SECTION("a leader that forfeited its term goes last") + { + // whatever the seeded order says, including when it would have put + // that node first + for (auto const& forfeiter : nodes) + { + auto skipped = closedLedger(101, forfeiter, false, prev); + auto schedule = computeNextLeaderSchedule(skipped, nodes, true); + REQUIRE(schedule.size() == nodes.size()); + REQUIRE(schedule.back() == forfeiter); + REQUIRE(schedule.front() != forfeiter); + auto rest = seeded; + rest.erase(std::find(rest.begin(), rest.end(), forfeiter)); + REQUIRE(std::vector(schedule.begin(), schedule.end() - 1) == + rest); + } + } + SECTION("a new term takes the seeded order as it is") + { + auto last = closedLedger(104, proposer, true, prev); + REQUIRE(computeNextLeaderSchedule(last, nodes, true) == seeded); + } + SECTION("unless the last ledger of the old term forfeited") + { + for (auto const& forfeiter : nodes) + { + auto last = closedLedger(104, forfeiter, false, prev); + auto schedule = computeNextLeaderSchedule(last, nodes, true); + REQUIRE(schedule.back() == forfeiter); + REQUIRE(schedule.front() != forfeiter); + } + } + SECTION("an incumbent or forfeiter outside the node set is ignored") + { + auto fewer = nodes; + fewer.erase(std::find(fewer.begin(), fewer.end(), proposer)); + REQUIRE(computeNextLeaderSchedule(header, fewer, true) == + computeLeaderSchedule(prev, fewer)); + auto skipped = closedLedger(101, proposer, false, prev); + REQUIRE(computeNextLeaderSchedule(skipped, fewer, true) == + computeLeaderSchedule(prev, fewer)); + } + SECTION("every node computes the same schedule") + { + for (bool applied : {true, false}) + { + auto closed = closedLedger(101, proposer, applied, prev); + auto schedule = computeNextLeaderSchedule(closed, nodes, true); + for (int i = 0; i < 10; i++) + { + auto shuffled = nodes; + stellar::shuffle(shuffled.begin(), shuffled.end(), + getGlobalRandomEngine()); + REQUIRE(computeNextLeaderSchedule(closed, shuffled, true) == + schedule); + } + } + } + SECTION("the prediction made during the ballot matches the schedule") + { + // While ledger 101 is in its ballot, its header does not exist yet: + // the seed (the hash of ledger 100) and the value do. + REQUIRE( + computeLeaderSchedule( + prev, nodes, leaderTermOutcome(101, header.scpValue, true)) == + computeNextLeaderSchedule(header, nodes, true)); + auto skipped = closedLedger(101, proposer, false, prev); + REQUIRE( + computeLeaderSchedule( + prev, nodes, leaderTermOutcome(101, skipped.scpValue, false)) == + computeNextLeaderSchedule(skipped, nodes, true)); + } +} + +TEST_CASE("leader terms over a chain of ledgers", + "[herder][leader-schedule][leader-term]") +{ + // Every ledger is closed by its round 1 leader, except where + // `roundOneFails` says the leader timed out and round 2's leader closed + // it, or `appliesNothing` says its set was skipped. + size_t const n = 10; + auto nodes = makeNodes(n); + auto run = [&](std::function roundOneFails, + std::function appliesNothing) { + std::map leaders; + std::map proposers; + LedgerHeader lcl = + closedLedger(1, nodes[0], false, sha256("genesis parent")); + for (uint32_t seq = 2; seq < 400; ++seq) + { + auto schedule = computeNextLeaderSchedule(lcl, nodes, true); + REQUIRE(schedule.size() == n); + leaders[seq] = schedule[0]; + auto const& proposer = + roundOneFails(seq) ? schedule[1] : schedule[0]; + proposers[seq] = proposer; + lcl = closedLedger(seq, proposer, !appliesNothing(seq), + xdrSha256(lcl)); + } + return std::make_pair(leaders, proposers); + }; + auto never = [](uint32_t) { return false; }; + + SECTION("a healthy leader leads a whole term, and no longer") + { + auto [leaders, proposers] = run(never, never); + std::set seen; + for (uint32_t seq = 5; seq + LEADER_TERM_LEDGERS < 400; + seq += LEADER_TERM_LEDGERS) + { + for (uint32_t i = 1; i < LEADER_TERM_LEDGERS; ++i) + { + REQUIRE(leaders[seq + i] == leaders[seq]); + } + seen.insert(leaders[seq]); + } + // terms are handed out by the seeded order: everyone gets some + REQUIRE(seen.size() == n); + } + SECTION("a timeout hands the rest of the term to the node that took over") + { + // round 1 fails in the second ledger of every term + auto fails = [](uint32_t seq) { + return seq % LEADER_TERM_LEDGERS == 1; + }; + auto [leaders, proposers] = run(fails, never); + for (uint32_t seq = 10; seq + LEADER_TERM_LEDGERS < 400; + seq += LEADER_TERM_LEDGERS) + { + // the term's first leader led two ledgers, and lost the second + REQUIRE(leaders[seq + 1] == leaders[seq]); + REQUIRE(proposers[seq + 1] != leaders[seq]); + // from the next ledger on, the node that closed it leads + for (uint32_t i = 2; i < LEADER_TERM_LEDGERS; ++i) + { + REQUIRE(leaders[seq + i] == proposers[seq + 1]); + REQUIRE(leaders[seq + i] != leaders[seq]); + } + } + } + SECTION("a ledger that applied nothing rotates the leader at once") + { + auto skipped = [](uint32_t seq) { + return seq % LEADER_TERM_LEDGERS == 2; + }; + auto [leaders, proposers] = run(never, skipped); + for (uint32_t seq = 10; seq + LEADER_TERM_LEDGERS < 400; + seq += LEADER_TERM_LEDGERS) + { + REQUIRE(leaders[seq + 1] == leaders[seq]); + REQUIRE(leaders[seq + 2] == leaders[seq]); + // The ledger after the skipped one is never led by the node + // whose set was skipped, and the node that takes over keeps the + // rest of the term. + REQUIRE(leaders[seq + 3] != leaders[seq]); + REQUIRE(leaders[seq + 4] == leaders[seq + 3]); + } + } +} diff --git a/src/herder/test/PendingEnvelopesTests.cpp b/src/herder/test/PendingEnvelopesTests.cpp index 6b8085ade..5a7ae2d82 100644 --- a/src/herder/test/PendingEnvelopesTests.cpp +++ b/src/herder/test/PendingEnvelopesTests.cpp @@ -123,6 +123,23 @@ createAppWithFollowerPeer(Config const& baseCfg) FollowerPeerApp res; res.mPeer = SecretKey::pseudoRandomForTesting(); Config cfg(baseCfg); + if (i > 0) + { + // A node's place in the leader order is decided by its key. The + // test configuration's key (derived from the instance number and + // the day's seed) can be one that sorts behind nearly every + // other key: no random peer would ever come after it. Try + // another local key as well, not only another peer. + auto const oldKey = cfg.NODE_SEED.getPublicKey(); + cfg.NODE_SEED = SecretKey::pseudoRandomForTesting(); + for (auto& validator : cfg.QUORUM_SET.validators) + { + if (validator == oldKey) + { + validator = cfg.NODE_SEED.getPublicKey(); + } + } + } cfg.QUORUM_SET.validators.emplace_back(res.mPeer.getPublicKey()); res.mClock = std::make_unique(); res.mApp = createTestApplication(*res.mClock, cfg); diff --git a/src/herder/test/ProposalPrevalidationTests.cpp b/src/herder/test/ProposalPrevalidationTests.cpp new file mode 100644 index 000000000..0de8f5eea --- /dev/null +++ b/src/herder/test/ProposalPrevalidationTests.cpp @@ -0,0 +1,1146 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// Tests of the leader's head start on the next transaction set: the state a +// speculative execution leaves is handed to the Herder +// (LedgerManager::startSpeculativeApply), the node that expects to lead the +// next ledger validates its candidates against it in the background +// (HerderImpl::startProposalPrevalidation), and the build of the next set +// takes those results instead of validating again (TxValidationCache). + +#include "crypto/SHA.h" +#include "herder/Herder.h" +#include "herder/HerderImpl.h" +#include "herder/LeaderSchedule.h" +#include "herder/LedgerCloseData.h" +#include "herder/TxSetFrame.h" +#include "herder/test/TestTxSetUtils.h" +#include "ledger/ImmutableLedgerView.h" +#include "ledger/LedgerHeaderUtils.h" +#include "ledger/LedgerManager.h" +#include "ledger/test/LedgerTestUtils.h" +#include "main/Application.h" +#include "medida/meter.h" +#include "simulation/Simulation.h" +#include "simulation/Topologies.h" +#include "test/Catch2.h" +#include "test/TestAccount.h" +#include "test/TestUtils.h" +#include "test/TxTests.h" +#include "test/test.h" +#include "util/MetricsRegistry.h" +#include "util/ProtocolVersion.h" +#include "util/XDROperators.h" +#include +#include +#include +#include + +using namespace stellar; +using namespace stellar::txtest; + +namespace stellar +{ +class ProposalPrevalidationTestAccess +{ + public: + // What HerderImpl::maybeStartSpeculativeApply records when the node + // votes for `value`. + static void + setVotedValue(HerderImpl& herder, uint64_t slot, StellarValue const& value) + { + herder.mSpeculativeApplyValue = + std::make_pair(slot, Value(xdr::xdr_to_opaque(value))); + } + + static void + start(HerderImpl& herder, std::shared_ptr state) + { + herder.startProposalPrevalidation(std::move(state)); + } + + static bool + running(HerderImpl& herder) + { + return herder.mProposalPrevalidation != nullptr; + } + + // Waits for the background tasks of the running pre-validation. + static void + waitUntilFinished(HerderImpl& herder) + { + auto job = herder.mProposalPrevalidation; + REQUIRE(job); + auto const deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (job->finishedAfterNs.load() < 0) + { + REQUIRE(std::chrono::steady_clock::now() < deadline); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + static std::pair + build(HerderImpl& herder, uint32_t ledgerSeq, ConsensusTime closeTime) + { + auto built = herder.buildTxSet(ledgerSeq, closeTime, false); + return std::make_pair(built.txSet, std::move(built.applicableTxSet)); + } + + static size_t + cachedResults(HerderImpl& herder) + { + return herder.mValidationCache.results.size(); + } + + static void + clearCache(HerderImpl& herder) + { + herder.mValidationCache = HerderImpl::ValidationCache{}; + } + + static bool + mayLeadAfter(HerderImpl& herder, uint64_t slot, StellarValue const& value, + bool appliedTransactions) + { + return herder.mayLeadAfter(slot, value, appliedTransactions); + } + + static std::chrono::milliseconds + prepareLeadTime(std::chrono::milliseconds expectedCloseTime, + std::chrono::milliseconds lastBuild) + { + return HerderImpl::prepareTxSetLeadTime(expectedCloseTime, lastBuild); + } + + // What the node expects its next build to take, given what it knows. + static std::chrono::milliseconds + expectedBuild(HerderImpl& herder) + { + return herder.expectedTxSetBuildDuration(); + } + + static void + setBuildHistory(HerderImpl& herder, std::chrono::milliseconds lastBuild, + std::chrono::nanoseconds costPerTx) + { + herder.mLastTxSetBuildDuration = lastBuild; + herder.mTxSetBuildCostPerTx = costPerTx; + } + + static void + setLastClosedTxSetSize(HerderImpl& herder, size_t txs) + { + herder.mLastClosedTxSetSize = txs; + } + + static size_t + lastClosedTxSetSize(HerderImpl& herder) + { + return herder.mLastClosedTxSetSize; + } +}; +} + +namespace +{ +using Access = ProposalPrevalidationTestAccess; + +uint32_t const GATE_PROTOCOL = + static_cast(APPLY_TIME_TX_SET_VALIDATION_PROTOCOL_VERSION); + +int64_t +meter(Application& app, std::string const& domain, std::string const& name, + std::string const& unit) +{ + return static_cast( + app.getMetrics().NewMeter({"herder", domain, name}, unit).count()); +} + +int64_t +applyCounter(Application& app, std::string const& name) +{ + return app.getMetrics().NewCounter({"ledger", "apply", name}).count(); +} + +struct CacheCounts +{ + int64_t mHit; + int64_t mMiss; +}; + +CacheCounts +cacheCounts(Application& app) +{ + return CacheCounts{meter(app, "txset", "validation-cache-hit", "tx"), + meter(app, "txset", "validation-cache-miss", "tx")}; +} + +TxSetXDRFrameConstPtr +makeUnvalidatedTxSet(Application& app, + std::vector const& txs) +{ + auto res = testtxset::makeNonValidatedGeneralizedTxSet( + {{std::make_pair(std::nullopt, txs)}, {}}, app, + app.getLedgerManager().getLastClosedLedgerHeader().hash); + REQUIRE(res.first); + return res.first; +} + +TransactionFrameBasePtr +paymentWithSeq(Application& app, TestAccount& from, SequenceNumber seq, + int64_t amount = 1) +{ + return transactionFromOperations( + app, from.getSecretKey(), seq, + {payment(app.getRoot()->getPublicKey(), amount)}, 100); +} + +std::set +txHashes(ApplicableTxSetFrameConstPtr const& txSet) +{ + std::set res; + for (auto const& phase : txSet->getPhases()) + { + for (auto const& tx : phase) + { + res.insert(tx->getFullHash()); + } + } + return res; +} + +std::set +hashesOf(std::vector const& txs) +{ + std::set res; + for (auto const& tx : txs) + { + res.insert(tx->getFullHash()); + } + return res; +} + +// A single validator (so it leads every ledger) whose ledgers apply on the +// ledger apply thread, with four funded accounts. Speculation and +// pre-validation are driven by the test, one step at a time. +struct PrevalidationFixture +{ + VirtualClock mClock; + Config mCfg; + Application::pointer mApp; + std::optional mA; + std::optional mB; + std::optional mC; + std::optional mD; + std::optional mE; + // What the mempool holds; the Herder pulls it in this order. + std::vector mMempool; + + PrevalidationFixture() + : mCfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT)) + { + mCfg.LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + mCfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + // The tests drive speculation themselves. + mCfg.EXPERIMENTAL_SPECULATIVE_LEDGER_APPLY = false; + REQUIRE(mCfg.backgroundLedgerApply()); + mApp = createTestApplication(mClock, mCfg); + auto root = mApp->getRoot(); + auto const balance = + mApp->getLedgerManager().getLastMinBalance(2) + 10'000'000; + mA.emplace(root->create("A", balance)); + mB.emplace(root->create("B", balance)); + mC.emplace(root->create("C", balance)); + mD.emplace(root->create("D", balance)); + mE.emplace(root->create("E", balance)); + herder().mGetTopTransactionsForTesting = [this](size_t) { + std::vector envelopes; + for (auto const& tx : mMempool) + { + envelopes.push_back(tx->getEnvelope()); + } + return envelopes; + }; + } + + ~PrevalidationFixture() + { + herder().mGetTopTransactionsForTesting = nullptr; + } + + HerderImpl& + herder() + { + return static_cast(mApp->getHerder()); + } + + LedgerManager& + lm() + { + return mApp->getLedgerManager(); + } + + uint32_t + nextSeq() + { + return lm().getLastClosedLedgerNum() + 1; + } + + ConsensusTime + nextCloseTime() + { + return ConsensusTime::fromApplyTime(ApplyTime::fromTimePoint( + lm().getLastClosedLedgerHeader().header.scpValue.closeTime + 1)); + } + + StellarValue + valueFor(TxSetXDRFrameConstPtr const& txSet, + xdr::xvector const& upgrades = emptyUpgradeSteps) + { + return herder().makeStellarValue(txSet->getContentsHash(), + nextCloseTime(), upgrades, + mCfg.NODE_SEED); + } + + // Executes the next ledger with `txSet` on the apply thread without + // closing it, as a node does when it votes for the value, and returns + // the state the execution left. + std::shared_ptr + speculate(TxSetXDRFrameConstPtr const& txSet, bool prevalidate, + xdr::xvector const& upgrades = emptyUpgradeSteps) + { + auto const value = valueFor(txSet, upgrades); + // By the time a node votes, SCP has asked for the slot's leaders, + // which brings the tracked quorum (the node set) up to date. + REQUIRE(herder().getHerderSCPDriver().getLeaderSchedule(nextSeq()) == + std::vector{mCfg.NODE_SEED.getPublicKey()}); + Access::setVotedValue(herder(), nextSeq(), value); + std::shared_ptr executed; + bool called = false; + lm().startSpeculativeApply( + LedgerCloseData(nextSeq(), txSet, value), + [&](std::shared_ptr state) { + called = true; + executed = state; + if (prevalidate) + { + Access::start(herder(), state); + } + }); + while (lm().isSpeculativelyApplying()) + { + mClock.crank(true); + } + REQUIRE(called); + return executed; + } + + // Externalizes the value `speculate` used for `txSet`, which closes the + // ledger. + void + externalize(TxSetXDRFrameConstPtr const& txSet) + { + mApp->getHerder().externalizeValue(txSet, nextSeq(), nextCloseTime(), + emptyUpgradeSteps); + } + + std::set + build() + { + auto built = Access::build(herder(), nextSeq(), nextCloseTime()); + REQUIRE(built.second); + return txHashes(built.second); + } +}; + +AccountEntry +accountIn(AbstractLedgerView const& view, TestAccount& acc) +{ + auto entry = view.getAccount(acc.getPublicKey()); + REQUIRE(entry); + return entry.current().data.account(); +} +} + +TEST_CASE("speculative apply hands over the state it executed", + "[ledger][speculative-apply][prevalidation]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& lm = f.lm(); + auto& a = *f.mA; + auto& b = *f.mB; + auto const aSeq = a.getLastSequenceNumber(); + auto const aBalance = a.getBalance(); + auto const bSeq = b.getLastSequenceNumber(); + auto const seq = f.nextSeq(); + auto const lclBefore = lm.getLastClosedLedgerHeader(); + + SECTION("a ledger that applies") + { + auto tx = paymentWithSeq(app, a, aSeq + 1); + auto txSet = makeUnvalidatedTxSet(app, {tx}); + auto const value = f.valueFor(txSet); + auto state = f.speculate(txSet, false); + REQUIRE(state); + REQUIRE(state->mLedgerSeq == seq); + REQUIRE(state->mValue == value); + REQUIRE(!state->mTxSetSkipped); + REQUIRE(state->mAppliedTxs == 1); + REQUIRE(state->mTxHashes->size() == 1); + REQUIRE(state->mTxHashes->count(tx->getFullHash()) == 1); + REQUIRE(state->mHeader->ledgerSeq == seq); + REQUIRE(state->mHeader->scpValue == value); + REQUIRE(state->mHeader->previousLedgerHash == lclBefore.hash); + + // The view shows the ledger as executed: A paid and its sequence + // number moved; B is read through from the previous ledger. + auto view = state->makeView(app.getMetrics()); + REQUIRE(view->getLedgerHeader().current().ledgerSeq == seq); + auto const executedA = accountIn(*view, a); + REQUIRE(executedA.seqNum == aSeq + 1); + REQUIRE(executedA.balance == aBalance - 100 - 1); + REQUIRE(accountIn(*view, b).seqNum == bSeq); + // ... while the last closed ledger is untouched. + REQUIRE(lm.getLastClosedLedgerHeader().hash == lclBefore.hash); + REQUIRE(accountIn(*lm.getLCLView(), a).seqNum == aSeq); + + // Once the value externalizes, the closed ledger holds what the + // captured state said, and the state itself did not move. + f.externalize(txSet); + REQUIRE(lm.getLastClosedLedgerNum() == seq); + auto const closedA = accountIn(*lm.getLCLView(), a); + REQUIRE(closedA.seqNum == executedA.seqNum); + REQUIRE(closedA.balance == executedA.balance); + REQUIRE(accountIn(*view, a).seqNum == aSeq + 1); + REQUIRE(applyCounter(app, "speculation-hit") == 1); + } + + SECTION("an erased entry") + { + auto merge = transactionFromOperations( + app, a.getSecretKey(), aSeq + 1, + {accountMerge(app.getRoot()->getPublicKey())}, 100); + auto state = f.speculate(makeUnvalidatedTxSet(app, {merge}), false); + REQUIRE(state); + REQUIRE(state->mAppliedTxs == 1); + auto view = state->makeView(app.getMetrics()); + REQUIRE(!view->getAccount(a.getPublicKey())); + REQUIRE(lm.getLCLView()->getAccount(a.getPublicKey())); + } + + SECTION("a ledger whose transaction set is invalid") + { + auto good = paymentWithSeq(app, a, aSeq + 1); + auto bad = paymentWithSeq(app, b, bSeq + 10); + auto state = f.speculate(makeUnvalidatedTxSet(app, {good, bad}), false); + REQUIRE(state); + REQUIRE(state->mTxSetSkipped); + REQUIRE(state->mAppliedTxs == 0); + REQUIRE(state->mTxHashes->empty()); + auto view = state->makeView(app.getMetrics()); + REQUIRE(accountIn(*view, a).seqNum == aSeq); + REQUIRE(accountIn(*view, a).balance == aBalance); + } + + SECTION("a ledger that is not the next one is not executed") + { + auto txSet = + makeUnvalidatedTxSet(app, {paymentWithSeq(app, a, aSeq + 1)}); + auto const value = f.valueFor(txSet); + bool called = false; + std::shared_ptr state; + lm.startSpeculativeApply( + LedgerCloseData(seq + 1, txSet, value), + [&](std::shared_ptr s) { + called = true; + state = s; + }); + while (lm.isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + REQUIRE(called); + REQUIRE(!state); + } +} + +TEST_CASE("leader pre-validates the candidates of the next ledger", + "[herder][txset][prevalidation]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto& c = *f.mC; + auto& d = *f.mD; + auto const aSeq = a.getLastSequenceNumber(); + auto const bSeq = b.getLastSequenceNumber(); + auto const cSeq = c.getLastSequenceNumber(); + auto const dSeq = d.getLastSequenceNumber(); + + // The ledger being closed (N) holds a payment of A and one of D. + auto closingA = paymentWithSeq(app, a, aSeq + 1); + auto closingD = paymentWithSeq(app, d, dSeq + 1); + // Candidates for the ledger after it: + // * A's next payment: only valid once N has applied, + // * another payment of D with the sequence number N consumes: valid + // before N, not after, + // * B's payment: valid either way, + // * C's payment with a sequence number too far ahead: never valid. + auto nextA = paymentWithSeq(app, a, aSeq + 2); + auto staleD = paymentWithSeq(app, d, dSeq + 1, 2); + auto plainB = paymentWithSeq(app, b, bSeq + 1); + auto futureC = paymentWithSeq(app, c, cSeq + 5); + REQUIRE(staleD->getFullHash() != closingD->getFullHash()); + + auto closing = makeUnvalidatedTxSet(app, {closingA, closingD}); + // N's transactions are still in the mempool while N is in its ballot. + f.mMempool = {closingA, closingD, nextA, staleD, plainB, futureC}; + auto const startedBefore = + meter(app, "prevalidate", "started", "prevalidation"); + auto const usedBefore = meter(app, "prevalidate", "used", "prevalidation"); + auto const discardedBefore = + meter(app, "prevalidate", "discarded", "prevalidation"); + + auto state = f.speculate(closing, true); + REQUIRE(state); + REQUIRE(Access::running(herder)); + REQUIRE(meter(app, "prevalidate", "started", "prevalidation") == + startedBefore + 1); + Access::waitUntilFinished(herder); + + SECTION("the build takes the results when the executed value closes") + { + f.externalize(closing); + // The overlay drops a closed ledger's transactions. + f.mMempool = {nextA, staleD, plainB, futureC}; + auto const before = cacheCounts(app); + auto const built = f.build(); + REQUIRE(built == hashesOf({nextA, plainB})); + auto const after = cacheCounts(app); + // nothing was validated on the main thread + REQUIRE(after.mHit - before.mHit == 4); + REQUIRE(after.mMiss - before.mMiss == 0); + REQUIRE(meter(app, "prevalidate", "used", "prevalidation") == + usedBefore + 1); + REQUIRE(!Access::running(herder)); + + // The same set comes out without the results... + Access::clearCache(herder); + auto const rebuiltBefore = cacheCounts(app); + REQUIRE(f.build() == built); + auto const rebuilt = cacheCounts(app); + REQUIRE(rebuilt.mHit - rebuiltBefore.mHit == 0); + REQUIRE(rebuilt.mMiss - rebuiltBefore.mMiss == 4); + // ... and a later build on the same ledger only validates what is + // new to it. + auto lateB = paymentWithSeq(app, c, cSeq + 1); + f.mMempool.push_back(lateB); + auto const lateBefore = cacheCounts(app); + // C now has two candidates; the lower sequence number is kept. + REQUIRE(f.build() == hashesOf({nextA, plainB, lateB})); + auto const late = cacheCounts(app); + REQUIRE(late.mHit - lateBefore.mHit == 3); + REQUIRE(late.mMiss - lateBefore.mMiss == 1); + } + + SECTION("the results are dropped when another value closes the ledger") + { + // B's payment closes the ledger instead: A and D did not move. + auto other = makeUnvalidatedTxSet(app, {plainB}); + f.externalize(other); + f.mMempool = {closingA, closingD, staleD, futureC, plainB}; + auto const before = cacheCounts(app); + // With the stale results, B's consumed payment would be proposed + // again. + auto const built = f.build(); + REQUIRE(built.size() == 2); + REQUIRE(built.count(closingA->getFullHash()) == 1); + REQUIRE(built.count(plainB->getFullHash()) == 0); + REQUIRE(built.count(futureC->getFullHash()) == 0); + auto const after = cacheCounts(app); + REQUIRE(after.mHit - before.mHit == 0); + REQUIRE(meter(app, "prevalidate", "used", "prevalidation") == + usedBefore); + REQUIRE(meter(app, "prevalidate", "discarded", "prevalidation") == + discardedBefore + 1); + REQUIRE(!Access::running(herder)); + } + + SECTION("the results are not kept past the ledger they are for") + { + f.externalize(closing); + f.mMempool = {nextA, staleD, plainB, futureC}; + REQUIRE(f.build() == hashesOf({nextA, plainB})); + REQUIRE(Access::cachedResults(herder) == 4); + // the next ledger closes; a build on top of it starts from scratch + f.externalize(makeUnvalidatedTxSet(app, {nextA})); + f.mMempool = {plainB, futureC}; + auto const before = cacheCounts(app); + REQUIRE(f.build() == hashesOf({plainB})); + auto const after = cacheCounts(app); + REQUIRE(after.mHit - before.mHit == 0); + REQUIRE(after.mMiss - before.mMiss == 2); + REQUIRE(Access::cachedResults(herder) == 2); + } +} + +TEST_CASE("pre-validation is not started when its results could not be used", + "[herder][txset][prevalidation]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto const aSeq = a.getLastSequenceNumber(); + auto const bSeq = b.getLastSequenceNumber(); + auto good = paymentWithSeq(app, a, aSeq + 1); + f.mMempool = {good, paymentWithSeq(app, b, bSeq + 1)}; + + SECTION("the executed set was invalid") + { + auto bad = paymentWithSeq(app, b, bSeq + 10); + auto state = f.speculate(makeUnvalidatedTxSet(app, {good, bad}), true); + REQUIRE(state); + REQUIRE(state->mTxSetSkipped); + REQUIRE(!Access::running(herder)); + } + SECTION("the node votes for another value by the time it has executed") + { + auto txSet = makeUnvalidatedTxSet(app, {good}); + auto const value = f.valueFor(txSet); + auto otherValue = value; + otherValue.closeTime += 1; + Access::setVotedValue(herder, f.nextSeq(), value); + std::shared_ptr executed; + f.lm().startSpeculativeApply( + LedgerCloseData(f.nextSeq(), txSet, value), + [&](std::shared_ptr state) { + executed = state; + Access::setVotedValue(herder, f.nextSeq(), otherValue); + Access::start(herder, state); + }); + while (f.lm().isSpeculativelyApplying()) + { + f.mClock.crank(true); + } + REQUIRE(executed); + REQUIRE(!Access::running(herder)); + } + SECTION("the executed value carries upgrades") + { + // Upgrades are applied when the ledger commits, after the state + // the candidates would be validated on. + LedgerUpgrade upgrade(LEDGER_UPGRADE_BASE_FEE); + upgrade.newBaseFee() = 200; + xdr::xvector upgrades; + upgrades.emplace_back(LedgerTestUtils::toUpgradeType(upgrade)); + auto state = + f.speculate(makeUnvalidatedTxSet(app, {good}), true, upgrades); + REQUIRE(state); + REQUIRE(!state->mValue.upgrades.empty()); + REQUIRE(!Access::running(herder)); + } + SECTION("there is nothing in the mempool") + { + auto txSet = makeUnvalidatedTxSet(app, {good}); + f.mMempool.clear(); + REQUIRE(f.speculate(txSet, true)); + REQUIRE(!Access::running(herder)); + } +} + +// What ledger N does to an account decides whether its next transaction is +// valid; the pre-validated results must say what validating on the closed +// ledger says, for every way N can touch a candidate's accounts. +TEST_CASE("builds with and without pre-validated results agree", + "[herder][txset][prevalidation]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto& c = *f.mC; + auto& d = *f.mD; + auto& e = *f.mE; + auto const root = app.getRoot()->getPublicKey(); + auto const aSeq = a.getLastSequenceNumber(); + auto const bSeq = b.getLastSequenceNumber(); + auto const dSeq = d.getLastSequenceNumber(); + auto const eSeq = e.getLastSequenceNumber(); + + // Ledger N: A pays; D pays away all it can spare; E merges into the root. + auto closingA = paymentWithSeq(app, a, aSeq + 1); + auto const spare = + d.getBalance() - app.getLedgerManager().getLastMinBalance(0) - 100; + REQUIRE(spare > 0); + auto drainD = paymentWithSeq(app, d, dSeq + 1, spare); + auto mergeE = transactionFromOperations(app, e.getSecretKey(), eSeq + 1, + {accountMerge(root)}, 100); + auto closing = makeUnvalidatedTxSet(app, {closingA, drainD, mergeE}); + + // Candidates for N+1: + // * A's next two payments (chained): the first is valid after N, the + // second waits for it, + // * D's next payment: its sequence number is right after N, but D can + // no longer pay the fee, + // * E's next payment: E does not exist after N, + // * B's payment with C paying the fee (a fee bump): valid. + auto nextA = paymentWithSeq(app, a, aSeq + 2); + auto laterA = paymentWithSeq(app, a, aSeq + 3); + auto brokeD = paymentWithSeq(app, d, dSeq + 2); + auto goneE = paymentWithSeq(app, e, eSeq + 2); + auto bumped = feeBump(app, c, paymentWithSeq(app, b, bSeq + 1), 1000); + REQUIRE(!txValidityDependsOnCloseTime(bumped->getEnvelope())); + + f.mMempool = {closingA, drainD, mergeE, nextA, + laterA, brokeD, goneE, bumped}; + auto state = f.speculate(closing, true); + REQUIRE(state); + REQUIRE(state->mAppliedTxs == 3); + REQUIRE(Access::running(herder)); + Access::waitUntilFinished(herder); + f.externalize(closing); + // N applied as the test assumes + REQUIRE(!f.lm().getLCLView()->getAccount(e.getPublicKey())); + REQUIRE(a.getLastSequenceNumber() == aSeq + 1); + REQUIRE(d.getLastSequenceNumber() == dSeq + 1); + + f.mMempool = {nextA, laterA, brokeD, goneE, bumped}; + auto const before = cacheCounts(app); + auto const withResults = f.build(); + auto const after = cacheCounts(app); + // A's later payment is set aside before validation (one transaction per + // source account); the other four come from the pre-validation. + REQUIRE(after.mHit - before.mHit == 4); + REQUIRE(after.mMiss - before.mMiss == 0); + REQUIRE(withResults == hashesOf({nextA, bumped})); + + Access::clearCache(herder); + auto const withoutBefore = cacheCounts(app); + auto const withoutResults = f.build(); + auto const withoutAfter = cacheCounts(app); + REQUIRE(withoutAfter.mHit - withoutBefore.mHit == 0); + REQUIRE(withoutAfter.mMiss - withoutBefore.mMiss == 4); + REQUIRE(withoutResults == withResults); +} + +TEST_CASE("a running pre-validation does not outlive its use", + "[herder][txset][prevalidation]") +{ + auto f = std::make_unique(); + auto& app = *f->mApp; + auto& herder = f->herder(); + auto& a = *f->mA; + auto const aSeq = a.getLastSequenceNumber(); + auto closing = + makeUnvalidatedTxSet(app, {paymentWithSeq(app, a, aSeq + 1)}); + // Enough candidates to keep the background tasks busy for a moment. + auto next = paymentWithSeq(app, a, aSeq + 2); + for (int i = 0; i < 400; ++i) + { + f->mMempool.push_back(next); + } + REQUIRE(f->speculate(closing, true)); + REQUIRE(Access::running(herder)); + + SECTION("the application shuts down") + { + // Destroyed with the job possibly mid-flight: no hang, no crash. + f.reset(); + } + SECTION("the build takes what is there and does not wait") + { + f->externalize(closing); + f->mMempool = {next}; + REQUIRE(f->build() == hashesOf({next})); + REQUIRE(!Access::running(herder)); + auto const counts = cacheCounts(app); + // whether or not the job had got to it, the candidate was validated + // exactly once + REQUIRE(counts.mHit + counts.mMiss >= 1); + } + SECTION("another ledger closes on top") + { + // Another leader's value closes the ledger and this node moves on + // without ever building on it. + f->externalize(closing); + f->externalize(makeUnvalidatedTxSet(app, {next})); + REQUIRE(!Access::running(herder)); + REQUIRE(Access::cachedResults(herder) == 0); + } +} + +TEST_CASE("transactions whose validity depends on the close time", + "[herder][txset][prevalidation]") +{ + TransactionEnvelope v1(ENVELOPE_TYPE_TX); + auto& cond = v1.v1().tx.cond; + REQUIRE(!txValidityDependsOnCloseTime(v1)); + + SECTION("time bounds") + { + cond.type(PRECOND_TIME); + // present but unbounded + REQUIRE(!txValidityDependsOnCloseTime(v1)); + cond.timeBounds().maxTime = 1000; + REQUIRE(txValidityDependsOnCloseTime(v1)); + cond.timeBounds().maxTime = 0; + cond.timeBounds().minTime = 5; + REQUIRE(txValidityDependsOnCloseTime(v1)); + } + SECTION("v2 preconditions") + { + cond.type(PRECOND_V2); + // ledger bounds and sequence conditions depend on the ledger, not + // on its close time + cond.v2().ledgerBounds.activate().maxLedger = 100; + cond.v2().minSeqNum.activate() = 7; + cond.v2().minSeqLedgerGap = 3; + REQUIRE(!txValidityDependsOnCloseTime(v1)); + SECTION("minimum sequence age") + { + cond.v2().minSeqAge = 10; + REQUIRE(txValidityDependsOnCloseTime(v1)); + } + SECTION("time bounds") + { + cond.v2().timeBounds.activate().maxTime = 99; + REQUIRE(txValidityDependsOnCloseTime(v1)); + } + } + SECTION("v0 envelopes") + { + TransactionEnvelope v0(ENVELOPE_TYPE_TX_V0); + REQUIRE(!txValidityDependsOnCloseTime(v0)); + v0.v0().tx.timeBounds.activate().maxTime = 1; + REQUIRE(txValidityDependsOnCloseTime(v0)); + } + SECTION("fee bumps follow their inner transaction") + { + TransactionEnvelope bump(ENVELOPE_TYPE_TX_FEE_BUMP); + bump.feeBump().tx.innerTx.type(ENVELOPE_TYPE_TX); + REQUIRE(!txValidityDependsOnCloseTime(bump)); + auto& inner = bump.feeBump().tx.innerTx.v1().tx.cond; + inner.type(PRECOND_TIME); + inner.timeBounds().maxTime = 1000; + REQUIRE(txValidityDependsOnCloseTime(bump)); + } +} + +TEST_CASE("time-bounded candidates are validated by the build itself", + "[herder][txset][prevalidation]") +{ + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto& b = *f.mB; + auto const aSeq = a.getLastSequenceNumber(); + + auto closing = + makeUnvalidatedTxSet(app, {paymentWithSeq(app, a, aSeq + 1)}); + auto plain = paymentWithSeq(app, a, aSeq + 2); + // valid for an hour from the last close + PreconditionsV2 cond; + cond.timeBounds.activate().maxTime = + f.lm().getLastClosedLedgerHeader().header.scpValue.closeTime + 3600; + auto bounded = transactionWithV2Precondition(app, b, 1, 100, cond); + REQUIRE(txValidityDependsOnCloseTime(bounded->getEnvelope())); + f.mMempool = {plain, bounded}; + + REQUIRE(f.speculate(closing, true)); + Access::waitUntilFinished(herder); + f.externalize(closing); + + auto const before = cacheCounts(app); + REQUIRE(f.build() == hashesOf({plain, bounded})); + auto const after = cacheCounts(app); + REQUIRE(after.mHit - before.mHit == 1); + REQUIRE(after.mMiss - before.mMiss == 1); + // and its result is not kept for a build with another close time + REQUIRE(Access::cachedResults(herder) == 1); +} + +TEST_CASE("the first-round leader builds shortly before the trigger", + "[herder][txset][prevalidation]") +{ + using namespace std::chrono; + auto lead = [](int closeTimeMs, int lastBuildMs) { + return Access::prepareLeadTime(milliseconds(closeTimeMs), + milliseconds(lastBuildMs)); + }; + SECTION("before the node has built anything") + { + // a quarter of the target close time + REQUIRE(lead(800, 0) == milliseconds(200)); + REQUIRE(lead(1000, 0) == milliseconds(250)); + // capped: a long close time does not make transactions wait longer + REQUIRE(lead(5000, 0) == milliseconds(250)); + REQUIRE(lead(0, 0) == milliseconds(0)); + } + SECTION("long enough for a build like the last one and its push") + { + // 180 ms + a quarter + 100 ms for the set to reach the peers + REQUIRE(lead(800, 180) == milliseconds(325)); + REQUIRE(lead(1000, 240) == milliseconds(400)); + // a quick build does not shorten it + REQUIRE(lead(800, 20) == milliseconds(200)); + REQUIRE(lead(5000, 20) == milliseconds(250)); + // on a long close time a slow build still gets its room + REQUIRE(lead(5000, 600) == milliseconds(850)); + } + SECTION("never more than half of the target close time") + { + REQUIRE(lead(800, 400) == milliseconds(400)); + REQUIRE(lead(800, 5000) == milliseconds(400)); + REQUIRE(lead(1000, 500) == milliseconds(500)); + } +} + +// A node that starts a term has not built for a while: its last build says +// little about the next one. What the last closed ledger held does. +TEST_CASE("the lead time of the first build of a term follows the size of the " + "last ledger", + "[herder][txset][prevalidation]") +{ + using namespace std::chrono; + PrevalidationFixture f; + auto& app = *f.mApp; + auto& herder = f.herder(); + auto& a = *f.mA; + auto lead = [&](int closeTimeMs) { + return Access::prepareLeadTime(milliseconds(closeTimeMs), + Access::expectedBuild(herder)); + }; + + SECTION("the herder learns the size of every ledger that closes") + { + auto const aSeq = a.getLastSequenceNumber(); + f.externalize( + makeUnvalidatedTxSet(app, {paymentWithSeq(app, a, aSeq + 1)})); + REQUIRE(Access::lastClosedTxSetSize(herder) == 1); + f.externalize(makeUnvalidatedTxSet(app, {})); + REQUIRE(Access::lastClosedTxSetSize(herder) == 0); + } + SECTION("a node that never built under load") + { + // its last build was of an empty set, long ago + Access::setBuildHistory(herder, milliseconds(1), nanoseconds(0)); + Access::setLastClosedTxSetSize(herder, 0); + REQUIRE(Access::expectedBuild(herder) == milliseconds(1)); + REQUIRE(lead(800) == milliseconds(200)); + // the network now closes ledgers of 9,300 transactions: at the + // default cost per transaction that is a build of 372 ms, more than + // the 200 ms a lead time sized for the old build would leave + Access::setLastClosedTxSetSize(herder, 9300); + REQUIRE(Access::expectedBuild(herder) == milliseconds(372)); + REQUIRE(lead(800) == milliseconds(400)); + // smaller ledgers, smaller head start + Access::setLastClosedTxSetSize(herder, 2000); + REQUIRE(Access::expectedBuild(herder) == milliseconds(80)); + REQUIRE(lead(800) == milliseconds(200)); + REQUIRE(lead(2000) == milliseconds(250)); + } + SECTION("a node that has measured its own builds") + { + // 180 ms for 6,400 transactions: about 28 us each + Access::setBuildHistory(herder, milliseconds(180), nanoseconds(28'125)); + Access::setLastClosedTxSetSize(herder, 6400); + REQUIRE(Access::expectedBuild(herder) == milliseconds(180)); + REQUIRE(lead(800) == milliseconds(325)); + // ledgers grew since its last build: sized for them, at its own rate + Access::setLastClosedTxSetSize(herder, 9600); + REQUIRE(Access::expectedBuild(herder) == milliseconds(270)); + REQUIRE(lead(800) == milliseconds(400)); + // ledgers shrank: the last build still counts until the next one + Access::setLastClosedTxSetSize(herder, 100); + REQUIRE(Access::expectedBuild(herder) == milliseconds(180)); + } + SECTION("a build records what it cost per transaction") + { + // a set too small to say anything leaves the rate alone + Access::setBuildHistory(herder, milliseconds(0), nanoseconds(777)); + f.mMempool = {paymentWithSeq(app, a, a.getLastSequenceNumber() + 1)}; + REQUIRE(f.build().size() == 1); + Access::setLastClosedTxSetSize(herder, 1'000'000); + REQUIRE(Access::expectedBuild(herder) == milliseconds(777)); + } +} + +// A network whose ledgers all carry a transaction: the node whose value +// closed a ledger leads the next one, for up to LEADER_TERM_LEDGERS ledgers, +// and leaders build from pre-validated candidates. +TEST_CASE("leaders keep their term and pre-validate in a network", + "[herder][leader-schedule][leader-term][prevalidation][simulation]") +{ + Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE); + auto simulation = Topologies::core(4, 0.75, networkID, [](int i) { + auto cfg = getTestConfig(i, Config::TESTDB_BUCKET_DB_PERSISTENT); + cfg.LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = GATE_PROTOCOL; + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + return cfg; + }); + auto nodes = simulation->getNodes(); + auto nodeIDs = simulation->getNodeIDs(); + simulation->startAllNodes(); + + // Every node's mempool holds the root account's next payments: one fits + // each ledger (one transaction per source account), and the one after it + // only becomes valid when that ledger has applied. + auto const rootKey = nodes[0]->getRoot()->getSecretKey(); + auto const rootID = rootKey.getPublicKey(); + auto const firstSeq = nodes[0]->getRoot()->getLastSequenceNumber(); + std::vector payments; + for (int i = 1; i <= 400; ++i) + { + payments.push_back(transactionFromOperations(*nodes[0], rootKey, + firstSeq + i, + {payment(rootID, 1)}, 100) + ->getEnvelope()); + } + auto rootSeq = [&](Application& app) { + auto root = app.getLedgerManager().getLCLView()->getAccount(rootID); + REQUIRE(root); + return root.current().data.account().seqNum; + }; + for (auto const& node : nodes) + { + auto* app = node.get(); + static_cast(app->getHerder()) + .mGetTopTransactionsForTesting = [&, app](size_t) { + std::vector top; + auto const next = static_cast(rootSeq(*app) - firstSeq); + for (size_t i = next; i < next + 4 && i < payments.size(); ++i) + { + top.push_back(payments[i]); + } + return top; + }; + } + + // Watch node 1 close ledgers. + auto& lm = nodes[1]->getLedgerManager(); + auto prev = lm.getLastClosedLedgerHeader(); + uint32_t const target = prev.header.ledgerSeq + 40; + Hash const emptyResults = xdrSha256(TransactionResultSet{}); + size_t withTransactions = 0; + size_t insideTerm = 0; + size_t incumbentLed = 0; + size_t incumbentClosed = 0; + std::set proposers; + simulation->crankUntil( + [&]() { + auto const cur = lm.getLastClosedLedgerHeader(); + if (cur.header.ledgerSeq == prev.header.ledgerSeq) + { + return false; + } + REQUIRE(cur.header.ledgerSeq == prev.header.ledgerSeq + 1); + auto const proposer = + getLcValueSignature(cur.header.scpValue).nodeID; + proposers.insert(proposer); + if (cur.header.txSetResultHash != emptyResults) + { + ++withTransactions; + } + auto const schedule = + computeNextLeaderSchedule(prev.header, nodeIDs, true); + REQUIRE(schedule.size() == 4); + // (the genesis ledger has no proposer) + bool const prevApplied = + prev.header.txSetResultHash != emptyResults && + prev.header.scpValue.ext.v() != STELLAR_VALUE_BASIC; + if (cur.header.ledgerSeq % LEADER_TERM_LEDGERS != 0 && prevApplied) + { + // inside a term: the node that closed the previous ledger + // leads this one + ++insideTerm; + auto const incumbent = + getLcValueSignature(prev.header.scpValue).nodeID; + if (schedule[0] == incumbent) + { + ++incumbentLed; + } + if (proposer == incumbent) + { + ++incumbentClosed; + } + } + else if (cur.header.ledgerSeq % LEADER_TERM_LEDGERS == 0 && + prevApplied) + { + // a new term: the seeded order, known a ledger early + REQUIRE(schedule == + computeLeaderSchedule(prev.header.previousLedgerHash, + nodeIDs)); + } + // The node itself derives the next ledger's leaders from the + // header it just closed. + auto& driver = static_cast(nodes[1]->getHerder()) + .getHerderSCPDriver(); + REQUIRE(driver.getLeaderSchedule(cur.header.ledgerSeq + 1) == + computeNextLeaderSchedule(cur.header, nodeIDs, true)); + prev = cur; + return cur.header.ledgerSeq >= target; + }, + 60 * simulation->getExpectedLedgerCloseTime(), false); + REQUIRE(prev.header.ledgerSeq >= target); + auto const paid = rootSeq(*nodes[1]) - firstSeq; + simulation->crankUntil( + [&]() { return simulation->haveAllExternalized(target, 2); }, + 10 * simulation->getExpectedLedgerCloseTime(), false); + + int64_t timeouts = 0; + int64_t prevalidationsUsed = 0; + int64_t cacheHits = 0; + int64_t speculationHits = 0; + for (auto const& node : nodes) + { + auto& metrics = node->getMetrics(); + timeouts += static_cast( + metrics.NewHistogram({"scp", "timeout", "leader"}).sum() + + metrics.NewHistogram({"scp", "timeout", "prepare"}).sum()); + prevalidationsUsed += + meter(*node, "prevalidate", "used", "prevalidation"); + cacheHits += meter(*node, "txset", "validation-cache-hit", "tx"); + speculationHits += applyCounter(*node, "speculation-hit"); + // no leader proposed a set that apply found invalid + REQUIRE(applyCounter(*node, "txset-skipped") == 0); + } + INFO(fmt::format("ledgers with txs {}, inside a term {}, incumbent led " + "{}, incumbent closed {}, timeouts {}, prevalidations " + "used {}, cache hits {}, speculation hits {}", + withTransactions, insideTerm, incumbentLed, + incumbentClosed, timeouts, prevalidationsUsed, cacheHits, + speculationHits)); + // Ledgers carried the payments, one each. + REQUIRE(withTransactions >= 30); + REQUIRE(paid == static_cast(withTransactions)); + // Inside a term the incumbent always leads round 1, and closes the + // ledger unless a round timed out (e.g. on a busy test machine). + REQUIRE(insideTerm >= 20); + REQUIRE(incumbentLed == insideTerm); + REQUIRE(static_cast(insideTerm - incumbentClosed) <= timeouts); + REQUIRE(incumbentClosed * 4 >= insideTerm * 3); + // Terms end: leadership still rotates. + REQUIRE(proposers.size() >= 2); + // Leaders executed their own ledgers speculatively and built the next + // set from candidates validated on that state. + REQUIRE(speculationHits > 0); + REQUIRE(prevalidationsUsed > 0); + REQUIRE(cacheHits > 0); + + for (auto const& node : nodes) + { + static_cast(node->getHerder()) + .mGetTopTransactionsForTesting = nullptr; + } +} diff --git a/src/ledger/ImmutableLedgerView.cpp b/src/ledger/ImmutableLedgerView.cpp index 46161215a..e76f08a91 100644 --- a/src/ledger/ImmutableLedgerView.cpp +++ b/src/ledger/ImmutableLedgerView.cpp @@ -454,4 +454,29 @@ ApplyLedgerView::ApplyLedgerView(ImmutableLedgerDataPtr state, : ImmutableLedgerView(std::move(state), metrics) { } + +std::unique_ptr +ExecutedLedgerState::makeView(MetricsRegistry& metrics) const +{ + releaseAssert(mHeader && mPreviousLedger && mChanges); + return std::make_unique( + mHeader, + [changes = mChanges](LedgerKey const& key) + -> std::optional> { + if (key.type() != ACCOUNT) + { + // Not captured: the previous ledger's version could be + // stale. + throw std::logic_error( + "executed ledger state only holds account entries"); + } + auto it = changes->find(key); + if (it == changes->end()) + { + return std::nullopt; + } + return it->second; + }, + ApplyLedgerView(mPreviousLedger, metrics)); +} } diff --git a/src/ledger/ImmutableLedgerView.h b/src/ledger/ImmutableLedgerView.h index 86e9d713c..eaf760ebe 100644 --- a/src/ledger/ImmutableLedgerView.h +++ b/src/ledger/ImmutableLedgerView.h @@ -274,6 +274,60 @@ class SorobanPreApplyLedgerView : public AbstractLedgerView ApplyLedgerView mLclView; }; +// The ledger state right after a ledger was executed (fees charged, +// transactions applied) and before it is committed: the snapshot of the +// ledger before it plus the entries the execution changed. The commit adds +// what the ledger's upgrades change, eviction, and the header's hashes; up to +// those, this is the state the ledger leaves behind. +// +// Everything in here is immutable and shared, so the state can be read from +// any thread, each through a view of its own (`makeView`). It is captured by +// the ledger apply thread from a speculative execution (see +// LedgerManager::startSpeculativeApply) for the node that expects to lead +// the next ledger, which validates its candidate transactions against it +// while the executed ledger is still going through consensus. +// +// Only what transaction validation reads is kept: the header and the account +// entries the execution changed. A view of this state refuses (throws on) any +// other kind of entry rather than answer with the previous ledger's version. +// The Soroban network configuration is the previous ledger's. What +// validation reads of it only changes with an upgrade, and upgrades are +// applied by the commit, so a state captured for a value that carries +// upgrades must not be used. (The commit also rewrites bookkeeping entries of +// the configuration on every ledger, the state size window and the eviction +// iterator; validation does not read those: from protocol 23 it uses the flat +// write fee.) +struct ExecutedLedgerState +{ + // Account entries the execution created or changed; nullptr for erased + // ones. + using Changes = UnorderedMap>; + + uint32_t mLedgerSeq{0}; + // The value the ledger was executed with. + StellarValue mValue; + // True if the value's transaction set was found invalid, so that the + // ledger applies nothing (see LedgerManagerImpl::validateTxSetForApply). + bool mTxSetSkipped{false}; + // Number of transactions the ledger applied. In the test-only + // overlay-only mode, which applies nothing: the size of its set. + size_t mAppliedTxs{0}; + // The ledger's header as execution left it: sequence number, value and + // fee pool are final; upgrades and the hashes computed by the commit are + // not in it. + std::shared_ptr mHeader; + // Snapshot of the ledger before. + ImmutableLedgerDataPtr mPreviousLedger; + std::shared_ptr mChanges; + // Full hashes of the transactions of the ledger's set. + std::shared_ptr const> mTxHashes; + + // A read-only view of the state, for validating transactions. Each view + // is for one thread. + std::unique_ptr + makeView(MetricsRegistry& metrics) const; +}; + // Immutable wrapper for a complete ledger state snapshot. // This object provides read-only access to all components of a full ledger // state at a specific ledger sequence. All components are instantiated together diff --git a/src/ledger/LedgerManager.h b/src/ledger/LedgerManager.h index 483820858..74d18659a 100644 --- a/src/ledger/LedgerManager.h +++ b/src/ledger/LedgerManager.h @@ -18,6 +18,7 @@ namespace stellar { class LedgerCloseData; +struct ExecutedLedgerState; class Database; class SorobanMetricsRegistry; struct SorobanApplyMetrics; @@ -440,7 +441,22 @@ class LedgerManager // commit the result. If another value externalizes, or another ledger is // handed in, the execution is dropped. Must not be called while a ledger // is applying. - virtual void startSpeculativeApply(LedgerCloseData const& ledgerData) = 0; + // + // With `onExecuted`, the apply thread captures the state the execution + // left (see ExecutedLedgerState) and hands it to `onExecuted` on the main + // thread once the execution has finished; nullptr if the ledger was not + // executed after all. The state stays valid whatever becomes of the + // execution: whether it describes the last closed ledger is for the + // caller to check, by comparing the value that externalized. + using SpeculativeApplyExecuted = + std::function)>; + virtual void startSpeculativeApply(LedgerCloseData const& ledgerData, + SpeculativeApplyExecuted onExecuted) = 0; + void + startSpeculativeApply(LedgerCloseData const& ledgerData) + { + startSpeculativeApply(ledgerData, SpeculativeApplyExecuted{}); + } // True while the apply thread may be executing a ledger speculatively. // Like isApplying(), this tells the main thread that the resources ledger // apply uses (e.g. the batch executor) are taken. diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index e29a98981..abf426eaa 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -1869,7 +1869,8 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, } void -LedgerManagerImpl::startSpeculativeApply(LedgerCloseData const& ledgerData) +LedgerManagerImpl::startSpeculativeApply(LedgerCloseData const& ledgerData, + SpeculativeApplyExecuted onExecuted) { releaseAssert(threadIsMain()); releaseAssert(mApp.getConfig().backgroundLedgerApply()); @@ -1878,12 +1879,18 @@ LedgerManagerImpl::startSpeculativeApply(LedgerCloseData const& ledgerData) ++mSpeculationsExecuting; mApplyState.getMetrics().mSpeculationStarted.inc(); mApp.postOnLedgerApplyThread( - [this, ledgerData]() { - speculativelyExecuteLedger(ledgerData); + [this, ledgerData, onExecuted = std::move(onExecuted)]() mutable { + auto state = speculativelyExecuteLedger( + ledgerData, static_cast(onExecuted)); mApp.postOnMainThread( - [this]() { + [this, state = std::move(state), + onExecuted = std::move(onExecuted)]() { releaseAssert(mSpeculationsExecuting > 0); --mSpeculationsExecuting; + if (onExecuted) + { + onExecuted(state); + } }, "speculative apply executed"); }, @@ -1919,8 +1926,63 @@ LedgerManagerImpl::abortSpeculativeApply() future.wait(); } -void -LedgerManagerImpl::speculativelyExecuteLedger(LedgerCloseData const& ledgerData) +std::shared_ptr +LedgerManagerImpl::captureExecutedState(LedgerExecution const& execution) const +{ + ZoneScoped; + auto state = std::make_shared(); + auto const& ledgerData = execution.mLedgerData; + state->mLedgerSeq = ledgerData.getLedgerSeq(); + state->mValue = ledgerData.getValue(); + state->mTxSetSkipped = execution.mTxSetSkipped; + { + // Not left active: the commit loads the header again. + auto header = execution.mLtx->loadHeader(); + state->mHeader = std::make_shared(header.current()); + } + state->mPreviousLedger = mApplyState.getLedgerState(); + + // Transaction validation only reads accounts (see ExecutedLedgerState). + auto changes = std::make_shared(); + execution.mLtx->forEachChangedEntry([&changes](LedgerKey const& key, + LedgerEntry const* entry) { + if (key.type() == ACCOUNT) + { + changes->emplace(key, + entry ? std::make_shared(*entry) + : nullptr); + } + }); + state->mChanges = std::move(changes); + + auto txHashes = std::make_shared>(); + if (!execution.mTxSetSkipped && execution.mApplicableTxSet) + { + txHashes->reserve(execution.mApplicableTxSet->sizeTxTotal()); + for (auto const& phase : execution.mApplicableTxSet->getPhases()) + { + for (auto const& tx : phase) + { + txHashes->insert(tx->getFullHash()); + } + } + } + // What the ledger applied: its results or, in overlay-only mode (which + // executes nothing and records no results), its transaction set. + state->mAppliedTxs = execution.mTxResultSet.results.size(); +#ifdef BUILD_TESTS + if (mApp.getRunInOverlayOnlyMode()) + { + state->mAppliedTxs = txHashes->size(); + } +#endif + state->mTxHashes = std::move(txHashes); + return state; +} + +std::shared_ptr +LedgerManagerImpl::speculativelyExecuteLedger(LedgerCloseData const& ledgerData, + bool captureState) { ZoneScoped; // At most one ledger is executed ahead of consensus: a newer request @@ -1928,7 +1990,7 @@ LedgerManagerImpl::speculativelyExecuteLedger(LedgerCloseData const& ledgerData) abortSpeculativeExecution(); if (mApp.isStopping()) { - return; + return nullptr; } { // The main thread asked for this on top of its LCL. Apply's own @@ -1939,7 +2001,7 @@ LedgerManagerImpl::speculativelyExecuteLedger(LedgerCloseData const& ledgerData) if (ledgerData.getLedgerSeq() != lcl.header.ledgerSeq + 1 || ledgerData.getTxSet()->previousLedgerHash() != lcl.hash) { - return; + return nullptr; } } try @@ -1953,6 +2015,29 @@ LedgerManagerImpl::speculativelyExecuteLedger(LedgerCloseData const& ledgerData) CLOG_ERROR(Ledger, "Speculative execution of ledger {} failed: {}", ledgerData.getLedgerSeq(), e.what()); mApplyState.markAbortedApplying(); + return nullptr; + } + if (!captureState) + { + return nullptr; + } + try + { + // On the apply thread, between the execution and its commit: timed, + // so that its cost is on record. + auto timer = mApp.getMetrics() + .NewTimer({"ledger", "apply", "capture-state"}) + .TimeScope(); + return captureExecutedState(*mSpeculativeExecution); + } + catch (std::exception const& e) + { + // Only costs the next leader its head start. + CLOG_WARNING(Ledger, + "Could not capture the state of the speculative " + "execution of ledger {}: {}", + ledgerData.getLedgerSeq(), e.what()); + return nullptr; } } diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h index 1b034aafd..fbc81def6 100644 --- a/src/ledger/LedgerManagerImpl.h +++ b/src/ledger/LedgerManagerImpl.h @@ -414,7 +414,13 @@ class LedgerManagerImpl : public LedgerManager // Apply thread only: the ledger executed ahead of consensus, if any. std::unique_ptr mSpeculativeExecution; - void speculativelyExecuteLedger(LedgerCloseData const& ledgerData); + // Returns the state the execution left if `captureState` is set and the + // ledger was executed. + std::shared_ptr + speculativelyExecuteLedger(LedgerCloseData const& ledgerData, + bool captureState); + std::shared_ptr + captureExecutedState(LedgerExecution const& execution) const; // Returns the speculative execution if it is one of `ledgerData`, and // drops it otherwise. std::unique_ptr @@ -696,7 +702,9 @@ class LedgerManagerImpl : public LedgerManager void applyLedger(LedgerCloseData const& ledgerData, bool calledViaExternalize) override; - void startSpeculativeApply(LedgerCloseData const& ledgerData) override; + using LedgerManager::startSpeculativeApply; + void startSpeculativeApply(LedgerCloseData const& ledgerData, + SpeculativeApplyExecuted onExecuted) override; bool isSpeculativelyApplying() const override; void abortSpeculativeApply() override; void completeLedgerClose(uint32_t ledgerSeq, bool calledViaExternalize, diff --git a/src/ledger/LedgerTxn.cpp b/src/ledger/LedgerTxn.cpp index d29d99627..807b79629 100644 --- a/src/ledger/LedgerTxn.cpp +++ b/src/ledger/LedgerTxn.cpp @@ -1516,6 +1516,36 @@ LedgerTxn::Impl::getDelta() return delta; } +void +LedgerTxn::forEachChangedEntry( + std::function const& f) const +{ + getImpl()->forEachChangedEntry(f); +} + +void +LedgerTxn::Impl::forEachChangedEntry( + std::function const& f) const +{ + abortIfWrongThread("forEachChangedEntry"); + throwIfChild(); + if (!mActive.empty()) + { + throw std::runtime_error( + "LedgerTxn has active entries: its changes can't be read"); + } + for (auto const& kv : mEntry) + { + auto const& key = kv.first; + if (key.type() != InternalLedgerEntryType::LEDGER_ENTRY) + { + continue; + } + f(key.ledgerKey(), + kv.second.isDeleted() ? nullptr : &kv.second->ledgerEntry()); + } +} + EntryIterator LedgerTxn::Impl::getEntryIterator(EntryMap const& entries) const { diff --git a/src/ledger/LedgerTxn.h b/src/ledger/LedgerTxn.h index ae05fae66..0802eabb4 100644 --- a/src/ledger/LedgerTxn.h +++ b/src/ledger/LedgerTxn.h @@ -832,6 +832,16 @@ class LedgerTxn : public AbstractLedgerTxn LedgerTxnDelta getDelta() override; + // Calls `f` for every ledger entry this LedgerTxn created, changed or + // erased, with its key and its current value (nullptr if erased). Unlike + // getChanges and getDelta this does not seal the LedgerTxn and does not + // update last-modified sequence numbers: it is for looking at the state + // of a LedgerTxn that will be worked on further. Requires that there is + // no child and that no entry is active. + void forEachChangedEntry( + std::function const& f) + const; + UnorderedMap getOffersByAccountAndAsset(AccountID const& account, Asset const& asset) override; diff --git a/src/ledger/LedgerTxnImpl.h b/src/ledger/LedgerTxnImpl.h index 54102cdbb..69eb29859 100644 --- a/src/ledger/LedgerTxnImpl.h +++ b/src/ledger/LedgerTxnImpl.h @@ -419,6 +419,11 @@ class LedgerTxn::Impl // - the entry cache may be, but is not guaranteed to be, cleared. LedgerTxnDelta getDelta(); + // forEachChangedEntry has the strong exception safety guarantee + void forEachChangedEntry( + std::function const& f) + const; + // getOffersByAccountAndAsset has the basic exception safety guarantee. UnorderedMap getOffersByAccountAndAsset(AccountID const& account, Asset const& asset); diff --git a/src/scp/BallotProtocol.cpp b/src/scp/BallotProtocol.cpp index 01bc51c56..7fb2a5e0a 100644 --- a/src/scp/BallotProtocol.cpp +++ b/src/scp/BallotProtocol.cpp @@ -549,6 +549,19 @@ BallotProtocol::progressRoundStep() /*proposal=*/true) != SCPDriver::kInvalidValue) { Value v = mCurrentBallot->getBallot().value; + // The application may put an equivalent value forward instead, + // but only while nothing ties the node to this exact one: once + // it has accepted a ballot for it as prepared, other nodes may + // have confirmed it and be bound to it, and a replacement could + // only split the round between the two. + if (!(mPrepared && mPrepared->getBallot().value == v)) + { + if (auto replacement = + driver.reproposeValue(slotIndex, round, v)) + { + v = replacement->getValue(); + } + } driver.proposedValue(slotIndex, round, v); return bumpState(v, round); } diff --git a/src/scp/SCPDriver.h b/src/scp/SCPDriver.h index a41af7d78..a66ae4973 100644 --- a/src/scp/SCPDriver.h +++ b/src/scp/SCPDriver.h @@ -251,6 +251,19 @@ class SCPDriver { } + // `reproposeValue` is called when the local node, as the leader of + // `round`, is about to put forward `value`, the value it already votes + // for, rather than building a new one. It may return an equivalent value + // to propose instead (e.g. the same content under the local node's own + // signature), or nullptr to propose `value` as it is. Not called once + // the node has accepted a ballot for `value` as prepared: it then keeps + // the value, like the nodes that may have confirmed it. + virtual ValueWrapperPtr + reproposeValue(uint64 slotIndex, uint32 round, Value const& value) + { + return nullptr; + } + // the following methods are used for monitoring of the SCP subsystem // most implementation don't really need to do anything with these diff --git a/src/scp/test/SCPTests.cpp b/src/scp/test/SCPTests.cpp index 6feb5d763..d6c7a93bb 100644 --- a/src/scp/test/SCPTests.cpp +++ b/src/scp/test/SCPTests.cpp @@ -216,6 +216,14 @@ class TestSCP : public SCPDriver mEnteredRounds.emplace_back(round, timedOut); } + ValueWrapperPtr + reproposeValue(uint64 slotIndex, uint32 round, Value const& value) override + { + mReproposeCalls.emplace_back(round, value); + return mReproposeValueOverride ? mReproposeValueOverride(round, value) + : ValueWrapperPtr(); + } + bool isProposalPending(uint64, NodeID const&) const override { @@ -228,6 +236,11 @@ class TestSCP : public SCPDriver int mValuesBuilt{0}; std::vector> mProposedValues; std::vector> mEnteredRounds; + // What the driver's reproposeValue was asked, and what it answers + // (nothing by default: the value is proposed as it is). + std::vector> mReproposeCalls; + std::function + mReproposeValueOverride; Hash getHashOf(std::vector> const& vals) const override @@ -3660,6 +3673,31 @@ TEST_CASE("leader rounds core5", "[scp][leaderschedule]") // x, not its own z: the transaction set of x is already out verifyPrepare(scp.mEnvs[1], v0SecretKey, qSetHash0, 0, x2); REQUIRE(scp.mValuesBuilt == 0); + // the application was offered to put an equivalent value forward + REQUIRE(scp.mReproposeCalls == + std::vector>{{2, xValue}}); + REQUIRE(scp.mProposedValues.back() == + std::make_pair(uint32(2), xValue)); + } + SECTION("to the local node, which proposes the application's " + "replacement for the value it voted for") + { + // e.g. the same content under the new leader's signature + schedule = {v1NodeID, v0NodeID, v2NodeID, v3NodeID, v4NodeID}; + scp.mReproposeValueOverride = [&](uint32 round, Value const& v) { + REQUIRE(round == 2); + REQUIRE(v == xValue); + return scp.wrapValue(kValue); + }; + scp.fireTimer(Slot::BALLOT_PROTOCOL_TIMER); + REQUIRE(scp.getLeaderRound(0) == 2); + REQUIRE(scp.mEnvs.size() == 2); + verifyPrepare(scp.mEnvs[1], v0SecretKey, qSetHash0, 0, + SCPBallot(2, kValue)); + REQUIRE(scp.mValuesBuilt == 0); + REQUIRE(scp.mReproposeCalls.size() == 1); + REQUIRE(scp.mProposedValues.back() == + std::make_pair(uint32(2), kValue)); } SECTION("to the local node, whose current value is not proposable") { @@ -3677,6 +3715,33 @@ TEST_CASE("leader rounds core5", "[scp][leaderschedule]") } } + SECTION("a leader tied to a value it accepted as prepared keeps it") + { + // v2 and v3 vote for the round 1 leader's x as the local node does: + // a quorum, so the local node accepts (1, x) as prepared. Other + // nodes may confirm it and be bound to x, so when the local node + // leads round 2 the application is not offered a replacement. + schedule = {v1NodeID, v0NodeID, v2NodeID, v3NodeID, v4NodeID}; + scp.mReproposeValueOverride = [&](uint32, Value const&) { + return scp.wrapValue(kValue); + }; + REQUIRE(!scp.startSlot(0, zValue)); + scp.receiveEnvelope(makePrepare(v1SecretKey, qSetHash, 0, x1)); + REQUIRE(scp.mEnvs.size() == 1); + scp.receiveEnvelope(makePrepare(v2SecretKey, qSetHash, 0, x1)); + scp.receiveEnvelope(makePrepare(v3SecretKey, qSetHash, 0, x1)); + REQUIRE(scp.mEnvs.size() == 2); + verifyPrepare(scp.mEnvs[1], v0SecretKey, qSetHash0, 0, x1, &x1); + REQUIRE(scp.hasBallotTimer()); + + scp.fireTimer(Slot::BALLOT_PROTOCOL_TIMER); + REQUIRE(scp.getLeaderRound(0) == 2); + REQUIRE(scp.mEnvs.size() == 3); + verifyPrepare(scp.mEnvs[2], v0SecretKey, qSetHash0, 0, x2, &x1); + REQUIRE(scp.mReproposeCalls.empty()); + REQUIRE(scp.mValuesBuilt == 0); + } + SECTION("a confirmed prepared value wins over the leader's") { schedule = {v1NodeID, v2NodeID, v0NodeID, v3NodeID, v4NodeID};