diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 1a4dbdcaae..389aa640fc 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -9,7 +9,7 @@ Describe what this pull request does, which issue it's resolving (usually applic
--->
# Checklist
-- [ ] Reviewed the [contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#submitting-changes) document
+- [ ] Reviewed the [contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#opening-a-pull-request) document
- [ ] Rebased on top of master (no merge commits)
- [ ] Ran `clang-format` v8.0.0 (via `make format` or the Visual Studio extension)
- [ ] Compiles
diff --git a/.github/workflows/overlay-min-version.yml b/.github/workflows/overlay-min-version.yml
new file mode 100644
index 0000000000..56b42ea4af
--- /dev/null
+++ b/.github/workflows/overlay-min-version.yml
@@ -0,0 +1,92 @@
+name: Overlay min version
+
+# Once pubnet has upgraded to a new ledger protocol, opens a PR raising
+# OVERLAY_PROTOCOL_MIN_VERSION to the lowest overlay version of the releases
+# that support that protocol (see scripts/check-overlay-min-version.sh).
+
+on:
+ schedule:
+ - cron: '0 15 * * *'
+ workflow_dispatch:
+
+permissions:
+ contents: write
+ pull-requests: write
+
+concurrency:
+ group: ${{ github.workflow }}
+ cancel-in-progress: false
+
+jobs:
+ bump-overlay-min-version:
+ if: github.repository == 'stellar/stellar-core'
+ runs-on: ubuntu-24.04
+ env:
+ BRANCH: bot/bump-overlay-min-version
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Get pubnet ledger protocol version
+ id: pubnet
+ run: |
+ PROTOCOL=$(curl -sSf --retry 3 https://horizon.stellar.org/ | jq -er '.current_protocol_version | numbers')
+ echo "Pubnet is on protocol $PROTOCOL"
+ echo "protocol=$PROTOCOL" >> "$GITHUB_OUTPUT"
+
+ - name: Raise OVERLAY_PROTOCOL_MIN_VERSION if possible
+ env:
+ PROTOCOL: ${{ steps.pubnet.outputs.protocol }}
+ run: |
+ set -o pipefail
+ ./scripts/check-overlay-min-version.sh --apply "$PROTOCOL" | tee "$RUNNER_TEMP/check.txt"
+
+ - name: Open or update PR
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PROTOCOL: ${{ steps.pubnet.outputs.protocol }}
+ BASE: ${{ github.ref_name }}
+ run: |
+ if git diff --quiet
+ then
+ echo "OVERLAY_PROTOCOL_MIN_VERSION is up to date"
+ exit 0
+ fi
+ MIN=$(sed -n 's/^[[:space:]]*OVERLAY_PROTOCOL_MIN_VERSION[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p' src/main/Config.cpp)
+
+ # Leave an open PR alone if it already makes the same change.
+ OPEN_PR=$(gh pr list --head "$BRANCH" --base "$BASE" --state open --json number --jq '.[].number')
+ if [ -n "$OPEN_PR" ] &&
+ git fetch --depth=1 origin "$BRANCH" 2>/dev/null &&
+ git diff --quiet FETCH_HEAD -- src/main/Config.cpp
+ then
+ echo "$BRANCH already raises OVERLAY_PROTOCOL_MIN_VERSION to $MIN"
+ exit 0
+ fi
+
+ git config user.name 'github-actions[bot]'
+ git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+ git checkout -B "$BRANCH"
+ git commit -qam "Bump OVERLAY_PROTOCOL_MIN_VERSION to $MIN" \
+ -m "Pubnet is on protocol $PROTOCOL, and no release supporting it has an overlay version below $MIN."
+ git push -f origin "$BRANCH"
+
+ BODY="$RUNNER_TEMP/body.md"
+ {
+ echo "Pubnet is on protocol $PROTOCOL, and no release supporting it has an overlay version below $MIN, so this raises \`OVERLAY_PROTOCOL_MIN_VERSION\` to $MIN. Opened by the \`${{ github.workflow }}\` workflow."
+ echo
+ echo "**Reviewer:** before merging, verify that pubnet is on protocol $PROTOCOL and that every release supporting it, including any not tagged in this repo, has \`OVERLAY_PROTOCOL_VERSION\` of at least $MIN. Nodes below $MIN will be disconnected."
+ echo
+ echo '```'
+ cat "$RUNNER_TEMP/check.txt"
+ echo '```'
+ } > "$BODY"
+ if [ -z "$OPEN_PR" ]
+ then
+ gh pr create --base "$BASE" --head "$BRANCH" \
+ --title "Bump OVERLAY_PROTOCOL_MIN_VERSION to $MIN" --body-file "$BODY"
+ else
+ gh pr edit "$OPEN_PR" \
+ --title "Bump OVERLAY_PROTOCOL_MIN_VERSION to $MIN" --body-file "$BODY"
+ fi
diff --git a/.github/workflows/socket-scan.yml b/.github/workflows/socket-scan.yml
new file mode 100644
index 0000000000..8607d5cd47
--- /dev/null
+++ b/.github/workflows/socket-scan.yml
@@ -0,0 +1,92 @@
+# Socket reachability scan for stellar-core.
+# For general Socket reachability documentation, see https://docs.socket.dev/docs/full-application-reachability
+# Multi-ecosystem: Rust (Cargo) + Python (scripts/pyproject.toml).
+#
+# --reach-ecosystems=cargo,pypi explicitly lists the ecosystems that get full Tier 1 reachability analysis. This repo's Coana-supported ecosystems are cargo (Cargo.toml) and pypi (scripts/pyproject.toml). The explicit list also excludes Coana's phantom NUGET detection: the repo has Builds/VisualStudio/stellar-core.sln (no actual C# code), which Coana otherwise mis-detects as a .NET project and aborts the scan demanding `dotnet`. Listing only cargo,pypi sidesteps that.
+#
+# >>> DEVELOPERS: if this repo adds a NEW Coana-supported ecosystem (e.g. an >>> npm package.json, a go.mod, a Gemfile), you MUST add that ecosystem to >>> this --reach-ecosystems list to get full Tier 1 reachability for it. >>> Supported ecosystem names: npm, pypi, cargo, golang, maven, gem, nuget. >>> Ecosystems NOT listed here still receive Tier 2 (precomputed) >>> reachability from Socket's backend automatically — but Tier 1 (the more >>> precise, your-code-aware analysis) only runs for the ecosystems listed.
+#
+# Schedule: Sun 12:24 UTC weekly. Use workflow_dispatch to run on demand.
+#
+# ============================================================================
+# Socket scan — reading the job status. (The scan step below produces this: an
+# exit code + an optional ::warning:: annotation, which GitHub Actions renders
+# as the job's state.)
+# ============================================================================
+# GREEN (exit 0, no warning): scan completed and every analyzed vulnerability
+# got full Tier 1 reachability (precise, your-code-aware). Nothing to do.
+# YELLOW (exit 0 + "::warning:: Socket scan completed with Tier 2 fallbacks"):
+# scan completed, but Tier 1 could NOT be computed for some/all
+# vulnerabilities, which fell back to Tier 2 (precomputed) reachability.
+# You still get CVE detection + Tier 2 results, just reduced precision
+# for the affected CVEs. The job is NOT failing.
+# RED (non-zero exit): scan did not complete. Do not assume any part
+# succeeded — could be reachability hard-failing, a missing language
+# toolchain, the runner out of memory, a network/API error, or even the
+# underlying CVE/SBOM detection failing. Check the logs and fix before
+# relying on results.
+# ----------------------------------------------------------------------------
+# THIS REPO STARTS YELLOW — a KNOWN upstream Coana bug, NOT your code or this
+# scan setup:
+# Coana's Rust analyzer hits "Maximum call stack size exceeded" on the cargo
+# root + src/rust (4 CVEs) -> Tier 2 fallback.
+# Reported to Socket; may be fixed upstream over time. Do NOT let this baseline
+# yellow train the team to ignore yellow — a *new* yellow (a different Tier 2
+# fallback that appears later) is a real signal worth investigating. After the
+# initial rollout, the team may resolve the baseline yellow at its discretion
+# (once Coana ships a fix, or by adjusting the scan) so GREEN becomes the
+# normal state and any future yellow stands out.
+# ============================================================================
+
+name: Socket reachability scan
+
+on:
+ schedule:
+ - cron: '24 12 * * 0'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ socket-scan:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ - run: rustup update
+ - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+ with:
+ python-version: "3.14.6"
+ - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+ - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+ with:
+ node-version: "24.18.0"
+
+ - name: Install Socket CLI
+ run: npm install -g socket
+
+ - name: Run Socket reachability scan
+ env:
+ SOCKET_SECURITY_API_TOKEN: ${{ secrets.SOCKET_SECURITY_API_TOKEN }}
+ run: |
+ # Stream the scan output through tee so the run log captures it AND
+ # we can grep it for Tier-2-fallback markers; capture the scan's
+ # exit code via ${PIPESTATUS[0]} (tee always exits 0). If the scan
+ # succeeded but logged a Tier 2 fallback, emit a ::warning::
+ # annotation that GitHub Actions renders as a yellow run-level
+ # warning without failing the job.
+ set +e
+ socket scan create --reach \
+ --org=stellar \
+ --no-interactive \
+ --reach-continue-on-no-source-files \
+ --reach-continue-on-analysis-errors \
+ --reach-continue-on-install-errors \
+ --reach-continue-on-missing-lock-files \
+ --reach-ecosystems=cargo,pypi \
+ . 2>&1 | tee /tmp/scan.log
+ rc=${PIPESTATUS[0]}
+ if [ $rc -eq 0 ] && grep -qE "Reachability falls back to Tier 2|fallback to the results from the pre-computed|Reachability falls back to precomputed" /tmp/scan.log; then
+ echo "::warning::Socket scan completed with Tier 2 fallbacks - some vulnerabilities used precomputed reachability instead of full Tier 1"
+ fi
+ exit $rc
diff --git a/.gitignore b/.gitignore
index 6191567e96..0e6535bdc4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -100,6 +100,7 @@ compile_commands.json
# tracy binaries
/tracy-capture
/tracy-gui
+/tracy-import-chrome
# test output
stellar-core-test-*
diff --git a/.gitmodules b/.gitmodules
index 4d03193577..7b736c102f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -52,6 +52,13 @@
[submodule "src/rust/soroban/p28"]
path = src/rust/soroban/p28
url = https://github.com/stellar/rs-soroban-env.git
+[submodule "src/rust/soroban/p29"]
+ path = src/rust/soroban/p29
+ url = https://github.com/stellar/rs-soroban-env.git
+ branch = release/v29.0.0-internal
+[submodule "src/rust/soroban/p30"]
+ path = src/rust/soroban/p30
+ url = https://github.com/stellar/rs-soroban-env.git
[submodule "lib/gperftools"]
path = lib/gperftools
url = https://github.com/gperftools/gperftools.git
diff --git a/Builds/VisualStudio/stellar-core.vcxproj b/Builds/VisualStudio/stellar-core.vcxproj
index a68c0845ab..b962590dc0 100644
--- a/Builds/VisualStudio/stellar-core.vcxproj
+++ b/Builds/VisualStudio/stellar-core.vcxproj
@@ -908,6 +908,7 @@ exit /b 0
+
@@ -1280,6 +1281,7 @@ exit /b 0
+
diff --git a/Builds/VisualStudio/stellar-core.vcxproj.filters b/Builds/VisualStudio/stellar-core.vcxproj.filters
index d1cf2654c5..0a2df18d3f 100644
--- a/Builds/VisualStudio/stellar-core.vcxproj.filters
+++ b/Builds/VisualStudio/stellar-core.vcxproj.filters
@@ -489,6 +489,9 @@
util
+
+ util
+
util
@@ -1737,6 +1740,9 @@
util
+
+ util
+
util
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0109554547..ace26329e7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -7,13 +7,53 @@ Guide](https://github.com/stellar/.github/blob/master/CONTRIBUTING.md) for detai
to stellar-core and Stellar's other repositories, especially with regard to our code of conduct and
contributor license agreement.
-## Submitting Changes
+The repository-specific requirements below take precedence where they are more restrictive than the
+general Stellar Contribution Guide.
-Fork the `stellar/stellar-core` repo on github and submit a pull request from a branch in your fork.
-Key things to keep in mind:
-* Keep your branch up-to-date by rebasing on `stellar/stellar-core` `master`
-* Ensure your branch focuses on a single issue at a time
-* Before merging, squash your commits and ensure you have a descriptive commit message
+## Before Contributing
+
+Stellar Core processes live financial transactions. Correctness is critical, and maintainer and
+reviewer time is limited. We welcome contributions that address a demonstrated, material problem or
+advance work that maintainers have agreed is a priority.
+
+### Start With an Issue
+
+Before creating an issue, search the existing issues and join a relevant discussion if an existing issue is present.
+When creating a new issue, you should establish:
+
+* The concrete problem that needs solving.
+* For a bug, include a minimal reproducer with the expected and actual behavior whenever possible.
+* Relevant alternatives, if any, and their tradeoffs.
+* For a performance change, measurements that show the current problem and provide a baseline for evaluating a solution.
+
+Personally verify findings before reporting them. Do not submit speculative findings or generated lists of possible issues.
+
+Opening an issue does not reserve the work or mean that a code change is wanted.
+An issue with the `help wanted` label is available for implementation, but contributors should
+still discuss their proposed approach on the issue before starting substantial work.
+
+### Participation and Review
+
+Keep your participation to issues and do not post comments on pull requests. Issue comments should be specific and evidence-based.
+Comments must not be about implementation strategies unless explicitly invited to do so.
+Maintainers will delete issue comments that are irrelevant to the issue.
+Proceed with a pull request only when invited by the maintainers, including for issues with the `help wanted` label.
+
+## Opening a Pull Request
+
+We are currently only accepting pull requests for issues with the `help wanted` label.
+Unsolicited pull requests will be closed without an explanation and reported as spam.
+Making changes to stellar-core requires deep architectural knowledge, a good understanding of system constraints, and
+an idea about the project roadmap. External contributors often do not have this context, and their pull requests
+typically focus on lower-priority issues. Reviewing those changes takes the maintainers' focus away from higher-priority work.
+
+Key things to keep in mind when creating a pull request:
+
+* Fork the `stellar/stellar-core` repository and submit the pull request from a branch in your fork.
+* Link the approved issue and keep the pull request within the agreed scope.
+* Keep your branch up-to-date by rebasing on `stellar/stellar-core` `master`.
+* Ensure your branch focuses on a single issue at a time.
+* Before merging, squash your commits and ensure you have a descriptive commit message.
## Stellar Core Contribution Specifics
@@ -294,6 +334,55 @@ files. You should then inspect to see that only the transactions you expected to
see change did so. If so, commit the changes as a new set of baselines for
future tests.
+## Running and updating LedgerCloseMeta checks
+
+Alongside the TxMeta hashes above, the unit tests can capture the full
+`LedgerCloseMeta` XDR of every ledger they close, and check it against golden
+data stored in the repository. Where the TxMeta baselines record a hash per
+transaction, this records the complete binary meta, so the files also serve as
+test data for downstream consumers such as Horizon and RPC. The two modes are:
+
+ * `--capture-lcm` which writes golden files, and
+ * `--check-lcm ` which checks against them, where `` is the
+ directory holding the golden trees (the source tree root).
+
+The golden data lives in `test-lcm-current` (for the current protocol) and
+`test-lcm-next` (for the next protocol), one subdirectory per test file, one
+file per leaf section, named by a truncated hash of the test and section names.
+Each subdirectory's `index.json` maps those hashes back to the names (test case
+and sections joined with `|`, as in the TxMeta baselines) and
+records the protocol version and rng seed that produced the data. Continuous
+integration runs `--check-lcm`, which fails fast if those headers do not match
+the running binary — so a protocol version bump requires re-capturing the
+data, even when no transaction semantics changed.
+
+To re-capture after an intentional change, after a protocol bump, or after
+merging in a `master` that changed the golden data, run from the source tree
+root:
+
+ stellar-core test [tx] --rng-seed 12345 --capture-lcm
+
+for a build with only the current protocol enabled, and the same command for a
+build configured with `--enable-next-protocol-version-unsafe-for-production`,
+which writes the `next` tier instead. Note there is no `--all-versions` here:
+the golden data is captured at the default (latest) protocol version only.
+
+Capture is a full-corpus operation: after a clean run it rebuilds each visited
+test file's `index.json` from what it captured and deletes golden files the run
+did not write, so leaves that are no longer produced do not accumulate. This is
+the same behaviour as `--record-test-tx-meta`. A run filtered to a subset of
+tests therefore deletes the goldens of sibling tests in the same file; that is
+fine while iterating locally, but always finish with the full `[tx]` run above
+before committing.
+
+Some tests are automatically skipped because their meta cannot serve as
+golden data: those that inject ledger entries straight
+into the bucket list (the meta never shows the entries being created), those
+that run a multi-node `Simulation`, and those using a config whose ledger
+content depends on thread scheduling or randomized nomination. If you add a
+test that needs a golden vector, close its ledgers through the ordinary
+single-node path.
+
## Fastdev and non-unified Rust builds
As of protocol 20, some components of stellar-core are written in Rust (notably
diff --git a/Cargo.lock b/Cargo.lock
index 7c6810fc92..23b4413290 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -154,7 +154,7 @@ dependencies = [
"ark-serialize 0.4.2",
"ark-std 0.4.0",
"derivative",
- "digest",
+ "digest 0.10.7",
"itertools 0.10.5",
"num-bigint",
"num-traits",
@@ -174,7 +174,7 @@ dependencies = [
"ark-serialize 0.5.0",
"ark-std 0.5.0",
"arrayvec",
- "digest",
+ "digest 0.10.7",
"educe",
"itertools 0.13.0",
"num-bigint",
@@ -265,7 +265,7 @@ checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5"
dependencies = [
"ark-serialize-derive 0.4.2",
"ark-std 0.4.0",
- "digest",
+ "digest 0.10.7",
"num-bigint",
]
@@ -278,7 +278,7 @@ dependencies = [
"ark-serialize-derive 0.5.0",
"ark-std 0.5.0",
"arrayvec",
- "digest",
+ "digest 0.10.7",
"num-bigint",
]
@@ -535,7 +535,7 @@ version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
- "digest",
+ "digest 0.10.7",
]
[[package]]
@@ -614,10 +614,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
dependencies = [
"cfg-if",
- "cpufeatures 0.3.0",
+ "cpufeatures 0.3.1",
"rand_core 0.10.1",
]
+[[package]]
+name = "cmov"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
+
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -633,6 +639,12 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
+[[package]]
+name = "const-oid"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
+
[[package]]
name = "const-str"
version = "0.4.3"
@@ -675,9 +687,9 @@ dependencies = [
[[package]]
name = "cpufeatures"
-version = "0.3.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
dependencies = [
"libc",
]
@@ -732,6 +744,24 @@ dependencies = [
"typenum",
]
+[[package]]
+name = "crypto-common"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
+dependencies = [
+ "hybrid-array",
+]
+
+[[package]]
+name = "ctutils"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
+dependencies = [
+ "cmov",
+]
+
[[package]]
name = "curve25519-dalek"
version = "4.1.3"
@@ -741,7 +771,7 @@ dependencies = [
"cfg-if",
"cpufeatures 0.2.12",
"curve25519-dalek-derive",
- "digest",
+ "digest 0.10.7",
"fiat-crypto",
"rustc_version",
"subtle",
@@ -820,7 +850,17 @@ version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
dependencies = [
- "const-oid",
+ "const-oid 0.9.6",
+ "zeroize",
+]
+
+[[package]]
+name = "der"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
+dependencies = [
+ "const-oid 0.10.2",
"zeroize",
]
@@ -873,11 +913,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
- "const-oid",
- "crypto-common",
+ "const-oid 0.9.6",
+ "crypto-common 0.1.6",
"subtle",
]
+[[package]]
+name = "digest"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
+dependencies = [
+ "crypto-common 0.2.2",
+]
+
[[package]]
name = "displaydoc"
version = "0.2.7"
@@ -907,11 +956,11 @@ version = "0.16.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
dependencies = [
- "der",
- "digest",
+ "der 0.7.10",
+ "digest 0.10.7",
"elliptic-curve",
"rfc6979",
- "signature",
+ "signature 2.2.0",
]
[[package]]
@@ -920,8 +969,8 @@ version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
dependencies = [
- "pkcs8",
- "signature",
+ "pkcs8 0.10.2",
+ "signature 2.2.0",
]
[[package]]
@@ -965,7 +1014,7 @@ checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47"
dependencies = [
"base16ct",
"crypto-bigint",
- "digest",
+ "digest 0.10.7",
"ff",
"generic-array",
"group",
@@ -1308,6 +1357,15 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "hash32"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "hashbrown"
version = "0.13.2"
@@ -1334,6 +1392,17 @@ version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+[[package]]
+name = "heapless"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4"
+dependencies = [
+ "hash32",
+ "stable_deref_trait",
+ "zeroize",
+]
+
[[package]]
name = "heck"
version = "0.5.0"
@@ -1419,7 +1488,7 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
- "digest",
+ "digest 0.10.7",
]
[[package]]
@@ -1467,6 +1536,16 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+[[package]]
+name = "hybrid-array"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214"
+dependencies = [
+ "ctutils",
+ "typenum",
+]
+
[[package]]
name = "hyper"
version = "0.14.32"
@@ -1745,6 +1824,16 @@ dependencies = [
"cpufeatures 0.2.12",
]
+[[package]]
+name = "keccak"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.1",
+]
+
[[package]]
name = "lazy_static"
version = "1.4.0"
@@ -2197,6 +2286,32 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "ml-dsa"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "add6b9d92e496f16f4526d68ff29da1483aba4b119baeab8bed3b9e3544a6f3d"
+dependencies = [
+ "crypto-common 0.2.2",
+ "ctutils",
+ "hybrid-array",
+ "module-lattice",
+ "pkcs8 0.11.0",
+ "shake",
+ "signature 3.0.0",
+]
+
+[[package]]
+name = "module-lattice"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe"
+dependencies = [
+ "ctutils",
+ "hybrid-array",
+ "num-traits",
+]
+
[[package]]
name = "multiaddr"
version = "0.18.2"
@@ -2526,8 +2641,18 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
- "der",
- "spki",
+ "der 0.7.10",
+ "spki 0.7.3",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
+dependencies = [
+ "der 0.8.2",
+ "spki 0.8.1",
]
[[package]]
@@ -3026,7 +3151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
dependencies = [
"base16ct",
- "der",
+ "der 0.7.10",
"generic-array",
"subtle",
"zeroize",
@@ -3096,7 +3221,7 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures 0.2.12",
- "digest",
+ "digest 0.10.7",
]
[[package]]
@@ -3105,8 +3230,19 @@ version = "0.10.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60"
dependencies = [
- "digest",
- "keccak",
+ "digest 0.10.7",
+ "keccak 0.1.5",
+]
+
+[[package]]
+name = "shake"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a"
+dependencies = [
+ "digest 0.11.3",
+ "keccak 0.2.2",
+ "sponge-cursor",
]
[[package]]
@@ -3140,10 +3276,19 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
dependencies = [
- "digest",
+ "digest 0.10.7",
"rand_core 0.6.4",
]
+[[package]]
+name = "signature"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5"
+dependencies = [
+ "digest 0.11.3",
+]
+
[[package]]
name = "slab"
version = "0.4.12"
@@ -3264,6 +3409,28 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "soroban-builtin-sdk-macros"
+version = "29.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5#a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5"
+dependencies = [
+ "itertools 0.13.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "soroban-builtin-sdk-macros"
+version = "30.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=6e6c4a87f46c7fdab0c2e2be82901b1e32378223#6e6c4a87f46c7fdab0c2e2be82901b1e32378223"
+dependencies = [
+ "itertools 0.13.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "soroban-env-common"
version = "21.2.2"
@@ -3274,7 +3441,7 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 21.2.2",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-xdr 21.2.0",
"wasmparser",
@@ -3290,7 +3457,7 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 22.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-xdr 22.0.0",
"wasmparser",
@@ -3306,7 +3473,7 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 23.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-xdr 23.0.0",
"wasmparser",
@@ -3322,7 +3489,7 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 24.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-xdr 24.0.0",
"wasmparser",
@@ -3338,7 +3505,7 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 25.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-xdr 24.0.1",
"wasmparser",
@@ -3354,7 +3521,7 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 26.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-xdr 26.0.0",
"wasmparser",
@@ -3371,9 +3538,9 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 27.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
- "stellar-xdr 27.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stellar-xdr 27.0.0",
"wasmparser",
]
@@ -3387,9 +3554,41 @@ dependencies = [
"num-derive",
"num-traits",
"soroban-env-macros 28.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
- "stellar-xdr 28.0.0",
+ "stellar-xdr 28.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasmparser",
+]
+
+[[package]]
+name = "soroban-env-common"
+version = "29.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5#a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5"
+dependencies = [
+ "crate-git-revision 0.0.9",
+ "ethnum",
+ "num-derive",
+ "num-traits",
+ "soroban-env-macros 29.0.0",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915)",
+ "static_assertions",
+ "stellar-xdr 28.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasmparser",
+]
+
+[[package]]
+name = "soroban-env-common"
+version = "30.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=6e6c4a87f46c7fdab0c2e2be82901b1e32378223#6e6c4a87f46c7fdab0c2e2be82901b1e32378223"
+dependencies = [
+ "crate-git-revision 0.0.9",
+ "ethnum",
+ "num-derive",
+ "num-traits",
+ "soroban-env-macros 30.0.0",
+ "soroban-wasmi 0.31.1-soroban.29.0.0",
+ "static_assertions",
+ "stellar-xdr 30.0.0",
"wasmparser",
]
@@ -3418,7 +3617,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 21.2.2",
"soroban-env-common 21.2.2",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.8",
"wasmparser",
@@ -3453,7 +3652,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 22.0.0",
"soroban-env-common 22.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.9",
"wasmparser",
@@ -3488,7 +3687,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 23.0.0",
"soroban-env-common 23.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.13",
"wasmparser",
@@ -3523,7 +3722,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 24.0.0",
"soroban-env-common 24.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.13",
"wasmparser",
@@ -3559,7 +3758,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 25.0.0",
"soroban-env-common 25.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.13",
"wasmparser",
@@ -3595,7 +3794,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 26.0.0",
"soroban-env-common 26.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.13",
"wasmparser",
@@ -3631,7 +3830,7 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 27.0.0",
"soroban-env-common 27.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.13",
"wasmparser",
@@ -3667,12 +3866,85 @@ dependencies = [
"sha3",
"soroban-builtin-sdk-macros 28.0.0",
"soroban-env-common 28.0.0",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"static_assertions",
"stellar-strkey 0.0.13",
"wasmparser",
]
+[[package]]
+name = "soroban-env-host"
+version = "29.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5#a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5"
+dependencies = [
+ "ark-bls12-381 0.5.0",
+ "ark-bn254 0.5.0",
+ "ark-ec 0.5.0",
+ "ark-ff 0.5.0",
+ "ark-serialize 0.5.0",
+ "curve25519-dalek",
+ "ecdsa",
+ "ed25519-dalek",
+ "elliptic-curve",
+ "generic-array",
+ "getrandom 0.2.11",
+ "hex-literal",
+ "hmac",
+ "k256",
+ "num-derive",
+ "num-integer",
+ "num-traits",
+ "p256",
+ "rand 0.8.5",
+ "rand_chacha",
+ "sec1",
+ "sha2",
+ "sha3",
+ "soroban-builtin-sdk-macros 29.0.0",
+ "soroban-env-common 29.0.0",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915)",
+ "static_assertions",
+ "stellar-strkey 0.0.13",
+ "wasmparser",
+]
+
+[[package]]
+name = "soroban-env-host"
+version = "30.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=6e6c4a87f46c7fdab0c2e2be82901b1e32378223#6e6c4a87f46c7fdab0c2e2be82901b1e32378223"
+dependencies = [
+ "ark-bls12-381 0.5.0",
+ "ark-bn254 0.5.0",
+ "ark-ec 0.5.0",
+ "ark-ff 0.5.0",
+ "ark-serialize 0.5.0",
+ "curve25519-dalek",
+ "ecdsa",
+ "ed25519-dalek",
+ "elliptic-curve",
+ "generic-array",
+ "getrandom 0.2.11",
+ "hex-literal",
+ "hmac",
+ "k256",
+ "ml-dsa",
+ "num-derive",
+ "num-integer",
+ "num-traits",
+ "p256",
+ "rand 0.8.5",
+ "rand_chacha",
+ "sec1",
+ "sha2",
+ "sha3",
+ "soroban-builtin-sdk-macros 30.0.0",
+ "soroban-env-common 30.0.0",
+ "soroban-wasmi 0.31.1-soroban.29.0.0",
+ "static_assertions",
+ "stellar-strkey 1.0.0",
+ "wasmparser",
+]
+
[[package]]
name = "soroban-env-macros"
version = "21.2.2"
@@ -3767,7 +4039,7 @@ dependencies = [
"quote",
"serde",
"serde_json",
- "stellar-xdr 27.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stellar-xdr 27.0.0",
"syn 2.0.119",
]
@@ -3781,7 +4053,35 @@ dependencies = [
"quote",
"serde",
"serde_json",
- "stellar-xdr 28.0.0",
+ "stellar-xdr 28.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "soroban-env-macros"
+version = "29.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5#a721944b5a4ea830a4bf5c2dbb8dbe8c1e1354c5"
+dependencies = [
+ "itertools 0.13.0",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "stellar-xdr 28.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "soroban-env-macros"
+version = "30.0.0"
+source = "git+https://github.com/stellar/rs-soroban-env?rev=6e6c4a87f46c7fdab0c2e2be82901b1e32378223#6e6c4a87f46c7fdab0c2e2be82901b1e32378223"
+dependencies = [
+ "itertools 0.13.0",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "stellar-xdr 30.0.0",
"syn 2.0.119",
]
@@ -3793,7 +4093,7 @@ dependencies = [
"arbitrary",
"soroban-env-host 27.0.0",
"soroban-synth-wasm",
- "soroban-wasmi",
+ "soroban-wasmi 0.31.1-soroban.20.0.1 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
"wasm-smith",
]
@@ -3805,7 +4105,7 @@ dependencies = [
"arbitrary",
"soroban-env-common 27.0.0",
"soroban-env-macros 27.0.0",
- "stellar-xdr 27.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stellar-xdr 27.0.0",
"wasm-encoder",
"wasmparser",
]
@@ -3822,8 +4122,33 @@ source = "git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a
dependencies = [
"smallvec",
"spin 0.9.8",
- "wasmi_arena",
- "wasmi_core",
+ "wasmi_arena 0.4.0 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
+ "wasmi_core 0.13.0 (git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0)",
+ "wasmparser-nostd",
+]
+
+[[package]]
+name = "soroban-wasmi"
+version = "0.31.1-soroban.20.0.1"
+source = "git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915#ff5aeeb286880d574bcabc100115126446dca915"
+dependencies = [
+ "smallvec",
+ "spin 0.9.8",
+ "wasmi_arena 0.4.0 (git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915)",
+ "wasmi_core 0.13.0 (git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915)",
+ "wasmparser-nostd",
+]
+
+[[package]]
+name = "soroban-wasmi"
+version = "0.31.1-soroban.29.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c596eadd9cbbbef8a33972972bd3c482de5ad44f42a3e8f2c69be21ec8e6d00"
+dependencies = [
+ "smallvec",
+ "spin 0.9.8",
+ "wasmi_arena 0.4.1",
+ "wasmi_core 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmparser-nostd",
]
@@ -3846,9 +4171,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
dependencies = [
"base64ct",
- "der",
+ "der 0.7.10",
+]
+
+[[package]]
+name = "spki"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9"
+dependencies = [
+ "base64ct",
+ "der 0.8.2",
]
+[[package]]
+name = "sponge-cursor"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620"
+
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -3883,6 +4224,8 @@ dependencies = [
"soroban-env-host 26.0.0",
"soroban-env-host 27.0.0",
"soroban-env-host 28.0.0",
+ "soroban-env-host 29.0.0",
+ "soroban-env-host 30.0.0",
"soroban-fuzz-targets",
"soroban-synth-wasm",
"soroban-test-wasms",
@@ -3895,7 +4238,7 @@ name = "stellar-overlay"
version = "0.1.0"
dependencies = [
"blake2",
- "digest",
+ "digest 0.10.7",
"futures",
"hostname",
"libp2p",
@@ -3905,7 +4248,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
- "stellar-xdr 26.0.0",
+ "stellar-xdr 30.0.0",
"tempfile",
"tokio",
"toml",
@@ -3916,14 +4259,14 @@ dependencies = [
[[package]]
name = "stellar-quorum-analyzer"
version = "0.1.0"
-source = "git+https://github.com/stellar/stellar-quorum-analyzer?rev=502a354eb9a31cf86098be84aa1b3081767fa3c7#502a354eb9a31cf86098be84aa1b3081767fa3c7"
+source = "git+https://github.com/stellar/stellar-quorum-analyzer?rev=811d19a1a523f035f9a442e9197a6fed609550f1#811d19a1a523f035f9a442e9197a6fed609550f1"
dependencies = [
"batsat",
"itertools 0.10.5",
"log",
"petgraph",
"stellar-strkey 0.0.13",
- "stellar-xdr 27.0.0 (git+https://github.com/stellar/rs-stellar-xdr?rev=3305b3e31f19fdb4e64e4b7a4354bb120cdf4415)",
+ "stellar-xdr 28.0.0 (git+https://github.com/stellar/rs-stellar-xdr?rev=28f34cf5a963e3cb13a25456eb9bbda7e0e1ad64)",
]
[[package]]
@@ -3958,6 +4301,29 @@ dependencies = [
"data-encoding",
]
+[[package]]
+name = "stellar-strkey"
+version = "0.0.18"
+source = "git+https://github.com/stellar/rs-stellar-strkey?rev=80085faa3e1f851fc28f3b2b941019265d46c314#80085faa3e1f851fc28f3b2b941019265d46c314"
+dependencies = [
+ "crate-git-revision 0.0.9",
+ "data-encoding",
+ "heapless",
+ "zeroize",
+]
+
+[[package]]
+name = "stellar-strkey"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5038ed2bf4e11842210d118f65791d8ca4747622daddd222338b3d06b6051bc1"
+dependencies = [
+ "crate-git-revision 0.0.9",
+ "data-encoding",
+ "heapless",
+ "zeroize",
+]
+
[[package]]
name = "stellar-xdr"
version = "21.2.0"
@@ -4065,9 +4431,11 @@ dependencies = [
[[package]]
name = "stellar-xdr"
-version = "27.0.0"
-source = "git+https://github.com/stellar/rs-stellar-xdr?rev=3305b3e31f19fdb4e64e4b7a4354bb120cdf4415#3305b3e31f19fdb4e64e4b7a4354bb120cdf4415"
+version = "28.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f93d09ff8b9f919b084f664003c4c546ac66a76affd5429460dbe29f4b326f8e"
dependencies = [
+ "base64 0.22.1",
"crate-git-revision 0.0.9",
"escape-bytes",
"ethnum",
@@ -4079,8 +4447,21 @@ dependencies = [
[[package]]
name = "stellar-xdr"
version = "28.0.0"
+source = "git+https://github.com/stellar/rs-stellar-xdr?rev=28f34cf5a963e3cb13a25456eb9bbda7e0e1ad64#28f34cf5a963e3cb13a25456eb9bbda7e0e1ad64"
+dependencies = [
+ "crate-git-revision 0.0.9",
+ "escape-bytes",
+ "ethnum",
+ "hex",
+ "sha2",
+ "stellar-strkey 0.0.18",
+]
+
+[[package]]
+name = "stellar-xdr"
+version = "30.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93d09ff8b9f919b084f664003c4c546ac66a76affd5429460dbe29f4b326f8e"
+checksum = "56f2fb2008b42fc746c0f6b46e375600068dd91f07748e9f1737c6341a2255f2"
dependencies = [
"base64 0.22.1",
"crate-git-revision 0.0.9",
@@ -4088,7 +4469,7 @@ dependencies = [
"ethnum",
"hex",
"sha2",
- "stellar-strkey 0.0.13",
+ "stellar-strkey 1.0.0",
]
[[package]]
@@ -4628,6 +5009,29 @@ name = "wasmi_arena"
version = "0.4.0"
source = "git+https://github.com/stellar/wasmi?rev=0ed3f3dee30dc41ebe21972399e0a73a41944aa0#0ed3f3dee30dc41ebe21972399e0a73a41944aa0"
+[[package]]
+name = "wasmi_arena"
+version = "0.4.0"
+source = "git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915#ff5aeeb286880d574bcabc100115126446dca915"
+
+[[package]]
+name = "wasmi_arena"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "104a7f73be44570cac297b3035d76b169d6599637631cf37a1703326a0727073"
+
+[[package]]
+name = "wasmi_core"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf1a7db34bff95b85c261002720c00c3a6168256dcb93041d3fa2054d19856a"
+dependencies = [
+ "downcast-rs",
+ "libm",
+ "num-traits",
+ "paste",
+]
+
[[package]]
name = "wasmi_core"
version = "0.13.0"
@@ -4639,6 +5043,17 @@ dependencies = [
"paste",
]
+[[package]]
+name = "wasmi_core"
+version = "0.13.0"
+source = "git+https://github.com/stellar/wasmi?rev=ff5aeeb286880d574bcabc100115126446dca915#ff5aeeb286880d574bcabc100115126446dca915"
+dependencies = [
+ "downcast-rs",
+ "libm",
+ "num-traits",
+ "paste",
+]
+
[[package]]
name = "wasmparser"
version = "0.116.1"
diff --git a/INSTALL.md b/INSTALL.md
index 7a5ded2f7f..cd74c5efcb 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -268,3 +268,15 @@ The GUI depends on the `capstone`, `freetype` and `glfw` libraries and their hea
# On MacOS
$ brew install capstone freetype2 glfw
+
+On Windows, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site.
+
+At a high level you need to install the required prerequisites to build clients, run in a shell:
+
+ vcpkg.exe integrate install
+ vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3
+
+Then build one of the servers. Solutions for servers compatible with the version of stellar-core can be found under:
+
+ * lib/tracy/profiler/build/win32 (GUI)
+ * lib/tracy/capture/build/win32
diff --git a/Makefile.am b/Makefile.am
index dad2d09aa5..2f0d26d22b 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -107,6 +107,20 @@ check-local: check-rust
endif # USE_RUST_OVERLAY
+if USE_TRACY_IMPORT_CHROME
+TRACY_IMPORT_CHROME_DIR=$(top_srcdir)/lib/tracy/import-chrome
+TRACY_IMPORT_CHROME_SOURCE=$(TRACY_IMPORT_CHROME_DIR)/src
+TRACY_IMPORT_CHROME_BUILD=$(TRACY_IMPORT_CHROME_DIR)/build/unix
+
+$(TRACY_IMPORT_CHROME_BUILD)/import-chrome-release: $(wildcard $(TRACY_IMPORT_CHROME_SOURCE)/*.*)
+ $(MAKE) -C $(TRACY_IMPORT_CHROME_BUILD) release CC="$(CC)" CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS) -DNO_PARALLEL_SORT=1" CFLAGS="$(CFLAGS)" TRACY_NO_ISA_EXTENSIONS=1 TRACY_NO_LTO=1 LEGACY=1
+
+tracy-import-chrome: $(TRACY_IMPORT_CHROME_BUILD)/import-chrome-release
+ cp -v $< $@
+
+bin_PROGRAMS += tracy-import-chrome
+endif # USE_TRACY_IMPORT_CHROME
+
EXTRA_DIST = stellar-core.supp test/testnet/multitail.conf \
test/testnet/run-test.sh README.md make-mks overlay/vendor
diff --git a/build-fuzz.sh b/build-fuzz.sh
index 21af59d041..46663ce58c 100755
--- a/build-fuzz.sh
+++ b/build-fuzz.sh
@@ -43,11 +43,11 @@ ccache -z
. "${HOME}/.cargo/env"
(cd "${SRC_DIR}" && ./autogen.sh)
-# NB: the oss-fuzz driver injects sanitizer flags to CFLAGS, CXXFLAGS
-# and RUSTFLAGS. This overlaps with our own support for sanitizers,
-# but not fatally. It does require us to enable the unified rust
-# build.
-"${SRC_DIR}/configure" --enable-fuzz --enable-unified-rust-unsafe-for-production --disable-postgres --enable-ccache
+# NB: the oss-fuzz driver injects sanitizer flags to CFLAGS and CXXFLAGS. This
+# overlaps with our own support for sanitizers, but not fatally. Note that this
+# requires --enable-fastdev-unsafe-for-production in order to pass sanitizer
+# flags through to rust and not provoke linking errors.
+"${SRC_DIR}/configure" --enable-fuzz --enable-sdfprefs --enable-fastdev-unsafe-for-production --disable-postgres --enable-ccache
make -j $(nproc)
make -C src fuzz-targets
cp src/fuzz_* "${OUT}"
diff --git a/ci-build.sh b/ci-build.sh
index e6e96dc9e6..5ea1569581 100755
--- a/ci-build.sh
+++ b/ci-build.sh
@@ -122,9 +122,10 @@ fi
config_flags="--enable-ccache --enable-sdfprefs ${PROTOCOL_CONFIG} ${DISABLE_POSTGRES}"
# NB: use `-gdwarf-3` (not -g or -gdwarf-4 or later) specifically as
# it produces the highest-fidelity backtraces with our current
-# rust/gimli-backed backtrace system.
-# export CFLAGS="-O2 -gdwarf-3 -fno-omit-frame-pointer -fsanitize-address-use-after-scope -fno-common"
-export CFLAGS="-O2 -gdwarf-3 -fno-omit-frame-pointer -fno-common"
+# rust/gimli-backed backtrace system. pass -g1 to limit the size of
+# debuginfo and also the memory cost to create it.
+# export CFLAGS="-O2 -gdwarf-3 -g1 -fno-omit-frame-pointer -fsanitize-address-use-after-scope -fno-common"
+export CFLAGS="-O2 -gdwarf-3 -g1 -fno-omit-frame-pointer -fno-common"
export CXXFLAGS="$CFLAGS"
# quarantine_size_mb / malloc_context_size : reduce memory usage to avoid
@@ -230,6 +231,15 @@ time make check
# export SKIP_SOROBAN_TESTS=true
# time make check
+# Reuse the same build to check the captured LedgerCloseMeta golden data
+# under test-lcm-current/ or test-lcm-next/ (the binary picks its tier).
+# No --all-versions: the golden data is captured at the default (latest)
+# protocol version only.
+echo Running fixed check-lcm tests
+export TEST_SPEC='[tx]'
+export STELLAR_CORE_TEST_PARAMS="--ll fatal -r simple --disable-dots --rng-seed 12345 --check-lcm ${SRC_DIR}"
+time make check
+
echo All done
date
diff --git a/common.mk b/common.mk
index 41ca4d835b..abae4f498a 100644
--- a/common.mk
+++ b/common.mk
@@ -31,6 +31,10 @@ if USE_TRACY
AM_CPPFLAGS += -DUSE_TRACY $(tracy_CFLAGS)
endif # USE_TRACY
+if USE_TRACY_MEMORY_TRACKING
+AM_CPPFLAGS += -DUSE_TRACY_MEMORY_TRACKING
+endif # USE_TRACY_MEMORY_TRACKING
+
if BUILD_TESTS
AM_CPPFLAGS += -DBUILD_TESTS=1
endif # BUILD_TESTS
diff --git a/configure.ac b/configure.ac
index 1ceb55d4ff..da4d0d60bd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -34,8 +34,9 @@ fi
# NB: use `-gdwarf-3` (not -g or -gdwarf-4 or later) specifically as
# it produces the highest-fidelity backtraces with our current
-# rust/gimli-backed backtrace system.
-test "${CFLAGS+set}" || CFLAGS="-gdwarf-3 -O2 -fno-omit-frame-pointer"
+# rust/gimli-backed backtrace system. pass -g1 to limit the size of
+# debuginfo and also the memory cost to create it.
+test "${CFLAGS+set}" || CFLAGS="-gdwarf-3 -g1 -O2 -fno-omit-frame-pointer"
test "${CXXFLAGS+set}" || CXXFLAGS="$CFLAGS"
AC_PROG_CC([clang gcc cc])
@@ -477,14 +478,24 @@ AM_CONDITIONAL(USE_POSTGRES, [test -n "$have_postgres"])
# tcmalloc_minimal - enabled by default on Linux for improved memory allocation performance
# Disabled when using sanitizers (asan, memcheck, tsan) as they replace malloc
# Uses vendored gperftools from lib/gperftools
+AC_ARG_ENABLE(tcmalloc,
+ AS_HELP_STRING([--disable-tcmalloc],
+ [Disable tcmalloc support]))
+# Accept options written by the old recursive-configure workaround so existing
+# build directories can recheck once and rewrite a clean config.status.
+AC_ARG_ENABLE(minimal)
+AC_ARG_ENABLE(cpu-profiler)
+AC_ARG_ENABLE(heap-profiler)
+AC_ARG_ENABLE(heap-checker)
+AC_ARG_ENABLE(debugalloc)
unset have_tcmalloc
-if test "x$enable_asan" = "xyes" -o "x$enable_memcheck" = "xyes" -o "x$enable_threadsanitizer" = "xyes"; then
+if test x"$enable_tcmalloc" = xno; then
+ AC_MSG_NOTICE([tcmalloc disabled by user])
+elif test "x$enable_asan" = "xyes" -o "x$enable_memcheck" = "xyes" -o "x$enable_threadsanitizer" = "xyes"; then
AC_MSG_NOTICE([tcmalloc disabled due to sanitizer being enabled])
else
case "${host_os}" in
linux*)
- # Configure gperftools for minimal tcmalloc only
- ac_configure_args="$ac_configure_args --enable-minimal --disable-cpu-profiler --disable-heap-profiler --disable-heap-checker --disable-debugalloc"
AC_CONFIG_SUBDIRS([lib/gperftools])
have_tcmalloc=1
AC_DEFINE([USE_TCMALLOC], [1], [Define to 1 if using tcmalloc])
@@ -502,6 +513,11 @@ AM_CONDITIONAL(USE_TCMALLOC, [test -n "$have_tcmalloc"])
AC_SUBST(libtcmalloc_CFLAGS)
AC_SUBST(libtcmalloc_LIBS)
+# AC_CONFIG_SUBDIRS disables option checking so options can be forwarded to
+# recursive configures. Re-enable it for the top-level configure by default;
+# generated recursion still passes --disable-option-checking to sub-configures.
+m4_divert_text([DEFAULTS], [enable_option_checking=fatal])
+
AC_ARG_ENABLE(tests,
AS_HELP_STRING([--disable-tests],
[Disable building test suite]))
@@ -536,7 +552,12 @@ AC_ARG_ENABLE(tracy-memory-tracking,
AS_HELP_STRING([--enable-tracy-memory-tracking],
[Enable 'tracy' profiler/tracer memory tracking code (slow)]))
AM_CONDITIONAL(USE_TRACY_MEMORY_TRACKING, [test x$enable_tracy_memory_tracking = xyes])
-
+if test x"$enable_tracy_memory_tracking" = xyes -a x"$have_tcmalloc" = x1; then
+ AC_MSG_ERROR([--enable-tracy-memory-tracking requires --disable-tcmalloc])
+fi
+if test x"$enable_tracy_memory_tracking" = xyes -a x"$enable_tracy" != xyes; then
+ AC_MSG_ERROR([--enable-tracy-memory-tracking requires --enable-tracy])
+fi
if test x"$enable_tracy" = xyes -a x"$enable_asan" = xyes; then
AC_MSG_ERROR([--enable-asan is not compatible with --enable-tracy])
fi
@@ -571,6 +592,11 @@ AC_ARG_ENABLE(tracy-csvexport,
[Enable 'tracy' profiler/tracer csvexport program]))
AM_CONDITIONAL(USE_TRACY_CSVEXPORT, [test x$enable_tracy_csvexport = xyes])
+AC_ARG_ENABLE(tracy-import-chrome,
+ AS_HELP_STRING([--enable-tracy-import-chrome],
+ [Enable 'tracy' profiler/tracer chrome-trace-format importer]))
+AM_CONDITIONAL(USE_TRACY_IMPORT_CHROME, [test x$enable_tracy_import_chrome = xyes])
+
AC_ARG_ENABLE(spdlog,
AS_HELP_STRING([--disable-spdlog],
[Disable spdlog]))
@@ -625,9 +651,55 @@ esc() {
echo $out
}
+# Remove gperftools-only options saved by the old workaround from future
+# top-level rechecks. Their values are fixed above before configuring gperftools.
+eval "set x $ac_configure_args"
+shift
+filtered_configure_args=
+for arg in "$@"; do
+ case "$arg" in
+ --enable-minimal | --disable-minimal | --enable-minimal=* | --disable-minimal=* | \
+ --enable-cpu-profiler | --disable-cpu-profiler | --enable-cpu-profiler=* | --disable-cpu-profiler=* | \
+ --enable-heap-profiler | --disable-heap-profiler | --enable-heap-profiler=* | --disable-heap-profiler=* | \
+ --enable-heap-checker | --disable-heap-checker | --enable-heap-checker=* | --disable-heap-checker=* | \
+ --enable-debugalloc | --disable-debugalloc | --enable-debugalloc=* | --disable-debugalloc=*)
+ ;;
+ *)
+ filtered_configure_args="$filtered_configure_args $(esc "$arg")"
+ ;;
+ esac
+done
+ac_configure_args=$filtered_configure_args
+
# explicitly propagate CFLAGS, CXXFLAGS and LDFLAGS in case they got modified by global options
ac_configure_args="$ac_configure_args $(esc "CC=$CC" "CXX=$CXX" "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" "LDFLAGS=$LDFLAGS" "RUSTC_WRAPPER=$RUSTC_WRAPPER" "LIB_FUZZING_ENGINE=$LIB_FUZZING_ENGINE")"
+# Configure gperftools separately so its generic option names do not affect
+# other recursive configures (in particular libsodium's --enable-minimal).
+AC_CONFIG_COMMANDS_POST([
+ if test -n "$have_tcmalloc"; then
+ saved_subdirs=$subdirs
+ saved_configure_args=$ac_configure_args
+
+ subdirs="lib/gperftools"
+ ac_configure_args="$saved_configure_args \
+ --enable-minimal \
+ --disable-cpu-profiler \
+ --disable-heap-profiler \
+ --disable-heap-checker \
+ --disable-debugalloc"
+ _AC_OUTPUT_SUBDIRS
+
+ ac_configure_args=$saved_configure_args
+ subdirs=
+ for subdir in $saved_subdirs; do
+ if test "x$subdir" != "xlib/gperftools"; then
+ subdirs="$subdirs $subdir"
+ fi
+ done
+ fi
+])
+
AC_CONFIG_HEADERS(config.h)
AC_CONFIG_FILES(lib/Makefile src/Makefile Makefile)
AC_OUTPUT
diff --git a/docs/apply-load-benchmark-sac.cfg b/docs/apply-load-benchmark-sac.cfg
index 2adcfd9458..6f822ad3fa 100644
--- a/docs/apply-load-benchmark-sac.cfg
+++ b/docs/apply-load-benchmark-sac.cfg
@@ -7,10 +7,13 @@
APPLY_LOAD_MODE="benchmark"
APPLY_LOAD_MODEL_TX="sac"
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate counts use Herder's queue size multipliers (2x ledger capacity by
+# default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The Soroban multiplier applies to transaction
+# envelopes after SAC batching.
# Whether to time the write part of the apply stage. This can be
# disabled to get less noisy results for non-write related changes,
diff --git a/docs/apply-load-benchmark-token.cfg b/docs/apply-load-benchmark-token.cfg
index ae02b67a4e..b75aefda05 100644
--- a/docs/apply-load-benchmark-token.cfg
+++ b/docs/apply-load-benchmark-token.cfg
@@ -7,10 +7,12 @@
APPLY_LOAD_MODE="benchmark"
APPLY_LOAD_MODEL_TX="custom_token"
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate counts use Herder's queue size multipliers (2x ledger capacity by
+# default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. Token transfers need two accounts per candidate.
# Whether to time the write part of the apply stage. This can be
# disabled to get less noisy results for non-write related changes,
@@ -61,4 +63,4 @@ NODE_SEED="SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2 self"
[QUORUM_SET]
THRESHOLD_PERCENT=100
-VALIDATORS=["$self"]
\ No newline at end of file
+VALIDATORS=["$self"]
diff --git a/docs/apply-load-for-meta.cfg b/docs/apply-load-for-meta.cfg
index 36c268d7df..3025a32700 100644
--- a/docs/apply-load-for-meta.cfg
+++ b/docs/apply-load-for-meta.cfg
@@ -9,10 +9,13 @@
# Select the apply-load mode.
APPLY_LOAD_MODE="ledger-limits"
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate generation uses Herder's queue size multipliers (2x ledger capacity
+# by default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The classic multiplier scales the candidate
+# count; the Soroban multiplier scales the candidate resource budget.
# Custom meta path - if not set it will be written to a temp directory and
# cleaned up after running the benchmark
diff --git a/docs/apply-load-ledger-limits.cfg b/docs/apply-load-ledger-limits.cfg
index 3dae22fb87..945f6e0e71 100644
--- a/docs/apply-load-ledger-limits.cfg
+++ b/docs/apply-load-ledger-limits.cfg
@@ -7,10 +7,13 @@
# Select the apply-load mode.
APPLY_LOAD_MODE="ledger-limits"
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate generation uses Herder's queue size multipliers (2x ledger capacity
+# by default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The classic multiplier scales the candidate
+# count; the Soroban multiplier scales the candidate resource budget.
# Medida metrics (histograms in particular) in apply path cause severe and
# non-deterministic performance degradation. While this has to be addressed
diff --git a/docs/apply-load-max-sac-tps.cfg b/docs/apply-load-max-sac-tps.cfg
index 43721f371e..a27e037ec1 100644
--- a/docs/apply-load-max-sac-tps.cfg
+++ b/docs/apply-load-max-sac-tps.cfg
@@ -1,15 +1,20 @@
# This is the Stellar Core configuration example for using the load generation
# (apply-load) tool for testing the theoretical max SAC (Stellar asset contract)
-# transfer TPS via binary search (measured based on apply time only).
+# transfer TPS via binary search driven by application timing.
# The core with this configuration should run using `./stellar-core apply-load`
# Select the apply-load mode.
APPLY_LOAD_MODE="max-sac-tps"
-# Which timing path to use. The max-sac-tps search targets apply-only close
-# time, so it only supports "apply".
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every search sample reports tx-set construction, validation, ledger close
+# and receive-to-close timings. Only application timing drives the search;
+# APPLY_LOAD_TIME_WRITES controls whether that timing includes writes.
+
+# Candidate counts use Herder's queue size multipliers (2x ledger capacity by
+# default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The Soroban multiplier applies to transaction
+# envelopes after SAC batching.
# Whether to time the write part of the apply stage. This can be
# disabled to get less noisy results for non-write related changes,
@@ -30,7 +35,7 @@ APPLY_LOAD_MAX_SAC_TPS_MIN_TPS = 1000
# Upper bound of the TPS in binary search
APPLY_LOAD_MAX_SAC_TPS_MAX_TPS = 15000
-# Number of seconds to apply the ledger for
+# Target application time per ledger, in milliseconds
APPLY_LOAD_TARGET_CLOSE_TIME_MS = 1000
# The only relevant network configuration parameter - number of transaction
@@ -59,9 +64,9 @@ APPLY_LOAD_BL_LAST_BATCH_LEDGERS = 0
ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true
# Diagnostic events should generally be disabled, but can be enabled for debug
ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = false
-# Set up a plenty of genesis accounts - benchmark will fail if the number is
-# not sufficient. This should be at least 2x of the maximum TPL, but can be set
-# higher than that.
+# Reserve one account per Soroban candidate at the rounded upper search bound,
+# plus the classic candidates and two setup accounts. SAC batching reduces the
+# envelope count before the Soroban queue multiplier is applied.
GENESIS_TEST_ACCOUNT_COUNT = 100000
# Minimal core config boilerplate
diff --git a/docs/metrics.md b/docs/metrics.md
index 09b92f5bb5..b37a5c24bc 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -62,6 +62,7 @@ crypto.verify.miss | meter | number of signature cach
crypto.verify.total | meter | sum of both hits and misses
crypto.verify.tx-valid-hit | meter | signature cache hits that occurred while validating transactions (outside of background signature validation)
crypto.verify.tx-valid-total | meter | sum of both hits and misses during transaction validation (outside of background signature validation)
+herder.txset.build | timer | time spent building the tx set proposed at nomination from the transaction queues
herder.txset.validate | timer | time spent turning a received tx set into an applicable tx set and validating it on a validity-cache miss
herder.pending[-soroban]-txs.age0 | counter | number of gen0 pending transactions
herder.pending[-soroban]-txs.age1 | counter | number of gen1 pending transactions
@@ -173,6 +174,7 @@ process.action.queue | counter | number of items waiting
process.action.overloaded | counter | 0-or-1 value indicating action-queue overloading
process.file.handles | counter | number of open file handles
process.memory.handles | counter | number of running processes in process manager
+process.memory.malloc-heap | counter | number of bytes used in the malloc heap
scp.envelope.emit | meter | SCP message sent
scp.envelope.invalidsig | meter | envelope failed signature verification
scp.envelope.receive | meter | SCP message received
@@ -195,6 +197,9 @@ scp.timing.externalized | timer | time spent in ballot pro
scp.timing.first-to-self-externalize-lag | timer | delay between first externalize message and local node externalizing
scp.timing.self-to-others-externalize-lag | timer | delay between local node externalizing and later externalize messages from other nodes
scp.timing.ballot-blocked-on-txset | timer | time balloting was blocked waiting for a txset download (milliseconds)
+scp.timing.trigger-to-ballot | timer | time from the local next-ledger trigger to first entry into the ballot protocol (proposal construction and nomination)
+scp.trigger.immediate | meter | next-ledger trigger was already due when scheduled (the node is behind the target block time)
+scp.trigger.lateness | timer | delay between the scheduled next-ledger trigger time and the trigger callback running
scp.trigger.prepare-start-fallback | meter | trigger timer fell back from the network-close-time anchor to the local prepare-start anchor
scp.value.invalid | meter | SCP value is invalid
scp.value.valid | meter | SCP value is valid
@@ -278,7 +283,8 @@ soroban.config.bucket-list-target-size-byte | counter | soroban config settin
soroban.module-cache.num-entries | counter | current number of entries in module cache
soroban.module-cache.compilation-time | timer | times each contract compilation when adding to module cache
soroban.module-cache.rebuild-time | timer | times each rebuild of module cache (including all compilations)
-soroban.module-cache.rebuild-bytes | counter | bytes of WASM bytecode compiled in last rebuild of module cache
+soroban.module-cache.rebuild-wasm-bytes | counter | bytes of WASM bytecode compiled in last rebuild of module cache
+soroban.module-cache.rebuild-heap-bytes | counter | bytes of heap memory allocated in last rebuild of module cache
soroban.in-memory-state.contract-code-size | counter | size in bytes of non-evicted ContractCode entries according to memory cost model
soroban.in-memory-state.contract-data-size | counter | size in bytes of ContractData entries in memory
soroban.in-memory-state.contract-code-entries | counter | number of ContractCode entries in memory
diff --git a/docs/software/commands.md b/docs/software/commands.md
index 8edf80399b..052437d90d 100644
--- a/docs/software/commands.md
+++ b/docs/software/commands.md
@@ -19,45 +19,35 @@ Common options can be placed at any place in the command line.
## Command line options
Command options can only by placed after command.
-* **apply-load**: Benchmarks Soroban transaction application time using
- synthetic transactions. By default the benchmark is isolated to mostly just
- executing the transactions and thus it omits a lot of the supporting
- mechanisms (such as overlay, SCP, mempool etc). It may also measure tx-set
- validation and consensus processing via `APPLY_LOAD_TIMING_PHASES` (see
- below). This command will generate enough
- transactions to fill up a synthetic transaction queue (it's just a list of
- transactions with the same limits as the real queue), and then create a
- transaction set off of that to apply. This can also be used to record the
- synthetic ledger close metadata emitted during the benchmark, and then use
- it for benchmarking the meta consumers.
- * This can only be used when `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true`
- * The mode is selected in the config file using `APPLY_LOAD_MODE`:
- - `APPLY_LOAD_MODE="ledger-limits"`: the default mode that measures the
- ledger close time for applying transactions.
- - `APPLY_LOAD_MODE="max-sac-tps"`: determines maximum TPS for the load
- consisting only of fast SAC transfer.
- - `APPLY_LOAD_MODE="benchmark"`: benchmarks a fixed-size ledger of model
- transactions. Use `APPLY_LOAD_MODEL_TX` to select the model transaction.
- * `APPLY_LOAD_TIMING_PHASES` selects one of two timing paths:
- - `"apply"`: the default apply-only benchmark. Its close helper still calls
- `checkValid`, but that happens before the recorded ledger-close timer and
- leaves the caches warm, as consensus validation would on a live node.
- - `"txset-validation-and-apply"`: simulates a non-leader receiving the tx
- set over the wire, validating it through local consensus (with the node
- as its own single-validator quorum), and then applying it. It reports
- validation, ledger close, and end-to-end time in addition to the
- apply-only output. It does not simulate network transport, peer fetching,
- or multi-node timing. Leader-side tx-set creation and signing happen
- before the measured span. The signature verification cache is cleared
- before validation, then retained so apply sees the warm cache produced by
- validation.
- `"txset-validation-and-apply"` is not supported with
- `APPLY_LOAD_MODE="max-sac-tps"`; that search targets apply-only close time.
- * Load generation is configured in the Core config file. The relevant settings
- all begin with `APPLY_LOAD_`. See full example configurations with
- per-setting documentation in the `docs` directory
- (all the `apply-load-*.cfg` files demonstrate different modes and use
- cases).
+* **apply-load**: Benchmarks tx-set construction, validation and application
+ using synthetic transactions. Every workload reports these phases plus
+ receive-to-close time. It can also record synthetic ledger close metadata
+ for benchmarking meta consumers.
+ * This can only be used when `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true`.
+ * `APPLY_LOAD_MODE` selects the workload:
+ - `"ledger-limits"`: generates load against configured resource limits for
+ a fixed number of ledgers.
+ - `"benchmark"`: benchmarks a fixed-size ledger of model transactions
+ selected by `APPLY_LOAD_MODEL_TX`.
+ - `"max-sac-tps"`: uses noisy binary search to find the SAC throughput that
+ meets `APPLY_LOAD_TARGET_CLOSE_TIME_MS`. Every sample reports all phases,
+ but only application timing drives the search. `APPLY_LOAD_TIME_WRITES`
+ selects ledger-close timing (including writes) or transaction-apply
+ timing (excluding writes). Phase summaries cover all sampled search loads.
+ * Construction uses an overfilled synthetic candidate list to simulate a busy
+ validator's queue. All modes use Herder's queue size multipliers (2× ledger
+ capacity by default). They scale model and classic candidate counts; in
+ `ledger-limits` mode the Soroban multiplier scales the candidate resource
+ budget. SAC batching is applied before scaling envelope counts.
+ * Each benchmark builds the set, reconstructs it from wire bytes, then
+ validates and applies it through local consensus with a single-validator
+ quorum. The signature cache is cleared before validation and retained for
+ application. Receive-to-close timing starts at wire decoding and excludes
+ construction. Candidate generation, serialization, signing and setup are
+ outside the reported phases. Network transport, peer fetching, transaction
+ queue submission and multi-node timing are not simulated.
+ * See `docs/apply-load-*.cfg` for example configurations and per-setting
+ documentation.
* **calculate-asset-supply**: Calculates total supply of an asset from the live and hot archive bucket lists IF the total supply fits in a 64 bit signed integer. Also validates against totalCoins for the native asset. Uses `--code ` and `--issuer ` to specify the asset. Uses the native asset if neither `--code` nor `--issuer` is given.
* **catchup **: Perform catchup from history
@@ -229,11 +219,40 @@ Command options can only by placed after command.
it cannot decode. Each file
is named with a truncated SHA-256 hash of the test/section path
(e.g. `a1b2c3d4e5f67890.xdr`), and an `index.json` in each
- directory maps hashes back to human-readable names. Each file contains
+ directory maps hashes back to the test case and section names, joined
+ with `|`. Each file contains
stream-framed `LedgerCloseMeta` entries that can be decoded with
`stellar-xdr decode --type LedgerCloseMeta --input stream-framed`.
- Meta is normalized (sorted) before writing so that output is
- deterministic given a fixed `--rng-seed`.
+ Non-deterministic diagnostic events are zeroed before writing, but
+ entries are otherwise written in their original order, which some
+ downstream consumers depend on. Comparisons — both the skip-rewrite
+ check during capture and `--check-lcm` — are done on normalized
+ (sorted) copies, so given a fixed `--rng-seed` files are only
+ rewritten or flagged on semantic changes. Each `index.json` is stamped
+ with the protocol version, rng seed and protocol-version list that
+ produced the data.
+ Capture is a full-corpus operation: after a clean run it rewrites each
+ visited test file's `index.json` from what the run captured and deletes
+ `.xdr` files it did not write, as `--record-test-tx-meta` does for its
+ baselines. A run filtered to a subset of tests therefore deletes the
+ goldens of sibling tests in the same file, so regenerate with the full
+ `[tx]` suite before committing.
+ * Tests whose `LedgerCloseMeta` cannot serve as golden data are skipped
+ automatically: those that inject ledger entries
+ straight into the bucket list (the meta never shows the entries being
+ created), run a multi-node `Simulation`, or use a config whose ledger
+ content depends on thread scheduling or randomized nomination.
+ `--check-lcm` fails if golden data still exists for such a test.
+ * `--check-lcm ` : check `LedgerCloseMeta` captured from tests
+ against the golden files under `DIRNAME/test-lcm-current/` (or
+ `test-lcm-next/` for vnext builds), where `DIRNAME` is the directory
+ containing the two trees (typically the source tree root). Fails fast
+ if the `index.json` headers don't match the running binary — e.g. after
+ a protocol version bump without regenerating the golden data — and
+ fails at the end of the run if any captured meta differs from the
+ corresponding golden file. Continuous integration runs this mode; after
+ intentional changes, regenerate with `--capture-lcm` under each build
+ configuration and commit the result.
* The network passphrase is set to `(V) (;,,;) (V)` for all captured meta.
* For [further info](https://github.com/philsquared/Catch/blob/master/docs/command-line.md)
on possible options for test.
@@ -291,27 +310,16 @@ Most commands return their results in JSON format.
* **bans**
List current active bans
-* **banaccounts**
- Manages the persistent list of banned accounts. Banned accounts are stored in
- the database and survive restarts. Any transaction whose source account,
- operation source account, fee-bump fee source, or (for Soroban transactions)
- write footprint account entry matches a banned address will be rejected from
- the transaction queue.
- * `banaccounts`
- Lists the currently banned account addresses as a JSON array.
- * `banaccounts?accountids=G_ADDRESS1,G_ADDRESS2,...`
- Adds the specified addresses to the persistent ban list. Existing bans are
- preserved (additive).
-
- Note: The `FILTERED_G_ADDRESSES` configuration option is deprecated. Any
- addresses configured there will be automatically migrated to the persistent
- ban list on startup.
-
-* **unbanaccounts**
- * `unbanaccounts`
- Clears all banned accounts.
- * `unbanaccounts?accountids=G_ADDRESS1,G_ADDRESS2,...`
- Removes the specified addresses from the persistent ban list.
+* **banaccounts** (deprecated)
+ Account banning has been removed; this endpoint is kept for backwards
+ compatibility and only returns a deprecation warning. See
+ [CAP-0077](https://github.com/stellar/stellar-protocol/blob/master/core/cap-0077.md)
+ for a more robust alternative. The `FILTERED_G_ADDRESSES` configuration
+ option is likewise deprecated and has no effect.
+
+* **unbanaccounts** (deprecated)
+ Account banning has been removed; this endpoint is kept for backwards
+ compatibility and only returns a deprecation warning.
* **checkdb**
Triggers the instance to perform a background check of the database's state.
@@ -399,10 +407,9 @@ Most commands return their results in JSON format.
* "FILTERED" - transaction rejected because it contains an operation type that Stellar Core filters out. See Stellar Core configuration `EXCLUDE_TRANSACTIONS_CONTAINING_OPERATION_TYPE` for more details.
Optional parameters:
- * `force=true` - bypasses banned account filtering (see `banaccounts`),
- allowing the transaction into the mempool even if its source account or
- fee source is on the ban list. Other filtering (operation type, Soroban
- key filtering) still applies. Example: `tx?blob=Base64&force=true`
+ * `force=true` - deprecated and has no effect (it used to bypass
+ banned account filtering, which has been removed). Accepted for
+ backwards compatibility; the response includes a deprecation warning.
* **upgrades**
* `upgrades?mode=get`
diff --git a/docs/software/security-protocol-release-notes.md b/docs/software/security-protocol-release-notes.md
index 2242613bb6..d112e7e154 100644
--- a/docs/software/security-protocol-release-notes.md
+++ b/docs/software/security-protocol-release-notes.md
@@ -59,6 +59,47 @@ It then follows that:
# List of releases
+## v29.0.0 (2026-10-05)
+* `Ledger` - security - Improve DEX offer crossing accuracy. Rounding during offer crossing could cause an asset pair to become permanently blocked from trading.
+ * exploited: no
+ * mitigation: code fix
+* `Ledger` - protocol - Liquidity pool hops are no longer counted against the path payment hop limit.
+* `Overlay` - protocol - Lower the maximum overlay message size.
+* `Overlay` - security - Drop messages exceeding the size limit early, before they are fully read and processed.
+ * exploited: no
+ * mitigation: code fix
+* `Overlay` - security - Fix outbound queue load shedding to prevent unbounded growth of queued messages.
+ * exploited: no
+ * mitigation: code fix
+* `Overlay` - security - Don't read messages from peers during the authentication handshake.
+ * exploited: no
+ * mitigation: code fix
+* `Overlay` - security - Remove the hash-of-hash optimization for flood message tracking.
+ * exploited: no
+ * mitigation: code fix
+
+## v28.0.1 (2026-08-14)
+* `Overlay` - security - Deduplicate query messages to prevent peers from inducing excessive resource consumption via repeated requests.
+ * exploited: no
+ * mitigation: code fix
+* `Overlay` - security - Skip background transaction signature verification for unauthenticated peers.
+ * exploited: no
+ * mitigation: code fix
+* `Herder` - security - Clamp additions in transaction set fee summing functions to prevent integer overflow.
+ * exploited: no
+ * mitigation: code fix
+
+## v28.0.0 (2026-08-13)
+* `Overlay` - security - Strengthen flow control and peer request processing limits to prevent excessive resource consumption.
+ * exploited: no
+ * mitigation: code fix
+* `Ledger` - security - Enforce transaction signature verification limits throughout transaction validation.
+ * exploited: no
+ * mitigation: code fix
+* `Ledger` - security - Ensure consistent validation of transaction XDR across protocol versions.
+ * exploited: no
+ * mitigation: code fix
+
## v26.1.0 (2026-05-07)
* `Ledger` - security - Replace block on transactions using `ed25519SignedPayload` signatures with an overall cap on signature verifications stellar-core will perform per transaction.
* exploited: no
diff --git a/docs/stellar-core_example.cfg b/docs/stellar-core_example.cfg
index d9663c28fe..c53968c5fc 100644
--- a/docs/stellar-core_example.cfg
+++ b/docs/stellar-core_example.cfg
@@ -563,7 +563,8 @@ ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=false
# incompatible with those of anyone else.
ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=false
-# ARTIFICIALLY_SET_CLOSE_TIME_FOR_TESTING (in seconds), defaults to no override
+# ARTIFICIALLY_SET_CLOSE_TIME_FOR_TESTING (in milliseconds), defaults to no
+# override
# Overrides the close time to the specified value but does not change checkpoint
# frequency - this may cause network instability.
# Do not use in production.
diff --git a/overlay/Cargo.toml b/overlay/Cargo.toml
index 97e48248bf..04b665747e 100644
--- a/overlay/Cargo.toml
+++ b/overlay/Cargo.toml
@@ -21,7 +21,7 @@ lru = "0.12"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
toml = "0.8"
-stellar-xdr = { version = "=26.0.0", default-features = true }
+stellar-xdr = { version = "=30.0.0", default-features = true }
# libp2p for networking
# QUIC transport only - provides encryption (TLS 1.3), multiplexing, and stream independence
diff --git a/overlay/src/flood/inv_messages.rs b/overlay/src/flood/inv_messages.rs
index 197dac956f..1c6f02cfc1 100644
--- a/overlay/src/flood/inv_messages.rs
+++ b/overlay/src/flood/inv_messages.rs
@@ -4,7 +4,7 @@
use std::io;
use std::sync::Arc;
-use stellar_xdr::curr::{
+use stellar_xdr::{
FloodAdvert, FloodDemand, Hash, Limits, ReadXdr, StellarMessage, TxAdvertVector,
TxDemandVector, WriteXdr, TX_DEMAND_VECTOR_MAX_SIZE,
};
diff --git a/overlay/src/libp2p_overlay.rs b/overlay/src/libp2p_overlay.rs
index ddda9b3957..3ec67a470a 100644
--- a/overlay/src/libp2p_overlay.rs
+++ b/overlay/src/libp2p_overlay.rs
@@ -1174,7 +1174,7 @@ async fn send_txset_response(
// built locally (trusted core). Keep the payload shared through admission
// and transmission, writing the small discriminant separately under the
// same stream lock as the payload.
- let discriminant = (stellar_xdr::curr::MessageType::GeneralizedTxSet as i32).to_be_bytes();
+ let discriminant = (stellar_xdr::MessageType::GeneralizedTxSet as i32).to_be_bytes();
let bytes = discriminant.len() + data.len();
match send_to_peer_stream_parts(&state, peer, StreamType::TxSet, &[&discriminant, &data]).await
@@ -1599,13 +1599,13 @@ async fn handle_inbound_scp_streams(
};
let scp_envelope = match message {
- stellar_xdr::curr::StellarMessage::GetTxSet(hash)
+ stellar_xdr::StellarMessage::GetTxSet(hash)
if accepts_txset_requests =>
{
handle_txset_request(&state, peer_id, hash.0);
continue;
}
- stellar_xdr::curr::StellarMessage::GetScpState(ledger_seq) => {
+ stellar_xdr::StellarMessage::GetScpState(ledger_seq) => {
info!(
"SCP_STATE_REQ: Peer {} requests SCP state for ledger >= {}",
peer_id, ledger_seq
@@ -1622,9 +1622,7 @@ async fn handle_inbound_scp_streams(
}
continue;
}
- stellar_xdr::curr::StellarMessage::ScpMessage(scp_envelope) => {
- scp_envelope
- }
+ stellar_xdr::StellarMessage::ScpMessage(scp_envelope) => scp_envelope,
other => {
warn!(
"SCP_PARSE_ERR: Dropping unexpected {} on SCP stream from {}",
@@ -2093,10 +2091,10 @@ async fn handle_inbound_txset_streams(mut incoming: IncomingStreams, state: Arc<
};
match message {
- stellar_xdr::curr::StellarMessage::GetTxSet(hash) => {
+ stellar_xdr::StellarMessage::GetTxSet(hash) => {
handle_txset_request(&state, peer_id, hash.0);
}
- stellar_xdr::curr::StellarMessage::GeneralizedTxSet(_tx_set) => {
+ stellar_xdr::StellarMessage::GeneralizedTxSet(_tx_set) => {
// The message was strict-decoded above, so the
// bytes after the 4-byte discriminant are the
// canonical tx set; hash the original bytes
@@ -2293,7 +2291,7 @@ fn blake2b_hash(data: &[u8]) -> [u8; 32] {
#[cfg(test)]
fn test_scp_envelope_xdr(slot_index: u64) -> Vec {
- use stellar_xdr::curr::{Limits, ScpEnvelope, WriteXdr};
+ use stellar_xdr::{Limits, ScpEnvelope, WriteXdr};
let mut envelope = ScpEnvelope::default();
envelope.statement.slot_index = slot_index;
@@ -2308,7 +2306,7 @@ fn test_tx(sequence: i64) -> Arc {
#[cfg(test)]
fn test_txset_xdr(seed: u8) -> ([u8; 32], Vec) {
- use stellar_xdr::curr::{GeneralizedTransactionSet, Hash, Limits, WriteXdr};
+ use stellar_xdr::{GeneralizedTransactionSet, Hash, Limits, WriteXdr};
let mut tx_set = GeneralizedTransactionSet::default();
let GeneralizedTransactionSet::V1(v1) = &mut tx_set;
@@ -4405,7 +4403,7 @@ async fn test_inv_getdata_tx_propagation() {
/// shows up as node1 processing zero demanded hashes.
#[tokio::test]
async fn test_getdata_retry_chunked_across_multiple_messages() {
- use stellar_xdr::curr::TX_DEMAND_VECTOR_MAX_SIZE;
+ use stellar_xdr::TX_DEMAND_VECTOR_MAX_SIZE;
let keypair1 = Keypair::generate_ed25519();
let keypair2 = Keypair::generate_ed25519();
diff --git a/overlay/src/libp2p_overlay/dispatch_tests.rs b/overlay/src/libp2p_overlay/dispatch_tests.rs
index 70204f615c..fde3941fc2 100644
--- a/overlay/src/libp2p_overlay/dispatch_tests.rs
+++ b/overlay/src/libp2p_overlay/dispatch_tests.rs
@@ -233,7 +233,7 @@ async fn scp_state_sends_preserve_slot_order_without_blocking_other_peers() {
}
fn large_txset() -> ([u8; 32], Vec) {
- use stellar_xdr::curr::{
+ use stellar_xdr::{
GeneralizedTransactionSet, Limits, ReadXdr, TransactionEnvelope, TransactionPhase,
TxSetComponent, TxSetComponentTxsMaybeDiscountedFee, WriteXdr,
};
diff --git a/overlay/src/libp2p_overlay/framed_io.rs b/overlay/src/libp2p_overlay/framed_io.rs
index 1fd45492b8..1332b066b0 100644
--- a/overlay/src/libp2p_overlay/framed_io.rs
+++ b/overlay/src/libp2p_overlay/framed_io.rs
@@ -65,7 +65,7 @@ mod tests {
async fn segmented_txset_frame_matches_xdr_with_partial_writes() {
let (_, data) = super::super::test_txset_xdr(7);
let expected = crate::xdr::frame_tx_set(&data);
- let discriminant = (stellar_xdr::curr::MessageType::GeneralizedTxSet as i32).to_be_bytes();
+ let discriminant = (stellar_xdr::MessageType::GeneralizedTxSet as i32).to_be_bytes();
let mut writer = ShortWriter::default();
write_frame_parts(&mut writer, &[&discriminant, &[], &data])
.await
diff --git a/overlay/src/main.rs b/overlay/src/main.rs
index 9b01219ffb..752ea0d62c 100644
--- a/overlay/src/main.rs
+++ b/overlay/src/main.rs
@@ -1517,7 +1517,7 @@ mod tests {
create_overlay(keypair, metrics, OverlayHandle::new(cmd_tx))
}
- use stellar_xdr::curr::{Limits, ScpEnvelope, WriteXdr};
+ use stellar_xdr::{Limits, ScpEnvelope, WriteXdr};
fn test_scp_envelope_xdr(slot_index: u64) -> Vec {
let mut envelope = ScpEnvelope::default();
@@ -1986,7 +1986,7 @@ mod tests {
/// A minimal valid GeneralizedTransactionSet whose content hash matches,
/// so it passes the CacheTxSet hash guard.
fn test_txset_xdr(seed: u8) -> ([u8; 32], Vec) {
- use stellar_xdr::curr::{GeneralizedTransactionSet, Hash};
+ use stellar_xdr::{GeneralizedTransactionSet, Hash};
let mut tx_set = GeneralizedTransactionSet::default();
let GeneralizedTransactionSet::V1(v1) = &mut tx_set;
@@ -1997,7 +1997,7 @@ mod tests {
}
fn test_mempool_tx(seq: i64) -> Arc {
- use stellar_xdr::curr::{
+ use stellar_xdr::{
Operation, SequenceNumber, Transaction, TransactionEnvelope, TransactionV1Envelope,
};
let envelope = TransactionEnvelope::Tx(TransactionV1Envelope {
diff --git a/overlay/src/wire.rs b/overlay/src/wire.rs
index 0b80bc5d52..6680c7fa43 100644
--- a/overlay/src/wire.rs
+++ b/overlay/src/wire.rs
@@ -20,7 +20,7 @@
use std::fmt;
use std::sync::Arc;
-use stellar_xdr::curr::{EnvelopeType, TransactionEnvelope};
+use stellar_xdr::{EnvelopeType, TransactionEnvelope};
use crate::xdr::{self, XdrError};
@@ -139,10 +139,10 @@ fn fee_and_ops(envelope: &TransactionEnvelope) -> Result<(i64, u32), XdrError> {
mod tests {
use super::*;
use crate::xdr::tests::valid_transaction_xdr;
- use stellar_xdr::curr::{Limits, TransactionEnvelope, WriteXdr};
+ use stellar_xdr::{Limits, TransactionEnvelope, WriteXdr};
fn decode(bytes: &[u8]) -> TransactionEnvelope {
- use stellar_xdr::curr::ReadXdr;
+ use stellar_xdr::ReadXdr;
TransactionEnvelope::from_xdr(bytes, Limits::none()).unwrap()
}
@@ -186,7 +186,7 @@ mod tests {
#[test]
fn to_flood_frame_matches_typed_stellar_message() {
- use stellar_xdr::curr::StellarMessage;
+ use stellar_xdr::StellarMessage;
let bytes = valid_transaction_xdr(1000, 1, 1);
let tx = ValidatedTx::from_network(&decode(&bytes), &bytes).unwrap();
diff --git a/overlay/src/xdr.rs b/overlay/src/xdr.rs
index 590d283457..8f0ded0bb0 100644
--- a/overlay/src/xdr.rs
+++ b/overlay/src/xdr.rs
@@ -7,10 +7,10 @@
use sha2::{Digest, Sha256};
use std::fmt;
-use stellar_xdr::curr as xdr;
+use stellar_xdr as xdr;
use xdr::{
Limits, MessageType, ReadXdr, ScpBallot, ScpEnvelope, ScpStatementPledges, StellarMessage,
- StellarValue,
+ StellarValue, StellarValueExt,
};
#[derive(Debug)]
@@ -143,6 +143,15 @@ fn collect_stellar_value_hash(hashes: &mut Vec<[u8; 32]>, value: &[u8]) {
let Ok(stellar_value) = StellarValue::from_xdr(value, Limits::none()) else {
return;
};
+ // Empty-tx-set values (both close-time formats) carry Core's synthetic
+ // empty tx set hash. Core builds that set locally, so there is nothing to
+ // fetch from the network.
+ if matches!(
+ stellar_value.ext,
+ StellarValueExt::EmptyTxSet(_) | StellarValueExt::EmptyTxSetMs(_)
+ ) {
+ return;
+ }
let hash: [u8; 32] = stellar_value.tx_set_hash.into();
if !hashes.contains(&hash) {
hashes.push(hash);
@@ -153,9 +162,11 @@ fn collect_stellar_value_hash(hashes: &mut Vec<[u8; 32]>, value: &[u8]) {
pub(crate) mod tests {
use super::*;
use xdr::{
- DecoratedSignature, GeneralizedTransactionSet, Hash, Limits, Operation, ScpNomination,
- ScpStatementPledges, SequenceNumber, StellarValueExt, TimePoint, Transaction,
- TransactionEnvelope, TransactionV1Envelope, Uint256, Value, VecM, WriteXdr,
+ DecoratedSignature, GeneralizedTransactionSet, Hash, LedgerCloseValueSignature, Limits,
+ Operation, ScpNomination, ScpStatementExternalize, ScpStatementPledges, SequenceNumber,
+ StellarValueExt, StellarValueProposedMsValue, StellarValueProposedValue,
+ StellarValueSignedMsValue, TimePoint, TimePointMs, Transaction, TransactionEnvelope,
+ TransactionV1Envelope, Uint256, Value, VecM, WriteXdr,
};
pub(crate) fn valid_transaction_xdr(fee: u32, sequence: i64, num_ops: usize) -> Vec {
@@ -268,4 +279,107 @@ pub(crate) mod tests {
let hashes = extract_txset_hashes_from_envelope(&envelope);
assert_eq!(hashes, vec![expected_hash]);
}
+
+ const VALUE_TX_SET_HASH: [u8; 32] = [0x42; 32];
+
+ fn stellar_value_xdr(ext: StellarValueExt) -> Vec {
+ StellarValue {
+ tx_set_hash: Hash(VALUE_TX_SET_HASH),
+ close_time: TimePoint(1_704_067_200),
+ upgrades: VecM::default(),
+ ext,
+ }
+ .to_xdr(Limits::none())
+ .unwrap()
+ }
+
+ fn nominate_envelope(value_xdr: Vec) -> ScpEnvelope {
+ let value = Value::try_from(value_xdr).unwrap();
+ let mut envelope = ScpEnvelope::default();
+ envelope.statement.pledges = ScpStatementPledges::Nominate(ScpNomination {
+ quorum_set_hash: Hash([0; 32]),
+ votes: VecM::try_from(vec![value.clone()]).unwrap(),
+ accepted: VecM::try_from(vec![value]).unwrap(),
+ });
+ envelope
+ }
+
+ fn signed_ms_ext() -> StellarValueExt {
+ StellarValueExt::SignedMs(StellarValueSignedMsValue {
+ close_time_ms: TimePointMs(1_704_067_200_123),
+ lc_value_signature: LedgerCloseValueSignature::default(),
+ })
+ }
+
+ // Every StellarValue format Core can produce, including the millisecond
+ // close time arms (protocol 30), must decode. Empty-tx-set values carry
+ // Core's synthetic hash and must never trigger a network fetch.
+ #[test]
+ fn extracts_txset_hashes_from_every_value_format() {
+ let proposed = |close_time_ms: Option| {
+ let tx_set_hash = Hash([0x11; 32]);
+ let previous_ledger_hash = Hash([0x22; 32]);
+ let lc_value_signature = LedgerCloseValueSignature::default();
+ match close_time_ms {
+ None => StellarValueExt::EmptyTxSet(StellarValueProposedValue {
+ tx_set_hash,
+ previous_ledger_hash,
+ previous_ledger_version: 29,
+ lc_value_signature,
+ }),
+ Some(ms) => StellarValueExt::EmptyTxSetMs(StellarValueProposedMsValue {
+ close_time_ms: TimePointMs(ms),
+ tx_set_hash,
+ previous_ledger_hash,
+ previous_ledger_version: 30,
+ lc_value_signature,
+ }),
+ }
+ };
+ let cases = [
+ (StellarValueExt::Basic, true),
+ (
+ StellarValueExt::Signed(LedgerCloseValueSignature::default()),
+ true,
+ ),
+ (signed_ms_ext(), true),
+ (proposed(None), false),
+ (proposed(Some(1_704_067_200_456)), false),
+ ];
+ for (ext, fetchable) in cases {
+ let envelope = nominate_envelope(stellar_value_xdr(ext.clone()));
+ let expected = if fetchable {
+ vec![VALUE_TX_SET_HASH]
+ } else {
+ vec![]
+ };
+ assert_eq!(
+ extract_txset_hashes_from_envelope(&envelope),
+ expected,
+ "{ext:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn extracts_txset_hash_from_ms_ballot() {
+ let value = Value::try_from(stellar_value_xdr(signed_ms_ext())).unwrap();
+ let mut envelope = ScpEnvelope::default();
+ envelope.statement.pledges = ScpStatementPledges::Externalize(ScpStatementExternalize {
+ commit: ScpBallot { counter: 1, value },
+ n_h: 1,
+ commit_quorum_set_hash: Hash([0; 32]),
+ });
+ assert_eq!(
+ extract_txset_hashes_from_envelope(&envelope),
+ vec![VALUE_TX_SET_HASH]
+ );
+ }
+
+ #[test]
+ fn skips_malformed_ms_values() {
+ let mut bytes = stellar_value_xdr(signed_ms_ext());
+ bytes.pop();
+ assert!(extract_txset_hashes_from_envelope(&nominate_envelope(bytes)).is_empty());
+ }
}
diff --git a/overlay/tests/e2e_binary.rs b/overlay/tests/e2e_binary.rs
index ef7b75d371..3a7d9bbf14 100644
--- a/overlay/tests/e2e_binary.rs
+++ b/overlay/tests/e2e_binary.rs
@@ -8,7 +8,7 @@ use std::path::PathBuf;
use std::process::{Child, Command, ExitStatus, Stdio};
use std::thread;
use std::time::{Duration, Instant};
-use stellar_xdr::curr::{Limits, ScpEnvelope, WriteXdr};
+use stellar_xdr::{Limits, ScpEnvelope, WriteXdr};
/// IPC message types (must match src/ipc/messages.rs)
mod ipc {
diff --git a/performance-eval/performance-eval.md b/performance-eval/performance-eval.md
index b3073f2f17..b72a5a0ec5 100644
--- a/performance-eval/performance-eval.md
+++ b/performance-eval/performance-eval.md
@@ -294,28 +294,84 @@ solution is, as root to run
Stellar-core has built-in support for Tracy traces.
-To install the visualizer, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site.
+To install the visualizer, follow the directions in [INSTALL.md](../INSTALL.md).
-At a high level you need to
+### General Visual Studio profiler
-install the required pre-requesites to build clients, run in a shell:
+The main page for the profiler built into Visual Studio Community Edition is located there: https://docs.microsoft.com/en-us/visualstudio/profiling/index
- vcpkg.exe integrate install
- vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3
+## All platforms
-Then build one of the servers.
+Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download
-Solutions for servers compatible with the version of stellar-core can be found under:
+# Memory-use profiling
- * lib/tracy/profiler/build/win32 (GUI)
- * lib/tracy/capture/build/win32
+Tools for memory use profiling are less well-developed than CPU profiling, but there are some options available.
-Note: when connecting, use `localhost` instead of `127.0.0.1` as Tracy binds by default to IPV6 addresses.
+## Tracy
-### General Visual Studio profiler
+Tracy has some built-in support for memory profiling, but turning it on will
+slow down core significantly and will use memory in the tracy client _very
+quickly_, easily overwhelming your workstation if you're not careful. So you can
+usually only turn it on for a brief period of time.
-The main page for the profiler built into Visual Studio Community Edition is located there: https://docs.microsoft.com/en-us/visualstudio/profiling/index
+It is most useful for examining a small part of the code for a short period of
+time, where you already have a fairly good idea of there being memory allocation
+issues that you want to see a precise accounting of. Allocations get linked to
+zones (as a list in each zone detail view) and are available for inspection in
+the "memory" window, along with a total map of memory and a list of all
+allocations.
-## All platforms
+Stellar-core has support for this mode separate from normal tracy tracing,
+because it is so performance intensive and memory hungry. You need to configure
+with --enable-tracy-memory-tracking and --disable-tcmalloc.
+
+## Heaptrack
+
+A better option for a high level "profile" of memory is the "heaptrack" tool,
+which is available on Linux.
+
+ $ sudo apt install heaptrack heaptrack-gui
+
+To use it you will also need to configure with --disable-tcmalloc, because it
+works by intercepting malloc/free calls underlying the default operator
+new/delete, and tcmalloc's operator new and delete will bypass that
+interception.
+
+Heaptrack should also be run only for a moderate amount of time, otherwise the
+recording will be huge. But it at least writes its recording to disk, and the
+recording is much more compact than tracy's in-memory structure, so it can run
+much longer than tracy in memory-recording mode without issue.
+
+Heaptrack can run a program as a subprocess or attach remotely. The remote
+attach mode allows you to avoid starting it until the program is close to the
+period you want to measure, so is recommended. You will need to enable ptrace
+permissions.
+
+Heaptrack's default recording mode is very slow as it symbolicates all the
+stacks while it runs. A better way is to record a _raw_ profile and then
+symbolicate the data after the fact.
+
+Combining these facts, the best execution we've found is like the following:
+
+ # in one terminal...
+ $ stellar-core ...
+
+ # in another terminal...
+ $ echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+ $ heaptrack --raw $(pidof stellar-core)
+
+ # switch back core and stop it with Ctrl-C when done
+
+ # heaptrack will exit and write a file like
+ # heaptrack.stellar-core.12345.raw.zst along with, hopefully, instructions
+ # to run something like this to post-process the raw file into a more
+ # compact and symbolicated form. This will run a long time:
+
+ $ zstd -dc < ".../heaptrack.stellar-core.12345.raw.zst" \
+ | /usr/lib/heaptrack/libexec/heaptrack_interpret \
+ | zstd -c > ".../heaptrack.stellar-core.12345.zst"
+
+ # finally load the profile into the visualization tool
+ $ heaptrack_gui ".../heaptrack.stellar-core.12345.zst"
-Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download
diff --git a/scripts/check-overlay-min-version.sh b/scripts/check-overlay-min-version.sh
new file mode 100755
index 0000000000..23cddd5035
--- /dev/null
+++ b/scripts/check-overlay-min-version.sh
@@ -0,0 +1,151 @@
+#!/bin/sh
+
+# This script checks OVERLAY_PROTOCOL_MIN_VERSION (in src/main/Config.cpp)
+# against the ledger protocol version a network is running.
+#
+# Once a network is on ledger protocol P, nodes need software supporting P to
+# stay in sync, so the minimum overlay version can be raised to the lowest
+# maximum overlay version (OVERLAY_PROTOCOL_VERSION) of any release that
+# supports P. It should never be higher than that, or it would disconnect
+# nodes running a release that can still follow the network.
+#
+# Usage: check-overlay-min-version.sh [--apply]
+#
+# Takes the releases supporting to be the release tags
+# v... with at least , relying on a release's
+# major version being the ledger protocol version it supports, and compares
+# OVERLAY_PROTOCOL_MIN_VERSION in the working tree against the lowest
+# OVERLAY_PROTOCOL_VERSION among them. Exits with a non-zero status if the
+# minimum is too high. If the minimum can be raised, says so, and with --apply
+# rewrites it in the working tree.
+#
+# Requires the release tags to have been fetched.
+
+set -e
+
+SRCDIR=$(realpath "$(dirname "$0")/..")
+CONFIG_CPP=src/main/Config.cpp
+
+APPLY=
+if [ "$1" = "--apply" ]
+then
+ APPLY=1
+ shift
+fi
+
+if [ $# -ne 1 ]
+then
+ echo "usage: $0 [--apply] " >&2
+ exit 2
+fi
+
+NETWORK_LEDGER=$1
+case "$NETWORK_LEDGER" in
+''|*[!0-9]*)
+ echo "error: network protocol must be a number, got '$NETWORK_LEDGER'" >&2
+ exit 2
+ ;;
+esac
+
+cd "$SRCDIR"
+
+# file_at : print the contents of at , or the
+# working tree copy if is empty.
+file_at()
+{
+ if [ -n "$1" ]
+ then
+ git show "$1:$2"
+ else
+ cat "$2"
+ fi
+}
+
+# extract : print the numeric value assigned
+# to in at , failing unless exactly one assignment is
+# found.
+extract()
+{
+ if ! CONTENTS=$(file_at "$1" "$2")
+ then
+ echo "error: failed to read $2 at ${1:-working tree}" >&2
+ exit 2
+ fi
+ VAL=$(printf '%s\n' "$CONTENTS" | sed -n "$4")
+ case "$VAL" in
+ ''|*[!0-9]*)
+ echo "error: expected exactly one numeric assignment to $3 in $2 at ${1:-working tree}, got '$VAL'" >&2
+ echo "(if the definition of $3 changed shape, update $0 to match)" >&2
+ exit 2
+ ;;
+ esac
+ echo "$VAL"
+}
+
+OVERLAY_EXPR='s/^[[:space:]]*OVERLAY_PROTOCOL_VERSION[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p'
+MIN_EXPR='s/^[[:space:]]*OVERLAY_PROTOCOL_MIN_VERSION[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p'
+
+# List the releases supporting ledger protocol $NETWORK_LEDGER, and set
+# TARGET_MIN to the lowest OVERLAY_PROTOCOL_VERSION among them and TARGET_TAG
+# to a release that has it.
+release_target()
+{
+ TARGET_MIN=
+ TARGET_TAG=
+ for T in $(git tag -l 'v[0-9]*' --sort=-v:refname)
+ do
+ MAJOR=${T#v}
+ MAJOR=${MAJOR%%[!0-9]*}
+ if [ "$MAJOR" -lt "$NETWORK_LEDGER" ]
+ then
+ break
+ fi
+ OVERLAY=$(extract "$T" "$CONFIG_CPP" OVERLAY_PROTOCOL_VERSION "$OVERLAY_EXPR")
+ echo " $T: OVERLAY_PROTOCOL_VERSION $OVERLAY"
+ if [ -z "$TARGET_MIN" ] || [ "$OVERLAY" -lt "$TARGET_MIN" ]
+ then
+ TARGET_MIN=$OVERLAY
+ TARGET_TAG=$T
+ fi
+ done
+ if [ -z "$TARGET_MIN" ]
+ then
+ echo "error: no release tag supports ledger protocol $NETWORK_LEDGER (tags not fetched?)" >&2
+ exit 2
+ fi
+}
+
+CODE_MIN=$(extract "" "$CONFIG_CPP" OVERLAY_PROTOCOL_MIN_VERSION "$MIN_EXPR")
+
+echo "Releases supporting ledger protocol $NETWORK_LEDGER:"
+release_target
+echo "Lowest OVERLAY_PROTOCOL_VERSION among them: $TARGET_MIN ($TARGET_TAG)"
+echo "OVERLAY_PROTOCOL_MIN_VERSION: $CODE_MIN"
+
+if [ "$CODE_MIN" -gt "$TARGET_MIN" ]
+then
+ echo "error: OVERLAY_PROTOCOL_MIN_VERSION ($CODE_MIN) is higher than the overlay version" >&2
+ echo "of $TARGET_TAG ($TARGET_MIN), which supports protocol $NETWORK_LEDGER, so it would" >&2
+ echo "disconnect nodes that can still follow the network." >&2
+ exit 1
+fi
+
+if [ "$CODE_MIN" -eq "$TARGET_MIN" ]
+then
+ echo "OK"
+ exit 0
+fi
+
+echo "OVERLAY_PROTOCOL_MIN_VERSION can be raised from $CODE_MIN to $TARGET_MIN"
+if [ -n "$APPLY" ]
+then
+ sed "s/^\([[:space:]]*OVERLAY_PROTOCOL_MIN_VERSION[[:space:]]*=[[:space:]]*\)$CODE_MIN;/\1$TARGET_MIN;/" "$CONFIG_CPP" > "$CONFIG_CPP.tmp"
+ mv "$CONFIG_CPP.tmp" "$CONFIG_CPP"
+ NEW_MIN=$(extract "" "$CONFIG_CPP" OVERLAY_PROTOCOL_MIN_VERSION "$MIN_EXPR")
+ if [ "$NEW_MIN" != "$TARGET_MIN" ]
+ then
+ echo "error: failed to update OVERLAY_PROTOCOL_MIN_VERSION in $CONFIG_CPP" >&2
+ exit 2
+ fi
+ echo "Updated $CONFIG_CPP"
+fi
diff --git a/scripts/check-protocol-overlay-versions.sh b/scripts/check-protocol-overlay-versions.sh
new file mode 100755
index 0000000000..2af3002cf0
--- /dev/null
+++ b/scripts/check-protocol-overlay-versions.sh
@@ -0,0 +1,96 @@
+#!/bin/sh
+
+# This script checks that a change that bumps the ledger protocol version
+# (MAX_SUPPORTED_PROTOCOL_VERSION in src/util/ProtocolVersion.h) also bumps
+# the maximum supported overlay protocol version
+# (OVERLAY_PROTOCOL_VERSION in src/main/Config.cpp), so that peers can tell
+# from the overlay handshake whether a node runs software that supports the
+# new ledger protocol.
+#
+# Usage: check-protocol-overlay-versions.sh []
+#
+# Compares (or the working tree if is omitted) against
+# . Exits with a non-zero status if OVERLAY_PROTOCOL_VERSION was
+# lowered, or if the ledger protocol version increased without
+# OVERLAY_PROTOCOL_VERSION increasing in the same change. An overlay bump that
+# landed separately (before or after an earlier protocol bump) doesn't count,
+# since it can't be told apart from one that belongs to the previous protocol.
+
+set -e
+
+SRCDIR=$(realpath "$(dirname "$0")/..")
+CONFIG_CPP=src/main/Config.cpp
+PROTOCOL_H=src/util/ProtocolVersion.h
+
+if [ $# -lt 1 ] || [ $# -gt 2 ]
+then
+ echo "usage: $0 []" >&2
+ exit 2
+fi
+
+BASE_REV=$1
+HEAD_REV=${2:-}
+
+cd "$SRCDIR"
+
+# file_at : print the contents of at , or the
+# working tree copy if is empty.
+file_at()
+{
+ if [ -n "$1" ]
+ then
+ git show "$1:$2"
+ else
+ cat "$2"
+ fi
+}
+
+# extract : print the numeric value assigned
+# to in at , failing unless exactly one assignment is
+# found.
+extract()
+{
+ if ! CONTENTS=$(file_at "$1" "$2")
+ then
+ echo "error: failed to read $2 at ${1:-working tree}" >&2
+ exit 2
+ fi
+ VAL=$(printf '%s\n' "$CONTENTS" | sed -n "$4")
+ case "$VAL" in
+ ''|*[!0-9]*)
+ echo "error: expected exactly one numeric assignment to $3 in $2 at ${1:-working tree}, got '$VAL'" >&2
+ echo "(if the definition of $3 changed shape, update $0 to match)" >&2
+ exit 2
+ ;;
+ esac
+ echo "$VAL"
+}
+
+LEDGER_EXPR='s/.*MAX_SUPPORTED_PROTOCOL_VERSION[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p'
+OVERLAY_EXPR='s/^[[:space:]]*OVERLAY_PROTOCOL_VERSION[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p'
+
+# The literal assigned to MAX_SUPPORTED_PROTOCOL_VERSION, i.e. the ledger
+# protocol version without the vnext +1.
+BASE_LEDGER=$(extract "$BASE_REV" "$PROTOCOL_H" MAX_SUPPORTED_PROTOCOL_VERSION "$LEDGER_EXPR")
+BASE_OVERLAY=$(extract "$BASE_REV" "$CONFIG_CPP" OVERLAY_PROTOCOL_VERSION "$OVERLAY_EXPR")
+HEAD_LEDGER=$(extract "$HEAD_REV" "$PROTOCOL_H" MAX_SUPPORTED_PROTOCOL_VERSION "$LEDGER_EXPR")
+HEAD_OVERLAY=$(extract "$HEAD_REV" "$CONFIG_CPP" OVERLAY_PROTOCOL_VERSION "$OVERLAY_EXPR")
+
+echo "Ledger protocol version: $BASE_LEDGER -> $HEAD_LEDGER"
+echo "OVERLAY_PROTOCOL_VERSION: $BASE_OVERLAY -> $HEAD_OVERLAY"
+
+if [ "$HEAD_OVERLAY" -lt "$BASE_OVERLAY" ]
+then
+ echo "error: OVERLAY_PROTOCOL_VERSION was lowered from $BASE_OVERLAY to $HEAD_OVERLAY" >&2
+ exit 1
+fi
+
+if [ "$HEAD_LEDGER" -gt "$BASE_LEDGER" ] && [ "$HEAD_OVERLAY" -le "$BASE_OVERLAY" ]
+then
+ echo "error: the ledger protocol version was bumped from $BASE_LEDGER to $HEAD_LEDGER" >&2
+ echo "but OVERLAY_PROTOCOL_VERSION was not bumped. A ledger protocol bump must be" >&2
+ echo "accompanied by a bump of the maximum overlay protocol version in $CONFIG_CPP." >&2
+ exit 1
+fi
+
+echo "OK"
diff --git a/src/Makefile.am b/src/Makefile.am
index 3c09fce682..937764f8f0 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -75,9 +75,11 @@ MAIN_FILES = $(GENERATED_VERSION_SOURCES) main/main.cpp
if BUILD_TESTS
stellar_core_SOURCES = $(MAIN_FILES) $(SRC_CXX_FILES) $(SRC_TEST_CXX_FILES)
+CARGO_FEATURE_FUZZ_TARGETS = --features fuzz_targets
CARGO_FEATURE_TESTUTILS = --features testutils
else # !BUILD_TESTS
stellar_core_SOURCES = $(MAIN_FILES) $(SRC_CXX_FILES)
+CARGO_FEATURE_FUZZ_TARGETS =
CARGO_FEATURE_TESTUTILS =
endif # !BUILD_TESTS
@@ -111,7 +113,7 @@ TEST_FILES = $(TESTDATA_DIR)/stellar-core_example.cfg $(TESTDATA_DIR)/stellar-co
BUILT_SOURCES = $(SRC_X_FILES:.x=.h) $(GENERATED_VERSION_SOURCES) $(TEST_FILES)
-$(SRC_X_FILES:.x=.h): $(XDRC)
+$(SRC_X_FILES:.x=.h): $(XDRC) Makefile
XDR_FEATURE_FLAGS =
SUFFIXES = .x .h .rs
@@ -144,10 +146,19 @@ stellar_core_SOURCES += $(GENERATED_XDRQUERY_SOURCES) util/xdrquery/XDRQueryPars
BUILT_SOURCES += rust/RustBridge.h $(GENERATED_RUST_SOURCES)
stellar_core_SOURCES += rust/RustBridge.h $(GENERATED_RUST_SOURCES)
+# Figure out if we ought to be passing -Zsanitizer=... RUSTFLAGS and friends
+SANITIZE_FLAGS := $(filter -fsanitize=%,$(CXXFLAGS))
+RUSTFLAGS_ASAN := $(if $(findstring address,$(SANITIZE_FLAGS)),-Zsanitizer=address,)
+RUSTFLAGS_TSAN := $(if $(findstring thread,$(SANITIZE_FLAGS)),-Zsanitizer=thread,)
+RUSTFLAGS_SANI := $(RUSTFLAGS_ASAN) $(RUSTFLAGS_TSAN)
+CARGOFLAGS_BUILDSTD := $(if $(findstring Zsanitizer,$(RUSTFLAGS_SANI)),-Zbuild-std,)
+RUSTFLAGS_CFGS := $(if $(findstring Zsanitizer,$(RUSTFLAGS_SANI)),--cfg curve25519_dalek_backend=\"serial\",)
+
+
RUST_TOOLCHAIN_FILE=$(top_srcdir)/rust-toolchain.toml
RUST_TOOLCHAIN_FILE_CHANNEL=$(shell sed -n 's/channel *= *"\([^"]*\)"/\1/p' $(RUST_TOOLCHAIN_FILE))
if ENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
-RUST_TOOLCHAIN_CHANNEL=$(if $(findstring -fsanitize=,$(CXXFLAGS)),nightly,$(RUST_TOOLCHAIN_FILE_CHANNEL))
+RUST_TOOLCHAIN_CHANNEL=$(if $(findstring Zsanitizer,$(RUSTFLAGS_SANI)),nightly,$(RUST_TOOLCHAIN_FILE_CHANNEL))
else
RUST_TOOLCHAIN_CHANNEL=$(RUST_TOOLCHAIN_FILE_CHANNEL)
endif
@@ -221,9 +232,11 @@ SOROBAN_BUILD_DIR=$(abspath $(RUST_BUILD_DIR))/soroban
# ALL_SOROBAN_PROTOCOLS as you see fit).
if ENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
-ALL_SOROBAN_PROTOCOLS=p27 p28
+# Only the two latest soroban modules belong here (keep in sync with the
+# fastdev feature in src/rust/Cargo.toml).
+ALL_SOROBAN_PROTOCOLS=p29 p30
else
-ALL_SOROBAN_PROTOCOLS=p21 p22 p23 p24 p25 p26 p27 p28
+ALL_SOROBAN_PROTOCOLS=p21 p22 p23 p24 p25 p26 p27 p28 p29 p30
endif
WIP_SOROBAN_PROTOCOL=
@@ -310,37 +323,69 @@ $(SOROBAN_BUILD_DIR)/%/target/git-state.txt: $(top_srcdir)/.git/modules/src/rust
printf '%s\n' "$$state" > $@.tmp; \
if cmp -s $@.tmp $@; then rm -f $@.tmp; else mv -f $@.tmp $@; fi
-# The fastdev rust build is a special non-production mode that builds all of
-# the rust dependencies of librust_stellar_core.a through a single cargo
-# invocation, which is actually the "normal" way cargo operates, but which also
-# has the negative side effect of resolving (merging) different point releases
-# and pre-release minor versions across transitive dependencies, which means we
+# The fastdev rust build is a special non-production mode that builds all of the
+# rust dependencies of librust_stellar_core.a through a single cargo invocation,
+# which is actually the "normal" way cargo operates, but which also has the
+# negative side effect of resolving (merging) different point releases and
+# pre-release minor versions across transitive dependencies, which means we
# can't control the _exact_ transitive dependencies as well as we'd like.
#
-# So we only use the fastdev rust build for certain special cases such as
-# testing with asan/tsan (they seem to only work well when built this way) and
-# use the non-unified build (with separate .a files for each separate soroban
-# version) for production builds.
+# So we only use the fastdev rust build for fast iteration while doing
+# development, or for special cases like testing with asan/tsan (they seem to
+# only work well when built this way), and use the non-unified build (with
+# separate .a files for each separate soroban version) for production builds.
+#
if ENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
+# Another delicate bit: the $(CARGO_FEATURE_FUZZ_TARGETS) feature.
+#
+# It pulls in a separate copy of the soroban host with `testutils` enabled,
+# which, in fastdev mode, winds up causing `testutils` to be enabled _on the
+# production soroban host_ as well (due to cargo feature unification), which
+# will cause the build to have slightly different costs and therefore diverge if
+# run against mainnet. Because of this, we are selective about it:
+#
+# - Regular build: turn on $(CARGO_FEATURE_FUZZ_TARGETS) for smoke-testing
+# - Fastdev build: turn off, to avoid divergence risk
+# - Fastdev + ENABLE_FUZZ: turn back on, to allow fuzzing
+#
+# This does mean that a fastdev + ENABLE_FUZZ build might diverge when run
+# against mainnet. But such a build is very slow, typically with multiple
+# sanitizers as well as the fuzzer instrumentation, and should not usually be
+# run against production traffic anyways. It exists just for running fuzzing
+# loops offline.
+
+if ENABLE_FUZZ
+FASTDEV_CARGO_FEATURE_FUZZ_TARGETS=$(CARGO_FEATURE_FUZZ_TARGETS)
+# Since we're manually invoking cargo rustc here, not running `cargo-fuzz`, we
+# have to manually pass some sancov flags along to get fuzz instrumentation in
+# the rust code. We also have to inhibit the trace-pc-guard instrumentation if
+# the fuzz engine is libfuzzer, because it explicitly refuses to run with that
+# mode (it just exits with a message to turn it off). The other fuzz engines,
+# of course, require it.
+TRACE_PC_GUARD_FLAG := $(if $(findstring -fsanitize=fuzzer,$(LIB_FUZZING_ENGINE)),,-Cllvm-args=-sanitizer-coverage-trace-pc-guard)
+RUSTFLAGS_FUZZ := -C passes=sancov-module \
+ -C llvm-args=-sanitizer-coverage-level=3 \
+ -C llvm-args=-sanitizer-coverage-inline-8bit-counters \
+ -C llvm-args=-sanitizer-coverage-pc-table \
+ $(TRACE_PC_GUARD_FLAG)
+else
+FASTDEV_CARGO_FEATURE_FUZZ_TARGETS=
+RUSTFLAGS_FUZZ :=
+endif
+
# In the fastdev build, we have to pass the --target flag. This actually breaks
# the non-unified build, so we wind up setting LIBRUST_STELLAR_CORE separately
# in fastdev and non-unified builds.
RUST_TARGET=$(shell rustc -vV | sed -n 's/host: //p')
LIBRUST_STELLAR_CORE=$(RUST_TARGET_DIR)/$(RUST_TARGET)/$(RUST_PROFILE_DIR)/librust_stellar_core.a
-RUSTFLAGS_ASAN := $(if $(findstring -fsanitize=address,$(CXXFLAGS)),-Zsanitizer=address,)
-RUSTFLAGS_TSAN := $(if $(findstring -fsanitize=thread,$(CXXFLAGS)),-Zsanitizer=thread,)
-RUSTFLAGS_SANI := $(RUSTFLAGS_ASAN) $(RUSTFLAGS_TSAN)
-CARGOFLAGS_BUILDSTD := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),-Zbuild-std,)
-RUSTFLAGS_CFGS := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),--cfg curve25519_dalek_backend=\"serial\",)
-
$(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(RUST_TOOLCHAIN_FILE)
cd $(abspath $(top_srcdir))/src/rust && \
CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \
RUSTC_WRAPPER="$(RUSTC_WRAPPER)" \
- RUSTFLAGS="$(RUSTFLAGS_SANI) $(RUSTFLAGS_CFGS)" \
+ RUSTFLAGS="$(RUSTFLAGS) $(RUSTFLAGS_SANI) $(RUSTFLAGS_CFGS) $(RUSTFLAGS_FUZZ)" \
CARGO_NET_GIT_FETCH_WITH_CLI=true \
$(CARGO) rustc \
$(CARGOFLAGS_BUILDSTD) \
@@ -350,6 +395,7 @@ $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(RUST
--locked \
--target-dir $(abspath $(RUST_TARGET_DIR)) \
$(CARGO_FEATURE_FASTDEV) $(CARGO_FEATURE_TRACY) $(CARGO_FEATURE_NEXT) $(CARGO_FEATURE_TESTUTILS) \
+ $(FASTDEV_CARGO_FEATURE_FUZZ_TARGETS) \
$(CARGO_FEATURE_TCMALLOC)
else # !ENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
@@ -416,7 +462,7 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) $(ALL_SOROBAN_GIT_
cd $(abspath $(top_srcdir))/src/rust/soroban/$$proto && \
CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \
RUSTC_WRAPPER="$(RUSTC_WRAPPER)" \
- RUSTFLAGS="-Cmetadata=$$proto" \
+ RUSTFLAGS="$(RUSTFLAGS) -Cmetadata=$$proto" \
CARGO_TARGET_DIR=$(SOROBAN_BUILD_DIR)/$$proto/target \
CARGO_NET_GIT_FETCH_WITH_CLI=true \
$(CARGO) build \
@@ -461,7 +507,7 @@ $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) $(ALL_SOROBAN_L
--locked \
--target-dir $(abspath $(RUST_TARGET_DIR)) \
$(CARGO_FEATURE_TRACY) $(CARGO_FEATURE_NEXT) $(CARGO_FEATURE_TESTUTILS) \
- $(CARGO_FEATURE_TCMALLOC) \
+ $(CARGO_FEATURE_TCMALLOC) $(CARGO_FEATURE_FUZZ_TARGETS) \
-- \
$(ALL_SOROBAN_EXTERN_ARGS) \
$(ALL_SOROBAN_DEPEND_ARGS)
diff --git a/src/bucket/BucketListBase.cpp b/src/bucket/BucketListBase.cpp
index 55dad3fe7f..ee7c04cfd3 100644
--- a/src/bucket/BucketListBase.cpp
+++ b/src/bucket/BucketListBase.cpp
@@ -890,7 +890,6 @@ template void BucketListBase::addBatchInternal(
template void BucketListBase::addBatchInternal(
Application& app, uint32_t currLedger, uint32_t currLedgerProtocol,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries);
+ LedgerEntryRefs const& initEntries, LedgerEntryRefs const& liveEntries,
+ LedgerKeyRefs const& deadEntries);
}
diff --git a/src/bucket/BucketManager.cpp b/src/bucket/BucketManager.cpp
index 232f9a874f..72e138661d 100644
--- a/src/bucket/BucketManager.cpp
+++ b/src/bucket/BucketManager.cpp
@@ -1024,9 +1024,9 @@ BucketManager::forgetUnreferencedBuckets(HistoryArchiveState const& has)
void
BucketManager::addLiveBatch(Application& app, LedgerHeader header,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries)
+ LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries)
{
ZoneScoped;
#ifdef BUILD_TESTS
@@ -1185,10 +1185,9 @@ BucketManager::resolveBackgroundEvictionScan(
ZoneScoped;
releaseAssert(mEvictionStatistics);
auto timer = mBucketListEvictionMetrics.blockingTime.TimeScope();
- LedgerTxnReadOnly ltxSnap(ltx);
- auto ledgerSeq = ltxSnap.getLedgerHeader().current().ledgerSeq;
- auto ledgerVers = ltxSnap.getLedgerHeader().current().ledgerVersion;
- auto networkConfig = SorobanNetworkConfig::loadFromLedger(ltxSnap);
+ auto ledgerSeq = ltx.loadHeader().current().ledgerSeq;
+ auto ledgerVers = ltx.loadHeader().current().ledgerVersion;
+ auto networkConfig = SorobanNetworkConfig::loadFromLedger(ltx);
releaseAssert(ledgerSeq == lclApplyView.getLedgerSeq() + 1);
if (!mEvictionFuture.valid())
diff --git a/src/bucket/BucketManager.h b/src/bucket/BucketManager.h
index 35d64cde1a..ebe02a839f 100644
--- a/src/bucket/BucketManager.h
+++ b/src/bucket/BucketManager.h
@@ -7,6 +7,7 @@
#include "bucket/BucketMergeMap.h"
#include "history/HistoryArchive.h"
#include "ledger/ImmutableLedgerView.h"
+#include "ledger/LedgerEntryRefs.h"
#include "ledger/NetworkConfig.h"
#include "main/Config.h"
#include "util/ThreadAnnotations.h"
@@ -315,9 +316,8 @@ class BucketManager : NonMovableOrCopyable
// `header` value should be taken from the ledger at which this batch is
// being added.
void addLiveBatch(Application& app, LedgerHeader header,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries);
+ LedgerEntryRefs initEntries, LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries);
void addHotArchiveBatch(Application& app, LedgerHeader header,
std::vector const& archivedEntries,
std::vector const& restoredEntries);
diff --git a/src/bucket/LiveBucket.cpp b/src/bucket/LiveBucket.cpp
index 0669c8b61b..e26adfc24a 100644
--- a/src/bucket/LiveBucket.cpp
+++ b/src/bucket/LiveBucket.cpp
@@ -377,11 +377,24 @@ LiveBucket::getRangeForType(LedgerEntryType type) const
return getIndex().getRangeForType(type);
}
+#ifdef BUILD_TESTS
std::vector
LiveBucket::convertToBucketEntry(bool useInit,
std::vector const& initEntries,
std::vector const& liveEntries,
std::vector const& deadEntries)
+{
+ auto initRefs = toRefs(initEntries);
+ auto liveRefs = toRefs(liveEntries);
+ auto deadRefs = toRefs(deadEntries);
+ return convertToBucketEntry(useInit, initRefs, liveRefs, deadRefs);
+}
+#endif
+
+std::vector
+LiveBucket::convertToBucketEntry(bool useInit, LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries)
{
ZoneScoped;
size_t totalSize =
@@ -393,21 +406,21 @@ LiveBucket::convertToBucketEntry(bool useInit,
std::vector sortedEntries;
sortedEntries.reserve(totalSize);
- for (auto const& e : initEntries)
+ for (LedgerEntry const& e : initEntries)
{
auto& ce = entries.emplace_back();
ce.type(useInit ? INITENTRY : LIVEENTRY);
ce.liveEntry() = e;
sortedEntries.push_back(&ce);
}
- for (auto const& e : liveEntries)
+ for (LedgerEntry const& e : liveEntries)
{
auto& ce = entries.emplace_back();
ce.type(LIVEENTRY);
ce.liveEntry() = e;
sortedEntries.push_back(&ce);
}
- for (auto const& e : deadEntries)
+ for (LedgerKey const& e : deadEntries)
{
auto& ce = entries.emplace_back();
ce.type(DEADENTRY);
@@ -433,12 +446,27 @@ LiveBucket::convertToBucketEntry(bool useInit,
return bucket;
}
+#ifdef BUILD_TESTS
std::shared_ptr
LiveBucket::fresh(BucketManager& bucketManager, uint32_t protocolVersion,
std::vector const& initEntries,
std::vector const& liveEntries,
std::vector const& deadEntries,
bool countMergeEvents, asio::io_context& ctx, bool doFsync)
+{
+ auto initRefs = toRefs(initEntries);
+ auto liveRefs = toRefs(liveEntries);
+ auto deadRefs = toRefs(deadEntries);
+ return fresh(bucketManager, protocolVersion, initRefs, liveRefs, deadRefs,
+ countMergeEvents, ctx, doFsync);
+}
+#endif
+
+std::shared_ptr
+LiveBucket::fresh(BucketManager& bucketManager, uint32_t protocolVersion,
+ LedgerEntryRefs initEntries, LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries, bool countMergeEvents,
+ asio::io_context& ctx, bool doFsync)
{
ZoneScoped;
// When building fresh buckets after protocol version 10 (i.e. version
@@ -480,10 +508,9 @@ LiveBucket::fresh(BucketManager& bucketManager, uint32_t protocolVersion,
std::shared_ptr
LiveBucket::freshInMemoryOnly(BucketManager& bucketManager,
uint32_t protocolVersion,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries,
- bool countMergeEvents)
+ LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries, bool countMergeEvents)
{
ZoneScoped;
// When building fresh buckets after protocol version 10 (i.e. version
diff --git a/src/bucket/LiveBucket.h b/src/bucket/LiveBucket.h
index fc68867b21..bc0ae0fb70 100644
--- a/src/bucket/LiveBucket.h
+++ b/src/bucket/LiveBucket.h
@@ -7,6 +7,7 @@
#include "bucket/BucketBase.h"
#include "bucket/BucketUtils.h"
#include "bucket/LiveBucketIndex.h"
+#include "ledger/LedgerEntryRefs.h"
namespace medida
{
@@ -82,11 +83,17 @@ class LiveBucket : public BucketBase,
static void checkProtocolLegality(BucketEntry const& entry,
uint32_t protocolVersion);
+ static std::vector
+ convertToBucketEntry(bool useInit, LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries);
+#ifdef BUILD_TESTS
static std::vector
convertToBucketEntry(bool useInit,
std::vector const& initEntries,
std::vector const& liveEntries,
std::vector const& deadEntries);
+#endif
template
static void mergeCasesWithEqualKeys(
@@ -116,22 +123,27 @@ class LiveBucket : public BucketBase,
// Create a fresh bucket from given vectors of init (created) and live
// (updated) LedgerEntries, and dead LedgerEntryKeys. The bucket will
// be sorted, hashed, and adopted in the provided BucketManager.
+ static std::shared_ptr
+ fresh(BucketManager& bucketManager, uint32_t protocolVersion,
+ LedgerEntryRefs initEntries, LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries, bool countMergeEvents,
+ asio::io_context& ctx, bool doFsync);
+#ifdef BUILD_TESTS
static std::shared_ptr
fresh(BucketManager& bucketManager, uint32_t protocolVersion,
std::vector const& initEntries,
std::vector const& liveEntries,
std::vector const& deadEntries, bool countMergeEvents,
asio::io_context& ctx, bool doFsync);
+#endif
// Create a fresh bucket that exists only in memory, without writing to
// disk, calculating a hash, or indexing. This should only be used for
// "level -1" snap buckets that are immediately merged into level 0.
static std::shared_ptr
freshInMemoryOnly(BucketManager& bucketManager, uint32_t protocolVersion,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries,
- bool countMergeEvents);
+ LedgerEntryRefs initEntries, LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries, bool countMergeEvents);
// Returns true if the given BucketEntry should be dropped in the bottom
// level bucket (i.e. DEADENTRY)
diff --git a/src/bucket/LiveBucketList.cpp b/src/bucket/LiveBucketList.cpp
index 0848db17b4..6bd34860df 100644
--- a/src/bucket/LiveBucketList.cpp
+++ b/src/bucket/LiveBucketList.cpp
@@ -14,9 +14,8 @@ namespace stellar
void
LiveBucketList::addBatch(Application& app, uint32_t currLedger,
uint32_t currLedgerProtocol,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries)
+ LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries, LedgerKeyRefs deadEntries)
{
ZoneScoped;
addBatchInternal(app, currLedger, currLedgerProtocol, initEntries,
@@ -26,6 +25,21 @@ LiveBucketList::addBatch(Application& app, uint32_t currLedger,
maybeInitializeCaches(app.getConfig());
}
+#ifdef BUILD_TESTS
+void
+LiveBucketList::addBatch(Application& app, uint32_t currLedger,
+ uint32_t currLedgerProtocol,
+ std::vector const& initEntries,
+ std::vector const& liveEntries,
+ std::vector const& deadEntries)
+{
+ auto initRefs = toRefs(initEntries);
+ auto liveRefs = toRefs(liveEntries);
+ auto deadRefs = toRefs(deadEntries);
+ addBatch(app, currLedger, currLedgerProtocol, initRefs, liveRefs, deadRefs);
+}
+#endif
+
BucketEntryCounters
LiveBucketList::sumBucketEntryCounters() const
{
diff --git a/src/bucket/LiveBucketList.h b/src/bucket/LiveBucketList.h
index 59f0e6d42e..9f72a98804 100644
--- a/src/bucket/LiveBucketList.h
+++ b/src/bucket/LiveBucketList.h
@@ -45,11 +45,16 @@ class LiveBucketList : public BucketListBase
// should have spilled due to passing through `currLedger`. The `currLedger`
// and `currProtocolVersion` values should be taken from the ledger at which
// this batch is being added.
+ void addBatch(Application& app, uint32_t currLedger,
+ uint32_t currLedgerProtocol, LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries, LedgerKeyRefs deadEntries);
+#ifdef BUILD_TESTS
void addBatch(Application& app, uint32_t currLedger,
uint32_t currLedgerProtocol,
std::vector const& initEntries,
std::vector const& liveEntries,
std::vector const& deadEntries);
+#endif
BucketEntryCounters sumBucketEntryCounters() const;
diff --git a/src/bucket/test/BucketIndexTests.cpp b/src/bucket/test/BucketIndexTests.cpp
index 4b14f0f881..f42be7629f 100644
--- a/src/bucket/test/BucketIndexTests.cpp
+++ b/src/bucket/test/BucketIndexTests.cpp
@@ -1780,8 +1780,8 @@ TEST_CASE("scanForLiveEntriesOfType loser tree unit tests",
HistoryArchiveState has;
has.currentLedger = lcl.header.ledgerSeq;
auto state = std::make_shared(
- bl, bm.getHotArchiveBucketList(), lcl, has, std::nullopt,
- app->getMetrics());
+ bl, bm.getHotArchiveBucketList(), lcl, has,
+ SorobanNetworkConfig::emptyConfig(), app->getMetrics());
return ImmutableLedgerView{state, app->getMetrics()};
};
diff --git a/src/bucket/test/BucketListTests.cpp b/src/bucket/test/BucketListTests.cpp
index e76064db3a..257d3f18f6 100644
--- a/src/bucket/test/BucketListTests.cpp
+++ b/src/bucket/test/BucketListTests.cpp
@@ -1112,16 +1112,17 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist][archival][soroban]")
LedgerKey stateArchivalKey(CONFIG_SETTING);
stateArchivalKey.configSetting().configSettingID =
ConfigSettingID::CONFIG_SETTING_STATE_ARCHIVAL;
- CheckValidLedgerViewWrapper ledgerView(*app);
+ auto ledgerView = app->getLedgerManager().getLCLView();
auto stateArchivalEntry =
- ledgerView.load(stateArchivalKey).current();
+ ledgerView->load(stateArchivalKey).current();
modifyStateArchivalFn(stateArchivalEntry.data.configSetting()
.stateArchivalSettings());
LedgerKey evictionIterKey(CONFIG_SETTING);
evictionIterKey.configSetting().configSettingID =
ConfigSettingID::CONFIG_SETTING_EVICTION_ITERATOR;
- auto evictionIterEntry = ledgerView.load(evictionIterKey).current();
+ auto evictionIterEntry =
+ ledgerView->load(evictionIterKey).current();
modifyEvictionIteratorFn(
evictionIterEntry.data.configSetting().evictionIterator());
@@ -1723,11 +1724,11 @@ TEST_CASE_VERSIONS("Searchable BucketListDB snapshots", "[bucketlist]")
}
closeLedger(*app);
- auto blLedgerView = app->getLedgerManager().copyImmutableLedgerView();
+ auto blLedgerView = app->getLedgerManager().getLCLView();
// Snapshot should automatically update with latest version
- auto loadedEntry = blLedgerView.loadLiveEntry(LedgerEntryKey(entry));
- REQUIRE((loadedEntry && *loadedEntry == entry));
+ auto loadedEntry = blLedgerView->load(LedgerEntryKey(entry));
+ REQUIRE((loadedEntry && loadedEntry.current() == entry));
}
}
diff --git a/src/bucket/test/BucketTestUtils.cpp b/src/bucket/test/BucketTestUtils.cpp
index 1e8da564ea..209dc73d58 100644
--- a/src/bucket/test/BucketTestUtils.cpp
+++ b/src/bucket/test/BucketTestUtils.cpp
@@ -12,6 +12,7 @@
#include "ledger/LedgerManager.h"
#include "ledger/LedgerTxn.h"
#include "main/Application.h"
+#include "test/TxTests.h"
#include "test/test.h"
#include "util/ProtocolVersion.h"
#include "xdr/Stellar-ledger.h"
@@ -37,9 +38,9 @@ getAppLedgerVersion(Application::pointer app)
void
addLiveBatchAndUpdateSnapshot(Application& app, LedgerHeader header,
- std::vector const& initEntries,
- std::vector const& liveEntries,
- std::vector const& deadEntries)
+ LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries)
{
auto& liveBl = app.getBucketManager().getLiveBucketList();
liveBl.addBatch(app, header.ledgerSeq, header.ledgerVersion, initEntries,
@@ -48,6 +49,18 @@ addLiveBatchAndUpdateSnapshot(Application& app, LedgerHeader header,
app.getLedgerManager().updateCanonicalStateForTesting(header);
}
+void
+addLiveBatchAndUpdateSnapshot(Application& app, LedgerHeader header,
+ std::vector const& initEntries,
+ std::vector const& liveEntries,
+ std::vector const& deadEntries)
+{
+ auto initRefs = toRefs(initEntries);
+ auto liveRefs = toRefs(liveEntries);
+ auto deadRefs = toRefs(deadEntries);
+ addLiveBatchAndUpdateSnapshot(app, header, initRefs, liveRefs, deadRefs);
+}
+
void
addHotArchiveBatchAndUpdateSnapshot(
Application& app, LedgerHeader header,
@@ -86,9 +99,10 @@ closeLedger(Application& app, std::optional skToSignValue,
CLOG_INFO(Bucket, "Artificially closing ledger {} with lcl={}, buckets={}",
ledgerNum, hexAbbrev(lcl.hash),
hexAbbrev(app.getBucketManager().getLiveBucketList().getHash()));
- app.getHerder().externalizeValue(TxSetXDRFrame::makeEmpty(lcl), ledgerNum,
- lcl.header.scpValue.closeTime, upgrades,
- skToSignValue);
+ app.getHerder().externalizeValue(
+ TxSetXDRFrame::makeEmpty(lcl), ledgerNum,
+ txtest::makeConsensusTime(lcl.header.scpValue.closeTime), upgrades,
+ skToSignValue);
while (lm.getLastClosedLedgerNum() < ledgerNum)
{
app.getClock().crank(true);
@@ -181,8 +195,8 @@ LedgerManagerForBucketTests::finalizeLedgerTxnChanges(
if (mUseTestEntries)
{
// Seal the ltx but throw its entries away.
- std::vector init, live;
- std::vector dead;
+ LedgerEntryRefVec init, live;
+ LedgerKeyRefVec dead;
// Any V20 features must be behind initialLedgerVers check, see comment
// in LedgerManagerImpl::ledgerClosed
@@ -262,7 +276,7 @@ LedgerManagerForBucketTests::finalizeLedgerTxnChanges(
// Seal the ltx and collect its entries, but don't load Soroban
// config yet -- test entries (including network config like eviction
// iterator) are added directly to the BucketList below, not via ltx.
- ltx.getAllEntries(init, live, dead);
+ ltx.sealAndBorrowAllEntries(init, live, dead);
// Add dead entries from ltx to entries that will be added to BucketList
// so we can test background eviction properly
@@ -280,7 +294,7 @@ LedgerManagerForBucketTests::finalizeLedgerTxnChanges(
// When the actual entries have the same key as test entries, we
// override the actual entries with the test entries (here we
// just don't add the actual entries if they're already present).
- for (auto const& liveEntry : live)
+ for (LedgerEntry const& liveEntry : live)
{
if (std::find_if(mTestLiveEntries.begin(),
mTestLiveEntries.end(),
@@ -295,12 +309,15 @@ LedgerManagerForBucketTests::finalizeLedgerTxnChanges(
}
// Use the testing values.
+ auto testInitRefs = toRefs(mTestInitEntries);
+ auto testLiveRefs = toRefs(mTestLiveEntries);
+ auto testDeadRefs = toRefs(mTestDeadEntries);
mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion,
- mTestInitEntries);
+ testInitRefs);
mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion,
- mTestLiveEntries);
- mApp.getBucketManager().addLiveBatch(
- mApp, lh, mTestInitEntries, mTestLiveEntries, mTestDeadEntries);
+ testLiveRefs);
+ mApp.getBucketManager().addLiveBatch(mApp, lh, testInitRefs,
+ testLiveRefs, testDeadRefs);
// Load the final Soroban config AFTER addLiveBatch so that test
// entries (e.g. stateArchivalSettings with eviction iterator) are
@@ -318,15 +335,14 @@ LedgerManagerForBucketTests::finalizeLedgerTxnChanges(
HistoryArchiveState tempHas;
tempHas.currentLedger = lh.ledgerSeq;
auto& bm = mApp.getBucketManager();
- auto tempState = ImmutableLedgerData::createAndMaybeLoadConfig(
+ auto tempState = std::make_shared(
bm.getLiveBucketList(), bm.getHotArchiveBucketList(), tempLcl,
- tempHas, mApp.getMetrics());
+ tempHas, std::nullopt, mApp.getMetrics());
finalSorobanConfig = tempState->getSorobanConfig();
}
mApplyState.updateInMemorySorobanState(
- mTestInitEntries, mTestLiveEntries, mTestDeadEntries, lh,
- finalSorobanConfig);
+ testInitRefs, testLiveRefs, testDeadRefs, lh, finalSorobanConfig);
mUseTestEntries = false;
mAlsoAddActualEntries = false;
diff --git a/src/bucket/test/BucketTestUtils.h b/src/bucket/test/BucketTestUtils.h
index b5eb8dba46..0a06d152a5 100644
--- a/src/bucket/test/BucketTestUtils.h
+++ b/src/bucket/test/BucketTestUtils.h
@@ -4,7 +4,9 @@
#pragma once
+#include "ledger/LedgerEntryRefs.h"
#include "test/TestUtils.h"
+#include "test/test.h"
#include "xdr/Stellar-ledger.h"
namespace stellar
@@ -12,6 +14,10 @@ namespace stellar
namespace BucketTestUtils
{
+void addLiveBatchAndUpdateSnapshot(Application& app, LedgerHeader header,
+ LedgerEntryRefs initEntries,
+ LedgerEntryRefs liveEntries,
+ LedgerKeyRefs deadEntries);
void addLiveBatchAndUpdateSnapshot(Application& app, LedgerHeader header,
std::vector const& initEntries,
std::vector const& liveEntries,
@@ -85,6 +91,10 @@ class LedgerManagerForBucketTests : public LedgerManagerImpl
std::vector const& deadEntries,
bool alsoAddActualEntries = false)
{
+ // Entries injected straight into the bucket list appear in the ledger
+ // without any transaction producing them, so the resulting meta is
+ // not a faithful record of how the state came to be.
+ disableLcmCapture();
mUseTestEntries = true;
mAlsoAddActualEntries = alsoAddActualEntries;
mTestInitEntries = initEntries;
@@ -97,6 +107,9 @@ class LedgerManagerForBucketTests : public LedgerManagerImpl
std::vector const& archiveEntries,
std::vector const& restoredEntries)
{
+ // As above: injected archive state has no transaction behind it, so
+ // the meta does not record how that state came to be.
+ disableLcmCapture();
mUseTestEntries = true;
mTestArchiveEntries = archiveEntries;
mTestRestoredEntries = restoredEntries;
@@ -124,5 +137,6 @@ class BucketTestApplication : public TestApplication
return std::make_unique(*this);
}
};
-}
-}
+
+} // namespace BucketTestUtils
+} // namespace stellar
diff --git a/src/catchup/ApplyCheckpointWork.cpp b/src/catchup/ApplyCheckpointWork.cpp
index ff5f8a5ebd..17c6fc7d05 100644
--- a/src/catchup/ApplyCheckpointWork.cpp
+++ b/src/catchup/ApplyCheckpointWork.cpp
@@ -12,6 +12,7 @@
#include "history/HistoryUtils.h"
#include "historywork/Progress.h"
#include "ledger/CheckpointRange.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "ledger/LedgerManager.h"
#include "main/Application.h"
#include "util/GlobalChecks.h"
@@ -154,6 +155,15 @@ ApplyCheckpointWork::getCurrentTxSet()
}
else
{
+ // The legacy field has to be empty when the generalized tx set
+ // is present.
+ if (!mTxHistoryEntry.txSet.txs.empty())
+ {
+ throw std::runtime_error(fmt::format(
+ FMT_STRING("txset entry for {:d} has both legacy and "
+ "generalized transactions"),
+ seq));
+ }
return TxSetXDRFrame::makeFromWire(
mTxHistoryEntry.ext.generalizedTxSet());
}
@@ -277,7 +287,7 @@ ApplyCheckpointWork::getNextLedgerCloseData()
// Empty-tx-set values should have the empty-tx-set hash (and vice versa)
if ((header.scpValue.txSetHash == Herder::EMPTY_TX_SET_HASH) !=
- (header.scpValue.ext.v() == STELLAR_VALUE_EMPTY_TX_SET))
+ isEmptyTxSetStellarValue(header.scpValue))
{
throw std::runtime_error(fmt::format(
FMT_STRING("ledger header for {:d} has mismatched empty-tx-set "
@@ -285,6 +295,43 @@ ApplyCheckpointWork::getNextLedgerCloseData()
header.ledgerSeq, static_cast(header.scpValue.ext.v())));
}
+ // Check that we use the correct time format in the ledger header.
+ if (!hasValidCloseTime(header.scpValue))
+ {
+ throw std::runtime_error(fmt::format(
+ FMT_STRING("ledger header for {:d} has inconsistent close time "
+ "fields: closeTime {:d}, consensus close time {}"),
+ header.ledgerSeq, header.scpValue.closeTime,
+ getConsensusTime(header.scpValue).toString()));
+ }
+
+ bool const hasMsCloseTime = isMsCloseTimeStellarValue(header.scpValue);
+ bool const protocolRequiresMsCloseTime =
+ protocolHasMsCloseTime(lclHeader.header.ledgerVersion);
+ if (hasMsCloseTime != protocolRequiresMsCloseTime)
+ {
+ throw std::runtime_error(fmt::format(
+ FMT_STRING("ledger header for {:d} has a {} close time "
+ "(StellarValue type {:d}) but protocol {:d} requires a "
+ "{} close time"),
+ header.ledgerSeq, hasMsCloseTime ? "millisecond" : "whole-second",
+ static_cast(header.scpValue.ext.v()),
+ lclHeader.header.ledgerVersion,
+ protocolRequiresMsCloseTime ? "millisecond" : "whole-second"));
+ }
+
+ // Close times must strictly increase from ledger to ledger
+ auto const previousCloseTime = getConsensusTime(lclHeader.header.scpValue);
+ auto const nextCloseTime = getConsensusTime(header.scpValue);
+ if (nextCloseTime <= previousCloseTime)
+ {
+ throw std::runtime_error(fmt::format(
+ FMT_STRING("ledger header for {:d} has a non-advancing close "
+ "time {} (previous ledger closed at {})"),
+ header.ledgerSeq, nextCloseTime.toString(),
+ previousCloseTime.toString()));
+ }
+
// We've verified the ledgerHeader (in the "trusted part of history"
// sense) in CATCHUP_VERIFY phase; we now need to check that the
// txhash we're about to apply is the one denoted by that ledger
@@ -311,6 +358,18 @@ ApplyCheckpointWork::getNextLedgerCloseData()
"hash"),
header.ledgerSeq, txset->sizeTxTotal()));
}
+
+ // Empty tx sets are not stored in history archives, so if no txs are
+ // present, the tx set must match the canonical empty tx set.
+ if (txset->getContentsHash() !=
+ TxSetXDRFrame::makeEmpty(lclHeader)->getContentsHash())
+ {
+ throw std::runtime_error(fmt::format(
+ FMT_STRING("replay txset for {:d} is not the canonical empty "
+ "tx set, but its ledger header carries the "
+ "empty-tx-set hash"),
+ header.ledgerSeq));
+ }
}
else if (header.scpValue.txSetHash != txset->getContentsHash())
{
diff --git a/src/catchup/CatchupWork.cpp b/src/catchup/CatchupWork.cpp
index 5866af3780..5af496f908 100644
--- a/src/catchup/CatchupWork.cpp
+++ b/src/catchup/CatchupWork.cpp
@@ -19,6 +19,7 @@
#include "historywork/DownloadVerifyTxResultsWork.h"
#include "historywork/GetAndUnzipRemoteFileWork.h"
#include "historywork/GetHistoryArchiveStateWork.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "ledger/LedgerManager.h"
#include "main/Application.h"
#include "main/PersistentState.h"
@@ -69,7 +70,7 @@ setHerderStateTo(FileTransferInfo const& ft, uint32_t ledger, Application& app)
app.getHerder().setTrackingSCPState(ledger, entry->header.scpValue,
/* isTrackingNetwork */ false);
CLOG_INFO(History, "Herder state is set! tracking={}, closeTime={}", ledger,
- entry->header.scpValue.closeTime);
+ getConsensusTime(entry->header.scpValue).toString());
return true;
}
@@ -439,9 +440,9 @@ CatchupWork::runCatchupStep()
if (mApplyBufferedLedgersWork)
{
// ApplyBufferedLedgersWork will try to apply
- // as many ledgers in mSyncingLedgers as possible.
+ // as many ledgers in mBufferedLedgers as possible.
// Note that it's not always possible to apply
- // _all_ the ledgers in mSyncingLedgers due to gaps.
+ // _all_ the ledgers in mBufferedLedgers due to gaps.
if (mApplyBufferedLedgersWork->getState() ==
State::WORK_SUCCESS)
{
diff --git a/src/catchup/LedgerApplyManagerImpl.cpp b/src/catchup/LedgerApplyManagerImpl.cpp
index 2ffa6885ea..c3970b2112 100644
--- a/src/catchup/LedgerApplyManagerImpl.cpp
+++ b/src/catchup/LedgerApplyManagerImpl.cpp
@@ -90,7 +90,7 @@ LedgerApplyManager::create(Application& app)
LedgerApplyManagerImpl::LedgerApplyManagerImpl(Application& app)
: mApp(app)
, mCatchupWork(nullptr)
- , mSyncingLedgersSize(
+ , mBufferedLedgersSize(
app.getMetrics().NewCounter({"ledger", "memory", "queued-ledgers"}))
, mLargestLedgerSeqHeard(0)
{
@@ -145,17 +145,17 @@ LedgerApplyManagerImpl::processLedger(LedgerCloseData const& ledgerData,
}
// Always add a newer ledger, maybe apply
- mSyncingLedgers.emplace(lastReceivedLedgerSeq, ledgerData);
+ mBufferedLedgers.emplace(lastReceivedLedgerSeq, ledgerData);
mLargestLedgerSeqHeard =
std::max(mLargestLedgerSeqHeard, lastReceivedLedgerSeq);
// 1. CatchupWork is not running yet
// 2. LedgerApplyManager received ledger that should be immediately applied
// by LedgerManager: check if we have any sequential ledgers. If so, attempt
- // to apply mSyncingLedgers and possibly get back in sync
+ // to apply mBufferedLedgers and possibly get back in sync
if (!mCatchupWork && lastReceivedLedgerSeq == *mLastQueuedToApply + 1)
{
- tryApplySyncingLedgers();
+ tryApplyBufferedLedgers();
return ProcessLedgerResult::PROCESSED_ALL_LEDGERS_SEQUENTIALLY;
}
@@ -170,8 +170,8 @@ LedgerApplyManagerImpl::processLedger(LedgerCloseData const& ledgerData,
// state machine to ledger L-1 (the end of the checkpoint covering K) and
// then replay buffered ledgers from L onwards.
CLOG_INFO(Ledger,
- "Close of ledger {} buffered. mSyncingLedgers has {} ledgers",
- ledgerData.getLedgerSeq(), mSyncingLedgers.size());
+ "Close of ledger {} buffered. mBufferedLedgers has {} ledgers",
+ ledgerData.getLedgerSeq(), mBufferedLedgers.size());
// First: if CatchupWork has started, just buffer and return early.
if (mCatchupWork)
@@ -184,14 +184,14 @@ LedgerApplyManagerImpl::processLedger(LedgerCloseData const& ledgerData,
return ProcessLedgerResult::WAIT_TO_APPLY_BUFFERED_OR_CATCHUP;
}
- trimSyncingLedgers();
+ trimBufferedLedgers();
logAndUpdateCatchupStatus(true);
return ProcessLedgerResult::WAIT_TO_APPLY_BUFFERED_OR_CATCHUP;
}
// Next, we buffer every out of sync ledger to allow us to get back in sync
// in case the ledgers we're missing are received.
- trimSyncingLedgers();
+ trimBufferedLedgers();
// Finally we wait some number of ledgers beyond the smallest buffered
// checkpoint ledger before we trigger the CatchupWork. This could be any
@@ -200,8 +200,8 @@ LedgerApplyManagerImpl::processLedger(LedgerCloseData const& ledgerData,
// we just check for any ledger larger than the checkpoint
std::string message;
- uint32_t firstLedgerInBuffer = mSyncingLedgers.begin()->first;
- uint32_t lastLedgerInBuffer = mSyncingLedgers.crbegin()->first;
+ uint32_t firstLedgerInBuffer = mBufferedLedgers.begin()->first;
+ uint32_t lastLedgerInBuffer = mBufferedLedgers.crbegin()->first;
if (mApp.getConfig().modeDoesCatchupWithBucketList() &&
HistoryManager::isFirstLedgerInCheckpoint(firstLedgerInBuffer,
mApp.getConfig()) &&
@@ -286,7 +286,7 @@ LedgerApplyManagerImpl::startCatchup(CatchupConfiguration configuration,
}
// Offline catchup isn't triggered by buffered ledgers
- releaseAssert(configuration.offline() == mSyncingLedgers.empty());
+ releaseAssert(configuration.offline() == mBufferedLedgers.empty());
releaseAssert(!mCatchupWork);
@@ -365,12 +365,12 @@ LedgerApplyManagerImpl::maybeGetNextBufferedLedgerToApply()
// Since we just applied a ledger, refresh mLastQueuedToApply
updateLastQueuedToApply();
- trimSyncingLedgers();
- if (!mSyncingLedgers.empty() &&
- mSyncingLedgers.begin()->first == *mLastQueuedToApply + 1)
+ trimBufferedLedgers();
+ if (!mBufferedLedgers.empty() &&
+ mBufferedLedgers.begin()->first == *mLastQueuedToApply + 1)
{
return std::make_optional(
- mSyncingLedgers.begin()->second);
+ mBufferedLedgers.begin()->second);
}
else
{
@@ -382,10 +382,10 @@ std::optional
LedgerApplyManagerImpl::maybeGetLargestBufferedLedger()
{
releaseAssert(threadIsMain());
- if (!mSyncingLedgers.empty())
+ if (!mBufferedLedgers.empty())
{
return std::make_optional(
- mSyncingLedgers.crbegin()->second);
+ mBufferedLedgers.crbegin()->second);
}
else
{
@@ -412,7 +412,7 @@ void
LedgerApplyManagerImpl::syncMetrics()
{
releaseAssert(threadIsMain());
- mSyncingLedgersSize.set_count(mSyncingLedgers.size());
+ mBufferedLedgersSize.set_count(mBufferedLedgers.size());
}
void
@@ -435,12 +435,12 @@ void
LedgerApplyManagerImpl::startOnlineCatchup()
{
releaseAssert(threadIsMain());
- releaseAssert(mSyncingLedgers.size() > 1);
+ releaseAssert(mBufferedLedgers.size() > 1);
// catchup just before first buffered ledger that way we will have a
// way to verify history consistency - compare previousLedgerHash of
// buffered ledger with last one downloaded from history
- auto const& lcd = mSyncingLedgers.begin()->second;
+ auto const& lcd = mBufferedLedgers.begin()->second;
auto firstBufferedLedgerSeq = lcd.getLedgerSeq();
auto hash = std::make_optional(lcd.getTxSet()->previousLedgerHash());
startCatchup({LedgerNumHashPair(firstBufferedLedgerSeq - 1, hash),
@@ -449,21 +449,21 @@ LedgerApplyManagerImpl::startOnlineCatchup()
}
void
-LedgerApplyManagerImpl::trimSyncingLedgers()
+LedgerApplyManagerImpl::trimBufferedLedgers()
{
releaseAssert(threadIsMain());
auto removeLedgersLessThan = [&](uint32_t ledger) {
// lower_bound returns an iterator pointing to the first element whose
// key is not considered to go before k. Thus we get the iterator to
// `ledger` if exists, or the first one after `ledger`.
- auto it = mSyncingLedgers.lower_bound(ledger);
+ auto it = mBufferedLedgers.lower_bound(ledger);
// This erases [begin, it).
- mSyncingLedgers.erase(mSyncingLedgers.begin(), it);
+ mBufferedLedgers.erase(mBufferedLedgers.begin(), it);
};
removeLedgersLessThan(*mLastQueuedToApply + 1);
- if (!mSyncingLedgers.empty())
+ if (!mBufferedLedgers.empty())
{
- auto const lastBufferedLedger = mSyncingLedgers.rbegin()->first;
+ auto const lastBufferedLedger = mBufferedLedgers.rbegin()->first;
if (HistoryManager::isFirstLedgerInCheckpoint(lastBufferedLedger,
mApp.getConfig()))
{
@@ -490,17 +490,17 @@ LedgerApplyManagerImpl::trimSyncingLedgers()
}
void
-LedgerApplyManagerImpl::tryApplySyncingLedgers()
+LedgerApplyManagerImpl::tryApplyBufferedLedgers()
{
ZoneScoped;
releaseAssert(threadIsMain());
- auto const parallelLedgerClose = mApp.getConfig().parallelLedgerClose();
+ auto const backgroundLedgerApply = mApp.getConfig().backgroundLedgerApply();
uint32_t nextToApply = *mLastQueuedToApply + 1;
// We can apply multiple ledgers here, which might be slow. This is a rare
// occurrence so we should be fine.
- auto it = mSyncingLedgers.cbegin();
- while (it != mSyncingLedgers.cend())
+ auto it = mBufferedLedgers.cbegin();
+ while (it != mBufferedLedgers.cend())
{
auto const& lcd = it->second;
@@ -512,7 +512,7 @@ LedgerApplyManagerImpl::tryApplySyncingLedgers()
// If we have too many ledgers queued to apply, just stop scheduling
// more and let the node gracefully go into catchup.
- if (parallelLedgerClose)
+ if (backgroundLedgerApply)
{
auto const lcl = mApp.getLedgerManager().getLastClosedLedgerNum();
releaseAssert(mLastQueuedToApply >= lcl);
@@ -526,7 +526,7 @@ LedgerApplyManagerImpl::tryApplySyncingLedgers()
}
// Notify LM that application has started
mApp.getLedgerManager().beginApply();
- mApp.postOnLedgerCloseThread(
+ mApp.postOnLedgerApplyThread(
[&app = mApp, lcd]() {
// No-op if app is shutting down
if (app.isStopping())
@@ -548,7 +548,7 @@ LedgerApplyManagerImpl::tryApplySyncingLedgers()
++nextToApply;
}
- mSyncingLedgers.erase(mSyncingLedgers.cbegin(), it);
+ mBufferedLedgers.erase(mBufferedLedgers.cbegin(), it);
}
void
diff --git a/src/catchup/LedgerApplyManagerImpl.h b/src/catchup/LedgerApplyManagerImpl.h
index 0b635963ce..baba5ee8e6 100644
--- a/src/catchup/LedgerApplyManagerImpl.h
+++ b/src/catchup/LedgerApplyManagerImpl.h
@@ -23,7 +23,7 @@ class Work;
class LedgerApplyManagerImpl : public LedgerApplyManager
{
// Maximum number of ledgers that can be queued to apply (this only applies
- // when Config.parallelLedgerClose() == true). If this number if exceeded,
+ // when Config.backgroundLedgerApply() == true). If this number is exceeded,
// core stops scheduling new ledgers to apply, and goes into catchup mode.
static uint32_t const MAX_EXTERNALIZE_LEDGER_APPLY_DRIFT;
@@ -31,7 +31,7 @@ class LedgerApplyManagerImpl : public LedgerApplyManager
std::shared_ptr mCatchupWork;
// key is ledgerSeq
- // mSyncingLedgers has the following invariants:
+ // mBufferedLedgers has the following invariants:
// (1) It is empty, or
// (2) It starts with LCL + 1 (the next ledger to apply), or
// (3) It contains at most 64 + 1 = 65 ledgers.
@@ -43,13 +43,13 @@ class LedgerApplyManagerImpl : public LedgerApplyManager
// L + 65 (or any ledger after that), we can delete {L, ..., L + 63} as it
// indicates that the checkpoint containing L + 64 has been published.
//
- // There are two methods that modify mSyncingLedgers
- // (trimSyncingLedgers, tryApplySyncingLedgers) and they both
- // maintain the invariants above.
- std::map mSyncingLedgers;
- medida::Counter& mSyncingLedgersSize;
+ // There are three methods that modify mBufferedLedgers
+ // (processLedger, trimBufferedLedgers, tryApplyBufferedLedgers) and they
+ // all maintain the invariants above.
+ std::map mBufferedLedgers;
+ medida::Counter& mBufferedLedgersSize;
- // These state variables track the flow of ledgers through mSyncingLedgers,
+ // These state variables track the flow of ledgers through mBufferedLedgers,
// they are the variables Q and L in the diagram in LedgerManager.h. See
// that diagram for details and discussion of the threading model and
// flow of ledgers between LedgerApplyManager and LedgerManager.
@@ -58,8 +58,8 @@ class LedgerApplyManagerImpl : public LedgerApplyManager
void updateLastQueuedToApply();
void startOnlineCatchup();
- void trimSyncingLedgers();
- void tryApplySyncingLedgers();
+ void trimBufferedLedgers();
+ void tryApplyBufferedLedgers();
uint32_t getCatchupCount();
CatchupMetrics mMetrics;
@@ -111,7 +111,7 @@ class LedgerApplyManagerImpl : public LedgerApplyManager
std::map const&
getBufferedLedgers() const
{
- return mSyncingLedgers;
+ return mBufferedLedgers;
}
std::shared_ptr
diff --git a/src/crypto/SecretKey.cpp b/src/crypto/SecretKey.cpp
index 1c92d1c090..ba36b1bb68 100644
--- a/src/crypto/SecretKey.cpp
+++ b/src/crypto/SecretKey.cpp
@@ -18,6 +18,7 @@
#include "util/Math.h"
#include "util/RandomEvictionCache.h"
#include
+#include
#include
#include
#include
@@ -41,16 +42,29 @@ namespace stellar
// to the state of the process; caching its results centrally
// makes all signature-verification in the program faster and
// has no effect on correctness.
+//
+// The cache is sharded across NUM_VERIFY_CACHE_SHARDS shards to
+// reduce mutex contention when multiple threads verify signatures
+// in parallel. Each shard has its own mutex and cache partition.
constexpr size_t VERIFY_SIG_CACHE_SIZE = 250'000;
-static std::mutex gVerifySigCacheMutex;
-static RandomEvictionCache gVerifySigCache(VERIFY_SIG_CACHE_SIZE);
-static uint64_t gVerifyCacheHit = 0;
-static uint64_t gVerifyCacheMiss = 0;
+constexpr size_t NUM_VERIFY_CACHE_SHARDS = 64;
+constexpr size_t VERIFY_SIG_CACHE_SHARD_SIZE =
+ VERIFY_SIG_CACHE_SIZE / NUM_VERIFY_CACHE_SHARDS;
+
+struct alignas(64) VerifySigCacheShard
+{
+ ANNOTATED_MUTEX(mMutex);
+ RandomEvictionCache mCache GUARDED_BY(mMutex);
+ uint64_t mHits GUARDED_BY(mMutex) = 0;
+ uint64_t mMisses GUARDED_BY(mMutex) = 0;
+ VerifySigCacheShard() : mCache(VERIFY_SIG_CACHE_SHARD_SIZE)
+ {
+ }
+};
-// Global flag to use Rust ed25519-dalek for signature verification
-// Protected by gVerifySigCacheMutex
-static bool gUseRustDalekVerify = false;
+static std::array
+ gVerifySigCacheShards;
static Hash
verifySigCacheKey(PublicKey const& key, Signature const& signature,
@@ -322,32 +336,37 @@ SecretKey::fromStrKeySeed(std::string const& strKeySeed)
void
PubKeyUtils::clearVerifySigCache()
{
- std::lock_guard guard(gVerifySigCacheMutex);
- gVerifySigCache.clear();
-}
-
-void
-PubKeyUtils::enableRustDalekVerify()
-{
- std::lock_guard guard(gVerifySigCacheMutex);
- gUseRustDalekVerify = true;
+ for (auto& shard : gVerifySigCacheShards)
+ {
+ LOCK_GUARD(shard.mMutex, guard);
+ shard.mCache.clear();
+ }
}
void
PubKeyUtils::seedVerifySigCache(unsigned int seed)
{
- std::lock_guard guard(gVerifySigCacheMutex);
- gVerifySigCache.seed(seed);
+ for (size_t i = 0; i < NUM_VERIFY_CACHE_SHARDS; ++i)
+ {
+ LOCK_GUARD(gVerifySigCacheShards[i].mMutex, guard);
+ gVerifySigCacheShards[i].mCache.seed(seed +
+ static_cast(i));
+ }
}
void
PubKeyUtils::flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses)
{
- std::lock_guard guard(gVerifySigCacheMutex);
- hits = gVerifyCacheHit;
- misses = gVerifyCacheMiss;
- gVerifyCacheHit = 0;
- gVerifyCacheMiss = 0;
+ hits = 0;
+ misses = 0;
+ for (auto& shard : gVerifySigCacheShards)
+ {
+ LOCK_GUARD(shard.mMutex, guard);
+ hits += shard.mHits;
+ misses += shard.mMisses;
+ shard.mHits = 0;
+ shard.mMisses = 0;
+ }
}
std::string
@@ -456,41 +475,34 @@ PubKeyUtils::verifySig(PublicKey const& key, Signature const& signature,
}
auto cacheKey = verifySigCacheKey(key, signature, bin);
- bool shouldUseRustDalekVerify;
+
+ // Cache key preimage is externally controlled, so pick the shard via the
+ // per-process keyed hash to prevent targeting a single shard.
+ // Also drop the lower bits of the hash to decouple per-shard hashing from
+ // shard selection.
+ auto shardIdx =
+ (std::hash{}(cacheKey) >> 8) % NUM_VERIFY_CACHE_SHARDS;
+ auto& shard = gVerifySigCacheShards[shardIdx];
{
- std::lock_guard guard(gVerifySigCacheMutex);
- if (gVerifySigCache.exists(cacheKey))
+ LOCK_GUARD(shard.mMutex, guard);
+ if (auto* cached = shard.mCache.maybeGet(cacheKey))
{
- ++gVerifyCacheHit;
- std::string hitStr("hit");
- ZoneText(hitStr.c_str(), hitStr.size());
- return {gVerifySigCache.get(cacheKey),
- VerifySigCacheLookupResult::HIT};
+ ++shard.mHits;
+ ZoneText("hit", 3);
+ return {*cached, VerifySigCacheLookupResult::HIT};
}
-
- shouldUseRustDalekVerify = gUseRustDalekVerify;
}
- std::string missStr("miss");
- ZoneText(missStr.c_str(), missStr.size());
+ ZoneText("miss", 4);
- bool ok;
- if (shouldUseRustDalekVerify)
+ bool ok = stellar::rust_bridge::verify_ed25519_signature_dalek(
+ key.ed25519().data(), signature.data(), bin.data(), bin.size());
{
- ok = stellar::rust_bridge::verify_ed25519_signature_dalek(
- key.ed25519().data(), signature.data(), bin.data(), bin.size());
+ LOCK_GUARD(shard.mMutex, guard);
+ ++shard.mMisses;
+ shard.mCache.put(cacheKey, ok);
}
- else
- {
- ok = (crypto_sign_verify_detached(signature.data(), bin.data(),
- bin.size(),
- key.ed25519().data()) == 0);
- }
-
- std::lock_guard guard(gVerifySigCacheMutex);
- ++gVerifyCacheMiss;
- gVerifySigCache.put(cacheKey, ok);
return {ok, VerifySigCacheLookupResult::MISS};
}
diff --git a/src/crypto/SecretKey.h b/src/crypto/SecretKey.h
index 9c3c96aeba..c30286e4d0 100644
--- a/src/crypto/SecretKey.h
+++ b/src/crypto/SecretKey.h
@@ -166,13 +166,6 @@ void clearVerifySigCache();
void seedVerifySigCache(unsigned int seed);
void flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses);
-// Enable Rust ed25519-dalek for signature verification
-// Once enabled, it cannot be disabled. It should be enabled at the protocol 24
-// boundary.
-// Note: This should be removed following the protocol 24 upgrade, rust ed25519
-// can be used unconditionally after upgrade, even for replay.
-void enableRustDalekVerify();
-
PublicKey random();
#ifdef BUILD_TESTS
PublicKey pseudoRandomForTesting();
diff --git a/src/crypto/StrKey.h b/src/crypto/StrKey.h
index 6162d423d9..02cda7986c 100644
--- a/src/crypto/StrKey.h
+++ b/src/crypto/StrKey.h
@@ -25,6 +25,7 @@ enum StrKeyVersionByte : uint8_t
STRKEY_HASH_X = 23, // 'X'
STRKEY_MUXED_ACCOUNT_ED25519 = 12, // 'M'
STRKEY_CONTRACT = 2, // 'C'
+ STRKEY_MUXED_CONTRACT = 22, // 'W'
};
// Encode a version byte and ByteSlice into StrKey
diff --git a/src/crypto/test/CryptoTests.cpp b/src/crypto/test/CryptoTests.cpp
index 9340c9e54c..9dc4eaaebf 100644
--- a/src/crypto/test/CryptoTests.cpp
+++ b/src/crypto/test/CryptoTests.cpp
@@ -1659,7 +1659,6 @@ ZcashTestVector const ZCASH_TEST_VECTORS[196] = {
TEST_CASE("Ed25519 test vectors from Zcash", "[crypto]")
{
- PubKeyUtils::enableRustDalekVerify();
for (auto const& tv : ZCASH_TEST_VECTORS)
{
PublicKey pk;
diff --git a/src/database/Database.cpp b/src/database/Database.cpp
index 64cb427557..e9a09a41ce 100644
--- a/src/database/Database.cpp
+++ b/src/database/Database.cpp
@@ -7,7 +7,6 @@
#include "database/DatabaseConnectionString.h"
#include "database/DatabaseTypeSpecificOperation.h"
#include "main/Application.h"
-#include "main/BannedAccountsPersistor.h"
#include "main/Config.h"
#include "overlay/StellarXDR.h"
#include "util/Decoder.h"
@@ -321,8 +320,13 @@ Database::applyMiscSchemaUpgrade(unsigned long vers)
getRawMiscSession() << "DETACH DATABASE source_db";
return;
case 2:
- // Add banned accounts table for persistent account filtering.
- BannedAccountsPersistor::maybeDropAndCreateNew(mMiscSession.session());
+ getRawMiscSession() << "DROP TABLE IF EXISTS bannedaccounts;";
+ getRawMiscSession() << "CREATE TABLE bannedaccounts (accountid "
+ "VARCHAR(56) PRIMARY KEY);";
+ break;
+ case 3:
+ // The account banning feature was removed; drop its table.
+ getRawMiscSession() << "DROP TABLE IF EXISTS bannedaccounts;";
break;
default:
throw std::runtime_error("Unknown DB schema version");
@@ -419,12 +423,17 @@ Database::applySchemaUpgrade(unsigned long vers)
// for Postgres and in-memory SQLite, create in the main DB.
if (!canUseMiscDB())
{
- BannedAccountsPersistor::maybeDropAndCreateNew(getRawSession());
+ getRawSession() << "DROP TABLE IF EXISTS bannedaccounts;";
+ getRawSession() << "CREATE TABLE bannedaccounts (accountid "
+ "VARCHAR(56) PRIMARY KEY);";
}
break;
case 28:
migrateLedgerHeadersToStoreState(*this);
break;
+ case 29:
+ getRawSession() << "DROP TABLE IF EXISTS bannedaccounts;";
+ break;
default:
throw std::runtime_error("Unknown DB schema version");
diff --git a/src/database/Database.h b/src/database/Database.h
index cb67e711c0..b0bad2efcf 100644
--- a/src/database/Database.h
+++ b/src/database/Database.h
@@ -29,11 +29,11 @@ class Application;
// smallest schema version supported
static constexpr unsigned long MIN_SCHEMA_VERSION = 25;
-static constexpr unsigned long SCHEMA_VERSION = 28;
+static constexpr unsigned long SCHEMA_VERSION = 29;
static constexpr unsigned long FIRST_MAIN_VERSION_WITH_MISC = 26;
// Misc schema version 0 means no misc table exists yet
static constexpr unsigned long MIN_MISC_SCHEMA_VERSION = 0;
-static constexpr unsigned long MISC_SCHEMA_VERSION = 2;
+static constexpr unsigned long MISC_SCHEMA_VERSION = 3;
/**
* Helper class for borrowing a SOCI prepared statement handle into a local
diff --git a/src/database/test/DatabaseTests.cpp b/src/database/test/DatabaseTests.cpp
index 618c3505ba..9132fd333b 100644
--- a/src/database/test/DatabaseTests.cpp
+++ b/src/database/test/DatabaseTests.cpp
@@ -12,7 +12,6 @@
#include "ledger/test/LedgerTestUtils.h"
#include "lib/util/stdrandom.h"
#include "main/Application.h"
-#include "main/BannedAccountsPersistor.h"
#include "main/Config.h"
#include "main/PersistentState.h"
#include "overlay/BanManager.h"
@@ -695,10 +694,6 @@ TEST_CASE("schema parity across DB backends", "[db][schematest]")
TmpDir tmpDir("schema-parity-test");
Config cfg1 = getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT);
cfg1.DATABASE = SecretValue{"sqlite3://" + tmpDir.getName() + "/test.db"};
- // Use non-empty FILTERED_G_ADDRESSES to test migration as well
- cfg1.FILTERED_G_ADDRESSES = {
- "GBO7VUL2TOKPWFAWKATIW7K3QYA7WQ63VDY5CAE6AFUUX6BHZBOC2WXC",
- "GATDQL767ZM2JQTBEG4BQ5WKOQNGAGWZDUN4GYT2UINPEU3RT2UAMVZH"};
VirtualClock clock1;
Application::pointer app1 = createTestApplication(clock1, cfg1);
@@ -724,9 +719,6 @@ TEST_CASE("schema parity across DB backends", "[db][schematest]")
// ---- PostgreSQL: compare tables and row counts ----
Config cfg2 = getTestConfig(1, Config::TESTDB_POSTGRESQL);
- cfg2.FILTERED_G_ADDRESSES = {
- "GBO7VUL2TOKPWFAWKATIW7K3QYA7WQ63VDY5CAE6AFUUX6BHZBOC2WXC",
- "GATDQL767ZM2JQTBEG4BQ5WKOQNGAGWZDUN4GYT2UINPEU3RT2UAMVZH"};
VirtualClock clock2;
Application::pointer app2 = createTestApplication(clock2, cfg2);
diff --git a/src/herder/Herder.h b/src/herder/Herder.h
index f34a2c9505..66207170ee 100644
--- a/src/herder/Herder.h
+++ b/src/herder/Herder.h
@@ -26,6 +26,7 @@ struct EmptyTxSet
};
using TxSetResult = std::variant;
class Application;
+class ConsensusTime;
class XDROutputFileStream;
class SorobanTransactionQueue;
@@ -151,12 +152,10 @@ class Herder
// generator, and therefore can skip certain expensive validity checks
virtual TxSubmitStatus recvTransaction(TransactionFrameBasePtr tx,
bool submittedFromSelf,
- bool force = false,
bool isLoadgenTx = false) = 0;
#else
virtual TxSubmitStatus recvTransaction(TransactionFrameBasePtr tx,
- bool submittedFromSelf,
- bool force = false) = 0;
+ bool submittedFromSelf) = 0;
#endif
virtual TxSetResult getTxSet(Hash const& hash) = 0;
virtual SCPQuorumSetPtr getQSet(Hash const& qSetHash) = 0;
@@ -177,7 +176,7 @@ class Herder
virtual void
externalizeValue(TxSetXDRFrameConstPtr txSet, uint32_t ledgerSeq,
- uint64_t closeTime,
+ ConsensusTime closeTime,
xdr::xvector const& upgrades,
std::optional skToSignValue = std::nullopt) = 0;
@@ -221,14 +220,11 @@ class Herder
// gets the upgrades that are scheduled by this node
virtual std::string getUpgradesJson() = 0;
- // Override the filtered accounts at runtime using pre-parsed AccountIDs.
- virtual void setFilteredAccounts(std::set const& accounts) = 0;
-
virtual void forceSCPStateIntoSyncWithLastClosedLedger() = 0;
// helper function to craft an SCPValue
virtual StellarValue
- makeStellarValue(Hash const& txSetHash, uint64_t closeTime,
+ makeStellarValue(Hash const& txSetHash, ConsensusTime closeTime,
xdr::xvector const& upgrades,
SecretKey const& s) = 0;
diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp
index 03e19fcb8c..9d25ad6929 100644
--- a/src/herder/HerderImpl.cpp
+++ b/src/herder/HerderImpl.cpp
@@ -18,6 +18,7 @@
#include "herder/TxSetFrame.h"
#include "herder/TxSetUtils.h"
#include "ledger/ImmutableLedgerView.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "ledger/LedgerManager.h"
#include "ledger/LedgerTxnImpl.h"
#include "ledger/P23HotArchiveBug.h"
@@ -52,6 +53,7 @@
#include "util/GlobalChecks.h"
#include
+#include
#include
#include
#include
@@ -63,7 +65,7 @@ namespace stellar
// Roughly ~10 minutes of consensus
constexpr uint32 const CLOSE_TIME_DRIFT_LEDGER_WINDOW_SIZE = 120;
// 10 seconds of drift threshold
-constexpr uint32 const CLOSE_TIME_DRIFT_SECONDS_THRESHOLD = 10;
+constexpr std::chrono::seconds const CLOSE_TIME_DRIFT_SECONDS_THRESHOLD{10};
std::unique_ptr
Herder::create(Application& app)
@@ -85,6 +87,11 @@ HerderImpl::SCPMetrics::SCPMetrics(Application& app)
{"scp", "envelope", "invalidsig"}, "envelope"))
, mTriggerPrepareStartFallback(app.getMetrics().NewMeter(
{"scp", "trigger", "prepare-start-fallback"}, "trigger"))
+ , mTxSetBuild(app.getMetrics().NewTimer({"herder", "txset", "build"}))
+ , mTriggerLateness(
+ app.getMetrics().NewTimer({"scp", "trigger", "lateness"}))
+ , mTriggerImmediate(
+ app.getMetrics().NewMeter({"scp", "trigger", "immediate"}, "trigger"))
{
}
@@ -153,7 +160,7 @@ void
HerderImpl::setTrackingSCPState(uint64_t index, StellarValue const& value,
bool isTrackingNetwork)
{
- mTrackingSCP = ConsensusData{index, value.closeTime};
+ mTrackingSCP = ConsensusData{index, getConsensusTime(value)};
if (isTrackingNetwork)
{
setState(Herder::HERDER_TRACKING_NETWORK_STATE);
@@ -183,7 +190,7 @@ HerderImpl::trackingConsensusLedgerIndex() const
return static_cast(mTrackingSCP.mConsensusIndex);
}
-TimePoint
+ConsensusTime
HerderImpl::trackingConsensusCloseTime() const
{
releaseAssert(getState() != Herder::State::HERDER_BOOTING_STATE);
@@ -331,9 +338,9 @@ HerderImpl::processExternalized(uint64 slotIndex, StellarValue const& value,
TxSetXDRFrameConstPtr externalizedSet;
if (std::holds_alternative(result))
{
- auto const& ov = value.ext.proposedValue();
- externalizedSet = TxSetXDRFrame::makeEmpty(ov.previousLedgerHash,
- ov.previousLedgerVersion);
+ externalizedSet =
+ TxSetXDRFrame::makeEmpty(getProposedPreviousLedgerHash(value),
+ getProposedPreviousLedgerVersion(value));
}
else
{
@@ -384,9 +391,9 @@ HerderImpl::processExternalized(uint64 slotIndex, StellarValue const& value,
// reflect upgrades with the ones included in this SCP round
{
bool updated;
- auto newUpgrades = mUpgrades.removeUpgrades(value.upgrades.begin(),
- value.upgrades.end(),
- value.closeTime, updated);
+ auto newUpgrades = mUpgrades.removeUpgrades(
+ value.upgrades.begin(), value.upgrades.end(), getApplyTime(value),
+ updated);
if (updated)
{
setUpgrades(newUpgrades);
@@ -454,12 +461,13 @@ HerderImpl::writeDebugTxSet(LedgerCloseData const& lcd)
static void
recordExternalizeAndCheckCloseTimeDrift(
uint64 slotIndex, StellarValue const& value,
- std::map>>& ctMap)
+ std::map>>&
+ ctMap)
{
auto it = ctMap.find(slotIndex);
if (it != ctMap.end())
{
- it->second.second = value.closeTime;
+ it->second.second = getConsensusTime(value);
}
if (ctMap.size() >= CLOSE_TIME_DRIFT_LEDGER_WINDOW_SIZE)
@@ -470,14 +478,21 @@ recordExternalizeAndCheckCloseTimeDrift(
auto const& [localCT, externalizedCT] = closeTimePair;
if (externalizedCT)
{
- h.Update(*externalizedCT - localCT);
+ h.Update(
+ std::chrono::duration_cast(
+ externalizedCT->toSystemTime() - localCT.toSystemTime())
+ .count());
}
}
- auto drift = static_cast(h.GetSnapshot().get75thPercentile());
- if (std::abs(drift) > CLOSE_TIME_DRIFT_SECONDS_THRESHOLD)
+ auto driftMs =
+ static_cast(h.GetSnapshot().get75thPercentile());
+ if (std::abs(driftMs) >
+ std::chrono::duration_cast(
+ CLOSE_TIME_DRIFT_SECONDS_THRESHOLD)
+ .count())
{
CLOG_WARNING(Herder, POSSIBLY_BAD_LOCAL_CLOCK);
- CLOG_WARNING(Herder, "Close time local drift is: {}", drift);
+ CLOG_WARNING(Herder, "Close time local drift is: {} ms", driftMs);
}
ctMap.clear();
@@ -543,7 +558,7 @@ HerderImpl::valueExternalized(uint64 slotIndex, StellarValue const& value,
// the overhead of externalization in the way we track SCP)
// Note: this only makes sense in the context of synchronous ledger
// application on the main thread.
- if (!mApp.getConfig().parallelLedgerClose())
+ if (!mApp.getConfig().backgroundLedgerApply())
{
trackingHeartBeat();
}
@@ -654,8 +669,7 @@ HerderImpl::emitEnvelope(SCPEnvelope const& envelope)
}
TxSubmitStatus
-HerderImpl::recvTransaction(TransactionFrameBasePtr tx, bool submittedFromSelf,
- bool force
+HerderImpl::recvTransaction(TransactionFrameBasePtr tx, bool submittedFromSelf
#ifdef BUILD_TESTS
,
bool isLoadgenTx
@@ -686,28 +700,30 @@ HerderImpl::checkCloseTime(SCPEnvelope const& envelope, bool enforceRecent)
using std::placeholders::_1;
auto const& st = envelope.statement;
- uint64_t ctCutoff = 0;
+ auto const& lcl = mLedgerManager.getLastClosedLedgerHeader().header;
+ ConsensusTime ctCutoff;
if (enforceRecent)
{
- auto now = VirtualClock::to_time_t(mApp.getClock().system_now());
- if (now >= mApp.getConfig().MAXIMUM_LEDGER_CLOSETIME_DRIFT)
+ auto const now = mApp.getClock().system_now();
+ std::chrono::seconds const maxDrift(
+ mApp.getConfig().MAXIMUM_LEDGER_CLOSETIME_DRIFT);
+ if (now.time_since_epoch() >= maxDrift)
{
- ctCutoff = now - mApp.getConfig().MAXIMUM_LEDGER_CLOSETIME_DRIFT;
+ ctCutoff = ConsensusTime::fromSystemTime(now - maxDrift,
+ lcl.ledgerVersion);
}
}
auto envLedgerIndex = envelope.statement.slotIndex;
auto& scpD = getHerderSCPDriver();
- auto const& lcl = mLedgerManager.getLastClosedLedgerHeader().header;
auto lastCloseIndex = lcl.ledgerSeq;
- auto lastCloseTime = lcl.scpValue.closeTime;
+ auto lastCloseTime = getConsensusTime(lcl.scpValue);
// see if we can get a better estimate of lastCloseTime for validating this
- // statement using consensus data:
- // update lastCloseIndex/lastCloseTime to be the highest possible but still
- // be less than envLedgerIndex
+ // statement using consensus data: update lastCloseIndex/lastCloseTime to
+ // be the highest possible but still be less than envLedgerIndex
if (getState() != HERDER_BOOTING_STATE)
{
auto trackingIndex = trackingConsensusLedgerIndex();
@@ -725,16 +741,16 @@ HerderImpl::checkCloseTime(SCPEnvelope const& envelope, bool enforceRecent)
return std::any_of(values.begin(), values.end(), [&](Value const& e) {
auto r = toStellarValue(e, sv);
// sv must be after cutoff
- r = r && sv.closeTime >= ctCutoff;
+ r = r && getConsensusTime(sv) >= ctCutoff;
if (r)
{
// statement received after the fact, only keep externalized
// value
r = (lastCloseIndex == envLedgerIndex &&
- lastCloseTime == sv.closeTime);
+ lastCloseTime == getConsensusTime(sv));
// for older messages, just ensure that they occurred before
r = r || (lastCloseIndex > envLedgerIndex &&
- lastCloseTime > sv.closeTime);
+ lastCloseTime > getConsensusTime(sv));
// for future message, perform the same validity check than
// within SCP
r = r || scpD.checkCloseTime(envLedgerIndex, lastCloseTime, sv);
@@ -951,7 +967,7 @@ HerderImpl::recvSCPEnvelope(SCPEnvelope const& envelope,
void
HerderImpl::externalizeValue(TxSetXDRFrameConstPtr txSet, uint32_t ledgerSeq,
- uint64_t closeTime,
+ ConsensusTime closeTime,
xdr::xvector const& upgrades,
std::optional skToSignValue)
{
@@ -1198,11 +1214,12 @@ HerderImpl::getUpgrades() const
#endif
std::chrono::milliseconds
-HerderImpl::ctValidityOffset(uint64_t ct, std::chrono::milliseconds maxCtOffset)
+HerderImpl::ctValidityOffset(ConsensusTime closeTime,
+ std::chrono::milliseconds maxCtOffset)
{
auto maxCandidateCt = mApp.getClock().system_now() + maxCtOffset +
Herder::MAX_TIME_SLIP_SECONDS;
- auto minCandidateCt = VirtualClock::from_time_t(ct);
+ auto minCandidateCt = closeTime.toSystemTime();
if (minCandidateCt > maxCandidateCt)
{
@@ -1289,7 +1306,10 @@ HerderImpl::triggerAnchorFromPrepareStart(
//
// To get a better sense of which, we also take into account
// nomination time, so the check becomes timeSinceNetworkLedgerStart > target
-// + nominationBudget, where nomination budget scales with timeouts.
+// + nominationBudget. The budget is measured on the steady clock from the
+// local trigger to entry into ballot. This includes proposal construction
+// and all nomination time, including unfinished rounds and delayed timer
+// callbacks.
//
// If we think we're drifting ahead after taking nomination into account, we
// fall back to prepare-start anchor, which is based on our local clock and
@@ -1326,8 +1346,8 @@ HerderImpl::triggerAnchorFromPrepareStart(
// not because our system clock drifted. The goal is to avoid falling back to
// a conservative timer and snowballing the real delay.
//
-// See scenario 1, but TL;DR we can look at the nomination timeouts and see
-// if the network is slow vs. the node drifting. If nomination is slow, we
+// See scenario 1: measured elapsed time accounts for slow nomination
+// independently of system clock drift. If nomination is slow, we
// can't fall back to the prepare-apply timer because it would compound the
// delay. If apply is slow, it doesn't matter which timer we use, they both
// will result in triggering immediately.
@@ -1342,22 +1362,17 @@ HerderImpl::triggerAnchorFromConsensusCloseTime(
};
auto consensusCloseTime = trackingConsensusCloseTime();
- if (consensusCloseTime == 0)
+ if (consensusCloseTime == ConsensusTime{})
{
CLOG_WARNING(Herder, "Consensus close time is 0, falling back to "
"prepare-start anchor");
return fallbackToPrepareStart();
}
- // Compare elapsed time on the externalized closeTime timeline with elapsed
- // time on our local prepare-start timeline.
- // Relation, with drift > 0 meaning our clock is ahead of network time:
- //
- // timeSinceNetworkLedgerStart
- // = nominationBudget + timeSinceLocalBallotStart + drift
- //
- // where nominationBudget is the slow-nomination allowance described above.
- auto externalizedSystemTime = VirtualClock::from_time_t(consensusCloseTime);
+ // Compare elapsed time since the externalized closeTime with local steady
+ // time. Their difference includes clock drift and how early or late our
+ // trigger fired relative to the proposer's close time.
+ auto externalizedSystemTime = consensusCloseTime.toSystemTime();
auto currentSystemTime = mApp.getClock().system_now();
auto timeSinceNetworkLedgerStart =
std::chrono::duration_cast(
@@ -1369,29 +1384,39 @@ HerderImpl::triggerAnchorFromConsensusCloseTime(
std::chrono::duration_cast(now -
localBallotStart);
+ auto nominationBudget =
+ mHerderSCPDriver.getTriggerToBallotDuration(lastIndex);
+ auto logFallback = [&](char const* reason) {
+ auto zero = std::chrono::milliseconds::zero();
+ CLOG_INFO(
+ Herder,
+ "Trigger fallback after ledger {}: {}, network elapsed {} "
+ "ms, trigger-to-ballot {} ms, ballot elapsed {} ms, local "
+ "wait {} ms, network wait {} ms",
+ lastIndex, reason, timeSinceNetworkLedgerStart.count(),
+ nominationBudget.count(), timeSinceLocalBallotStart.count(),
+ std::max(zero, expectedClose - timeSinceLocalBallotStart).count(),
+ std::max(zero, expectedClose - timeSinceNetworkLedgerStart)
+ .count());
+ return fallbackToPrepareStart();
+ };
+
// Scenario 2: if system time is behind the local prepare-start timer, the
// network-based anchor can wedge the node, so use the local fallback.
if (timeSinceLocalBallotStart > timeSinceNetworkLedgerStart)
{
- return fallbackToPrepareStart();
- }
-
- // Scenario 1: widen the ahead-drift bound by the slow nomination we can
- // explain from the previous slot's timeout count.
- auto nominationTimeouts =
- mHerderSCPDriver.getNominationTimeouts(lastIndex).value_or(0);
- auto nominationBudget = std::chrono::milliseconds::zero();
- for (int64_t round = 1; round <= nominationTimeouts; ++round)
- {
- nominationBudget += mHerderSCPDriver.computeTimeout(
- static_cast(round), /*isNomination=*/true);
+ return logFallback("clock behind");
}
+ // A timeout counts only after its callback runs, and none are counted
+ // after entry into ballot. A slow fetch or busy main thread can therefore
+ // delay nomination without increasing the count. Measure elapsed time
+ // directly instead of approximating it with completed timeout durations.
// Scenario 1: if elapsed system time exceeds target plus explainable
// nomination delay, treat it as clock-ahead drift and use the fallback.
if (timeSinceNetworkLedgerStart > expectedClose + nominationBudget)
{
- return fallbackToPrepareStart();
+ return logFallback("clock ahead or slow ballot/apply");
}
return now - timeSinceNetworkLedgerStart;
@@ -1445,6 +1470,7 @@ HerderImpl::setupTriggerNextLedger()
// This ensures that next value to nominate is valid
auto triggerTime = lastLedgerStartingPoint + milliseconds;
+ bool const triggerDue = triggerTime <= now;
if (triggerTime < now)
{
triggerTime = now;
@@ -1453,7 +1479,9 @@ HerderImpl::setupTriggerNextLedger()
auto triggerOffset = std::chrono::duration_cast(
triggerTime - now);
- auto minCandidateCt = lcl.header.scpValue.closeTime + 1;
+ // The smallest close time the next nomination may propose
+ auto minCandidateCt =
+ getConsensusTime(lcl.header.scpValue).next(lcl.header.ledgerVersion);
auto ctOffset = ctValidityOffset(minCandidateCt, triggerOffset);
if (ctOffset > std::chrono::milliseconds::zero())
@@ -1469,10 +1497,18 @@ HerderImpl::setupTriggerNextLedger()
if (!mApp.getConfig().MANUAL_CLOSE)
{
- mTriggerTimer.async_wait(std::bind(&HerderImpl::triggerNextLedger, this,
- static_cast(nextIndex),
- true),
- &VirtualTimer::onFailureNoop);
+ if (triggerDue)
+ {
+ mSCPMetrics.mTriggerImmediate.Mark();
+ }
+ mTriggerTimer.async_wait(
+ [this, nextIndex, triggerTime]() {
+ mSCPMetrics.mTriggerLateness.Update(
+ std::max(VirtualClock::duration::zero(),
+ mApp.getClock().now() - triggerTime));
+ triggerNextLedger(static_cast(nextIndex), true);
+ },
+ &VirtualTimer::onFailureNoop);
if (getSCP().isValidator())
{
@@ -1482,13 +1518,14 @@ HerderImpl::setupTriggerNextLedger()
{
// Validate against the close time we expect at the trigger,
// not the earlier wall-clock time at which we start work.
- auto closeTime = std::max(
+ auto closeTime = std::max(
minCandidateCt,
- VirtualClock::to_time_t(
+ ConsensusTime::fromSystemTime(
mApp.getClock().system_now() +
- std::chrono::duration_cast<
- VirtualClock::system_time_point::duration>(
- triggerTime - now)));
+ std::chrono::duration_cast<
+ VirtualClock::system_time_point::duration>(
+ triggerTime - now),
+ lcl.header.ledgerVersion));
// Yield out of ledger close before touching its read view.
// This is main-thread work; only its scheduling changes.
mPrepareTxSetTimer.expires_at(now);
@@ -1641,7 +1678,7 @@ HerderImpl::discardPreparedTxSet()
}
void
-HerderImpl::prepareTxSet(uint32_t ledgerSeq, uint64_t closeTime)
+HerderImpl::prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime)
{
if (!isTracking() || !mLedgerManager.isSynced() ||
mLedgerManager.isApplying() ||
@@ -1653,18 +1690,20 @@ HerderImpl::prepareTxSet(uint32_t ledgerSeq, uint64_t closeTime)
CLOG_INFO(Herder,
"Prepared TX set for ledger {} before trigger: {} transactions, "
"close time {}",
- ledgerSeq, mPreparedTxSet->txSet->sizeTxTotal(), closeTime);
+ ledgerSeq, mPreparedTxSet->txSet->sizeTxTotal(),
+ closeTime.toString());
}
HerderImpl::PreparedTxSet
-HerderImpl::buildTxSet(uint32_t ledgerSeq, uint64_t closeTime)
+HerderImpl::buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime)
{
+ auto const txSetBuildStart = std::chrono::steady_clock::now();
auto const lcl = mLedgerManager.getLastClosedLedgerHeader();
releaseAssert(ledgerSeq == lcl.header.ledgerSeq + 1);
- releaseAssert(closeTime > lcl.header.scpValue.closeTime);
- auto const lowerBoundCloseTimeOffset =
- closeTime - lcl.header.scpValue.closeTime;
- auto const upperBoundCloseTimeOffset = lowerBoundCloseTimeOffset;
+ releaseAssert(closeTime > getConsensusTime(lcl.header.scpValue));
+ // Apply-time offset: see the comment in triggerNextLedger
+ auto const closeTimeOffset =
+ closeTime.toApplyTime() - getApplyTime(lcl.header.scpValue);
TxSetXDRFrameConstPtr proposedSet;
ApplicableTxSetFrameConstPtr applicableProposedSet;
@@ -1756,9 +1795,8 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, uint64_t closeTime)
PerPhaseTransactionList invalidTxPhases;
invalidTxPhases.resize(txPhases.size());
- std::tie(proposedSet, applicableProposedSet) =
- makeTxSetFromTransactions(txPhases, mApp, lowerBoundCloseTimeOffset,
- upperBoundCloseTimeOffset, invalidTxPhases);
+ std::tie(proposedSet, applicableProposedSet) = makeTxSetFromTransactions(
+ txPhases, mApp, closeTimeOffset, invalidTxPhases);
CLOG_INFO(Herder, "Proposed TX set has {} transactions",
proposedSet->sizeTxTotal());
@@ -1771,12 +1809,12 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, uint64_t closeTime)
std::vector invalidTxHashes;
if (!invalidTxPhases.empty())
{
- CheckValidLedgerViewWrapper ledgerView(mApp);
+ auto ledgerView = mLedgerManager.getLCLView();
for (auto const& phase : invalidTxPhases)
{
for (auto const& tx : phase)
{
- auto acc = ledgerView.getAccount(tx->getSourceID());
+ auto acc = ledgerView->getAccount(tx->getSourceID());
if (acc &&
tx->getSeqNum() > acc.current().data.account().seqNum + 1)
{
@@ -1802,6 +1840,9 @@ HerderImpl::buildTxSet(uint32_t ledgerSeq, uint64_t closeTime)
validCandidates += txPhases[i].size() - invalidTxPhases[i].size();
}
bool capacityLimited = validCandidates > proposedSet->sizeTxTotal();
+ mSCPMetrics.mTxSetBuild.Update(
+ std::chrono::duration_cast(
+ std::chrono::steady_clock::now() - txSetBuildStart));
return PreparedTxSet{lcl.hash,
ledgerSeq,
closeTime,
@@ -1833,7 +1874,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
if (mLedgerManager.isApplying())
{
// This can only happen when closing ledgers in parallel
- releaseAssert(mApp.getConfig().parallelLedgerClose());
+ releaseAssert(mApp.getConfig().backgroundLedgerApply());
CLOG_DEBUG(Herder, "triggerNextLedger: skipping (applying) : {}",
mApp.getStateHuman());
return;
@@ -1853,8 +1894,8 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
// We pick as next close time the current time unless it's before the last
// close time. We don't know how much time it will take to reach consensus
// so this is the most appropriate value to use as closeTime.
- uint64_t nextCloseTime =
- VirtualClock::to_time_t(mApp.getClock().system_now());
+ auto nextCloseTime = ConsensusTime::fromSystemTime(
+ mApp.getClock().system_now(), lcl.header.ledgerVersion);
if (ledgerSeqToTrigger == lcl.header.ledgerSeq + 1)
{
auto it = mDriftCTSlidingWindow.find(ledgerSeqToTrigger);
@@ -1878,9 +1919,10 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
}
}
- if (nextCloseTime <= lcl.header.scpValue.closeTime)
+ auto const lclCloseTime = getConsensusTime(lcl.header.scpValue);
+ if (nextCloseTime <= lclCloseTime)
{
- nextCloseTime = lcl.header.scpValue.closeTime + 1;
+ nextCloseTime = lclCloseTime.next(lcl.header.ledgerVersion);
}
// Ensure we're about to nominate a value with valid close time
@@ -1891,10 +1933,15 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
{
CLOG_WARNING(Herder,
"Invalid close time selected ({}), skipping nomination",
- nextCloseTime);
+ nextCloseTime.toString());
return;
}
+ // Start the trigger-to-ballot interval only once this node is going to
+ // build a proposal and nominate. A skipped trigger must not credit the
+ // time spent waiting for peers to the drift allowance.
+ mHerderSCPDriver.recordNominationTrigger(ledgerSeqToTrigger);
+
// Consume the private snapshot only for the ledger it was built against.
// Keep its close time: changing it would invalidate time-bound transactions
// and the cached validation result.
@@ -1915,9 +1962,15 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
ledgerSeqToTrigger);
}
nextCloseTime = prepared->closeTime;
- auto const upperBoundCloseTimeOffset =
- nextCloseTime - lcl.header.scpValue.closeTime;
- auto const& proposedSet = prepared->txSet;
+ // The offset is computed in *apply* time rather than consensus time:
+ // transaction time bounds are whole-second quantities compared against
+ // StellarValue::closeTime, which CAP-0088 leaves at whole seconds. With ms
+ // close times the next ledger may therefore apply in the same second as the
+ // LCL (a zero offset), and a transaction whose time bounds are met at that
+ // second is valid in both ledgers.
+ auto const closeTimeOffset =
+ nextCloseTime.toApplyTime() - getApplyTime(lcl.header.scpValue);
+ auto proposedSet = prepared->txSet;
auto const& applicableProposedSet = prepared->applicableTxSet;
Hash txSetHash;
if (!prepared->invalidTxHashes.empty())
@@ -1935,16 +1988,26 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
return;
}
+ // New proposed tx set must be valid, so we explicitly populate tx set
+ // validity cache so SCP can reuse the result.
+ mHerderSCPDriver.cacheValidTxSet(*applicableProposedSet, lcl,
+ closeTimeOffset);
+
+#ifdef BUILD_TESTS
+ if (mNominatedTxSetOverride)
+ {
+ proposedSet = mNominatedTxSetOverride(lcl);
+ CLOG_INFO(Herder, "Nominating overridden tx set {} for slot {}",
+ hexAbbrev(proposedSet->getContentsHash()),
+ lcl.header.ledgerSeq + 1);
+ }
+#endif
+
txSetHash = proposedSet->getContentsHash();
CLOG_INFO(Herder, "Built TX set: hash={}",
binToHex(txSetHash).substr(0, 8));
- // New proposed tx set must be valid, so we explicitly populate tx set
- // validity cache so SCP can reuse the result.
- mHerderSCPDriver.cacheValidTxSet(*applicableProposedSet, lcl,
- upperBoundCloseTimeOffset);
-
// Inform the item fetcher so queries from other peers about his txSet
// can be answered. Note this can trigger SCP callbacks, externalize, etc
// if we happen to build a txset that we were trying to download.
@@ -1984,8 +2047,8 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
// see if we need to include some upgrades
std::vector upgrades;
{
- CheckValidLedgerViewWrapper ledgerView(mApp);
- upgrades = mUpgrades.createUpgradesFor(lcl.header, ledgerView,
+ auto ledgerView = mApp.getLedgerManager().getLCLView();
+ upgrades = mUpgrades.createUpgradesFor(lcl.header, *ledgerView,
mApp.getConfig());
}
for (auto const& upgrade : upgrades)
@@ -2065,13 +2128,8 @@ HerderImpl::setUpgrades(Upgrades::UpgradeParameters const& upgrades)
std::string
HerderImpl::getUpgradesJson()
{
- auto ledgerView = CheckValidLedgerViewWrapper(mApp);
- return mUpgrades.getParameters().toDebugJson(ledgerView);
-}
-
-void
-HerderImpl::setFilteredAccounts(std::set const& accounts)
-{
+ auto ledgerView = mApp.getLedgerManager().getLCLView();
+ return mUpgrades.getParameters().toDebugJson(*ledgerView);
}
void
@@ -2799,21 +2857,6 @@ HerderImpl::start()
startTxSetGCTimer();
startCheckForDeadNodesInterval();
- auto& bap = mApp.getBannedAccountsPersistor();
- if (!mApp.getConfig().FILTERED_G_ADDRESSES.empty())
- {
- CLOG_WARNING(
- Herder,
- "FILTERED_G_ADDRESSES is deprecated and will be removed in a "
- "future release. The current {} address(es) will be stored in the "
- "database. You can safely remove FILTERED_G_ADDRESSES from the "
- "config. Use 'banaccounts'/'unbanaccounts' HTTP commands to manage "
- "banned accounts going forward.",
- mApp.getConfig().FILTERED_G_ADDRESSES.size());
- bap.addBannedAccounts(mApp.getConfig().FILTERED_G_ADDRESSES);
- }
-
- setFilteredAccounts(bap.getBannedAccounts());
// RustOverlayManager is started automatically in OverlayManager::start()
// which is called by ApplicationImpl::start() before Herder::start()
}
@@ -3005,49 +3048,66 @@ bool
HerderImpl::verifyStellarValueSignature(StellarValue const& sv)
{
ZoneScoped;
- switch (sv.ext.v())
- {
- case STELLAR_VALUE_BASIC:
- // This function should never be called with an unsigned value
- releaseAssert(false);
- case STELLAR_VALUE_SIGNED:
- return PubKeyUtils::verifySig(sv.ext.lcValueSignature().nodeID,
- sv.ext.lcValueSignature().signature,
- xdr::xdr_to_opaque(mApp.getNetworkID(),
- ENVELOPE_TYPE_SCPVALUE,
- sv.txSetHash,
- sv.closeTime))
- .valid;
- case STELLAR_VALUE_EMPTY_TX_SET:
+ // This function should never be called with an unsigned value
+ releaseAssert(isSignedStellarValue(sv) || isEmptyTxSetStellarValue(sv));
+
+ auto const& signature = getLcValueSignature(sv);
+ auto const& txSetHash =
+ isEmptyTxSetStellarValue(sv) ? getProposedTxSetHash(sv) : sv.txSetHash;
+ if (isMsCloseTimeStellarValue(sv))
{
- auto const& ov = sv.ext.proposedValue();
return PubKeyUtils::verifySig(
- ov.lcValueSignature.nodeID, ov.lcValueSignature.signature,
- xdr::xdr_to_opaque(mApp.getNetworkID(),
- ENVELOPE_TYPE_SCPVALUE, ov.txSetHash,
- sv.closeTime))
+ signature.nodeID, signature.signature,
+ xdr::xdr_to_opaque(
+ mApp.getNetworkID(), ENVELOPE_TYPE_SCPVALUE, txSetHash,
+ sv.closeTime, getConsensusTime(sv).milliseconds()))
.valid;
}
- default:
- releaseAssert(false);
- }
+
+ return PubKeyUtils::verifySig(signature.nodeID, signature.signature,
+ xdr::xdr_to_opaque(mApp.getNetworkID(),
+ ENVELOPE_TYPE_SCPVALUE,
+ txSetHash, sv.closeTime))
+ .valid;
}
StellarValue
-HerderImpl::makeStellarValue(Hash const& txSetHash, uint64_t closeTime,
+HerderImpl::makeStellarValue(Hash const& txSetHash, ConsensusTime closeTime,
xdr::xvector const& upgrades,
SecretKey const& s)
{
ZoneScoped;
+ bool const useMsCloseTime = getHerderSCPDriver().protocolUsesMsCloseTime();
+ // Whole-second value types can only carry whole-second close times
+ releaseAssert(useMsCloseTime || closeTime.isWholeSecond());
+
StellarValue sv;
- sv.ext.v(STELLAR_VALUE_SIGNED);
sv.txSetHash = txSetHash;
- sv.closeTime = closeTime;
+ // closeTime carries the whole-second apply time in both value formats
+ sv.closeTime = closeTime.toApplyTime().timePoint();
sv.upgrades = upgrades;
- sv.ext.lcValueSignature().nodeID = s.getPublicKey();
- sv.ext.lcValueSignature().signature =
- s.sign(xdr::xdr_to_opaque(mApp.getNetworkID(), ENVELOPE_TYPE_SCPVALUE,
- sv.txSetHash, sv.closeTime));
+ if (useMsCloseTime)
+ {
+ sv.ext.v(STELLAR_VALUE_SIGNED_MS);
+ sv.ext.signedMsValue().closeTimeMs = closeTime.milliseconds();
+ auto& signature = getLcValueSignature(sv);
+ signature.nodeID = s.getPublicKey();
+ // The signature also covers the full ms close time (CAP-0088). Sign
+ // exactly what verifyStellarValueSignature reads back out of the value.
+ signature.signature = s.sign(xdr::xdr_to_opaque(
+ mApp.getNetworkID(), ENVELOPE_TYPE_SCPVALUE, sv.txSetHash,
+ sv.closeTime, getConsensusTime(sv).milliseconds()));
+ }
+ else
+ {
+ sv.ext.v(STELLAR_VALUE_SIGNED);
+ auto& signature = getLcValueSignature(sv);
+ signature.nodeID = s.getPublicKey();
+ signature.signature = s.sign(
+ xdr::xdr_to_opaque(mApp.getNetworkID(), ENVELOPE_TYPE_SCPVALUE,
+ sv.txSetHash, sv.closeTime));
+ }
+ releaseAssert(hasValidCloseTime(sv));
return sv;
}
diff --git a/src/herder/HerderImpl.h b/src/herder/HerderImpl.h
index 662eb74c02..bebbce06d6 100644
--- a/src/herder/HerderImpl.h
+++ b/src/herder/HerderImpl.h
@@ -10,6 +10,7 @@
#include "herder/PendingEnvelopes.h"
#include "herder/QuorumIntersectionChecker.h"
#include "herder/Upgrades.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "overlay/NetworkConstants.h"
#include "util/Timer.h"
#include "util/UnorderedMap.h"
@@ -42,7 +43,7 @@ class HerderImpl : public Herder
struct ConsensusData
{
uint64_t mConsensusIndex{0};
- TimePoint mConsensusCloseTime{0};
+ ConsensusTime mConsensusCloseTime;
};
void setTrackingSCPState(uint64_t index, StellarValue const& value,
@@ -52,7 +53,7 @@ class HerderImpl : public Herder
// in fully booted state
uint32 trackingConsensusLedgerIndex() const override;
- TimePoint trackingConsensusCloseTime() const;
+ ConsensusTime trackingConsensusCloseTime() const;
// the ledger index that we expect to externalize next
uint32
@@ -101,12 +102,11 @@ class HerderImpl : public Herder
#ifdef BUILD_TESTS
TxSubmitStatus recvTransaction(TransactionFrameBasePtr tx,
- bool submittedFromSelf, bool force = false,
+ bool submittedFromSelf,
bool isLoadgenTx = false) override;
#else
TxSubmitStatus recvTransaction(TransactionFrameBasePtr tx,
- bool submittedFromSelf,
- bool force = false) override;
+ bool submittedFromSelf) override;
#endif
EnvelopeStatus recvSCPEnvelope(SCPEnvelope const& envelope) override;
@@ -119,7 +119,7 @@ class HerderImpl : public Herder
StellarMessage const& txset) override;
void externalizeValue(TxSetXDRFrameConstPtr txSet, uint32_t ledgerSeq,
- uint64_t closeTime,
+ ConsensusTime closeTime,
xdr::xvector const& upgrades,
std::optional skToSignValue) override;
@@ -155,6 +155,11 @@ class HerderImpl : public Herder
{
mFlowControlExtraBuffer = std::make_optional(bytes);
}
+
+ // When set, this node nominates the tx set returned by this function
+ // (given the LCL) instead of the one it builds.
+ std::function
+ mNominatedTxSetOverride;
#endif
std::vector getSCPStateForPeer(uint32 ledgerSeq) override;
@@ -200,8 +205,6 @@ class HerderImpl : public Herder
void setUpgrades(Upgrades::UpgradeParameters const& upgrades) override;
std::string getUpgradesJson() override;
- void setFilteredAccounts(std::set const& accounts) override;
-
void forceSCPStateIntoSyncWithLastClosedLedger() override;
bool resolveNodeID(std::string const& s, PublicKey& retKey) override;
@@ -216,7 +219,7 @@ class HerderImpl : public Herder
QuorumTracker::QuorumMap const& getCurrentlyTrackedQuorum() const override;
virtual StellarValue
- makeStellarValue(Hash const& txSetHash, uint64_t closeTime,
+ makeStellarValue(Hash const& txSetHash, ConsensusTime closeTime,
xdr::xvector const& upgrades,
SecretKey const& s) override;
@@ -246,8 +249,9 @@ class HerderImpl : public Herder
// Given a candidate close time, determine an offset needed to make it
// valid (at current system time). Returns 0 if ct is already valid
std::chrono::milliseconds
- ctValidityOffset(uint64_t ct, std::chrono::milliseconds maxCtOffset =
- std::chrono::milliseconds::zero());
+ ctValidityOffset(ConsensusTime closeTime,
+ std::chrono::milliseconds maxCtOffset =
+ std::chrono::milliseconds::zero());
void setupTriggerNextLedger();
@@ -255,15 +259,15 @@ class HerderImpl : public Herder
{
Hash previousLedgerHash;
uint32_t ledgerSeq;
- uint64_t closeTime;
+ ConsensusTime closeTime;
TxSetXDRFrameConstPtr txSet;
ApplicableTxSetFrameConstPtr applicableTxSet;
std::vector invalidTxHashes;
bool capacityLimited;
};
std::optional mPreparedTxSet;
- PreparedTxSet buildTxSet(uint32_t ledgerSeq, uint64_t closeTime);
- void prepareTxSet(uint32_t ledgerSeq, uint64_t closeTime);
+ PreparedTxSet buildTxSet(uint32_t ledgerSeq, ConsensusTime closeTime);
+ void prepareTxSet(uint32_t ledgerSeq, ConsensusTime closeTime);
void discardPreparedTxSet();
// Compute the trigger-timer anchor point using the local node's
@@ -321,7 +325,7 @@ class HerderImpl : public Herder
// Map SCP slots to local time of nomination and the time slot was
// externalized by the network
- std::map>>
+ std::map>>
mDriftCTSlidingWindow;
// saves upgrade parameters
@@ -367,6 +371,17 @@ class HerderImpl : public Herder
// network-close-time anchor to the local prepare-start anchor.
medida::Meter& mTriggerPrepareStartFallback;
+ // Time spent building the tx set proposed at nomination.
+ medida::Timer& mTxSetBuild;
+
+ // How long after its scheduled time the trigger timer callback ran
+ // (main-thread congestion)
+ medida::Timer& mTriggerLateness;
+
+ // Marked when the next trigger is already due when it is scheduled,
+ // i.e. the node is behind the target block time
+ medida::Meter& mTriggerImmediate;
+
SCPMetrics(Application& app);
};
diff --git a/src/herder/HerderSCPDriver.cpp b/src/herder/HerderSCPDriver.cpp
index 9355745690..d487c7d0bd 100644
--- a/src/herder/HerderSCPDriver.cpp
+++ b/src/herder/HerderSCPDriver.cpp
@@ -10,6 +10,7 @@
#include "herder/HerderImpl.h"
#include "herder/LedgerCloseData.h"
#include "herder/PendingEnvelopes.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "ledger/LedgerManager.h"
#include "main/Application.h"
#include "main/ErrorMessages.h"
@@ -34,15 +35,6 @@
namespace stellar
{
-namespace
-{
-bool
-isEmptyTxSetStellarValue(StellarValue const& sv)
-{
- return sv.ext.v() == STELLAR_VALUE_EMPTY_TX_SET;
-}
-}
-
uint32_t const TXSETVALID_CACHE_SIZE = 1000;
Hash
@@ -68,6 +60,8 @@ HerderSCPDriver::SCPMetrics::SCPMetrics(Application& app)
app.getMetrics().NewTimer({"scp", "timing", "nominated"}))
, mPrepareToExternalize(
app.getMetrics().NewTimer({"scp", "timing", "externalized"}))
+ , mTriggerToBallot(
+ app.getMetrics().NewTimer({"scp", "timing", "trigger-to-ballot"}))
, mFirstToSelfExternalizeLag(app.getMetrics().NewTimer(
{"scp", "timing", "first-to-self-externalize-lag"}))
, mSelfToOthersExternalizeLag(app.getMetrics().NewTimer(
@@ -261,6 +255,13 @@ HerderSCPDriver::protocolAllowsEmptyTxSetValues() const
EMPTY_TX_SET_PROTOCOL_VERSION);
}
+bool
+HerderSCPDriver::protocolUsesMsCloseTime() const
+{
+ return protocolHasMsCloseTime(
+ mLedgerManager.getLastClosedLedgerHeader().header.ledgerVersion);
+}
+
bool
HerderSCPDriver::isParallelTxSetDownloadEnabled() const
{
@@ -271,24 +272,30 @@ HerderSCPDriver::isParallelTxSetDownloadEnabled() const
// value validation
bool
-HerderSCPDriver::checkCloseTime(uint64_t slotIndex, uint64_t lastCloseTime,
+HerderSCPDriver::checkCloseTime(uint64_t slotIndex, ConsensusTime lastCloseTime,
StellarValue const& b) const
{
+ auto const closeTime = getConsensusTime(b);
+
// Check closeTime (not too old)
- if (b.closeTime <= lastCloseTime)
+ if (closeTime <= lastCloseTime)
{
CLOG_TRACE(Herder, "Close time too old for slot {}, got {} vs {}",
- slotIndex, b.closeTime, lastCloseTime);
+ slotIndex, closeTime.toString(), lastCloseTime.toString());
return false;
}
// Check closeTime (not too far in future)
- uint64_t timeNow = mApp.timeNow();
- if (b.closeTime > timeNow + Herder::MAX_TIME_SLIP_SECONDS.count())
+ auto const protocolVersion =
+ mLedgerManager.getLastClosedLedgerHeader().header.ledgerVersion;
+ auto const maxCloseTime = ConsensusTime::fromSystemTime(
+ mApp.getClock().system_now() + Herder::MAX_TIME_SLIP_SECONDS,
+ protocolVersion);
+ if (closeTime > maxCloseTime)
{
CLOG_TRACE(Herder,
"Close time too far in future for slot {}, got {} vs {}",
- slotIndex, b.closeTime, timeNow);
+ slotIndex, closeTime.toString(), maxCloseTime.toString());
return false;
}
return true;
@@ -301,14 +308,16 @@ HerderSCPDriver::validatePastOrFutureValue(
{
ZoneScoped;
releaseAssert(slotIndex != lcl.header.ledgerSeq + 1);
+ auto const closeTime = getConsensusTime(b);
+ auto const lclCloseTime = getConsensusTime(lcl.header.scpValue);
if (slotIndex == lcl.header.ledgerSeq)
{
// previous ledger
- if (b.closeTime != lcl.header.scpValue.closeTime)
+ if (closeTime != lclCloseTime)
{
- CLOG_TRACE(Herder,
- "Got a bad close time for ledger {}, got {} vs {}",
- slotIndex, b.closeTime, lcl.header.scpValue.closeTime);
+ CLOG_TRACE(
+ Herder, "Got a bad close time for ledger {}, got {} vs {}",
+ slotIndex, closeTime.toString(), lclCloseTime.toString());
return SCPDriver::kInvalidValue;
}
if (isEmptyTxSetStellarValue(b))
@@ -318,13 +327,13 @@ HerderSCPDriver::validatePastOrFutureValue(
return SCPDriver::kInvalidValue;
}
- auto const& ov = b.ext.proposedValue();
// We can check previousLedgerHash because the LCL header
// contains the hash of its parent. We cannot check
// previousLedgerVersion because the LCL header only has
// its own version, and a protocol upgrade on the LCL
// could make it differ from its parent's version.
- if (ov.previousLedgerHash != lcl.header.previousLedgerHash)
+ if (getProposedPreviousLedgerHash(b) !=
+ lcl.header.previousLedgerHash)
{
CLOG_TRACE(Herder,
"Got a bad previousLedgerHash for empty-tx-set "
@@ -337,15 +346,15 @@ HerderSCPDriver::validatePastOrFutureValue(
else if (slotIndex < lcl.header.ledgerSeq)
{
// basic sanity check on older value
- if (b.closeTime >= lcl.header.scpValue.closeTime)
+ if (closeTime >= lclCloseTime)
{
- CLOG_TRACE(Herder,
- "Got a bad close time for ledger {}, got {} vs {}",
- slotIndex, b.closeTime, lcl.header.scpValue.closeTime);
+ CLOG_TRACE(
+ Herder, "Got a bad close time for ledger {}, got {} vs {}",
+ slotIndex, closeTime.toString(), lclCloseTime.toString());
return SCPDriver::kInvalidValue;
}
}
- else if (!checkCloseTime(slotIndex, lcl.header.scpValue.closeTime, b))
+ else if (!checkCloseTime(slotIndex, lclCloseTime, b))
{
// future messages must be valid compared to lastCloseTime
return SCPDriver::kInvalidValue;
@@ -412,7 +421,8 @@ HerderSCPDriver::validateValueAgainstLocalState(uint64_t slotIndex,
if (isCurrentLedger)
{
// The value is for LCL+1, perform all possible checks
- if (!checkCloseTime(slotIndex, lcl.header.scpValue.closeTime, b))
+ if (!checkCloseTime(slotIndex, getConsensusTime(lcl.header.scpValue),
+ b))
{
return SCPDriver::kInvalidValue;
}
@@ -437,9 +447,8 @@ HerderSCPDriver::validateValueAgainstLocalState(uint64_t slotIndex,
slotIndex);
return SCPDriver::kInvalidValue;
}
- auto const& ov = b.ext.proposedValue();
- if (ov.previousLedgerHash != lcl.hash ||
- ov.previousLedgerVersion != lcl.header.ledgerVersion)
+ if (getProposedPreviousLedgerHash(b) != lcl.hash ||
+ getProposedPreviousLedgerVersion(b) != lcl.header.ledgerVersion)
{
CLOG_DEBUG(Herder,
"HerderSCPDriver::validateValue i: {} empty-tx-set "
@@ -456,7 +465,8 @@ HerderSCPDriver::validateValueAgainstLocalState(uint64_t slotIndex,
TxSetXDRFrameConstPtr txSet = std::get(
mPendingEnvelopes.getTxSet(txSetHash));
- auto closeTimeOffset = b.closeTime - lcl.header.scpValue.closeTime;
+ auto closeTimeOffset =
+ getApplyTime(b) - getApplyTime(lcl.header.scpValue);
if (!txSet)
{
@@ -513,7 +523,8 @@ HerderSCPDriver::validateValueAgainstLocalState(uint64_t slotIndex,
}
bool
-HerderSCPDriver::deserializeAndValidateStellarValue(Value const& value,
+HerderSCPDriver::deserializeAndValidateStellarValue(uint64_t slotIndex,
+ Value const& value,
StellarValue& sv) const
{
ZoneScoped;
@@ -527,22 +538,55 @@ HerderSCPDriver::deserializeAndValidateStellarValue(Value const& value,
return false;
}
- bool const emptyTxSetsAllowed = protocolAllowsEmptyTxSetValues();
- if (sv.ext.v() != STELLAR_VALUE_SIGNED)
+ // An ms value's closeTime must agree with its closeTimeMs
+ if (!hasValidCloseTime(sv))
{
- if (!emptyTxSetsAllowed)
- {
- // Empty-tx-set values are not allowed, and the value is not a
- // signed value, so it is invalid.
- return false;
- }
+ return false;
+ }
- if (!isEmptyTxSetStellarValue(sv))
- {
- // The value is not a signed value or an empty-tx-set value, so it
- // is invalid.
- return false;
- }
+ // Whether a slot uses whole-second or ms close times is decided by the
+ // protocol of the ledger before it (CAP-0088). We only know that protocol
+ // for certain for the next slot, LCL+1; for any other slot the network may
+ // have upgraded somewhere between our LCL and that slot, so we accept
+ // every format we cannot rule out:
+ //
+ // LCL before ms upgrade LCL at/after ms upgrade
+ // slot <= LCL whole-second only either (may predate it)
+ // slot == LCL+1 whole-second only ms only
+ // slot > LCL+1 either (may have upgraded) ms only
+ //
+ // Prior to the upgrade, if we are behind, it's possible our peers our
+ // sending us valid future slots after an upgrade we have not yet applied.
+ // Similarly for past slots, it's possible these predate the upgrade that we
+ // have already applied.
+ auto const& lclHeader = mLedgerManager.getLastClosedLedgerHeader().header;
+ bool const lclProtocolHasMsCloseTime =
+ protocolHasMsCloseTime(lclHeader.ledgerVersion);
+ bool const slotAlreadyClosed = slotIndex <= lclHeader.ledgerSeq;
+ bool const slotBeyondNext = slotIndex > lclHeader.ledgerSeq + 1;
+ bool const valueHasMsCloseTime = isMsCloseTimeStellarValue(sv);
+
+ bool const msCloseTimeAllowed = lclProtocolHasMsCloseTime || slotBeyondNext;
+ bool const wholeSecondCloseTimeAllowed =
+ !lclProtocolHasMsCloseTime || slotAlreadyClosed;
+ bool const valueFormatAllowed =
+ valueHasMsCloseTime ? msCloseTimeAllowed : wholeSecondCloseTimeAllowed;
+ if (!valueFormatAllowed)
+ {
+ return false;
+ }
+
+ // Values must be signed or empty-tx-set values
+ bool const isSigned = isSignedStellarValue(sv);
+ bool const isEmpty = isEmptyTxSetStellarValue(sv);
+ if (!isSigned && !isEmpty)
+ {
+ return false;
+ }
+ // Empty-tx-set values are only valid once the protocol allows them
+ if (isEmpty && !protocolAllowsEmptyTxSetValues())
+ {
+ return false;
}
// Empty-tx-set values must have the empty-tx-set hash, and
@@ -597,7 +641,7 @@ HerderSCPDriver::validateValue(uint64_t slotIndex, Value const& value,
releaseAssert(threadIsMain());
StellarValue b;
- if (!deserializeAndValidateStellarValue(value, b))
+ if (!deserializeAndValidateStellarValue(slotIndex, value, b))
{
mSCPMetrics.mValueInvalid.Mark();
return SCPDriver::kInvalidValue;
@@ -635,7 +679,7 @@ HerderSCPDriver::extractValidValue(uint64_t slotIndex, Value const& value)
{
ZoneScoped;
StellarValue b;
- if (!deserializeAndValidateStellarValue(value, b))
+ if (!deserializeAndValidateStellarValue(slotIndex, value, b))
{
return nullptr;
}
@@ -685,14 +729,26 @@ HerderSCPDriver::makeEmptyTxSetValueFromValue(Value const& v) const
{
ZoneScoped;
StellarValue proposedValue = toStellarValueOrThrow(v);
- releaseAssert(proposedValue.ext.v() == STELLAR_VALUE_SIGNED);
+ releaseAssert(isSignedStellarValue(proposedValue));
auto const& lcl = mLedgerManager.getLastClosedLedgerHeader();
StellarValue sv;
- sv.ext.v(STELLAR_VALUE_EMPTY_TX_SET);
sv.txSetHash = Herder::EMPTY_TX_SET_HASH;
sv.closeTime = proposedValue.closeTime;
sv.upgrades = proposedValue.upgrades;
+ if (proposedValue.ext.v() == STELLAR_VALUE_SIGNED_MS)
+ {
+ sv.ext.v(STELLAR_VALUE_EMPTY_TX_SET_MS);
+ auto& ov = sv.ext.proposedMsValue();
+ ov.closeTimeMs = proposedValue.ext.signedMsValue().closeTimeMs;
+ ov.txSetHash = proposedValue.txSetHash;
+ ov.previousLedgerHash = lcl.hash;
+ ov.previousLedgerVersion = lcl.header.ledgerVersion;
+ ov.lcValueSignature =
+ proposedValue.ext.signedMsValue().lcValueSignature;
+ return xdr::xdr_to_opaque(sv);
+ }
+ sv.ext.v(STELLAR_VALUE_EMPTY_TX_SET);
sv.ext.proposedValue().txSetHash = proposedValue.txSetHash;
sv.ext.proposedValue().previousLedgerHash = lcl.hash;
sv.ext.proposedValue().previousLedgerVersion = lcl.header.ledgerVersion;
@@ -1464,6 +1520,16 @@ HerderSCPDriver::getExternalizeLag(NodeID const& id) const
return n->second.GetSnapshot().get75thPercentile();
}
+void
+HerderSCPDriver::recordNominationTrigger(uint64_t slotIndex)
+{
+ auto& timing = mSCPExecutionTimes[slotIndex];
+ if (!timing.mTriggerStart && !timing.mNominationStart)
+ {
+ timing.mTriggerStart = mApp.getClock().now();
+ }
+}
+
void
HerderSCPDriver::recordSCPEvent(uint64_t slotIndex, bool isNomination)
{
@@ -1473,11 +1539,19 @@ HerderSCPDriver::recordSCPEvent(uint64_t slotIndex, bool isNomination)
if (isNomination)
{
- timing.mNominationStart =
- std::make_optional(start);
+ if (!timing.mNominationStart)
+ {
+ timing.mNominationStart = start;
+ }
}
else
{
+ // The first entry into ballot ends the trigger-to-ballot interval
+ if (!timing.mPrepareStart && timing.mTriggerStart &&
+ start >= *timing.mTriggerStart)
+ {
+ mSCPMetrics.mTriggerToBallot.Update(start - *timing.mTriggerStart);
+ }
timing.mPrepareStart =
std::make_optional(start);
}
@@ -1817,15 +1891,17 @@ HerderSCPDriver::wrapStellarValue(StellarValue const& sv)
void
HerderSCPDriver::cacheValidTxSet(ApplicableTxSetFrame const& txSet,
LedgerHeaderHistoryEntry const& lcl,
- uint64_t closeTimeOffset) const
+ ApplyTimeOffset closeTimeOffset) const
{
- auto key = TxSetValidityKey{lcl.hash, txSet.getContentsHash(),
- closeTimeOffset, closeTimeOffset};
+ auto key =
+ TxSetValidityKey{lcl.hash, txSet.getContentsHash(),
+ closeTimeOffset.seconds(), closeTimeOffset.seconds()};
bool* pRes = mTxSetValidCache.maybeGet(key);
if (pRes == nullptr)
{
#ifdef SCP_DEBUGGING
- releaseAssert(txSet.checkValid(mApp, closeTimeOffset, closeTimeOffset));
+ releaseAssert(txSet.checkValid(mApp, closeTimeOffset.seconds(),
+ closeTimeOffset.seconds()));
#endif
mTxSetValidCache.put(key, true);
}
@@ -1843,12 +1919,13 @@ HerderSCPDriver::cacheValidTxSet(ApplicableTxSetFrame const& txSet,
bool
HerderSCPDriver::checkAndCacheTxSetValid(TxSetXDRFrame const& txSet,
LedgerHeaderHistoryEntry const& lcl,
- uint64_t closeTimeOffset) const
+ ApplyTimeOffset closeTimeOffset) const
{
ZoneScoped;
- auto key = TxSetValidityKey{lcl.hash, txSet.getContentsHash(),
- closeTimeOffset, closeTimeOffset};
+ auto key =
+ TxSetValidityKey{lcl.hash, txSet.getContentsHash(),
+ closeTimeOffset.seconds(), closeTimeOffset.seconds()};
bool* pRes = mTxSetValidCache.maybeGet(key);
if (pRes == nullptr)
@@ -1877,8 +1954,8 @@ HerderSCPDriver::checkAndCacheTxSetValid(TxSetXDRFrame const& txSet,
}
else
{
- res = applicableTxSet->checkValid(mApp, closeTimeOffset,
- closeTimeOffset);
+ res = applicableTxSet->checkValid(mApp, closeTimeOffset.seconds(),
+ closeTimeOffset.seconds());
}
mTxSetValidCache.put(key, res);
@@ -1981,6 +2058,25 @@ HerderSCPDriver::getNominationTimeouts(uint64_t slotIndex) const
return std::nullopt;
}
+std::chrono::milliseconds
+HerderSCPDriver::getTriggerToBallotDuration(uint64_t slotIndex) const
+{
+ auto it = mSCPExecutionTimes.find(slotIndex);
+ if (it != mSCPExecutionTimes.end())
+ {
+ auto const& timing = it->second;
+ if (timing.mTriggerStart && timing.mPrepareStart &&
+ *timing.mPrepareStart > *timing.mTriggerStart)
+ {
+ // Stop at ballot even if a local proposal is still being built.
+ // Ballot and apply are already covered by the fallback anchor.
+ return std::chrono::duration_cast(
+ *timing.mPrepareStart - *timing.mTriggerStart);
+ }
+ }
+ return std::chrono::milliseconds::zero();
+}
+
void
HerderSCPDriver::markSlotAsRestored(uint64_t slotIndex)
{
diff --git a/src/herder/HerderSCPDriver.h b/src/herder/HerderSCPDriver.h
index d4906b64db..ec3b1bc9a2 100644
--- a/src/herder/HerderSCPDriver.h
+++ b/src/herder/HerderSCPDriver.h
@@ -24,6 +24,7 @@ class Histogram;
namespace stellar
{
class Application;
+class ConsensusTime;
class HerderImpl;
class LedgerManager;
class PendingEnvelopes;
@@ -56,6 +57,7 @@ class HerderSCPDriver : public SCPDriver
}
void recordSCPExecutionMetrics(uint64_t slotIndex);
+ void recordNominationTrigger(uint64_t slotIndex);
void recordSCPEvent(uint64_t slotIndex, bool isNomination);
void recordSCPExternalizeEvent(uint64_t slotIndex, NodeID const& id,
bool forceUpdateSelf);
@@ -80,7 +82,7 @@ class HerderSCPDriver : public SCPDriver
// Construct a CAP-0083 empty-tx-set value from `v`. A few caveats about
// this function:
- // * `v` must be a STELLAR_VALUE_SIGNED value.
+ // * `v` must be a STELLAR_VALUE_SIGNED or STELLAR_VALUE_SIGNED_MS value.
// * This function should only be called from slots with slot indices equal
// to LCL+1
Value makeEmptyTxSetValueFromValue(Value const& v) const override;
@@ -95,6 +97,9 @@ class HerderSCPDriver : public SCPDriver
// Returns true iff the protocol allows CAP-0083 empty-tx-set values
bool protocolAllowsEmptyTxSetValues() const override;
+ // Returns true iff the protocol uses millisecond-resolution close times.
+ bool protocolUsesMsCloseTime() const;
+
// timer handling
void setupTimer(uint64_t slotIndex, int timerID,
std::chrono::milliseconds timeout,
@@ -156,7 +161,7 @@ class HerderSCPDriver : public SCPDriver
std::optional getPrepareStart(uint64_t slotIndex);
// validate close time as much as possible
- bool checkCloseTime(uint64_t slotIndex, uint64_t lastCloseTime,
+ bool checkCloseTime(uint64_t slotIndex, ConsensusTime lastCloseTime,
StellarValue const& b) const;
// wraps a *valid* StellarValue (throws if it can't find txSet/qSet)
@@ -209,11 +214,17 @@ class HerderSCPDriver : public SCPDriver
};
void cacheValidTxSet(ApplicableTxSetFrame const& txSet,
LedgerHeaderHistoryEntry const& lcl,
- uint64_t closeTimeOffset) const;
+ ApplyTimeOffset closeTimeOffset) const;
// Get the number of nomination timeouts that occurred for a given slot
std::optional getNominationTimeouts(uint64_t slotIndex) const;
+ // Elapsed local time from the trigger through entry into ballot, measured
+ // on the steady clock. Includes construction and incomplete nomination
+ // rounds. Missing timing history does not establish any allowance.
+ std::chrono::milliseconds
+ getTriggerToBallotDuration(uint64_t slotIndex) const;
+
#ifdef BUILD_TESTS
RandomEvictionCache&
getTxSetValidityCache()
@@ -260,6 +271,10 @@ class HerderSCPDriver : public SCPDriver
medida::Timer& mNominateToPrepare;
medida::Timer& mPrepareToExternalize;
+ // Local trigger to first entry into ballot (proposal construction
+ // plus nomination); the drift allowance of the trigger anchor
+ medida::Timer& mTriggerToBallot;
+
// Timers tracking externalize messages
medida::Timer& mFirstToSelfExternalizeLag;
medida::Timer& mSelfToOthersExternalizeLag;
@@ -294,6 +309,7 @@ class HerderSCPDriver : public SCPDriver
struct SCPTiming
{
+ std::optional mTriggerStart;
std::optional mNominationStart;
std::optional mPrepareStart;
@@ -355,9 +371,10 @@ class HerderSCPDriver : public SCPDriver
bool checkAndCacheTxSetValid(TxSetXDRFrame const& txSet,
LedgerHeaderHistoryEntry const& lcl,
- uint64_t closeTimeOffset) const;
+ ApplyTimeOffset closeTimeOffset) const;
- bool deserializeAndValidateStellarValue(Value const& value,
+ bool deserializeAndValidateStellarValue(uint64_t slotIndex,
+ Value const& value,
StellarValue& sv) const;
void extractValidUpgrades(StellarValue& sv, bool nomination) const;
};
diff --git a/src/herder/LedgerCloseData.cpp b/src/herder/LedgerCloseData.cpp
index 7de432ff1e..305201c03f 100644
--- a/src/herder/LedgerCloseData.cpp
+++ b/src/herder/LedgerCloseData.cpp
@@ -3,6 +3,7 @@
#include "crypto/Hex.h"
#include "herder/Herder.h"
#include "herder/Upgrades.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "main/Application.h"
#include "util/GlobalChecks.h"
#include "util/Logging.h"
@@ -63,12 +64,22 @@ stellarValueToString(Config const& c, StellarValue const& sv)
res << " EMPTY_TXSET@"
<< c.toShortString(sv.ext.proposedValue().lcValueSignature.nodeID);
break;
+ case STELLAR_VALUE_SIGNED_MS:
+ res << " SIGNED_MS@"
+ << c.toShortString(sv.ext.signedMsValue().lcValueSignature.nodeID);
+ break;
+ case STELLAR_VALUE_EMPTY_TX_SET_MS:
+ res << " EMPTY_TXSET_MS@"
+ << c.toShortString(
+ sv.ext.proposedMsValue().lcValueSignature.nodeID);
+ break;
default:
res << " UNKNOWN";
break;
}
- res << " txH: " << hexAbbrev(sv.txSetHash) << ", ct: " << sv.closeTime
- << ", upgrades: [";
+ res << " txH: " << hexAbbrev(sv.txSetHash)
+ << ", ct: " << getConsensusTime(sv).toString();
+ res << ", upgrades: [";
for (auto const& upgrade : sv.upgrades)
{
if (upgrade.empty())
diff --git a/src/herder/PendingEnvelopes.cpp b/src/herder/PendingEnvelopes.cpp
index 0d188f8a05..204fa49860 100644
--- a/src/herder/PendingEnvelopes.cpp
+++ b/src/herder/PendingEnvelopes.cpp
@@ -6,6 +6,7 @@
#include "herder/HerderPersistence.h"
#include "herder/HerderUtils.h"
#include "herder/TxSetFrame.h"
+#include "ledger/LedgerManager.h"
#include "main/Application.h"
#include "main/Config.h"
#include "overlay/RustOverlayManager.h"
@@ -350,7 +351,15 @@ PendingEnvelopes::recvSCPEnvelope(SCPEnvelope const& envelope)
}
auto const& values = maybeValues.value();
- if (std::any_of(values.begin(), values.end(), [this](auto const& value) {
+ auto const& scpDriver = mHerder.getHerderSCPDriver();
+ // The protocol gates below are judged against the local LCL, which only
+ // governs slots up to lcl+1. A node that has fallen behind a protocol
+ // upgrade cannot know which protocol governs later slots, so it must not
+ // drop protocol-gated values for future slots.
+ bool const beyondLocalProtocol =
+ envelope.statement.slotIndex >
+ mApp.getLedgerManager().getLastClosedLedgerNum() + 1;
+ if (std::any_of(values.begin(), values.end(), [&](auto const& value) {
switch (value.ext.v())
{
case STELLAR_VALUE_BASIC:
@@ -360,8 +369,17 @@ PendingEnvelopes::recvSCPEnvelope(SCPEnvelope const& envelope)
// Signed values are allowed
return false;
case STELLAR_VALUE_EMPTY_TX_SET:
- return !mHerder.getHerderSCPDriver()
- .protocolAllowsEmptyTxSetValues();
+ return !beyondLocalProtocol &&
+ !scpDriver.protocolAllowsEmptyTxSetValues();
+ case STELLAR_VALUE_SIGNED_MS:
+ // Millisecond close times are only permitted once the
+ // protocol uses them
+ return !beyondLocalProtocol &&
+ !scpDriver.protocolUsesMsCloseTime();
+ case STELLAR_VALUE_EMPTY_TX_SET_MS:
+ return !beyondLocalProtocol &&
+ !(scpDriver.protocolAllowsEmptyTxSetValues() &&
+ scpDriver.protocolUsesMsCloseTime());
default:
releaseAssert(false);
}
diff --git a/src/herder/TxSetFrame.cpp b/src/herder/TxSetFrame.cpp
index 63bb6bbb38..03b3914223 100644
--- a/src/herder/TxSetFrame.cpp
+++ b/src/herder/TxSetFrame.cpp
@@ -22,7 +22,11 @@
#include "util/ProtocolVersion.h"
#include "util/XDRCereal.h"
#include "util/XDROperators.h"
+#include "util/numeric.h"
#include "xdrpp/marshal.h"
+#ifdef BUILD_TESTS
+#include "test/Catch2.h"
+#endif
#include
#include
@@ -100,8 +104,20 @@ validateSequentialPhaseXDRStructure(TransactionPhase const& phase)
}
bool
-validateParallelComponent(ParallelTxsComponent const& component)
-{
+validateParallelComponent(ParallelTxsComponent const& component,
+ uint32_t ledgerVersion)
+{
+ // A base fee without any transactions to apply it to has no effect, so
+ // base fee should be 0, (see
+ // CANONICAL_EMPTY_SOROBAN_PHASE_PROTOCOL_VERSION) comment in
+ // ProtocolVersion.h for more details).
+ if (component.executionStages.empty() && component.baseFee &&
+ protocolVersionStartsFrom(
+ ledgerVersion, CANONICAL_EMPTY_SOROBAN_PHASE_PROTOCOL_VERSION))
+ {
+ CLOG_DEBUG(Herder, "Got bad txSet: base fee without stages");
+ return false;
+ }
for (auto const& stage : component.executionStages)
{
if (stage.empty())
@@ -122,7 +138,8 @@ validateParallelComponent(ParallelTxsComponent const& component)
}
bool
-validateTxSetXDRStructure(GeneralizedTransactionSet const& txSet)
+validateTxSetXDRStructure(GeneralizedTransactionSet const& txSet,
+ uint32_t ledgerVersion)
{
int const MAX_PHASE = 1;
@@ -160,7 +177,8 @@ validateTxSetXDRStructure(GeneralizedTransactionSet const& txSet)
phase.v());
return false;
}
- if (!validateParallelComponent(phase.parallelTxsComponent()))
+ if (!validateParallelComponent(phase.parallelTxsComponent(),
+ ledgerVersion))
{
return false;
}
@@ -795,21 +813,22 @@ TxSetXDRFrame::makeFromStoredTxSet(StoredTransactionSet const& storedSet)
std::pair
makeTxSetFromTransactions(
PerPhaseTransactionList const& txPhases, Application& app,
- uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset
+ ApplyTimeOffset closeTimeOffset
#ifdef BUILD_TESTS
,
- bool skipValidation,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder
+ bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario
#endif
)
{
PerPhaseTransactionList invalidTxs;
invalidTxs.resize(txPhases.size());
- return makeTxSetFromTransactions(txPhases, app, lowerBoundCloseTimeOffset,
- upperBoundCloseTimeOffset, invalidTxs
+ return makeTxSetFromTransactions(txPhases, app, closeTimeOffset, invalidTxs
#ifdef BUILD_TESTS
,
- skipValidation, parallelSorobanOrder
+ enforceTxsApplyOrder, parallelSorobanOrder,
+ disableTxValidationForLegacyScenario
#endif
);
}
@@ -817,12 +836,12 @@ makeTxSetFromTransactions(
std::pair
makeTxSetFromTransactions(
PerPhaseTransactionList const& txPhases, Application& app,
- uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset,
- PerPhaseTransactionList& invalidTxs
+ ApplyTimeOffset closeTimeOffset, PerPhaseTransactionList& invalidTxs
#ifdef BUILD_TESTS
,
- bool skipValidation,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder
+ bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario
#endif
)
{
@@ -849,17 +868,36 @@ makeTxSetFromTransactions(
auto& invalid = invalidTxs[i];
TxFrameList validatedTxs;
#ifdef BUILD_TESTS
- if (skipValidation)
+ if (disableTxValidationForLegacyScenario)
{
+ REQUIRE(protocolVersionIsBefore(app.getLedgerManager()
+ .getLastClosedLedgerHeader()
+ .header.ledgerVersion,
+ ProtocolVersion::V_20));
validatedTxs = phaseTxs;
}
else
{
#endif
validatedTxs = TxSetUtils::trimInvalid(
- phaseTxs, app, accountFeeMap, lowerBoundCloseTimeOffset,
- upperBoundCloseTimeOffset, invalid);
+ phaseTxs, app, accountFeeMap, closeTimeOffset.seconds(),
+ closeTimeOffset.seconds(), invalid);
#ifdef BUILD_TESTS
+ // In tests we shouldn't be really trying to close ledgers with
+ // invalid transactions trimmed (which is unfortunately a very
+ // common footgun that many tests have hit).
+ // However, enforcing that generically is quite involved as test
+ // builds are used for e.g. loadgen scenarios. This method is also
+ // expectedly used in the tests that test tx trimming.
+ // So for now we only ensure that a subset of test-only scenarios
+ // which also historically had a lot of issues due to skipping
+ // validation only deals with the valid transactions now.
+ // Ideally we clean this up further in the future and ensure that
+ // tests always close ledgers with valid transactions.
+ if (enforceTxsApplyOrder)
+ {
+ REQUIRE(invalid.empty());
+ }
}
#endif
auto phaseType = static_cast(i);
@@ -867,7 +905,7 @@ makeTxSetFromTransactions(
applySurgePricing(phaseType, validatedTxs, app
#ifdef BUILD_TESTS
,
- skipValidation, parallelSorobanOrder
+ enforceTxsApplyOrder, parallelSorobanOrder
#endif
);
auto inclusionFeeMap = inclusionFeeMapBinding;
@@ -907,7 +945,7 @@ makeTxSetFromTransactions(
// for nomination.
auto outputTxSet = preliminaryApplicableTxSet->toWireTxSetFrame();
#ifdef BUILD_TESTS
- if (skipValidation)
+ if (enforceTxsApplyOrder)
{
// Fill in the contents hash if we're skipping the normal roundtrip
// and validation flow.
@@ -949,7 +987,7 @@ makeTxSetFromTransactions(
// We already trimmed invalid transactions in an earlier call to
// `trimInvalid`, so skip transaction validation here
auto validationResult = outputApplicableTxSet->checkValidInternalWithResult(
- app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, true);
+ app, closeTimeOffset.seconds(), closeTimeOffset.seconds(), true);
if (validationResult != TxSetValidationResult::VALID)
{
throw std::runtime_error(fmt::format(
@@ -1002,22 +1040,23 @@ TxSetXDRFrame::makeFromHistoryTransactions(Hash const& previousLedgerHash,
#ifdef BUILD_TESTS
std::pair
makeTxSetFromTransactions(
- TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset, bool enforceTxsApplyOrder,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder)
+ TxFrameList txs, Application& app, ApplyTimeOffset closeTimeOffset,
+ bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario)
{
TxFrameList invalid;
- return makeTxSetFromTransactions(
- txs, app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, invalid,
- enforceTxsApplyOrder, parallelSorobanOrder);
+ return makeTxSetFromTransactions(txs, app, closeTimeOffset, invalid,
+ enforceTxsApplyOrder, parallelSorobanOrder,
+ disableTxValidationForLegacyScenario);
}
std::pair
makeTxSetFromTransactions(
- TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs,
- bool enforceTxsApplyOrder,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder)
+ TxFrameList txs, Application& app, ApplyTimeOffset closeTimeOffset,
+ TxFrameList& invalidTxs, bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario)
{
releaseAssert(threadIsMain());
releaseAssert(!app.getLedgerManager().isApplying());
@@ -1041,8 +1080,8 @@ makeTxSetFromTransactions(
PerPhaseTransactionList invalid;
invalid.resize(perPhaseTxs.size());
auto res = makeTxSetFromTransactions(
- perPhaseTxs, app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset,
- invalid, enforceTxsApplyOrder, parallelSorobanOrder);
+ perPhaseTxs, app, closeTimeOffset, invalid, enforceTxsApplyOrder,
+ parallelSorobanOrder, disableTxValidationForLegacyScenario);
if (enforceTxsApplyOrder)
{
auto const& resPhases = res.second->getPhases();
@@ -1108,7 +1147,7 @@ TxSetXDRFrame::prepareForApply(Application& app,
if (isGeneralizedTxSet())
{
auto const& xdrTxSet = std::get(mXDRTxSet);
- if (!validateTxSetXDRStructure(xdrTxSet))
+ if (!validateTxSetXDRStructure(xdrTxSet, lclHeader.ledgerVersion))
{
CLOG_DEBUG(Herder,
"Got bad generalized txSet with invalid XDR structure");
@@ -2277,7 +2316,8 @@ ApplicableTxSetFrame::getTotalFees(LedgerHeader const& lh) const
{
for (auto const& tx : phaseTxs)
{
- total += tx->getFee(lh, getTxBaseFee(tx), true);
+ total =
+ saturatingAdd(total, tx->getFee(lh, getTxBaseFee(tx), true));
}
}
return total;
@@ -2292,7 +2332,7 @@ ApplicableTxSetFrame::getTotalInclusionFees() const
{
for (auto const& tx : phaseTxs)
{
- total += tx->getInclusionFee();
+ total = saturatingAdd(total, tx->getInclusionFee());
}
}
return total;
diff --git a/src/herder/TxSetFrame.h b/src/herder/TxSetFrame.h
index 5f1fdb8e65..fbf5e182c2 100644
--- a/src/herder/TxSetFrame.h
+++ b/src/herder/TxSetFrame.h
@@ -6,6 +6,7 @@
#include "herder/SurgePricingUtils.h"
#include "ledger/LedgerHashUtils.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "overlay/StellarXDR.h"
#include "transactions/TransactionFrame.h"
#include "util/NonCopyable.h"
@@ -108,6 +109,12 @@ using PerPhaseTransactionList = std::vector;
// Not all the transactions will be included in the result: invalid
// transactions are trimmed and optionally returned via `invalidTxs` and if
// there are too many remaining transactions surge pricing is applied.
+// `closeTimeOffset` is how far ahead of the last closed ledger's apply time the
+// ledger this set is built for applies. Transaction time bounds are
+// whole-second quantities checked against that apply time (used as both the
+// lower and the upper bound, since the close time is known exactly), which is
+// why the offset is in apply-time units rather than derived from the consensus
+// close time.
// The result is guaranteed to pass `checkValid` check with the same
// arguments as in this method, so additional validation is not needed.
//
@@ -116,44 +123,50 @@ using PerPhaseTransactionList = std::vector;
std::pair
makeTxSetFromTransactions(
PerPhaseTransactionList const& txPhases, Application& app,
- uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset
+ ApplyTimeOffset closeTimeOffset
#ifdef BUILD_TESTS
- // Skips the tx set validation and preserves the pointers
- // to the passed-in transactions - use in conjunction with
- // `enforceTxsApplyOrder` argument in test-only overrides.
+ // `enforceTxsApplyOrder` forces the transactions to be ordered the same
+ // way as in the provided container. This is used by the test `closeLedger`
+ // helpers with the `strictOrder` flag enabled.
+ // `disableTxValidationForLegacyScenario` is used in the same helpers
+ // and it disables the input transaction validation.
,
- bool skipValidation = false,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}
+ bool enforceTxsApplyOrder = false,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder = {},
+ bool disableTxValidationForLegacyScenario = false
#endif
);
std::pair
makeTxSetFromTransactions(
PerPhaseTransactionList const& txPhases, Application& app,
- uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset,
+ ApplyTimeOffset closeTimeOffset,
PerPhaseTransactionList& invalidTxsPerPhase
#ifdef BUILD_TESTS
- // Skips the tx set validation and preserves the pointers
- // to the passed-in transactions - use in conjunction with
- // `enforceTxsApplyOrder` argument in test-only overrides.
+ // `enforceTxsApplyOrder` forces the transactions to be ordered the same
+ // way as in the provided container. This is used by the test `closeLedger`
+ // helpers with the `strictOrder` flag enabled.
+ // `disableTxValidationForLegacyScenario` is used in the same helpers
+ // and it disables the input transaction validation.
,
- bool skipValidation = false,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}
+ bool enforceTxsApplyOrder = false,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder = {},
+ bool disableTxValidationForLegacyScenario = false
#endif
);
#ifdef BUILD_TESTS
std::pair
makeTxSetFromTransactions(
- TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset, bool enforceTxsApplyOrder = false,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder = {});
+ TxFrameList txs, Application& app, ApplyTimeOffset closeTimeOffset,
+ bool enforceTxsApplyOrder = false,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder = {},
+ bool disableTxValidationForLegacyScenario = false);
std::pair
makeTxSetFromTransactions(
- TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs,
- bool enforceTxsApplyOrder = false,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder = {});
+ TxFrameList txs, Application& app, ApplyTimeOffset closeTimeOffset,
+ TxFrameList& invalidTxs, bool enforceTxsApplyOrder = false,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder = {},
+ bool disableTxValidationForLegacyScenario = false);
#endif
// `TxSetFrame` is a wrapper around `TransactionSet` or
@@ -374,21 +387,22 @@ class TxSetPhaseFrame
friend std::pair
makeTxSetFromTransactions(
PerPhaseTransactionList const& txPhases, Application& app,
- uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset,
+ ApplyTimeOffset closeTimeOffset,
PerPhaseTransactionList& invalidTxsPerPhase
#ifdef BUILD_TESTS
,
- bool skipValidation,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder
+ bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario
#endif
);
#ifdef BUILD_TESTS
friend std::pair
makeTxSetFromTransactions(
- TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs,
- bool enforceTxsApplyOrder,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder);
+ TxFrameList txs, Application& app, ApplyTimeOffset closeTimeOffset,
+ TxFrameList& invalidTxs, bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario);
#endif
TxSetPhaseFrame(TxSetPhase phase, TxFrameList const& txs,
std::shared_ptr inclusionFeeMap);
@@ -508,11 +522,12 @@ class ApplicableTxSetFrame
return mPhases.size();
}
- // Returns the sum of all fees that this transaction set would take.
+ // Returns the sum of all fees that this transaction set would take. Clamps
+ // at INT64_MAX.
int64_t getTotalFees(LedgerHeader const& lh) const;
// Returns the sum of all _inclusion fee_ bids for all transactions in this
- // set.
+ // set. Clamps at INT64_MAX.
int64_t getTotalInclusionFees() const;
// Returns whether this transaction set is generalized, i.e. representable
@@ -546,21 +561,22 @@ class ApplicableTxSetFrame
friend std::pair
makeTxSetFromTransactions(
PerPhaseTransactionList const& txPhases, Application& app,
- uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset,
+ ApplyTimeOffset closeTimeOffset,
PerPhaseTransactionList& invalidTxsPerPhase
#ifdef BUILD_TESTS
,
- bool skipValidation,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder
+ bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario
#endif
);
#ifdef BUILD_TESTS
friend std::pair
makeTxSetFromTransactions(
- TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset,
- uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs,
- bool enforceTxsApplyOrder,
- txtest::ParallelSorobanOrder const& parallelSorobanOrder);
+ TxFrameList txs, Application& app, ApplyTimeOffset closeTimeOffset,
+ TxFrameList& invalidTxs, bool enforceTxsApplyOrder,
+ txtest::ParallelSorobanOrder const& parallelSorobanOrder,
+ bool disableTxValidationForLegacyScenario);
#endif
ApplicableTxSetFrame(Application& app,
diff --git a/src/herder/TxSetUtils.cpp b/src/herder/TxSetUtils.cpp
index 2a98d38b32..80df5f4d63 100644
--- a/src/herder/TxSetUtils.cpp
+++ b/src/herder/TxSetUtils.cpp
@@ -12,11 +12,13 @@
#include "ledger/LedgerTxn.h"
#include "ledger/LedgerTxnEntry.h"
#include "ledger/LedgerTxnHeader.h"
+#include "main/AppConnector.h"
#include "main/Application.h"
#include "main/Config.h"
#include "main/ErrorMessages.h"
#include "transactions/MutableTransactionResult.h"
#include "transactions/TransactionUtils.h"
+#include "util/BatchExecutor.h"
#include "util/GlobalChecks.h"
#include "util/Logging.h"
#include "util/ProtocolVersion.h"
@@ -165,91 +167,137 @@ TxSetUtils::buildAccountTxQueues(TxFrameList const& txs)
template
TxFrameListWithErrors
TxSetUtils::getInvalidTxListWithErrors(
- T const& txs, Application& app,
+ T const& inTxs, Application& app,
UnorderedMap& accountFeeMap,
uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset)
{
ZoneScoped;
releaseAssert(threadIsMain());
- CheckValidLedgerViewWrapper ledgerView(app);
-#ifdef BUILD_TESTS
- // See TransactionQueue::canAdd for the overlay-only-mode rationale.
- ledgerView.mSkipSeqNumCheck = app.getRunInOverlayOnlyMode();
-#endif
+ TxFrameList txs(inTxs.begin(), inTxs.end());
+
+ auto ledgerView = app.getLedgerManager().getLCLView();
// Validate minSeqLedgerGap and LedgerBounds against the next ledgerSeq,
// which is what will be used at apply time.
std::optional validationLedgerSeq;
if (protocolVersionStartsFrom(
- ledgerView.getLedgerHeader().current().ledgerVersion,
+ ledgerView->getLedgerHeader().current().ledgerVersion,
ProtocolVersion::V_19))
{
validationLedgerSeq =
app.getLedgerManager().getLastClosedLedgerNum() + 1;
}
+ // Parallelize transaction validation using the batch executor with
+ // `taskCount` batches.
+ auto taskCount = app.getBatchExecutor().preferredTaskCount();
+
+ // In some cases we might process a transaction set for ledger N while
+ // applying ledger N, in which case we won't be able to re-use the batch
+ // executor and thus need to fall back to single-threaded execution.
+ // This is really an edge case: it requires several tx sets to exist for the
+ // same ledger (which is rare), and the timing must be very specific for
+ // processing to happen after the vote and application start. Thus in
+ // practice we may consider this rare enough to not worry about the
+ // performance impact.
+ // NB: Both this method and `isApplying` must happen on the main thread,
+ // so there is no race condition risk.
+ if (app.getLedgerManager().isApplying())
+ {
+ taskCount = 1;
+ }
+#ifdef BUILD_TESTS
+ // In in-memory mode we use a raw LTX in validation, which is not safe to
+ // share across multiple threads. That's avoidable, but requires changes to
+ // LTX and validation logic, so it's not worth for fixing this just for
+ // tests.
+ if (app.getConfig().MODE_USES_IN_MEMORY_LEDGER)
+ {
+ taskCount = 1;
+ }
+#endif
+
+ std::vector> ledgerViews;
+ ledgerViews.emplace_back(std::move(ledgerView));
+
+ for (size_t i = 1; i < taskCount; ++i)
+ {
+ ledgerViews.emplace_back(app.getLedgerManager().getLCLView());
+ }
+
+ std::vector>> txValidationResult(
+ txs.size());
+ auto& appConnector = app.getAppConnector();
+
+ app.getBatchExecutor().executeBatchOverRanges(
+ txs.size(), taskCount,
+ [&appConnector, &txs, &ledgerViews, &txValidationResult,
+ lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset,
+ validationLedgerSeq](size_t begin, size_t end, size_t rangeIndex) {
+ auto const& view = *ledgerViews.at(rangeIndex);
+ auto const header = view.getLedgerHeader().current();
+ auto diagnostics = DiagnosticEventManager::createDisabled();
+ for (size_t i = begin; i < end; ++i)
+ {
+ auto res = txs[i]->checkValid(appConnector, view, 0,
+ lowerBoundCloseTimeOffset,
+ upperBoundCloseTimeOffset,
+ diagnostics, validationLedgerSeq);
+ txValidationResult[i].first = res->isSuccess();
+ if (!res->isSuccess())
+ {
+ continue;
+ }
+ auto feeSource = view.getAccount(txs[i]->getFeeSourceID());
+ if (feeSource)
+ {
+ txValidationResult[i].second =
+ getAvailableBalance(header, feeSource.current());
+ }
+ }
+ });
+
TxFrameListWithErrors invalidTxsWithError;
- auto& invalidTxs = invalidTxsWithError.first;
- auto& errorCode = invalidTxsWithError.second;
+ auto& [invalidTxs, errorCode] = invalidTxsWithError;
errorCode = TxSetValidationResult::VALID;
- std::unordered_set seenInvalidTxs;
- auto diagnostics = DiagnosticEventManager::createDisabled();
- for (auto const& tx : txs)
+ for (size_t i = 0; i < txs.size(); ++i)
{
- auto txResult = tx->checkValid(
- app.getAppConnector(), ledgerView, 0, lowerBoundCloseTimeOffset,
- upperBoundCloseTimeOffset, diagnostics, validationLedgerSeq);
- if (!txResult->isSuccess())
+ auto const& tx = txs[i];
+ auto const& [txIsValid, feeSourceBalance] = txValidationResult[i];
+ if (!txIsValid)
{
invalidTxs.emplace_back(tx);
- seenInvalidTxs.emplace(tx->getFullHash());
errorCode = TxSetValidationResult::TX_VALIDATION_FAILED;
+ continue;
}
- else
+ int64_t& accFee = accountFeeMap[tx->getFeeSourceID()];
+ if (INT64_MAX - accFee < tx->getFullFee())
{
- int64_t& accFee = accountFeeMap[tx->getFeeSourceID()];
- if (INT64_MAX - accFee < tx->getFullFee())
- {
- accFee = INT64_MAX;
- }
- else
- {
- accFee += tx->getFullFee();
- }
+ accFee = INT64_MAX;
}
- }
-
- auto header = ledgerView.getLedgerHeader().current();
- for (auto const& tx : txs)
- {
- // Already added invalid tx
- if (seenInvalidTxs.find(tx->getFullHash()) != seenInvalidTxs.end())
+ else
{
- continue;
+ accFee += tx->getFullFee();
}
-
- auto feeSourceID = tx->getFeeSourceID();
- auto feeSource = ledgerView.getAccount(feeSourceID);
- // feeSource should exist since we've already run checkValid, log
- // internal bug
- if (!feeSource)
+ // `feeSourceBalance` should exist as transaction must be valid, log
+ // an internal error and skip the transaction otherwise.
+ if (!feeSourceBalance)
{
CLOG_ERROR(Herder,
"Account not found when checking TxSet validity");
CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG);
+ invalidTxs.emplace_back(tx);
+ errorCode = TxSetValidationResult::TX_VALIDATION_FAILED;
continue;
}
- auto it = accountFeeMap.find(feeSourceID);
- auto totFee = it->second;
- if (getAvailableBalance(header, feeSource.current()) < totFee)
+ if (*feeSourceBalance < accFee)
{
invalidTxs.push_back(tx);
- // Only override the error code if it wasn't already set
+ // Only override the error code if it wasn't already set.
if (errorCode == TxSetValidationResult::VALID)
{
errorCode = TxSetValidationResult::ACCOUNT_CANT_PAY_FEE;
}
- releaseAssert(seenInvalidTxs.insert(tx->getFullHash()).second);
CLOG_DEBUG(
Herder, "Got bad txSet: account can't pay fee tx: {}",
xdrToCerealString(tx->getEnvelope(), "TransactionEnvelope"));
diff --git a/src/herder/TxSetUtils.h b/src/herder/TxSetUtils.h
index d744dc9476..160ae09eb9 100644
--- a/src/herder/TxSetUtils.h
+++ b/src/herder/TxSetUtils.h
@@ -47,6 +47,9 @@ class TxSetUtils
// Returns transactions from a TxSet that are invalid along with the
// aggregate validation result for the set.
+ // NB: this assumes that all the transaction envelopes are unique in `txs`
+ // (which is always the case, as the envelopes come either from tx queue,
+ // or generated from TxSet XDR).
template
static TxFrameListWithErrors
getInvalidTxListWithErrors(TxContainer const& txs, Application& app,
diff --git a/src/herder/Upgrades.cpp b/src/herder/Upgrades.cpp
index a4224cd320..2e4eb633fe 100644
--- a/src/herder/Upgrades.cpp
+++ b/src/herder/Upgrades.cpp
@@ -176,7 +176,7 @@ namespace stellar
namespace
{
uint32_t
-readMaxSorobanTxSetSize(CheckValidLedgerViewWrapper const& ledgerView)
+readMaxSorobanTxSetSize(AbstractLedgerView const& ledgerView)
{
LedgerKey key(LedgerEntryType::CONFIG_SETTING);
key.configSetting().configSettingID =
@@ -214,7 +214,7 @@ Upgrades::UpgradeParameters::toJson() const
std::string
Upgrades::UpgradeParameters::toDebugJson(
- CheckValidLedgerViewWrapper const& ledgerView) const
+ AbstractLedgerView const& ledgerView) const
{
Json::Value upgradesJson;
Json::Reader reader;
@@ -280,11 +280,11 @@ Upgrades::getParameters() const
std::vector
Upgrades::createUpgradesFor(LedgerHeader const& lclHeader,
- CheckValidLedgerViewWrapper const& ledgerView,
+ AbstractLedgerView const& ledgerView,
Config const& appCfg) const
{
auto result = std::vector{};
- if (!timeForUpgrade(lclHeader.scpValue.closeTime))
+ if (!timeForUpgrade(getApplyTime(lclHeader.scpValue)))
{
return result;
}
@@ -370,9 +370,8 @@ Upgrades::applyTo(LedgerUpgrade const& upgrade, Application& app,
break;
case LEDGER_UPGRADE_CONFIG:
{
- CheckValidLedgerViewWrapper ltxState(ltx);
- auto cfgUpgrade =
- ConfigUpgradeSetFrame::makeFromKey(ltxState, upgrade.newConfig());
+ auto cfgUpgrade = ConfigUpgradeSetFrame::makeFromKey(
+ ltx.makeView(), upgrade.newConfig());
if (!cfgUpgrade)
{
throw std::runtime_error(
@@ -472,7 +471,7 @@ Upgrades::toString() const
Upgrades::UpgradeParameters
Upgrades::removeUpgrades(std::vector::const_iterator beginUpdates,
std::vector::const_iterator endUpdates,
- uint64_t closeTime, bool& updated)
+ ApplyTime closeTime, bool& updated)
{
updated = false;
UpgradeParameters res = mParams;
@@ -482,7 +481,7 @@ Upgrades::removeUpgrades(std::vector::const_iterator beginUpdates,
// upgrades don't attempt to change the network
if (res.mUpgradeTime + res.mExpirationMinutes.value_or(
DEFAULT_UPGRADE_EXPIRATION_MINUTES) <=
- VirtualClock::from_time_t(closeTime))
+ VirtualClock::from_time_t(closeTime.timePoint()))
{
auto resetParamIfSet = [&](auto& o) {
if (o)
@@ -570,7 +569,7 @@ Upgrades::removeUpgrades(std::vector::const_iterator beginUpdates,
Upgrades::UpgradeValidity
Upgrades::isValidForApply(UpgradeType const& opaqueUpgrade,
LedgerUpgrade& upgrade, Application& app,
- CheckValidLedgerViewWrapper const& ledgerView)
+ AbstractLedgerView const& ledgerView)
{
try
{
@@ -643,12 +642,11 @@ Upgrades::isValidForApply(UpgradeType const& opaqueUpgrade,
}
bool
-Upgrades::isValidForNomination(
- LedgerUpgrade const& upgrade,
- CheckValidLedgerViewWrapper const& ledgerView) const
+Upgrades::isValidForNomination(LedgerUpgrade const& upgrade,
+ AbstractLedgerView const& ledgerView) const
{
if (!timeForUpgrade(
- ledgerView.getLedgerHeader().current().scpValue.closeTime))
+ getApplyTime(ledgerView.getLedgerHeader().current().scpValue)))
{
return false;
}
@@ -695,13 +693,13 @@ Upgrades::isValid(UpgradeType const& upgrade, LedgerUpgradeType& upgradeType,
bool nomination, Application& app) const
{
LedgerUpgrade lupgrade;
- auto ledgerView = CheckValidLedgerViewWrapper(app);
- bool res = isValidForApply(upgrade, lupgrade, app, ledgerView) ==
+ auto ledgerView = app.getLedgerManager().getLCLView();
+ bool res = isValidForApply(upgrade, lupgrade, app, *ledgerView) ==
UpgradeValidity::VALID;
if (nomination)
{
- res = res && isValidForNomination(lupgrade, ledgerView);
+ res = res && isValidForNomination(lupgrade, *ledgerView);
}
if (res)
@@ -712,9 +710,9 @@ Upgrades::isValid(UpgradeType const& upgrade, LedgerUpgradeType& upgradeType,
}
bool
-Upgrades::timeForUpgrade(uint64_t time) const
+Upgrades::timeForUpgrade(ApplyTime time) const
{
- return mParams.mUpgradeTime <= VirtualClock::from_time_t(time);
+ return mParams.mUpgradeTime <= VirtualClock::from_time_t(time.timePoint());
}
void
@@ -870,7 +868,8 @@ updateOffer(
// liabilities for this offer, so the only applicable limit is the
// offer amount. We then use adjustOffer to check that it will
// satisfy thresholds.
- if (!erase && adjustOffer(offer.price, offer.amount, INT64_MAX) == 0)
+ if (!erase && adjustOffer(header.current().ledgerVersion, offer.price,
+ offer.amount, INT64_MAX) == 0)
{
erase = true;
res = UpdateOfferResult::AdjustedToZero;
@@ -881,7 +880,8 @@ updateOffer(
// The same logic for adjustOffer discussed above applies here,
// except that we now actually update the offer to reflect the
// adjustment.
- auto adjAmount = adjustOffer(offer.price, offer.amount, INT64_MAX);
+ auto adjAmount = adjustOffer(header.current().ledgerVersion,
+ offer.price, offer.amount, INT64_MAX);
if (adjAmount != offer.amount)
{
offer.amount = adjAmount;
@@ -1251,7 +1251,6 @@ Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx,
}
if (needUpgradeToVersion(ProtocolVersion::V_25, prevVersion, newVersion))
{
- PubKeyUtils::enableRustDalekVerify();
SorobanNetworkConfig::createCostTypesForV25(ltx, app);
}
if (needUpgradeToVersion(ProtocolVersion::V_26, prevVersion, newVersion))
@@ -1259,6 +1258,10 @@ Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx,
SorobanNetworkConfig::updateCostTypesForV26(ltx, app);
SorobanNetworkConfig::createLedgerEntriesForV26(ltx, app);
}
+ if (needUpgradeToVersion(ProtocolVersion::V_30, prevVersion, newVersion))
+ {
+ SorobanNetworkConfig::createCostTypesForV30(ltx, app);
+ }
if (protocolVersionEquals(prevVersion, ProtocolVersion::V_23) &&
protocolVersionEquals(newVersion, ProtocolVersion::V_24) &&
@@ -1298,9 +1301,8 @@ Upgrades::applyReserveUpgrade(AbstractLedgerTxn& ltx, uint32_t newReserve)
}
ConfigUpgradeSetFrameConstPtr
-ConfigUpgradeSetFrame::makeFromKey(
- CheckValidLedgerViewWrapper const& ledgerView,
- ConfigUpgradeSetKey const& key)
+ConfigUpgradeSetFrame::makeFromKey(AbstractLedgerView const& ledgerView,
+ ConfigUpgradeSetKey const& key)
{
auto lk = ConfigUpgradeSetFrame::getLedgerKey(key);
auto ltxe = ledgerView.load(lk);
@@ -1424,8 +1426,7 @@ ConfigUpgradeSetFrame::getLedgerKey(ConfigUpgradeSetKey const& upgradeKey)
}
bool
-ConfigUpgradeSetFrame::upgradeNeeded(
- CheckValidLedgerViewWrapper const& ledgerView) const
+ConfigUpgradeSetFrame::upgradeNeeded(AbstractLedgerView const& ledgerView) const
{
if (protocolVersionIsBefore(
ledgerView.getLedgerHeader().current().ledgerVersion,
diff --git a/src/herder/Upgrades.h b/src/herder/Upgrades.h
index b0bc8feb7b..515f496555 100644
--- a/src/herder/Upgrades.h
+++ b/src/herder/Upgrades.h
@@ -16,11 +16,12 @@
namespace stellar
{
class AbstractLedgerTxn;
+class ApplyTime;
class Config;
class Database;
struct LedgerHeader;
struct LedgerUpgrade;
-class CheckValidLedgerViewWrapper;
+class AbstractLedgerView;
class ConfigUpgradeSetFrame;
using ConfigUpgradeSetFrameConstPtr =
@@ -70,8 +71,7 @@ class Upgrades
std::string toJson() const;
void fromJson(std::string const& s);
- std::string
- toDebugJson(CheckValidLedgerViewWrapper const& ledgerView) const;
+ std::string toDebugJson(AbstractLedgerView const& ledgerView) const;
};
Upgrades()
@@ -86,7 +86,7 @@ class Upgrades
// create upgrades for given ledger
std::vector
createUpgradesFor(LedgerHeader const& lclHeader,
- CheckValidLedgerViewWrapper const& ledgerView,
+ AbstractLedgerView const& ledgerView,
Config const& appCfg) const;
// apply upgrade to ledger header
@@ -110,8 +110,7 @@ class Upgrades
// If the upgrade could be deserialized then lupgrade is set
static UpgradeValidity
isValidForApply(UpgradeType const& upgrade, LedgerUpgrade& lupgrade,
- Application& app,
- CheckValidLedgerViewWrapper const& ledgerView);
+ Application& app, AbstractLedgerView const& ledgerView);
// returns true if upgrade is a valid upgrade step
// in which case it also sets upgradeType
@@ -126,20 +125,19 @@ class Upgrades
UpgradeParameters
removeUpgrades(std::vector::const_iterator beginUpdates,
std::vector::const_iterator endUpdates,
- uint64_t time, bool& updated);
+ ApplyTime closeTime, bool& updated);
static void maybeDropAndCreateNew(Database& db);
private:
UpgradeParameters mParams;
- bool timeForUpgrade(uint64_t time) const;
+ bool timeForUpgrade(ApplyTime time) const;
// returns true if upgrade is a valid upgrade step
// in which case it also sets lupgrade
- bool
- isValidForNomination(LedgerUpgrade const& upgrade,
- CheckValidLedgerViewWrapper const& ledgerView) const;
+ bool isValidForNomination(LedgerUpgrade const& upgrade,
+ AbstractLedgerView const& ledgerView) const;
static void applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx,
uint32_t newVersion);
@@ -159,7 +157,7 @@ class ConfigUpgradeSetFrame
{
public:
static ConfigUpgradeSetFrameConstPtr
- makeFromKey(CheckValidLedgerViewWrapper const& ledgerView,
+ makeFromKey(AbstractLedgerView const& ledgerView,
ConfigUpgradeSetKey const& key);
static LedgerKey getLedgerKey(ConfigUpgradeSetKey const& upgradeKey);
@@ -168,7 +166,7 @@ class ConfigUpgradeSetFrame
ConfigUpgradeSetKey const& getKey() const;
- bool upgradeNeeded(CheckValidLedgerViewWrapper const& ledgerView) const;
+ bool upgradeNeeded(AbstractLedgerView const& ledgerView) const;
void applyTo(AbstractLedgerTxn& ltx, Application& app) const;
diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp
index 1b78c6fe3b..4fc6d21658 100644
--- a/src/herder/test/HerderTests.cpp
+++ b/src/herder/test/HerderTests.cpp
@@ -26,9 +26,11 @@
#include "crypto/SHA.h"
#include "database/Database.h"
#include "herder/HerderUtils.h"
+#include "ledger/LedgerHeaderUtils.h"
#include "ledger/LedgerManager.h"
#include "ledger/LedgerTxn.h"
#include "ledger/LedgerTxnHeader.h"
+#include "lib/json/json.h"
#include "main/CommandHandler.h"
#include "main/PersistentState.h"
#include "overlay/OverlayMetrics.h"
@@ -55,6 +57,7 @@
#include "xdrpp/autocheck.h"
#include "xdrpp/marshal.h"
#include
+#include
#include
#include
#include
@@ -150,7 +153,8 @@ testTxSet(uint32 protocolVersion)
SECTION("valid set")
{
- auto txSet = makeTxSetFromTransactions(txs, *app, 0, 0).second;
+ auto txSet =
+ makeTxSetFromTransactions(txs, *app, ApplyTimeOffset{}).second;
REQUIRE(txSet->sizeTxTotal() == nbAccounts);
REQUIRE(txSet->checkValidWithResult(*app, 0, 0) ==
TxSetValidationResult::VALID);
@@ -162,7 +166,8 @@ testTxSet(uint32 protocolVersion)
{
genTx();
}
- auto txSet = makeTxSetFromTransactions(txs, *app, 0, 0).second;
+ auto txSet =
+ makeTxSetFromTransactions(txs, *app, ApplyTimeOffset{}).second;
REQUIRE(txSet->sizeTxTotal() == cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE);
REQUIRE(txSet->checkValidWithResult(*app, 0, 0) ==
TxSetValidationResult::VALID);
@@ -170,7 +175,7 @@ testTxSet(uint32 protocolVersion)
SECTION("invalid tx")
{
auto diagnostics = DiagnosticEventManager::createDisabled();
- CheckValidLedgerViewWrapper ledgerView(*app);
+ auto ledgerView = app->getLedgerManager().getLCLView();
SECTION("no user")
{
@@ -180,15 +185,16 @@ testTxSet(uint32 protocolVersion)
// Individual tx check: account doesn't exist
REQUIRE(badTx
- ->checkValid(app->getAppConnector(), ledgerView, 0, 0,
+ ->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
0, diagnostics)
->getResultCode() == txNO_ACCOUNT);
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions(txs, *app, 0, 0, removed).second;
+ auto txSet = makeTxSetFromTransactions(
+ txs, *app, ApplyTimeOffset{}, removed)
+ .second;
REQUIRE(removed.size() == 1);
REQUIRE(removed.back() == badTx);
REQUIRE(txSet->sizeTxTotal() == nbAccounts);
@@ -210,15 +216,16 @@ testTxSet(uint32 protocolVersion)
// Individual tx check: bad sequence number
REQUIRE(badTx
- ->checkValid(app->getAppConnector(), ledgerView, 0, 0,
+ ->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
0, diagnostics)
->getResultCode() == txBAD_SEQ);
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions(txs, *app, 0, 0, removed).second;
+ auto txSet = makeTxSetFromTransactions(
+ txs, *app, ApplyTimeOffset{}, removed)
+ .second;
REQUIRE(removed.size() == 1);
REQUIRE(removed.back() == badTx);
REQUIRE(txSet->sizeTxTotal() == nbAccounts - 1);
@@ -241,17 +248,18 @@ testTxSet(uint32 protocolVersion)
// Individual tx check: insufficient balance
// Need fresh snapshot after account creation
- CheckValidLedgerViewWrapper lsNew(*app);
+ auto lsNew = app->getLedgerManager().getLCLView();
REQUIRE(badTx
- ->checkValid(app->getAppConnector(), lsNew, 0, 0, 0,
+ ->checkValid(app->getAppConnector(), *lsNew, 0, 0, 0,
diagnostics)
->getResultCode() == txINSUFFICIENT_BALANCE);
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions(txs, *app, 0, 0, removed).second;
+ auto txSet = makeTxSetFromTransactions(
+ txs, *app, ApplyTimeOffset{}, removed)
+ .second;
REQUIRE(removed.size() == 1);
REQUIRE(removed.back() == badTx);
REQUIRE(txSet->sizeTxTotal() == nbAccounts - 1);
@@ -275,15 +283,16 @@ testTxSet(uint32 protocolVersion)
// Individual tx check: bad auth (signature invalidated by maxTime
// change)
REQUIRE(badTx
- ->checkValid(app->getAppConnector(), ledgerView, 0, 0,
+ ->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
0, diagnostics)
->getResultCode() == txBAD_AUTH);
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions(txs, *app, 0, 0, removed).second;
+ auto txSet = makeTxSetFromTransactions(
+ txs, *app, ApplyTimeOffset{}, removed)
+ .second;
REQUIRE(removed.size() == 1);
REQUIRE(removed.back() == badTx);
REQUIRE(txSet->sizeTxTotal() == nbAccounts - 1);
@@ -447,7 +456,7 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
};
auto diagnostics = DiagnosticEventManager::createDisabled();
- CheckValidLedgerViewWrapper ledgerView(*app);
+ auto ledgerView = app->getLedgerManager().getLCLView();
SECTION("invalid transaction")
{
@@ -457,15 +466,15 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
auto fb1 = feeBump(*app, account2, tx1, minBalance2);
// Individual tx check: fee bump exceeds fee source balance
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->getResultCode() == txINSUFFICIENT_BALANCE);
SECTION("build block")
{
TxFrameList invalidTxs;
- auto txSet =
- makeTxSetFromTransactions({fb1}, *app, 0, 0, invalidTxs);
+ auto txSet = makeTxSetFromTransactions(
+ {fb1}, *app, ApplyTimeOffset{}, invalidTxs);
compareTxs(invalidTxs, {fb1});
}
SECTION("validate block")
@@ -483,18 +492,18 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
auto fb2 = feeBump(*app, account2, tx2, 200);
// Individual tx checks: first exceeds balance, second is valid
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->getResultCode() == txINSUFFICIENT_BALANCE);
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb2->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
SECTION("build block")
{
TxFrameList invalidTxs;
- auto txSet = makeTxSetFromTransactions({fb1, fb2}, *app, 0, 0,
- invalidTxs);
+ auto txSet = makeTxSetFromTransactions(
+ {fb1, fb2}, *app, ApplyTimeOffset{}, invalidTxs);
// fb1 is rejected
compareTxs(invalidTxs, {fb1});
}
@@ -511,18 +520,18 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
auto tx2 = transaction(*app, account3, 1, 1, 100);
auto fb2 = feeBump(*app, account2, tx2, minBalance2);
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb2->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->getResultCode() == txINSUFFICIENT_BALANCE);
SECTION("build block")
{
TxFrameList invalidTxs;
- auto txSet = makeTxSetFromTransactions({fb1, fb2}, *app, 0, 0,
- invalidTxs);
+ auto txSet = makeTxSetFromTransactions(
+ {fb1, fb2}, *app, ApplyTimeOffset{}, invalidTxs);
compareTxs(invalidTxs, {fb2});
}
SECTION("validate block")
@@ -540,18 +549,18 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
auto fb2 = feeBump(*app, account2, tx2, 200);
// Individual tx checks
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb2->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->getResultCode() == txFEE_BUMP_INNER_FAILED);
SECTION("build block")
{
TxFrameList invalidTxs;
- auto txSet = makeTxSetFromTransactions({fb1, fb2}, *app, 0, 0,
- invalidTxs);
+ auto txSet = makeTxSetFromTransactions(
+ {fb1, fb2}, *app, ApplyTimeOffset{}, invalidTxs);
compareTxs(invalidTxs, {fb2});
}
SECTION("validate block")
@@ -572,23 +581,23 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
feeBump(*app, account2, tx3, minBalance2 - minBalance0 - 199);
// Individual tx checks
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb2->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->getResultCode() == txFEE_BUMP_INNER_FAILED);
// Individually, fb2 is valid, but with fb1 it would exceed balance
- REQUIRE(fb3->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb3->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
SECTION("build block")
{
TxFrameList invalidTxs;
- auto txSet = makeTxSetFromTransactions({fb1, fb2, fb3}, *app, 0,
- 0, invalidTxs);
- compareTxs(invalidTxs, {fb1, fb2, fb3});
+ auto txSet = makeTxSetFromTransactions(
+ {fb1, fb2, fb3}, *app, ApplyTimeOffset{}, invalidTxs);
+ compareTxs(invalidTxs, {fb2, fb3});
}
SECTION("validate block")
{
@@ -600,10 +609,10 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
SECTION("two fee bumps, same fee source, valid individually, combined "
"exceed balance")
{
- CheckValidLedgerViewWrapper ledgerView(*app);
+ auto ledgerView = app->getLedgerManager().getLCLView();
auto balanceOfFbAccount = getAvailableBalance(
- ledgerView.getLedgerHeader().current(),
- ledgerView.getAccount(account2.getPublicKey()).current());
+ ledgerView->getLedgerHeader().current(),
+ ledgerView->getAccount(account2.getPublicKey()).current());
// Enforce balance invariance
int64_t fee1 = 200;
@@ -619,21 +628,19 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
auto fb2 = feeBump(*app, account2, tx2, fee2);
// Individual txs are valid
auto diagnostics = DiagnosticEventManager::createDisabled();
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb2->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
TxFrameList invalidTxs;
SECTION("build block")
{
- auto txSet = makeTxSetFromTransactions({fb1, fb2}, *app, 0, 0,
- invalidTxs);
- // Both are marked invalid because their combined fees exceed
- // account2's balance
- compareTxs(invalidTxs, {fb1, fb2});
+ auto txSet = makeTxSetFromTransactions(
+ {fb1, fb2}, *app, ApplyTimeOffset{}, invalidTxs);
+ compareTxs(invalidTxs, {fb2});
}
SECTION("validate block")
{
@@ -672,10 +679,10 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
createUploadWasmTx(*app, sorobanAccount2, 100,
DEFAULT_TEST_RESOURCE_FEE, resources);
- CheckValidLedgerViewWrapper ledgerView(*app);
+ auto ledgerView = app->getLedgerManager().getLCLView();
auto balanceOfFbAccount = getAvailableBalance(
- ledgerView.getLedgerHeader().current(),
- ledgerView.getAccount(feeSourceAccount.getPublicKey())
+ ledgerView->getLedgerHeader().current(),
+ ledgerView->getAccount(feeSourceAccount.getPublicKey())
.current());
// Set fees so that each is valid individually but combined they
@@ -694,22 +701,20 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
// Individual txs are valid
auto diagnostics = DiagnosticEventManager::createDisabled();
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb1->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
+ REQUIRE(fb2->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
+ 0, diagnostics)
->isSuccess());
PerPhaseTransactionList invalidPerPhase;
invalidPerPhase.resize(2);
SECTION("build block")
{
- auto txSet = makeTxSetFromTransactions({{}, {fb1, fb2}}, *app,
- 0, 0, invalidPerPhase);
- // Both are marked invalid because their combined fees exceed
- // feeSourceAccount's balance
- compareTxs(invalidPerPhase[1], {fb1, fb2});
+ auto txSet = makeTxSetFromTransactions(
+ {{}, {fb1, fb2}}, *app, ApplyTimeOffset{}, invalidPerPhase);
+ compareTxs(invalidPerPhase[1], {fb2});
}
SECTION("validate block")
{
@@ -766,11 +771,11 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
auto diagnostics = DiagnosticEventManager::createDisabled();
REQUIRE(classicFb
- ->checkValid(app->getAppConnector(), ledgerView, 0, 0,
+ ->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
0, diagnostics)
->isSuccess());
REQUIRE(sorobanFb
- ->checkValid(app->getAppConnector(), ledgerView, 0, 0,
+ ->checkValid(app->getAppConnector(), *ledgerView, 0, 0,
0, diagnostics)
->isSuccess());
@@ -779,7 +784,8 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
SECTION("build block")
{
auto txSet = makeTxSetFromTransactions(
- {{classicFb}, {sorobanFb}}, *app, 0, 0, invalidPerPhase);
+ {{classicFb}, {sorobanFb}}, *app, ApplyTimeOffset{},
+ invalidPerPhase);
compareTxs(invalidPerPhase[0], {});
compareTxs(invalidPerPhase[1], {sorobanFb});
}
@@ -808,91 +814,128 @@ testTxSetWithFeeBumps(uint32 protocolVersion)
}
}
-TEST_CASE("getInvalidTxListWithErrors returns no duplicates")
+TEST_CASE("tx set validation rejects txs that fee source cannot pay for",
+ "[txset]")
{
Config cfg(getTestConfig());
VirtualClock clock;
Application::pointer app = createTestApplication(clock, cfg);
- auto const minBalance2 = app->getLedgerManager().getLastMinBalance(2);
auto root = app->getRoot();
- // Create accounts for tx sources and fee source
- auto account1 = root->create("a1", minBalance2);
- auto account2 = root->create("a2", minBalance2);
- auto account3 = root->create("a3", minBalance2);
- auto account4 = root->create("a4", minBalance2);
-
- CheckValidLedgerViewWrapper ledgerView(*app);
- auto balanceOfFeeSource = getAvailableBalance(
- ledgerView.getLedgerHeader().current(),
- ledgerView.getAccount(account2.getPublicKey()).current());
-
- // Create three fee bumps from account2 (fee source):
- // - fb1: fails checkValid (bad sequence number)
- // - fb2: passes checkValid
- // - fb3: passes checkValid
- // Combined fees of fb2 + fb3 exceed balance, so both should be invalid
- // fb1 is invalid due to checkValid failure
- // This tests that fb1 doesn't appear twice (once from checkValid fail,
- // once from fee check)
- int64_t fee1 = 200;
- int64_t fee2 = balanceOfFeeSource / 2 + 100;
- int64_t fee3 = balanceOfFeeSource / 2 + 100;
-
- // fb1: Bad seqNum to ensure it fails checkValid
- auto tx1 = transactionFromOperations(
- *app, account1, 555, {payment(account1.getPublicKey(), 1)}, 100);
- auto fb1 = feeBump(*app, account2, tx1, fee1);
-
- // fb2 and fb3: Valid transactions
- auto tx2 = transactionFromOperations(
- *app, account3, account3.getLastSequenceNumber() + 1,
- {payment(account3.getPublicKey(), 1)}, 100);
- auto fb2 = feeBump(*app, account2, tx2, fee2);
-
- auto tx3 = transactionFromOperations(
- *app, account4, account4.getLastSequenceNumber() + 1,
- {payment(account4.getPublicKey(), 1)}, 100);
- auto fb3 = feeBump(*app, account2, tx3, fee3);
-
- // Verify fb1 fails checkValid - inner tx has bad sequence number
- auto diagnostics = DiagnosticEventManager::createDisabled();
- REQUIRE(fb1->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
- ->getResultCode() == txFEE_BUMP_INNER_FAILED);
- // Verify fb2 and fb3 pass checkValid individually
- REQUIRE(fb2->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
- ->isSuccess());
- REQUIRE(fb3->checkValid(app->getAppConnector(), ledgerView, 0, 0, 0,
- diagnostics)
- ->isSuccess());
-
- // Verify combined fees of fb2 + fb3 exceed balance
- REQUIRE(fb2->getFullFee() + fb3->getFullFee() > balanceOfFeeSource);
- // But each individual fee is payable
- REQUIRE(fb2->getFullFee() < balanceOfFeeSource);
- REQUIRE(fb3->getFullFee() < balanceOfFeeSource);
-
- TxFrameList txs = {fb1, fb2, fb3};
- UnorderedMap accountFeeMap;
- auto invalidTxs =
- TxSetUtils::getInvalidTxListWithErrors(txs, *app, accountFeeMap, 0, 0)
- .first;
-
- // Check for no duplicates by comparing size with unique count
- std::unordered_set uniqueHashes;
- for (auto const& tx : invalidTxs)
- {
- uniqueHashes.insert(tx->getFullHash());
- }
- REQUIRE(invalidTxs.size() == uniqueHashes.size());
-
- // All 3 txs should be invalid:
- // - fb1: fails checkValid
- // - fb2, fb3: can't pay combined fees
- REQUIRE(invalidTxs.size() == 3);
+ auto feeSource = root->create(
+ "feeSource", app->getLedgerManager().getLastMinBalance(1) * 10);
+ auto account1 =
+ root->create("a1", app->getLedgerManager().getLastMinBalance(1) * 20);
+ auto account2 =
+ root->create("a2", app->getLedgerManager().getLastMinBalance(1) * 30);
+
+ auto ledgerView = app->getLedgerManager().getLCLView();
+ int64_t const feeSourceBalance = getAvailableBalance(
+ ledgerView->getLedgerHeader().current(),
+ ledgerView->getAccount(feeSource.getPublicKey()).current());
+
+ auto makePayment = [&](TestAccount& source, int64_t fee,
+ bool isValid = true) {
+ SequenceNumber seq = isValid ? source.getLastSequenceNumber() + 1
+ : source.getLastSequenceNumber() + 100;
+ return transactionFromOperations(*app, source.getSecretKey(), seq,
+ {payment(source.getPublicKey(), 1)},
+ static_cast(fee));
+ };
+ auto makeFeeBump = [&](TestAccount& source, int64_t fee,
+ bool isValid = true) {
+ return feeBump(*app, feeSource, makePayment(source, 100, isValid), fee);
+ };
+
+ // NB: the exact contents of the invalid transaction list are partially an
+ // implementation detail. Protocol only observes whether the list is
+ // non-empty, so any implementation that returns at least one invalid
+ // transaction for every invalid tx set is consistent (as long as it agrees
+ // on what an 'invalid' transaction is).
+ // Nomination logic requires `getInvalidTxListWithErrors` to return at least
+ // the minimum subset of invalid transactions (such that the rest of the
+ // transactions are valid), but returning more than that is not observable
+ // by the protocol.
+ auto checkInvalidTxs = [&](TxFrameList const& txs,
+ UnorderedMap accountFeeMap,
+ TxFrameList const& expectedInvalidTxs,
+ TxSetValidationResult expectedResult) {
+ auto const [invalidTxs, result] =
+ TxSetUtils::getInvalidTxListWithErrors(txs, *app, accountFeeMap, 0,
+ 0);
+ REQUIRE(result == expectedResult);
+ REQUIRE(invalidTxs.size() == expectedInvalidTxs.size());
+ for (size_t i = 0; i < expectedInvalidTxs.size(); ++i)
+ {
+ REQUIRE(invalidTxs[i]->getFullHash() ==
+ expectedInvalidTxs[i]->getFullHash());
+ }
+ };
+
+ SECTION("txs that fit into the balance are accepted")
+ {
+ auto tx1 = makePayment(feeSource, feeSourceBalance / 2);
+ auto tx2 = makeFeeBump(account1, feeSourceBalance / 2);
+ checkInvalidTxs({tx1, tx2}, {}, {}, TxSetValidationResult::VALID);
+ }
+
+ SECTION("tx that is both invalid and can't pay its fee is reported once")
+ {
+ auto tx = makePayment(feeSource, feeSourceBalance + 100,
+ /* isValid */ false);
+ checkInvalidTxs({tx}, {}, {tx},
+ TxSetValidationResult::TX_VALIDATION_FAILED);
+ }
+
+ SECTION("fee bumps with a shared fee source")
+ {
+ auto invalidFb = makeFeeBump(account1, 200, /* isValid */ false);
+ auto fb1 = makeFeeBump(account1, feeSourceBalance / 2 + 1);
+ // fb2 doesn't fit into the fee source balance.
+ auto fb2 = makeFeeBump(account2, feeSourceBalance / 2);
+
+ checkInvalidTxs({fb1, fb2}, {}, {fb2},
+ TxSetValidationResult::ACCOUNT_CANT_PAY_FEE);
+ checkInvalidTxs({invalidFb, fb1, fb2}, {}, {invalidFb, fb2},
+ TxSetValidationResult::TX_VALIDATION_FAILED);
+ checkInvalidTxs({fb2, invalidFb, fb1}, {}, {invalidFb, fb1},
+ TxSetValidationResult::TX_VALIDATION_FAILED);
+ checkInvalidTxs({fb1, fb2, invalidFb}, {}, {fb2, invalidFb},
+ TxSetValidationResult::TX_VALIDATION_FAILED);
+ }
+
+ SECTION("fee bumps mixed with regular txs")
+ {
+ auto tx = makePayment(feeSource, feeSourceBalance - 1000);
+ auto fb = makeFeeBump(account1, 1001);
+ REQUIRE(tx->getFeeSourceID() == fb->getFeeSourceID());
+
+ checkInvalidTxs({tx, fb}, {}, {fb},
+ TxSetValidationResult::ACCOUNT_CANT_PAY_FEE);
+ checkInvalidTxs({fb, tx}, {}, {tx},
+ TxSetValidationResult::ACCOUNT_CANT_PAY_FEE);
+ }
+
+ SECTION("account fee map is accounted for")
+ {
+ auto tx = makePayment(feeSource, 200);
+
+ SECTION("tx still fits into the balance")
+ {
+ UnorderedMap accountFeeMap = {
+ {feeSource.getPublicKey(), feeSourceBalance - 200}};
+ checkInvalidTxs({tx}, accountFeeMap, {},
+ TxSetValidationResult::VALID);
+ }
+ SECTION("tx no longer fits into the balance")
+ {
+ UnorderedMap accountFeeMap = {
+ {feeSource.getPublicKey(), feeSourceBalance - 199}};
+ checkInvalidTxs({tx}, accountFeeMap, {tx},
+ TxSetValidationResult::ACCOUNT_CANT_PAY_FEE);
+ }
+ }
}
TEST_CASE("txset", "[herder][txset]")
@@ -962,9 +1005,9 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
auto txInvalid = transactionWithV2Precondition(
*app, a1, 1, 100, minSeqAgeCond(minGap + 1));
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({txInvalid}, *app, 0, 0, removed)
- .second;
+ auto txSet = makeTxSetFromTransactions({txInvalid}, *app,
+ ApplyTimeOffset{}, removed)
+ .second;
REQUIRE(removed.back() == txInvalid);
REQUIRE(txSet->sizeTxTotal() == 0);
@@ -994,14 +1037,14 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
removed.clear();
if (minSeqNumTxIsFeeBump)
{
- txSet = makeTxSetFromTransactions({fb1, fb2Invalid}, *app, 0, 0,
- removed)
+ txSet = makeTxSetFromTransactions({fb1, fb2Invalid}, *app,
+ ApplyTimeOffset{}, removed)
.second;
}
else
{
- txSet = makeTxSetFromTransactions({tx1, tx2Invalid}, *app, 0, 0,
- removed)
+ txSet = makeTxSetFromTransactions({tx1, tx2Invalid}, *app,
+ ApplyTimeOffset{}, removed)
.second;
}
@@ -1063,8 +1106,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
SECTION("build block")
{
TxFrameList removed;
- auto txSet = makeTxSetFromTransactions({tx1, txInvalid}, *app,
- 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx1, txInvalid}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.back() == txInvalid);
}
SECTION("validate block")
@@ -1086,8 +1129,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
auto tx2 = transactionWithV2Precondition(
*app, a2, 1, 100, ledgerBoundsCond(lclNum + 1, 0));
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx1, tx2}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions({tx1, tx2}, *app,
+ ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
SECTION("maxLedger")
@@ -1097,8 +1140,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
SECTION("build block")
{
TxFrameList removed;
- auto txSet = makeTxSetFromTransactions({tx1, txInvalid}, *app,
- 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx1, txInvalid}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.back() == txInvalid);
}
SECTION("validate block")
@@ -1120,8 +1163,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
auto tx2 = transactionWithV2Precondition(
*app, a2, 1, 100, ledgerBoundsCond(0, lclNum + 2));
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx1, tx2}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions({tx1, tx2}, *app,
+ ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
}
@@ -1141,8 +1184,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
{
tx->addSignature(root->getSecretKey());
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
SECTION("fail")
@@ -1150,8 +1193,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.back() == tx);
}
SECTION("validate block")
@@ -1183,8 +1226,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
{
tx->addSignature(a2.getSecretKey());
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
SECTION("fail")
@@ -1192,8 +1235,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.back() == tx);
}
SECTION("validate block")
@@ -1220,8 +1263,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
SECTION("build block")
{
TxFrameList removed;
- auto txSet = makeTxSetFromTransactions({txDupeSigner}, *app, 0,
- 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {txDupeSigner}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.back() == txDupeSigner);
REQUIRE(txDupeSigner->getResultCode() == txMALFORMED);
}
@@ -1243,8 +1286,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
auto rootTx =
transactionWithV2Precondition(*app, *root, 1, 100, cond);
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({rootTx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions({rootTx}, *app,
+ ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
SECTION("signer overlap with added account signer")
@@ -1258,8 +1301,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
tx->addSignature(root->getSecretKey());
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
SECTION("signature missing")
@@ -1267,8 +1310,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
SECTION("build block")
{
TxFrameList removed;
- auto txSet =
- makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions(
+ {tx}, *app, ApplyTimeOffset{}, removed);
REQUIRE(removed.back() == tx);
}
SECTION("validate block")
@@ -1297,7 +1340,8 @@ TEST_CASE("txset with PreconditionsV2", "[herder][txset]")
{*root}, cond);
TxFrameList removed;
- auto txSet = makeTxSetFromTransactions({tx}, *app, 0, 0, removed);
+ auto txSet = makeTxSetFromTransactions({tx}, *app,
+ ApplyTimeOffset{}, removed);
REQUIRE(removed.empty());
}
}
@@ -1359,7 +1403,7 @@ TEST_CASE("txset base fee", "[herder][txset]")
txs.push_back(tx);
}
auto [txSet, applicableTxSet] =
- makeTxSetFromTransactions(txs, *app, 0, 0);
+ makeTxSetFromTransactions(txs, *app, ApplyTimeOffset{});
REQUIRE(applicableTxSet->size(lhCopy) == lim);
REQUIRE(extraAccounts >= 2);
@@ -1587,17 +1631,42 @@ TEST_CASE("tx set hits overlay byte limit during construction",
}
};
- auto testPhaseWithOverlayLimit = [&](TxSetPhase const& phase) {
+ // Builds transactions for `phase` from consecutive genesis accounts,
+ // starting at `nextAccount`, until their total envelope size reaches
+ // `targetBytes`.
+ auto buildTxs = [&](TxSetPhase phase, uint32_t& nextAccount,
+ size_t targetBytes) {
TxFrameList txs;
size_t totalSize = 0;
- int txCount = 0;
-
- while (totalSize < MAX_TX_SET_ALLOWANCE)
+ while (totalSize < targetBytes)
{
- auto a = txtest::getGenesisAccount(*app, txCount++);
+ auto a = txtest::getGenesisAccount(*app, nextAccount++);
txs.emplace_back(makeTx(a, phase));
totalSize += xdr::xdr_size(txs.back()->getEnvelope());
}
+ return txs;
+ };
+
+ // Checks that `phase` was trimmed down to its byte allowance
+ auto checkPhaseTrimmed = [&](ApplicableTxSetFrameConstPtr const& txSet,
+ TxSetPhase phase) {
+ auto const& phaseTxs = txSet->getPhase(phase);
+ auto trimmedSize =
+ std::accumulate(phaseTxs.begin(), phaseTxs.end(), size_t(0),
+ [&](size_t a, TransactionFrameBasePtr const& tx) {
+ return a += xdr::xdr_size(tx->getEnvelope());
+ });
+
+ auto byteAllowance = phase == TxSetPhase::SOROBAN
+ ? app->getConfig().getSorobanByteAllowance()
+ : app->getConfig().getClassicByteAllowance();
+ REQUIRE(trimmedSize > byteAllowance - conf().txMaxSizeBytes());
+ REQUIRE(trimmedSize <= byteAllowance);
+ };
+
+ auto testPhaseWithOverlayLimit = [&](TxSetPhase const& phase) {
+ uint32_t nextAccount = 0;
+ auto txs = buildTxs(phase, nextAccount, MAX_TX_SET_ALLOWANCE);
PerPhaseTransactionList invalidPhases;
invalidPhases.resize(static_cast(TxSetPhase::PHASE_COUNT));
@@ -1612,23 +1681,15 @@ TEST_CASE("tx set hits overlay byte limit during construction",
phases = PerPhaseTransactionList{txs, {}};
}
- auto [txSet, applicableTxSet] =
- makeTxSetFromTransactions(phases, *app, 0, 0, invalidPhases);
+ auto [txSet, applicableTxSet] = makeTxSetFromTransactions(
+ phases, *app, ApplyTimeOffset{}, invalidPhases);
+ // Overlay v2 keeps its own tx set allowance (see
+ // overlay/NetworkConstants.h), which fits the Rust overlay's message
+ // size cap.
REQUIRE(txSet->encodedSize() <= MAX_TX_SET_ALLOWANCE);
REQUIRE(invalidPhases[static_cast