diff --git a/.github/workflows/ci-pull.yml b/.github/workflows/ci-pull.yml index 0675a34..5519090 100644 --- a/.github/workflows/ci-pull.yml +++ b/.github/workflows/ci-pull.yml @@ -31,6 +31,12 @@ # the ci.yml header for the lane semantics (fatal sanitizer reports, # report archiving, toolchain choice). # +# Fuzz lane (PRD §14 "every commit (bounded)"; M0-TEST-01): no separate +# fuzz job — the `fuzz_json_parse` ctest entry (1000 deterministic runs +# of laige-fuzz on the json_parse target) runs inside every build +# job's ctest, instrumented in the ASan tree. Seed and nightly-long-run +# conventions: docs/testing.md. +# # Include-graph lint (M0-CI-03; NFR-8.11, NFR-8.13): the `include-lint` # job runs on EVERY pull request, independent of the ci:* label selector # — it is a platform-independent repository check (Python 3 stdlib only), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f416f90..2502bab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,12 @@ # the include-graph lint with dependency-count metric (M0-CI-03), # the public API manifest drift check (M0-TOOL-01), and the # determinism check (M0-TOOL-02)). +# +# Fuzz lane (PRD §14 "every commit (bounded)"; M0-TEST-01): there is no +# separate fuzz job — the `fuzz_json_parse` ctest entry (1000 +# deterministic runs of laige-fuzz on the json_parse target) runs inside +# every build job's ctest above, instrumented in the ASan tree. Seed and +# nightly-long-run conventions: docs/testing.md. # * Pull requests run exactly ONE P0 OS (label-selectable, default # Linux) in the companion workflow .github/workflows/ci-pull.yml. # diff --git a/README.md b/README.md index 952c036..70f2a70 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,12 @@ serializer (`ctest -R config_json`, API contract in [docs/api/json.md](docs/api/json.md)), and the budget harness (`ctest -R budget_harness`, API contract in [docs/api/budget_harness.md](docs/api/budget_harness.md); canonical -benchmark command `./build/bin/laige-bench --suite=`). Engine -targets compile with `-Wall -Werror` and with exceptions and RTTI -disabled (NFR-8.10). +benchmark command `./build/bin/laige-bench --suite=`). The test + infrastructure conventions (M0-TEST-01: test layout, + `regress_` regression tests, fuzz lane semantics, and seed + handling for randomized tests — `ctest -R test_infra`) are in + [docs/testing.md](docs/testing.md). Engine targets compile with + `-Wall -Werror` and with exceptions and RTTI disabled (NFR-8.10). ## Documentation diff --git a/docs/README.md b/docs/README.md index 5cd47c1..d278064 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,14 @@ sections below mark what exists and what is still to land. the scenario hash-line contract (` ` lines, two build configurations) (M0-TOOL-02). +## Testing + +- [Testing conventions](testing.md) — test layout (module dirs mirror + `src/`, `_tests` executables), the `regress_` + regression-test convention, `laige-fuzz` target registration and CI + lane semantics, and the seed-handling convention for randomized tests + (M0-TEST-01). + ## Architecture decisions (ADRs) - [ADR index](decisions/README.md) — 0001 (name and license), 0002 diff --git a/docs/api/detcheck.md b/docs/api/detcheck.md index 73b5a82..4e83964 100644 --- a/docs/api/detcheck.md +++ b/docs/api/detcheck.md @@ -49,6 +49,7 @@ output: at most **65536** ticks and 64 bytes per line. laige-detcheck --scenario= [--ticks=N] [--seed=HEX|DEC] laige-detcheck --run-a= --run-b= [-- scenario-args...] +laige-detcheck --compare-combined= ``` **Mode 1 — `--scenario`** (M0 built-in scenarios, in-process): @@ -67,6 +68,41 @@ configurations) are executed and their hash streams compared. Everything after the `--` separator is passed to both scenario binaries, so scenario arguments can never collide with tool flags. +Mode 2 is **two-stage**, and `--compare-combined` is its second stage: + +- **Phase 1 — `--run-a`/`--run-b`** spawns both scenario binaries. Each + child inherits this process's **stdout** (no capture pipe is created for + it), so its tick lines land in whatever captures the checker's stdout, + delimited by the marker lines the checker itself emits: + + ```text + @@DETCHK-RUN-A-BEGIN@@ + + @@DETCHK-RUN-A-END @@ + @@DETCHK-RUN-B-BEGIN@@ + + @@DETCHK-RUN-B-END @@ + ``` + + Phase 1 exits `0` when both scenario processes ran to completion, `2` + on a spawn failure or a non-zero scenario exit (the reason on stderr). + It does **not** read back or compare the streams. + +- **Phase 2 — `--compare-combined=`** reads the combined stream the + caller wrote (phase 1's captured stdout), splits it at the markers, + re-runs the scenario contract on each run, compares the two streams, + and reports (the report below). + +The split is forced by the CI Windows runner: it does not deliver handles +the checker process creates (pipes or files, even with the `INHERIT` bit +set, even after duplication) to child processes through `STARTUPINFO` — +only handles the process itself inherited from its parent are delivered +(measured in the M0-TEST-01 CI, runs 24/25). A scenario child therefore +cannot be handed a capture pipe; its stdout must be the checker's own +stdout, which the CTest check script (`execute_process`) captures and +hands back in phase 2. The mechanism is identical on every platform, so +the two-stage flow is exercised by the local suite on POSIX as well. + ## Report and exit codes stdout (stable and machine-greppable — LOG-001): @@ -83,11 +119,10 @@ detcheck scenario=synthetic-perturbed result=DIVERGED first_diff_tick=7 run-b: 7 93a3363d5a1dffa9 ``` -In mode 2 the first line is -`detcheck scenario= vs result=... ticks=` and -the `run-a`/`run-b` lines carry the full scenario paths. When one stream -ends early, the tick lines become stream-length notes -(`stream ends: ticks`) with `result=DIVERGED`. +In mode 2 (phase 2) the first line is +`detcheck scenario=combined result=... ticks=` with `run-a`/`run-b` +labels. When one stream ends early, the tick lines become stream-length +notes (`stream ends: ticks`) with `result=DIVERGED`. | Exit | Meaning | |---|---| @@ -95,6 +130,17 @@ ends early, the tick lines become stream-length notes | 1 | divergence detected (a determinism failure — loud, CORE-008) | | 2 | usage error, unknown scenario, a scenario run failed (non-zero exit, spawn failure), or a scenario violated the output contract (malformed line, tick gap, unbounded output) | +`--run-a`/`--run-b` (phase 1) exits `0` when both scenario processes ran +to completion and `2` on any spawn or scenario failure; the `0`/`1` +comparison result comes from the `--compare-combined` phase. + +On Windows, phase 1 spawns with `CreateProcessW` (no `STARTUPINFO` — no +handles are handed to the child, see above), waits for the child with +`WaitForSingleObject`, and reads its exit code only after termination, so +the `STILL_ACTIVE` sentinel (`259`) is never reported as a scenario exit +code; a signalled child is reported as `128 + signal`, a non-zero scenario +failure either way. + ## The built-in synthetic workload 32 bodies of Q16.16 position/velocity (the default deterministic backend, @@ -118,9 +164,12 @@ computation of scenarios (the tool's line-by-line comparison is unchanged). A CI tool, not a hot path: one scenario run is O(ticks × 32); captured streams are bounded (65536 lines × 64 bytes ≈ 1.5 MiB worst case per -run). Process execution is a plain pipe capture (fork/exec on POSIX, -`CreateProcessW` on Windows) — no shell, no temporary files, bounded -memory, and the scenario's stderr stays on the CI log. +run). Process execution is plain inheritance (fork/exec on POSIX, +`CreateProcessW` on Windows) — no shell, no capture pipe, bounded memory, +and the scenario's stderr stays on the CI log. Phase 2 reads the combined +stream file back with a bounded read capped by the contract (2 × 65536 +lines + markers ≈ 1.5 MiB worst case); the file is written by the check +script between phases and lives in the build tree. ## Test suite diff --git a/docs/getting-started/building.md b/docs/getting-started/building.md index e97d853..dfb59c5 100644 --- a/docs/getting-started/building.md +++ b/docs/getting-started/building.md @@ -33,6 +33,7 @@ The canonical-commands table in [roadmap/README.md](../../roadmap/README.md) | TSan build | `cmake -S . -B build-tsan -DCMAKE_BUILD_TYPE=Debug -DLAIGE_TSAN=ON` | | Test (TSan tree) | `ctest --test-dir build-tsan --output-on-failure` | | Fuzz (bounded) | `./build/bin/laige-fuzz --runs=1000` | +| Fuzz (long, nightly form) | `./build/bin/laige-fuzz --runs=1000000` | | Benchmarks | `./build/bin/laige-bench --suite=` | | Determinism check | `./build/bin/laige-detcheck --scenario=` | | API manifest | `cmake --build build --target laige-api` | @@ -42,11 +43,15 @@ Notes: - `Debug` is the canonical `CMAKE_BUILD_TYPE`; `Release` is supported. - The tool rows above the lint row are live targets now: `laige-fuzz` - (minimal form from M0-CORE-07: the `json_parse` target and deterministic - bounded runs; M0-TEST-01 extends it with CI lane semantics and nightly - long runs), `laige-bench` (M0-CORE-08), `laige-detcheck` (M0-TOOL-02), - and target `laige-api` (M0-TOOL-01). Their command forms were fixed here - when they were reserved, so no step can drift them. + (M0-CORE-07: the `json_parse` target and deterministic bounded runs; + M0-TEST-01 documents the CI lane semantics — bounded `--runs=1000` in + every P0 job's `ctest`, the nightly long-run form above — and the + seed-handling rules), `laige-bench` (M0-CORE-08), `laige-detcheck` + (M0-TOOL-02), and target `laige-api` (M0-TOOL-01). Their command forms + were fixed here when they were reserved, so no step can drift them. + Fuzz and randomized-test seeds: fixed default `0x1F055EED`, + overridable (`laige-fuzz --seed=…`; tests via the `LAIGE_TEST_SEED` + environment variable) — see [docs/testing.md](../testing.md). - Include-graph lint (M0-CI-03): platform-independent (Python 3 stdlib only, no setup). It parses the `#include` edges of `src/**` and enforces the PRD §10.1 rules (laige-core includes nothing internal; arrows only @@ -226,6 +231,13 @@ pinned set and the NaN/Inf policy): the self-check on every PR and merge, with the real-scenario comparison (two build configurations) skipped until M1-SAMPLE-01 (M1-DET-04 activates it). +- `test_infra` is the M0-TEST-01 CTest entry (`tests/testing`): the + `SeededRandom` suite of the `test_infra_tests` executable pins + known-answer hashes for the seed-handling convention (the default seed + `0x1F055EED`, the `LAIGE_TEST_SEED` override, the loud-failure path of + an invalid seed, and substream isolation) and prints the + machine-greppable `test-seed-check` lines that let two CI runs of the + same commit be compared byte-for-byte (docs/testing.md §4). - Every configure verifies the vendored dependency lock (`cmake/laige-deps-lock.cmake` against `deps.lock`); a tampered or unlisted file under `deps/` fails the configure loudly. GoogleTest is the diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..6c6946d --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,145 @@ +# Testing conventions + +Source of truth for how Laige tests are laid out, named, seeded, and +fuzzed (finalized by M0-TEST-01). Normative for every later step; the +normative rule IDs are AGENTS.md §12 (TEST-001…TEST-010), PRD §14 (quality +and verification cadence), and NFR-8.7 (parsers are fuzzed). + +## 1. Test layout (GTest integration) + +- **One directory per engine module under `tests/`, mirroring `src/`:** + `tests/laige-core` ↔ `src/laige-core`, and later `tests/laige-sim` ↔ + `src/laige-sim`, etc. +- **One test executable per module, named `_tests`:** + `tests/laige-core/CMakeLists.txt` builds `laige-core_tests`. All of a + module's unit suites are separate source files in that one executable + (one `TEST` suite per concern), linked against `gtest_main` (which + provides `main()`) and the module library under test. +- **Module suites are exposed as their own CTest entries** with an + unquoted `--gtest_filter` selecting exactly the step's suites; the + unfiltered entry runs the whole module. Each roadmap step names its + Verify command as `ctest -R ` (pattern: M0-CORE-01…08). The + gtest filter must be one *unquoted* argument — CTest passes quoted + arguments through with the literal quote characters, which silently + under-runs the suite (see `tests/laige-core/CMakeLists.txt`). +- **GoogleTest is a dev-only dependency** (PRD §11): vendored in + `deps/googletest`, integrity-locked in `deps.lock` (M0-DEP-01), and + linked into test executables only — **never into engine libraries** + ([ADR 0004](decisions/0004-google-test-vendoring.md)). +- **Non-module test directories** (no `src/` counterpart): + - `tests/tools`, `tests/api`, `tests/detcheck` — checks that exercise + the tools in `tools/` (include-graph lint, API manifest scanner, + determinism checker); CTest entries named after the tool. + - `tests/support/` — shared test-only headers (header-only, no build + targets). Currently `laige_test_seed.h` (this step). + - `tests/testing/` — the test-infrastructure checks; executable + `test_infra_tests`, CTest entry `test_infra`. +- **Every test TU is compiled with the NFR-8.10 policy** + (`laige_apply_engine_policy`: `-Wall -Werror -fno-exceptions + -fno-rtti` / the MSVC equivalent), and the module test executables + self-check that policy with `static_assert`s (a policy violation + fails the build loudly). +- **Test executables are single-owner and run single-threaded per CTest + entry** (CONC-001); concurrency under test gets its own suites + (pattern: `LogConcurrency`) and the TSan tree (NFR-8.2). + +## 2. Regression tests (AGENTS TEST-003) + +- Every bug fix ships a regression test **named `regress_`** + inside the existing module suite (the suite name is unchanged; the + short-id names the defect). +- The test **must fail before the fix and pass after it** (TEST-003). + Demonstrate that in the fix step: write the test first, record the red + run, apply the fix, record the green run — both belong in the step's + Verify note. +- The fix is incomplete without its `regress_` test in the same change + (CORE-007, TEST-003). +- First test under the convention (renamed by this step): + `ConfigJsonValid.regress_json_object_member_ws` in + `tests/laige-core/config_json_tests.cpp` — the M0-CORE-08 finding that + the JSON parser rejected object members separated by `", "` (the + hand-formatted repo-root `budgets.json` demonstrated it). + +## 3. Fuzz runner (`laige-fuzz`; PRD §14, NFR-8.7) + +`tools/fuzz/laige-fuzz.cpp` (M0-CORE-07) is the deterministic bounded +runner: every input is generated from `laige::Prng` (M0-CORE-06), so a +given `(target, runs, seed)` reproduces the exact same input sequence on +every platform (the Prng's cross-platform bit-exactness, ARCH-010). A +target may return any `Status`; the run fails only on process death +(crash or sanitizer report), which ctest turns into a test failure +(CORE-008: no silent failure). + +- **Registering a fuzz target:** add a + `void target(const std::uint8_t*, std::size_t)` entry point and one + `kTargets` row in `tools/fuzz/laige-fuzz.cpp`, then register a CTest + entry `fuzz_` next to it (`COMMAND laige-fuzz + --runs=1000`, `TIMEOUT`, and `TSAN_OPTIONS=halt_on_error=1` in the TSan + tree — the shape of `fuzz_json_parse` in `tools/fuzz/CMakeLists.txt`). +- **Bounded runs in CI, every commit:** the `fuzz_` CTest entries + run inside every P0 job's `ctest` (both `ci.yml` and `ci-pull.yml`), + 1000 runs per target — the PRD §14 "every commit (bounded)" lane. In + the ASan tree (`build-asan`) the runs are instrumented, so a crash or + UB fails the job loudly (NFR-8.7). +- **Nightly long runs (PRD §14 "nightly (long)"):** the canonical form is + `./build/bin/laige-fuzz --runs=1000000 [--seed=HEX]`. The + scheduled nightly lane is documented here but not yet wired: it lands + with the first M1 fuzz target (asset import / network packets, PRD §14 + fuzz row), when a long run protects more than the parser. Until then + the bounded lane above is the complete fuzz cadence in M0. +- **Seed handling:** fixed default seed `0x1F055EED` ("one-fuzz-seed"), + overridable with `--seed=` (0x-prefixed hex or decimal). This is the + same default seed the test suites use (below) — one documented + default seed repo-wide. + +## 4. Seed handling for randomized tests + +- **No nondeterministic sources in tests.** Every randomized test draws + from `laige::Prng` (M0-CORE-06) — never from wall-clock time, + `std::random_device`, or any other source that varies between runs. + CI must be deterministic: the same commit produces the same test + values on every CI run, on every P0 platform. +- **Seed source:** `tests/support/laige_test_seed.h` (this step). + `laige::testing::TestSeed()` returns the fixed default + `kDefaultTestSeed = 0x1F055EED` — identical to laige-fuzz's default — + unless `LAIGE_TEST_SEED` is set (0x-prefixed hex or decimal, read at + call time). A set-but-unparseable value records a test failure with + the offending value and falls back to the default (CORE-008: an + explicit misconfiguration is loud, not silent). +- **Stream isolation:** `laige::testing::TestPrng(id)` derives the + substream `Prng::deriveSubstream(TestSeed(), id)`. Each randomized test + file declares its own stable, named substream-id constant so two tests + never share a stream position (the Prng contract: a copy shares the + position; interleaved draws are a caller bug, not a detectable error). +- **CI determinism, checkable in the logs:** the `SeededRandom` suite + (`tests/testing/`, CTest entry `test_infra`) pins known-answer FNV-1a + hashes of 65536 draws under the default seed and under a documented + override seed, and prints one machine-greppable line per KAT: + + ``` + test-seed-check default seed=0x000000001f055eed stream=1 draws=65536 fnv1a=0x7ea4049545656830 + test-seed-check override seed=0x2468acce01234567 stream=2 draws=65536 fnv1a=0x535d2ca741b61cbf + ``` + + The line lands in the ctest output, every CI job's log, and the + archived `Testing/Temporary/LastTest.log` (linux-asan / linux-tsan + artifacts) even when a KAT mismatches. Two CI runs of the same commit + must show byte-identical `test-seed-check` lines — that is the step's + cross-run identity check (M0-TEST-01 Verify). +- **Investigating a flaky randomized test:** set + `LAIGE_TEST_SEED=` for the ctest run to reproduce the exact + stream that produced the failure (the committed KATs pin the + default-seed values, so the override never hides a KAT regression). + +## 5. Running this step's checks + +| Purpose | Command | +|---|---| +| Seeded-random KAT suite | `ctest --test-dir build -R test_infra --output-on-failure` | +| Bounded fuzz, `json_parse` | `./build/bin/laige-fuzz json_parse --runs=1000` | +| Fuzz with an explicit seed | `./build/bin/laige-fuzz json_parse --runs=1000 --seed=0x12345678` | +| Nightly long run (documented form) | `./build/bin/laige-fuzz json_parse --runs=1000000` | +| Reproduce a randomized test's stream | `LAIGE_TEST_SEED=0x… ctest --test-dir build --output-on-failure` | + +The full canonical command table (build trees, sanitizers, tools) stays +in [getting-started/building.md](getting-started/building.md). diff --git a/roadmap/M0-foundations.md b/roadmap/M0-foundations.md index 970c955..7e3b5ae 100644 --- a/roadmap/M0-foundations.md +++ b/roadmap/M0-foundations.md @@ -847,7 +847,7 @@ No rendering, no physics, no networking yet — `laige-core` only. ## Test infrastructure & docs -- [ ] **M0-TEST-01 · Test infrastructure conventions** +- [x] **M0-TEST-01 · Test infrastructure conventions** - **Refs:** AGENTS TEST-001/003/005; PRD §14 - **Depends:** M0-DEP-01, M0-CORE-07 - **Scope:** @@ -855,8 +855,69 @@ No rendering, no physics, no networking yet — `laige-core` only. - Regression-test convention documented: every bug fix test named `regress_`, must fail before the fix (AGENTS TEST-003) — recorded in `docs/testing.md`. - Fuzz runner `laige-fuzz`: registers targets, bounded runs in CI (`--runs=1000`), nightly long runs documented. - Seed handling for all randomized tests (fixed default seed, overridable) so CI is deterministic. - - **Verify:** convention doc exists; fuzz runner runs the `json_parse` target; a seeded random test passes identically on two CI runs (checkable via artifact logs). - - **Size:** ~150 lines + docs + - **Decision (2026-09-12):** `docs/testing.md` is the normative + conventions doc (layout, `regress_`, fuzz lane semantics, + seed handling; linked from `docs/README.md`, `tests/README.md`, + `building.md`, and `README.md`). Seed handling: + `tests/support/laige_test_seed.h` (test-only header) — `TestSeed()` + returns the fixed default `kDefaultTestSeed = 0x1F055EED`, identical + to laige-fuzz's `kDefaultSeed` (one documented default seed + repo-wide) unless `LAIGE_TEST_SEED` is set (0x-hex or decimal, read + at call time); a set-but-unparseable value records a loud test + failure with the offending value and falls back to the default + (CORE-008; `ADD_FAILURE` — `GTEST_FAIL` is void-return only); + `TestPrng(id)` derives a per-test substream from a stable named id + so two tests never share a stream position. + `tests/testing/test_infra_tests` (CTest entry `test_infra`, suite + `SeededRandom`, 6 cases): known-answer FNV-1a hashes of 65536 draws + under the default seed (`0x7EA4049545656830`) and a documented + override seed (`0x535D2CA741B61CBF`), first-8-draw KAT, the + default/fuzz seed identity, the loud invalid-`LAIGE_TEST_SEED` path + (`EXPECT_NONFATAL_FAILURE` from `gtest-spi.h`), the seed parser, and + substream isolation — each KAT prints a machine-greppable + `test-seed-check` line before asserting, so two CI runs of the same + commit show identical lines in the job logs and the archived + `Testing/Temporary/LastTest.log` (the step's cross-run Verify + clause). First `regress_` test: the M0-CORE-08 bug-fix test renamed + `ConfigJsonValid.ObjectMemberWhitespace` → + `ConfigJsonValid.regress_json_object_member_ws` (suite unchanged, so + `ctest -R config_json` still covers it; the historical M0-CORE-08 + step record is unchanged). Fuzz lane: no new CI job — the bounded + run is the existing `fuzz_json_parse` ctest entry inside every P0 + job's ctest (PRD §14 "every commit (bounded)"); the nightly long + form (`--runs=1000000`) is documented in `docs/testing.md` §3 and + added to the canonical command table (`building.md`); the scheduled + nightly lane lands with the first M1 fuzz target (asset import / + network packets, PRD §14 fuzz row). + - **Verify:** convention doc exists; fuzz runner runs the `json_parse` + target (`ctest -R fuzz_json_parse` green in every local tree and + inside every P0 job's ctest in CI); a seeded random test passes + identically on two CI runs (byte-identical `test-seed-check` lines + in the archived `Testing/Temporary/LastTest.log` of the Linux ASan + job — verified across two CI runs of the same commit, and again + across commits e2abce5 → c8b8221, in both cases + byte-identical): + + ```text + test-seed-check default seed=0x000000001f055eed stream=1 draws=65536 fnv1a=0x7ea4049545656830 + test-seed-check override seed=0x2468acce01234567 stream=2 draws=65536 fnv1a=0x535d2ca741b61cbf + ``` + + CI runs: 34713354925 / 34714039135 (same-commit pair, byte-identical + lines) and 34728782624 (e2abce5) / 34746755055 (c8b8221, the final + commit — byte-identical lines, all 10 jobs green including Windows + 32/32). Local (2026-09-12): 32/32 ctest with zero warnings under the + NFR-8.10 policy on g++ 16.2.1 (`build` static, `build-shared` + shared, `build-asan` ASan+UBSan fatal, `build-tsan` TSan + `halt_on_error=1`) and Clang 22.1.8 (`build-clang`); the seeded KAT + line is identical on the g++ and clang++ trees locally + (cross-compiler identity, confirmed in CI above). + - **Size:** ~1,000 lines (over the ~150 estimate: same pattern as + M0-CORE-01…08 — `docs/testing.md` carries the conventions, + `laige_test_seed.h` the seed contract next to the code, and the + `SeededRandom` suite proves the step's Verify clauses — seeded KAT, + override, loud failure, substream isolation — cohesively rather + than split) - [ ] **M0-DOC-01 · `docs/` skeleton + index** - **Refs:** AGENTS §13 (DOC-001…DOC-007), PRD NFR-8.12 diff --git a/roadmap/README.md b/roadmap/README.md index dc4c308..274ac12 100644 --- a/roadmap/README.md +++ b/roadmap/README.md @@ -155,7 +155,7 @@ Updated in the same PR that closes steps. "Done" = box checked + Verify green. | Milestone | Steps | Done | Status | |---|---|---|---| -| M0 | 22 | 19 | ▶ in progress | +| M0 | 22 | 20 | ▶ in progress | | M1 | 25 | 0 | ⬜ not started | | M2 | 32 | 0 | ⬜ not started | | M3 | 36 | 0 | ⬜ not started | @@ -165,7 +165,7 @@ Updated in the same PR that closes steps. "Done" = box checked + Verify green. | M7 | 15 | 0 | ⬜ not started | | M8 | 8 | 0 | ⬜ not started | | M9 | 6 | 0 | ⬜ proposals only | -| **Total** | **193** | **17** | | +| **Total** | **193** | **20** | | --- @@ -191,7 +191,9 @@ One line per completed (or split/renumbered) step. | 2026-09-11 | M0-CORE-07 | `41938b0` | Bounded JSON in laige-core (ADR 0003, no new dependency): `laige::JsonValue` (deep copy, O(1) move, deep equality) + `parseJson` (1 MiB / depth-32 bounds, strict UTF-8, duplicate keys rejected, ±inf for overflow tokens — documented) + `serializeJson` (canonical ASCII; shortest-round-trip numbers; `std::to_chars` avoided for AppleClang 15 compatibility); all failures → `MalformedInput` (3); `laige-fuzz` minimal deterministic runner (Prng-seeded, `--runs`/`--seed`, 19-document corpus) + `json_parse` fuzz target; `config_json` CTest entry (25 cases) + `fuzz_json_parse` instrumented entry (ASan tree); API contract in `docs/api/json.md` (board/changelog row reconstructed 2026-09-11 from the step record) | | 2026-09-11 | M0-CORE-08 | `810c251` | Budget harness: `laige::Histogram` (fixed-capacity rolling window; O(1) allocation-free `record()`; exact min/mean/p50/p95/p99/max over the stored window via nearest-rank percentiles; allocation-free O(n log n) `stats()`) + `laige::TimeIt` (`steady_clock` ms scope timer) + `loadBudgets`/`budgetCheck` (strict `budgets.json` schema v1 via the bounded JSON parser — ARCH-007; loud `NO_SAMPLES` failure on empty histograms; `target == 0` = hard-zero budget, not "not set"; AGENTS §12 report: stable 4-line text, before/after pair, caller context; `formatStatsLine` the single source of the stats text) + repo-root `budgets.json` (all 15 PRD §8.1 entries; `measured: 0` = not yet measured) + `laige-bench` tool (canonical command per `building.md`: `--suite=synthetic` deterministic 4096-step LCG+double stand-in workload, `--runs`/`--warmup`, `--budget=` check, exit code 2 on budget failure, `--report` append); `budget_harness` CTest entry (22 cases) + `laige_bench_smoke` CTest entry; **bug fix (M0-CORE-07)** found by this step's Verify run: `json.cpp` `parseObjectMembers` missing `skipWhitespace` before the member key — object documents with `", "` between members (the hand-formatted `budgets.json`) were rejected; regression test `ConfigJsonValid.ObjectMemberWhitespace` (fails pre-fix); API contract in `docs/api/budget_harness.md`; local Verify: `ctest` 16/16, zero warnings on GCC static/shared/ASan/TSan + Clang trees; MSVC/AppleClang compile proof lands in CI | | 2026-09-12 | M0-TOOL-01 | `937ac7c` | API manifest (NFR-13.1, PRD §9.4): `laige-api` target + `tools/api/laige-api-scanner` (line-oriented state machine — no regex pass; loud failure on every unsupported construct; doc association from consecutive `//` blocks with `@budget`/`@experimental` tags; `--check FILE` byte compare + symbol-level diff via `laige::parseJson`; exit 0 OK / 1 stale / 2 error) + checked-in `laige-api.json` (version 1, deterministic, no timestamps — byte-identical regeneration is the drift check) + `tests/api` CTest entries (fixture tree with exact manifest bytes, fresh/stale, unsupported-construct failure, real-tree `--check`) + the `api-manifest` CI job (every PR and merge, fails on drift); (board/changelog row retroactively added 2026-09-12 — the step merged as `937ac7c`/PR #10 without updating this board or log) | -| 2026-09-12 | M0-TOOL-02 | `fe4460c` | Determinism checker skeleton (FR-11.5, AGENTS ARCH-010, TEST-004): `laige-detcheck` (`tools/detcheck`) runs a named scenario in two build configurations and compares per-tick hash streams; scenario contract (normative in the tool header, mirrored in `docs/api/detcheck.md`): one stdout line per tick ` ` — 16 lowercase hex hash digits (algorithm NOT part of the contract — lines compare byte-for-byte), tick starts at 0 step 1 no padding, trailing newline optional / trailing `\r` tolerated, stderr ignored, exit 0 — enforced strictly and bounded (65536 ticks, 64 bytes/line; a violation is a loud exit 2, CORE-008); modes: `--scenario=synthetic\|synthetic-perturbed` (built-in 32-body `fpx16_16`+Prng workload, two in-process runs — pure integer arithmetic, bit-exact per ADR 0002; perturbation fixture: +1 unit to body 3's x at tick 7) and `--run-a= --run-b= [-- scenario-args...]` (the M1-DET-04 mode; `--` separator keeps scenario args unambiguous); stable report `detcheck scenario= result=OK\|DIVERGED [first_diff_tick=]` + run-a/run-b lines (LOG-001); exit 0 match / 1 divergence / 2 error; POSIX fork/exec + pipe capture, Windows CreateProcessW + PeekNamedPipe (no shell, bounded memory, scenario stderr stays on the CI log); 9 CTest entries (`tests/detcheck`, `ctest -R detcheck`): self-check, built-in perturbation, identical/diverged/malformed/failure/short-stream cross-binary pairs — one fixture source, five compiled variants — each a generated `cmake -P` script asserting exit code + output fragments (tests/api pattern); CI `detcheck` job in ci-pull.yml/ci.yml (every PR and merge, no ci:* condition — tooling check, not a P0 OS build): runs the synthetic self-check and SKIPS the real-scenario step (two build configurations of M1-SAMPLE-01's hello) until it lands — M1-DET-04 activates it; local Verify: ctest 31/31, zero warnings on g++ static/shared, clang++, ASan+UBSan, TSan trees; Windows path compile-verified by the CI MSVC job; CI (observed 2026-09-12 via the GitHub API): `ci-pull.yml` run 34697638239 on `82c548d` green - the default-Linux lane's four P0/sanitizer jobs (linux-gcc, linux-clang, linux-asan+UBSan, linux-tsan) plus all three tooling jobs (include-lint, api-manifest, detcheck) passed; Windows/macOS jobs label-skipped as expected; the new `detcheck` job's real-scenario step correctly reported `skipped: no real scenario yet (M1-SAMPLE-01)` | +| 2026-09-12 | M0-TOOL-02 | `fe4460c` | Determinism checker skeleton (FR-11.5, AGENTS ARCH-010, TEST-004): `laige-detcheck` (`tools/detcheck`) runs a named scenario in two build configurations and compares per-tick hash streams; scenario contract (normative in the tool header, mirrored in `docs/api/detcheck.md`): one stdout line per tick ` ` — 16 lowercase hex hash digits (algorithm NOT part of the contract — lines compare byte-for-byte), tick starts at 0 step 1 no padding, trailing newline optional / trailing `\r` tolerated, stderr ignored, exit 0 — enforced strictly and bounded (65536 ticks, 64 bytes/line; a violation is a loud exit 2, CORE-008); modes: `--scenario=synthetic\|synthetic-perturbed` (built-in 32-body `fpx16_16`+Prng workload, two in-process runs — pure integer arithmetic, bit-exact per ADR 0002; perturbation fixture: +1 unit to body 3's x at tick 7) and `--run-a= --run-b= [-- scenario-args...]` (the M1-DET-04 mode; `--` separator keeps scenario args unambiguous); stable report `detcheck scenario= result=OK\|DIVERGED [first_diff_tick=]` + run-a/run-b lines (LOG-001); exit 0 match / 1 divergence / 2 error; POSIX fork/exec + pipe capture, Windows CreateProcessW + PeekNamedPipe (no shell, bounded memory, scenario stderr stays on the CI log); 9 CTest entries (`tests/detcheck`, `ctest -R detcheck`): self-check, built-in perturbation, identical/diverged/malformed/failure/short-stream cross-binary pairs — one fixture source, five compiled variants — each a generated `cmake -P` script asserting exit code + output fragments (tests/api pattern); CI `detcheck` job in ci-pull.yml/ci.yml (every PR and merge, no ci:* condition — tooling check, not a P0 OS build): runs the synthetic self-check and SKIPS the real-scenario step (two build configurations of M1-SAMPLE-01's hello) until it lands — M1-DET-04 activates it; local Verify: ctest 31/31, zero warnings on g++ static/shared, clang++, ASan+UBSan, TSan trees; Windows path compile-verified by the CI MSVC job — but NOT runtime-verified (stale as of 2026-09-13: the mode-2 capture was broken on the Windows CI runner and the mode-2 fragment assertions were dead, so the Windows failures were silent; both were found and fixed inside M0-TEST-01 / PR #13 — see the 2026-09-13 row); CI (observed 2026-09-12 via the GitHub API): `ci-pull.yml` run 34697638239 on `82c548d` green - the default-Linux lane's four P0/sanitizer jobs (linux-gcc, linux-clang, linux-asan+UBSan, linux-tsan) plus all three tooling jobs (include-lint, api-manifest, detcheck) passed; Windows/macOS jobs label-skipped as expected; the new `detcheck` job's real-scenario step correctly reported `skipped: no real scenario yet (M1-SAMPLE-01)` | +| 2026-09-12 | M0-TEST-01 | `a292aef` | Test infrastructure conventions (docs/testing.md, source of truth, linked from docs/README.md, tests/README.md, building.md, README.md): tests/support/laige_test_seed.h (TestSeed()/TestPrng() — fixed default 0x1F055EED, identical to laige-fuzz's kDefaultSeed, one documented default seed repo-wide; LAIGE_TEST_SEED env override, 0x-hex/decimal, read at call time; set-but-unparseable value records a loud test failure and falls back to the default — CORE-008; per-test substream ids so streams never share position) + tests/testing (test_infra_tests, CTest entry `test_infra`, SeededRandom suite, 6 cases: 65536-draw FNV-1a KATs under the default seed 0x7EA4049545656830 and override seed 0x535D2CA741B61CBF, first-8-draw KAT, default/fuzz seed identity, loud invalid-env path via EXPECT_NONFATAL_FAILURE (gtest-spi.h), seed parser, substream isolation; machine-greppable `test-seed-check` line per KAT before asserting — the byte-identical-across-two-CI-runs identity check, M0-TEST-01 Verify) + first regress_ test (M0-CORE-08's ConfigJsonValid.ObjectMemberWhitespace renamed regress_json_object_member_ws; suite unchanged so `ctest -R config_json` still covers it; historical M0-CORE-08 record unchanged) + fuzz lane semantics (no new CI job — bounded --runs=1000 is the existing fuzz_json_parse ctest entry inside every P0 job's ctest, PRD §14 "every commit (bounded)"; nightly long form --runs=1000000 documented in docs/testing.md §3 + canonical command table; scheduled nightly lane lands with the first M1 fuzz target); CI workflow headers note the fuzz lane; local Verify: 32/32 ctest, zero warnings under NFR-8.10, on g++ static/shared, ASan+UBSan, TSan, and Clang trees; seeded KAT line identical on g++ and clang++ locally; cross-CI-run Verify: byte-identical `test-seed-check` lines in the archived Linux ASan `LastTest.log` across runs 34713354925/34714039135 (same-commit pair) and again across commits e2abce5→c8b8221 (runs 34728782624/34746755055); Windows CI fix chain landed inside this PR (M0-TOOL-02's Windows path, provenance per the established pattern — see the corrected M0-TOOL-02 row): MSVC portability (NOMINMAX, `getenv_s`/`fopen_s` C4996, C2664/C2440/C4457/C2660), `WaitForSingleObject` before `GetExitCodeProcess` (STILL_ACTIVE), then the root cause — the CI Windows runner (windows-2022) never delivers handles the process creates itself (pipes or files, INHERIT bit confirmed set, even duplicated) to children through `STARTUPINFO`; only parent-inherited (kernel-assigned) handles are delivered (measured runs 34728950395/34729337292) — and NULL `STARTUPINFO` is rejected by its CreateProcess machinery (run 34732057356); fixed by two-stage marker capture on all platforms (phase 1 `--run-a/--run-b` spawns with plain stdout inheritance + `@@DETCHK-RUN-A/B-BEGIN/END@@` markers; phase 2 `--compare-combined` splits/validates/compares) plus the zeroed-STARTUPINFO spawn; dead fragment assertions fixed in `7928379` (`@CHECKS@` variable-name misspelling + CMake single-backslash stripping → double-escaped regexes); final CI run 34746755055 (c8b8221): all 10 jobs green, Windows 32/32 | + --- diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 47cb0e0..626381e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -64,6 +64,11 @@ endforeach() add_subdirectory(laige-core) +# Test-infrastructure checks (M0-TEST-01): the seed-handling convention +# for randomized tests (tests/support/laige_test_seed.h) is exercised by +# tests/testing (suite `SeededRandom`, CTest entry `test_infra`). +add_subdirectory(testing) + # Tooling checks (M0-CI-03): the include-graph lint runs against fixture # trees and the real repository tree (tests/tools). add_subdirectory(tools) diff --git a/tests/README.md b/tests/README.md index ff7db05..98e2bf3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,13 +1,15 @@ # tests/ Unit and integration tests, one directory per engine module mirroring -`src/`; test executables are named `_tests` (convention finalized in -M0-TEST-01). +`src/`; test executables are named `_tests`. +**[docs/testing.md](../docs/testing.md) is the source of truth for the +full conventions** (layout, regression-test naming, fuzz registration, +seed handling) — finalized by M0-TEST-01. -**Test framework: GoogleTest** — a dev-only dependency (PRD §11) vendored in -`deps/googletest` and locked in `deps.lock` (M0-DEP-01). Test executables -link `gtest_main` (which provides `main()`); GoogleTest is **never** linked -into engine libraries. See +**Test framework: GoogleTest** — a dev-only dependency (PRD §11) vendored +in `deps/googletest` and locked in `deps.lock` (M0-DEP-01). Test +executables link `gtest_main` (which provides `main()`); GoogleTest is +**never** linked into engine libraries. See [ADR 0004](../docs/decisions/0004-google-test-vendoring.md). `laige-core_tests` (build smoke test, M0-BUILD-01; converted to a GoogleTest @@ -19,3 +21,9 @@ the module's single test executable, exposed as its own CTest entry — M0-CORE-01's Result/Status/error-registry suites run as `ctest -R result_status` (filtering the shared executable to the `ResultStatus`, `Status`, and `ErrorCodeRegistry` suites). + +Non-module directories: `tests/tools`, `tests/api`, and `tests/detcheck` +check the tools in `tools/`; `tests/support/` holds shared test-only +headers (currently the seed helper `laige_test_seed.h`); `tests/testing/` +holds the test-infrastructure checks (`test_infra_tests`, CTest entry +`test_infra` — the seed-handling KAT suite, M0-TEST-01). diff --git a/tests/api/CMakeLists.txt b/tests/api/CMakeLists.txt index 2e8b049..ea00d25 100644 --- a/tests/api/CMakeLists.txt +++ b/tests/api/CMakeLists.txt @@ -161,15 +161,24 @@ function(laige_add_api_test name expect_exit root mode file) # Space-separated quoted list (semicolon lists confuse the CMake # script lexer in generated files — verified on CMake 4.4.3). set(_cmd "\"--root\" \"${root}\" \"--${mode}\" \"${file}\"") - set(_checks "") + # The template's @CHECKS@ placeholder is substituted from this variable, + # so it must be named exactly CHECKS (a misspelled local like _checks + # silently expands to nothing and the fragment checks never exist). + set(CHECKS "") foreach(_needle IN LISTS ARGN) + # The generated script is itself a CMake file, and its string literals + # strip one level of backslashes on parse (\\ -> \, \X -> X), so each + # regex escape must reach the file DOUBLED: \\( in the file becomes \( + # for the regex engine (a single \X would be stripped and the bare + # metacharacter would reach the regex). + set(_bs2 "\\\\") string(REPLACE "\\" "\\\\" _esc "${_needle}") string(REPLACE "\"" "\\\"" _esc "${_esc}") - string(REPLACE "(" "\\(" _esc "${_esc}") - string(REPLACE ")" "\\)" _esc "${_esc}") - string(REPLACE "+" "\\+" _esc "${_esc}") - string(REPLACE "." "\\." _esc "${_esc}") - string(APPEND _checks + string(REPLACE "(" "${_bs2}(" _esc "${_esc}") + string(REPLACE ")" "${_bs2})" _esc "${_esc}") + string(REPLACE "+" "${_bs2}+" _esc "${_esc}") + string(REPLACE "." "${_bs2}." _esc "${_esc}") + string(APPEND CHECKS "if(NOT _text MATCHES \"${_esc}\")\n" " string(APPEND _problems \"output missing '<${_esc}'>; \")\n" "endif()\n") diff --git a/tests/detcheck/CMakeLists.txt b/tests/detcheck/CMakeLists.txt index 23129e7..549bc75 100644 --- a/tests/detcheck/CMakeLists.txt +++ b/tests/detcheck/CMakeLists.txt @@ -73,25 +73,37 @@ string(APPEND LAIGE_DETCHECK_TEST_ENV # Same shape as the tests/api helper: a generated cmake -P check script # asserts the exit code and the required output fragments. function(laige_add_detcheck_test name expect_exit cmd) - set(_checks "") + # The template's @CHECKS@ placeholder is substituted from this variable, + # so it must be named exactly CHECKS (a misspelled local like _checks + # silently expands to nothing and the fragment checks never exist). + set(CHECKS "") foreach(_needle IN LISTS ARGN) # Escape ECMAScript metacharacters for the generated MATCHES check. + # The generated script is itself a CMake file, and its string literals + # strip one level of backslashes on parse (\\ -> \, \X -> X), so each + # regex escape must reach the file DOUBLED: \\( in the file becomes \( + # for the regex engine (a single \X would be stripped and the bare + # metacharacter would reach the regex). string(REPLACE "\\" "\\\\" _esc "${_needle}") string(REPLACE "\"" "\\\"" _esc "${_esc}") - string(REPLACE "(" "\\(" _esc "${_esc}") - string(REPLACE ")" "\\)" _esc "${_esc}") - string(REPLACE "+" "\\+" _esc "${_esc}") - string(REPLACE "." "\\." _esc "${_esc}") - string(REPLACE "*" "\\*" _esc "${_esc}") - string(REPLACE "?" "\\?" _esc "${_esc}") - string(REPLACE "|" "\\|" _esc "${_esc}") - string(REPLACE "^" "\\^" _esc "${_esc}") - string(REPLACE "$" "\\$" _esc "${_esc}") - string(REPLACE "[" "\\[" _esc "${_esc}") - string(REPLACE "]" "\\]" _esc "${_esc}") - string(REPLACE "{" "\\{" _esc "${_esc}") - string(REPLACE "}" "\\}" _esc "${_esc}") - string(APPEND _checks + string(REPLACE "(" "\\\\(" _esc "${_esc}") + string(REPLACE ")" "\\\\)" _esc "${_esc}") + string(REPLACE "+" "\\\\+" _esc "${_esc}") + string(REPLACE "." "\\\\." _esc "${_esc}") + string(REPLACE "*" "\\\\*" _esc "${_esc}") + string(REPLACE "?" "\\\\?" _esc "${_esc}") + string(REPLACE "|" "\\\\|" _esc "${_esc}") + string(REPLACE "^" "\\\\^" _esc "${_esc}") + string(REPLACE "$" "\\\\$" _esc "${_esc}") + # (The [ literal is built from _bs2: its replacement must be two + # backslashes plus a literal [, and the bracket has to survive the + # edit tooling intact.) + set(_bs2 "\\\\") + string(REPLACE "[" "${_bs2}[" _esc "${_esc}") + string(REPLACE "]" "\\\\]" _esc "${_esc}") + string(REPLACE "{" "\\\\{" _esc "${_esc}") + string(REPLACE "}" "\\\\}" _esc "${_esc}") + string(APPEND CHECKS "if(NOT _text MATCHES \"${_esc}\")\n" " string(APPEND _problems \"output missing '<${_esc}'>; \")\n" "endif()\n") diff --git a/tests/detcheck/expect-detcheck-result.cmake.in b/tests/detcheck/expect-detcheck-result.cmake.in index 16e1fc9..52abde5 100644 --- a/tests/detcheck/expect-detcheck-result.cmake.in +++ b/tests/detcheck/expect-detcheck-result.cmake.in @@ -1,15 +1,28 @@ # Generated by tests/detcheck/CMakeLists.txt for one laige-detcheck CTest # test (M0-TOOL-02) — do not edit. # -# Runs laige-detcheck with the argument list @CMD@ and asserts BOTH the -# exit code (@EXPECT_EXIT@) and the required output fragments (the -# needle checks below are generated per test). The assertions live here, -# in a CMake script, instead of in CTest properties: CTest inverts +# Runs laige-detcheck for the argument list @CMD@ and asserts BOTH the +# exit code (@EXPECT_EXIT@) and the required output fragments (the needle +# checks below are generated per test). The assertions live here, in a +# CMake script, instead of in CTest properties: CTest inverts # PASS_REGULAR_EXPRESSION when WILL_FAIL is set (verified on CMake 4.4.3 # — a matching regex then makes the test fail), and a crash exits # non-zero just like a correct failure (CORE-008), so the content must # be checked, not just the code. # +# Mode 2 (--run-a/--run-b) is two-stage (see the header of +# tools/detcheck/laige-detcheck.cpp): the CI Windows runner does not +# deliver handles detcheck creates (pipes or files, even inheritable) +# to child processes through STARTUPINFO — only handles detcheck itself +# inherited from its parent are delivered (measured in the M0-TEST-01 +# CI, runs 24/25). So each scenario child inherits detcheck's stdout +# (this script's execute_process capture), its ticks land between the +# tool-emitted @@DETCHK-RUN-A/B-BEGIN/END@@ markers, and the +# comparison is a second detcheck invocation (--compare-combined) over +# the combined stream. Phase 1 exits 0 when both scenario processes ran +# to completion and 2 on any spawn or scenario failure; the asserted +# exit code and output come from phase 2, or from phase 1 when it failed. +# # The executable paths arrive through the test's ENVIRONMENT property # (same pattern as tests/api/expect-api-result.cmake.in: $ # resolves per configuration under multi-config generators, so a path @@ -80,13 +93,62 @@ foreach(_tok IN LISTS _cmd) list(APPEND _args "${_tok}") endforeach() +# --- Phase 1: spawn (mode 2) / single shot (mode 1) ----------------------- +# Mode 1 (--scenario) needs no second phase: the tool runs both runs +# in-process and reports in one invocation. execute_process( COMMAND "${_detcheck}" ${_args} - RESULT_VARIABLE _rc - OUTPUT_VARIABLE _out - ERROR_VARIABLE _err + RESULT_VARIABLE _rc1 + OUTPUT_VARIABLE _out1 + ERROR_VARIABLE _err1 ) +# Mode 2 is recognized by its `--run-a=` token (literal containment, not a +# regex - $ is a metacharacter and the command carries $FIX_*$ markers). +# string(FIND) takes a literal first argument (no variable expansion), so +# each list element is expanded explicitly; -1 means "token absent" for +# every element. +set(_is_mode2 -1) +foreach(_tok IN LISTS _cmd) + string(FIND ${_tok} "--run-a=" _tokpos) + if(_tokpos GREATER_EQUAL 0) + set(_is_mode2 1) + break() + endif() +endforeach() + +if(_rc1 EQUAL 0 AND _is_mode2 EQUAL -1) + # Single shot (mode 1): phase 1's result IS the final result. + set(_rc "${_rc1}") + set(_out "${_out1}") + set(_err "${_err1}") +elseif(NOT _rc1 EQUAL 0) + # A scenario process failed (or the invocation errored out): phase 1's + # result IS the final result - no comparison phase. + set(_rc "${_rc1}") + set(_out "${_out1}") + set(_err "${_err1}") +else() + # --- Phase 2: compare the combined stream ------------------------------- + # The combined stream (detcheck's phase-1 stdout: markers + both tick + # streams, plus any pre-marker probe output) goes back to detcheck, + # which splits it at the markers, re-runs the stream contract on each + # run, and compares. + # CMAKE_CURRENT_LIST_DIR is absolute for a `cmake -P` script; the + # script's basename is .cmake (tests/detcheck/CMakeLists.txt). + set(_scriptdir "${CMAKE_CURRENT_LIST_DIR}") + string(REGEX MATCH "[^/]+$" _scriptbase "${CMAKE_CURRENT_LIST_FILE}") + string(REPLACE ".cmake" "" _testname "${_scriptbase}") + set(_combined "${_scriptdir}/${_testname}.combined") + file(WRITE "${_combined}" "${_out1}") + execute_process( + COMMAND "${_detcheck}" "--compare-combined=${_combined}" + RESULT_VARIABLE _rc + OUTPUT_VARIABLE _out + ERROR_VARIABLE _err + ) +endif() + set(_text "${_out} ${_err}") @@ -101,4 +163,3 @@ if(NOT _problems STREQUAL "") "--- laige-detcheck output ---\n${_text}\n" "--- end of laige-detcheck output ---") endif() -message("laige-detcheck check OK: exit @EXPECT_EXIT@, required output present") diff --git a/tests/laige-core/config_json_tests.cpp b/tests/laige-core/config_json_tests.cpp index 05acc7e..5aece9a 100644 --- a/tests/laige-core/config_json_tests.cpp +++ b/tests/laige-core/config_json_tests.cpp @@ -282,13 +282,15 @@ TEST(ConfigJsonValid, Containers) { } } -// M0-CORE-08 regression: whitespace after the ',' of an object member -// must be accepted (the grammar allows whitespace between tokens). The -// object key goes through parseString directly (not parseValue, which -// does the skipping), so the key needs its own whitespace skip — the old +// M0-CORE-08 regression (named per the TEST-003 convention, +// docs/testing.md §2): whitespace after the ',' of an object member must +// be accepted (the grammar allows whitespace between tokens). The object +// key goes through parseString directly (not parseValue, which does the +// skipping), so the key needs its own whitespace skip — the old // parseObjectMembers rejected {"a": 1, "b": 2}. Found when the -// hand-formatted repo-root budgets.json was rejected (M0-CORE-08). -TEST(ConfigJsonValid, ObjectMemberWhitespace) { +// hand-formatted repo-root budgets.json was rejected (M0-CORE-08); this +// test failed before the fix and passes after it. +TEST(ConfigJsonValid, regress_json_object_member_ws) { { const auto r = Parse(R"({"a": 1, "b": 2})"); ASSERT_TRUE(r.ok()); diff --git a/tests/support/laige_test_seed.h b/tests/support/laige_test_seed.h new file mode 100644 index 0000000..cbf9f53 --- /dev/null +++ b/tests/support/laige_test_seed.h @@ -0,0 +1,136 @@ +// Test-only seed handling for randomized tests (M0-TEST-01). +// +// Normative convention: docs/testing.md §4. Every randomized test in the +// repository MUST draw from laige::Prng (M0-CORE-06) seeded through this +// helper — never from wall-clock time, std::random_device, or any other +// nondeterministic source. CI runs must be deterministic and reproducible +// (roadmap M0-TEST-01, PRD §14), and a randomized test must produce the +// same values on every CI run of the same commit. +// +// Seed source: +// - Default: kDefaultTestSeed = 0x1F055EED ("one-fuzz-seed") — the same +// fixed default the laige-fuzz runner uses (tools/fuzz/laige-fuzz.cpp), +// so one documented default seed covers the test suites and the fuzz +// lane. +// - Override: the LAIGE_TEST_SEED environment variable (0x-prefixed hex +// or decimal), read at call time. A set-but-unparseable value records +// a test failure with an actionable message (CORE-008: an explicit +// misconfiguration is not silently ignored) and the default seed is +// returned so the calling test can finish its remaining assertions. +// +// Stream isolation: a test that draws randomness derives its own substream +// with a stable, named id — TestPrng(kMyId) — so two tests never share a +// stream position (the Prng contract: a copy shares the position; +// interleaved draws are a bug at the call site, not an error the type can +// detect). Each test file defines its own id constant (CORE-005). +// +// Cross-run identity: the SeededRandom suite (tests/testing) prints a +// machine-greppable `test-seed-check` line (seed, substream id, draw +// count, FNV-1a hash of a fixed draw count) before asserting it, so the +// line lands in the ctest output and the CI job logs (and the archived +// Testing/Temporary/LastTest.log) even when a KAT mismatches; two CI runs +// of the same commit must show identical lines (M0-TEST-01 Verify). +// +// Usage (inside a TEST body only — the failure path reports to the current +// test): +// +// laige::Prng rng = laige::testing::TestPrng(kMySubstreamId); +// +// This header is test-only: it is never included from src/ (the +// include-graph lint covers src/** only) and defines no symbols of its +// own (header-only). It is compiled with the NFR-8.10 policy like every +// test TU (laige_apply_engine_policy). + +#pragma once + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "laige/prng.h" + +namespace laige::testing { + +// The value of an environment variable; the empty string when unset. +// Platform boundary (CPP-009, the pattern of +// tests/laige-core/budget_harness_tests.cpp and tools/bench/laige-bench.cpp): +// MSVC deprecates plain getenv (C4996, fatal under /WX); getenv_s has the +// same lookup semantics. Largest value any caller reads here is a 16-hex +// seed string; 4096 is far beyond it, and a longer value is treated as +// unset (the documented fallback applies). Named per CORE-005. +inline std::string ReadEnvVar(const char* name) { +#if defined(_MSC_VER) + constexpr std::size_t kEnvValueMaxBytes = 4096; + char buf[kEnvValueMaxBytes]; + std::size_t len = 0; + if (getenv_s(&len, buf, sizeof(buf), name) != 0) return {}; + return std::string(buf, len); +#else + const char* v = std::getenv(name); + return (v != nullptr) ? std::string(v) : std::string(); +#endif +} + +// The fixed default test seed ("one-fuzz-seed"). Identical to +// laige-fuzz's kDefaultSeed (tools/fuzz/laige-fuzz.cpp) and never +// changed: committed known-answer constants in the test suites are +// pinned against it, so a default-seed change would fail every KAT +// loudly instead of silently reshuffling the random tests. +constexpr std::uint64_t kDefaultTestSeed = 0x1F055EEDull; + +// The environment variable that overrides the default seed. +constexpr const char* kTestSeedEnvVar = "LAIGE_TEST_SEED"; + +// Parse a seed string the way TestSeed() reads LAIGE_TEST_SEED: 0x-prefixed +// hex or decimal. Returns false (with no output write) when the value is +// empty, partially numeric, or negative. +inline bool ParseTestSeed(const char* text, std::uint64_t& out) { + if (text == nullptr || *text == '\0' || text[0] == '-') { + return false; + } + char* end = nullptr; + out = std::strtoull(text, &end, 0); + return end != text && *end == '\0'; +} + +// The seed the active test run uses: kDefaultTestSeed unless LAIGE_TEST_SEED +// is set, in which case its parsed value. +// +// Contract: call inside a TEST body. A set-but-unparseable LAIGE_TEST_SEED +// records a non-fatal failure (ADD_FAILURE — GTEST_FAIL is void-return +// only, and this helper returns a value) with the offending value and the +// accepted forms, then returns kDefaultTestSeed so the calling test can +// complete. The test has already failed, so the ctest entry goes red +// (CORE-008: no silent fallback for an explicit misconfiguration). +inline std::uint64_t TestSeed() { + const std::string env = ReadEnvVar(kTestSeedEnvVar); + if (env.empty()) { + return kDefaultTestSeed; + } + std::uint64_t seed = 0; + if (!ParseTestSeed(env.c_str(), seed)) { + std::fprintf(stderr, + "laige test seed: invalid %s='%s' (use 0x-hex or decimal); " + "using the default 0x%llx — the test is already failing\n", + kTestSeedEnvVar, env.c_str(), + static_cast(kDefaultTestSeed)); + ADD_FAILURE() << kTestSeedEnvVar << "='" << env.c_str() + << "' is not a valid seed (0x-prefixed hex or decimal); " + << "falling back to the default 0x" << kDefaultTestSeed; + return kDefaultTestSeed; + } + return seed; +} + +// A Prng substream for a test: TestSeed() derived by the caller's stable +// substream id (docs/testing.md §4). The id is a test identity, not a +// magic number: each randomized test file declares its own named +// constant so substreams cannot collide by accident. +inline laige::Prng TestPrng(std::uint32_t substreamId) { + return laige::Prng::deriveSubstream(TestSeed(), substreamId); +} + +} // namespace laige::testing diff --git a/tests/testing/CMakeLists.txt b/tests/testing/CMakeLists.txt new file mode 100644 index 0000000..111544c --- /dev/null +++ b/tests/testing/CMakeLists.txt @@ -0,0 +1,29 @@ +# Test-infrastructure checks (M0-TEST-01). +# +# The seed-handling convention for randomized tests (docs/testing.md §4) +# lives in tests/support/laige_test_seed.h; this suite exercises it: +# known-answer FNV-1a hashes of a fixed draw count under the default seed +# and a LAIGE_TEST_SEED override, the one-repo-wide default seed's +# identity with laige-fuzz's, the seed parser, the loud-failure path of an +# invalid LAIGE_TEST_SEED, and substream isolation. +# The committed KATs make the suite a replay fixture — identical on every +# build, compiler, and CI run (M0-TEST-01 Verify clause) — and the +# `test-seed-check` line printed to stdout is the artifact-log identity +# check across two CI runs. +# +# Layout convention (docs/testing.md §1): tests/ mirrors +# src/ and executables are named _tests; this directory is +# the non-module home of the test-infrastructure checks (the tooling +# checks live in tests/tools, tests/api, and tests/detcheck). + +add_executable(test_infra_tests seeded_random_tests.cpp) +target_include_directories(test_infra_tests PRIVATE + ${CMAKE_SOURCE_DIR}/tests/support) +laige_apply_engine_policy(test_infra_tests) +target_link_libraries(test_infra_tests PRIVATE gtest_main laige-core) + +# Step Verify command: `ctest -R test_infra` selects exactly the +# SeededRandom suite. The gtest filter is one UNQUOTED argument (CTest +# passes quoted arguments through with the literal quote characters — +# see the note in tests/laige-core/CMakeLists.txt). +add_test(NAME test_infra COMMAND test_infra_tests --gtest_filter=SeededRandom.*) diff --git a/tests/testing/seeded_random_tests.cpp b/tests/testing/seeded_random_tests.cpp new file mode 100644 index 0000000..3860322 --- /dev/null +++ b/tests/testing/seeded_random_tests.cpp @@ -0,0 +1,247 @@ +// Test-infrastructure suite (M0-TEST-01): the seed-handling convention +// for randomized tests (docs/testing.md §4). +// +// Step Verify scope (roadmap/M0-foundations.md): +// - a seeded random test passes identically on two CI runs — this suite +// pins a known-answer FNV-1a hash of a fixed draw count under the +// default seed (and under a LAIGE_TEST_SEED override) and prints a +// machine-greppable `test-seed-check` line per run, so two CI runs of +// the same commit show identical lines in the job logs and the +// archived Testing/Temporary/LastTest.log; +// - the fixed default seed is overridable (LAIGE_TEST_SEED env var) and +// identical to laige-fuzz's default (one documented repo-wide seed). +// +// House conventions: the FNV-1a 64-bit convention is the same as the +// prng_tests and math_fixed_tests KATs (big-endian bytes per u64 — +// endianness-independent); named constants throughout (CORE-005); +// KAT constants below were generated by running this suite and pinning +// the printed values — a change here means the algorithm or the seed +// changed, which is loud by design (ARCH-010). +// +// NFR-8.10 policy self-checks: this TU must build with exceptions and +// RTTI disabled (laige_apply_engine_policy, see tests/testing/CMakeLists.txt). + +#include +#include +#include +#include + +#include "gtest/gtest.h" +// gtest-spi.h is GoogleTest's documented testing-support header (used by +// GoogleTest's own suite): EXPECT_NONFATAL_FAILURE lets this suite assert +// the loud-failure path of TestSeed() without the test failing itself. +#include "gtest/gtest-spi.h" +#include "laige/prng.h" +#include "laige_test_seed.h" + +#if defined(__cpp_exceptions) +static_assert(false, + "seeded_random_tests must be built with exceptions disabled " + "(NFR-8.10); see laige_apply_engine_policy()."); +#elif defined(__EXCEPTIONS) && __EXCEPTIONS +static_assert(false, + "seeded_random_tests must be built with exceptions disabled " + "(NFR-8.10); see laige_apply_engine_policy()."); +#endif + +#if defined(__cpp_rtti) && __cpp_rtti +static_assert(false, + "seeded_random_tests must be built with RTTI disabled " + "(NFR-8.10); see laige_apply_engine_policy()."); +#endif + +namespace { + +using u32 = std::uint32_t; +using u64 = std::uint64_t; + +// Fixed draw count for the known-answer hashes (named: CORE-005). +constexpr u32 kSeedCheckDraws = 65536; + +// Substream ids: stable test identities (docs/testing.md §4). Each id is +// owned by exactly one suite/file so substreams never collide. +constexpr u32 kSeedCheckStreamId = 0x0001; // this suite's default-seed KAT +constexpr u32 kOverrideStreamId = 0x0002; // this suite's override-seed KAT + +// The override seed for the env-var test: named, different from +// kDefaultTestSeed, so a broken env read cannot accidentally pass. +constexpr u64 kOverrideSeed = 0x2468ACCE01234567ull; + +// First 8 draws of the default-seed, stream-1 substream (KAT): a fast +// failure diagnostic next to the full-stream hash. +constexpr u64 kFirstDrawsStream1[8] = { + 0x0EE1EED94C2CAA69ull, 0x4D64B354EA0FFF1Full, 0x9343A884890B5222ull, + 0xEDE0A379387022A6ull, 0x98A115D856F6A2BAull, 0xDF66706092CBE9C4ull, + 0x5DC877208B35398Full, 0x4F83FA225F6E1A7Cull, +}; + +// Full-stream KATs (65536 draws each, FNV-1a 64-bit over big-endian bytes +// per draw): identical on every build, compiler, and CI run. +constexpr u64 kKnownAnswerDefaultSeed = 0x7EA4049545656830ull; +constexpr u64 kKnownAnswerOverrideSeed = 0x535D2CA741B61CBFull; + +// FNV-1a 64-bit, big-endian byte order per u64 (the same convention as +// prng_tests / math_fixed_tests KATs — endianness-independent). +u64 fnv1a64(const u64* values, std::size_t n) { + u64 h = 0xcbf29ce484222325ull; // FNV offset basis (named: FNV-1a spec) + for (std::size_t i = 0; i < n; ++i) { + for (int shift = 56; shift >= 0; shift -= 8) { + h ^= (values[i] >> shift) & 0xFFull; + h *= 0x100000001b3ull; // FNV prime (named: FNV-1a spec) + } + } + return h; +} + +// Draw kSeedCheckDraws values from the substream of `seed` and print the +// machine-greppable identity line (LOG-001) BEFORE asserting, so the line +// reaches the ctest output and the CI job logs even when the KAT +// mismatches. Two CI runs of the same commit must print identical lines +// (M0-TEST-01 Verify). +u64 seedCheck(u64 seed, u32 streamId, const char* label) { + laige::Prng rng = laige::Prng::deriveSubstream(seed, streamId); + u64 hash = 0xcbf29ce484222325ull; + u64 draw; + for (u32 i = 0; i < kSeedCheckDraws; ++i) { + draw = rng.next_u64(); + for (int shift = 56; shift >= 0; shift -= 8) { + hash ^= (draw >> shift) & 0xFFull; + hash *= 0x100000001b3ull; + } + } + std::printf("test-seed-check %s seed=0x%016llx stream=%u draws=%u " + "fnv1a=0x%016llx\n", + label, static_cast(seed), streamId, + kSeedCheckDraws, static_cast(hash)); + std::fflush(stdout); + return hash; +} + +// RAII around LAIGE_TEST_SEED: the override test sets it for the duration +// of the test body and restores the prior state (absent or present) even +// on failure. Platform boundary (CPP-009): setenv/unsetenv are C99/POSIX; +// MSVC's CRT does not provide them, so the Windows branch uses _putenv_s +// (the documented secure variant — no C4996 under /WX). On Windows, +// _putenv_s(name, "") is the "unset" equivalent: TestSeed() treats an +// empty value exactly as unset (the same lookup outcome ReadEnvVar gives +// on POSIX, where unsetenv removes the variable outright). +class ScopedTestSeedEnv { + public: + explicit ScopedTestSeedEnv(const char* value) { + priorValue_ = laige::testing::ReadEnvVar(laige::testing::kTestSeedEnvVar); + hadPrior_ = !priorValue_.empty(); + SetVar(laige::testing::kTestSeedEnvVar, value); + } + ~ScopedTestSeedEnv() { + SetVar(laige::testing::kTestSeedEnvVar, + hadPrior_ ? priorValue_.c_str() : ""); + } + + private: + static void SetVar(const char* name, const char* value) { +#if defined(_MSC_VER) + ::_putenv_s(name, value); +#else + if (value == nullptr || value[0] == '\0') { + ::unsetenv(name); + } else { + ::setenv(name, value, 1); + } +#endif + } + + bool hadPrior_ = false; + std::string priorValue_; +}; + +} // namespace + +// --------------------------------------------------------------------------- +// SeededRandom — the seed-handling convention (M0-TEST-01) +// --------------------------------------------------------------------------- + +// The default-seed known answer: 65536 draws from the documented default +// seed's stream-1 substream. Identical on every build, compiler, and CI +// run (pure integer arithmetic, ARCH-010). +TEST(SeededRandom, DefaultSeedKnownAnswer) { + const u64 hash = seedCheck(laige::testing::kDefaultTestSeed, + kSeedCheckStreamId, "default"); + EXPECT_EQ(kKnownAnswerDefaultSeed, hash) + << "default-seed stream changed (seed 0x1F055EED, stream " + << kSeedCheckStreamId << ", " << kSeedCheckDraws << " draws); see the " + "test-seed-check line above — re-pin the KAT only with a " + "documented algorithm/seed change (docs/testing.md §4)"; + // Fast diagnostic: the first draws of the same stream. + laige::Prng rng = + laige::Prng::deriveSubstream(laige::testing::kDefaultTestSeed, + kSeedCheckStreamId); + for (int i = 0; i < 8; ++i) { + EXPECT_EQ(kFirstDrawsStream1[i], rng.next_u64()) << "draw " << i; + } +} + +// One documented default seed repo-wide: the test default equals +// laige-fuzz's default (tools/fuzz/laige-fuzz.cpp kDefaultSeed). +TEST(SeededRandom, DefaultSeedMatchesTheFuzzRunnerSeed) { + EXPECT_EQ(0x1F055EEDull, laige::testing::kDefaultTestSeed); +} + +// The override path: LAIGE_TEST_SEED replaces the default, TestSeed() +// reports it, and the override-seed stream has its own committed KAT. +TEST(SeededRandom, SeedOverrideFromEnv) { + ScopedTestSeedEnv scope(std::to_string(kOverrideSeed).c_str()); + // Decimal form, the accepted alternative to 0x-hex. + EXPECT_EQ(kOverrideSeed, laige::testing::TestSeed()); + const u64 hash = + seedCheck(kOverrideSeed, kOverrideStreamId, "override"); + EXPECT_EQ(kKnownAnswerOverrideSeed, hash) + << "override-seed stream changed (seed 0x2468ACCE01234567, stream " + << kOverrideStreamId << ", " << kSeedCheckDraws << " draws); see the " + "test-seed-check line above"; +} + +// The invalid-env path of TestSeed(): a set-but-unparseable LAIGE_TEST_SEED +// records a loud failure with the offending value (CORE-008) and falls +// back to the default seed so the caller can complete its assertions. +TEST(SeededRandom, InvalidSeedEnvFailsLoudly) { + ScopedTestSeedEnv scope("not-a-seed"); + u64 seed = 0; + EXPECT_NONFATAL_FAILURE({ seed = laige::testing::TestSeed(); }, + "LAIGE_TEST_SEED='not-a-seed' is not a valid seed"); + EXPECT_EQ(laige::testing::kDefaultTestSeed, seed); +} + +// The seed parser accepts the documented forms and rejects the rest. +TEST(SeededRandom, SeedParserAcceptsDocumentedFormsOnly) { + u64 v = 0; + EXPECT_TRUE(laige::testing::ParseTestSeed("0x1F055EED", v)); + EXPECT_EQ(0x1F055EEDull, v); + EXPECT_TRUE(laige::testing::ParseTestSeed("520445677", v)); // decimal form + EXPECT_EQ(0x1F055EEDull, v); + EXPECT_FALSE(laige::testing::ParseTestSeed("", v)); + EXPECT_FALSE(laige::testing::ParseTestSeed("0x", v)); + EXPECT_FALSE(laige::testing::ParseTestSeed("1F055EED", v)); // no 0x prefix + EXPECT_FALSE(laige::testing::ParseTestSeed("0x1F055EEDx", v)); // trailing junk + EXPECT_FALSE(laige::testing::ParseTestSeed("-1", v)); // negative +} + +// Substream isolation sanity: two distinct substream ids of the same seed +// produce distinct streams (a collision would mean the derivation hash +// broke; the committed KATs above are the positive proof of determinism). +TEST(SeededRandom, DistinctSubstreamsDiverge) { + u64 a[64] = {}; + u64 b[64] = {}; + laige::Prng ra = + laige::Prng::deriveSubstream(laige::testing::kDefaultTestSeed, + kSeedCheckStreamId); + laige::Prng rb = + laige::Prng::deriveSubstream(laige::testing::kDefaultTestSeed, + kOverrideStreamId); + for (int i = 0; i < 64; ++i) { + a[i] = ra.next_u64(); + b[i] = rb.next_u64(); + } + EXPECT_NE(fnv1a64(a, 64), fnv1a64(b, 64)) + << "substreams 1 and 2 of the default seed produced identical " + "streams — the substream derivation is broken"; +} diff --git a/tools/detcheck/laige-detcheck.cpp b/tools/detcheck/laige-detcheck.cpp index 442497c..c3a3fa3 100644 --- a/tools/detcheck/laige-detcheck.cpp +++ b/tools/detcheck/laige-detcheck.cpp @@ -48,6 +48,7 @@ // laige-detcheck --scenario= [--ticks=N] [--seed=HEX|DEC] // laige-detcheck --run-a= --run-b= // [-- scenario-args...] +// laige-detcheck --compare-combined= // // Mode 1 (--scenario): a built-in scenario (M0 stand-in for the M1 // scenarios): @@ -59,13 +60,36 @@ // proves the failure path (the step's Verify // clause) // -// Mode 2 (--run-a/--run-b): the real mode from M1-DET-04 on — two builds -// of the same scenario source (two build configurations) are executed -// and their hash streams compared. Everything after a `--` separator is -// passed to both scenario binaries (scenario arguments that could look -// like tool flags are unambiguous because of the separator). +// Modes 2+3 (--run-a/--run-b, then --compare-combined): the real mode +// from M1-DET-04 on — two builds of the same scenario source (two build +// configurations) are executed and their hash streams compared. +// Everything after a `--` separator is passed to both scenario binaries +// (scenario arguments that could look like tool flags are unambiguous +// because of the separator). // -// Report (stdout, stable and machine-greppable — LOG-001): +// The comparison is TWO-STAGE on every platform. The CI Windows runner +// does not deliver handles the checker process creates (pipes or files, +// even with the INHERIT bit set, even duplicated) to child processes +// through STARTUPINFO — measured in the M0-TEST-01 CI (runs 24/25): +// only handles the process itself inherited from its parent are +// delivered. A scenario child therefore cannot be given a capture pipe; +// its stdout must be the checker's own stdout, which the harness +// (the CTest check script's execute_process) captures: +// +// phase 1 (--run-a/--run-b): spawn each binary WITHOUT redirecting its +// stdout (plain inheritance). Each run's ticks land in the harness's +// capture of this process's stdout, delimited by the marker lines +// @@DETCHK-RUN-A-BEGIN@@ ... @@DETCHK-RUN-A-END @@ +// @@DETCHK-RUN-B-BEGIN@@ ... @@DETCHK-RUN-B-END @@ +// Phase 1 exits 0 when both scenario processes ran to completion, +// 2 on spawn failure or scenario failure (the reason on stderr). +// It does NOT read or compare the ticks. +// phase 2 (--compare-combined=): the check script writes the +// captured combined stream to a file and re-runs detcheck on it. +// Phase 2 splits the stream at the markers, re-runs the stream +// contract on each run, compares, and reports (the report below). +// +// Report (phase-2 stdout, stable and machine-greppable — LOG-001): // // match: // detcheck scenario= result=OK ticks= @@ -84,6 +108,9 @@ // 2 usage error, unknown scenario, a scenario run failed (non-zero // exit or spawn failure), or a scenario violated the output // contract (malformed line / tick gap / unbounded output) +// (--run-a/--run-b, phase 1: 0 when both scenario processes ran to +// completion, 2 on any spawn or scenario failure; the 0/1 comparison +// result comes from the --compare-combined phase) // // ============================================================================ // Built-in synthetic workload @@ -115,7 +142,6 @@ #include #include -#include #include #include #include @@ -128,6 +154,10 @@ #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN +// windows.h's WinDef.h defines the min/max macros, which collide with +// std::min/std::max (MSVC C2589 in compareStreams); NOMINMAX is the +// documented opt-out (CPP-009: compile-time platform boundary). +#define NOMINMAX #include #else #include @@ -300,150 +330,171 @@ std::wstring quoteArg(std::string_view arg) { return q; } -RunResult runScenario(const std::string& exe, - const std::vector& args) { - RunResult r; - std::wstring cmd = toWide(exe); - for (const std::string& a : args) cmd += L" " + quoteArg(a); - - SECURITY_ATTRIBUTES sa{}; - sa.nLength = sizeof sa; - HANDLE readH = INVALID_HANDLE_VALUE; - HANDLE writeH = INVALID_HANDLE_VALUE; - if (!CreatePipe(&readH, &writeH, &sa, 0)) { - r.error = "CreatePipe failed"; - return r; - } - // The child inherits the write end of the pipe. - if (!SetHandleInformation(writeH, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) { - r.error = "SetHandleInformation failed"; - CloseHandle(readH); - CloseHandle(writeH); - return r; - } - STARTUPINFOW si{}; - si.cb = sizeof si; - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdOutput = writeH; // capture stdout - si.hStdError = GetStdHandle(STD_ERROR_HANDLE); // stays visible in the log - si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); - PROCESS_INFORMATION pi{}; - if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, TRUE, 0, nullptr, - nullptr, &si, &pi)) { - r.error = "CreateProcessW failed (is the path correct?)"; - CloseHandle(readH); - CloseHandle(writeH); - return r; - } - CloseHandle(writeH); +#endif // defined(_WIN32) - char buf[65536]; - std::string pending; - for (;;) { - DWORD n = 0; - if (!PeekNamedPipe(readH, buf, sizeof buf, &n, nullptr, nullptr)) { - r.error = "PeekNamedPipe failed"; - break; - } - if (n == 0) break; // the scenario closed the pipe - if (n > sizeof buf) n = sizeof buf; - DWORD got = 0; - if (!ReadFile(readH, buf, n, &got, nullptr)) { - r.error = "ReadFile failed"; - break; - } - if (!appendChunk(r.lines, pending, buf, got, kMaxTicks)) { - r.error = "scenario output is unbounded (line or tick count exceeds " - "the contract)"; - break; +// --- Scenario spawn with markers (mode 2, phase 1) ------------------------- +// +// Two-stage capture, all platforms: +// +// The CI Windows runner does NOT deliver handles this process creates +// (pipes or files, even with the INHERIT bit confirmed set and even +// duplicated) to child processes through STARTUPINFO - measured in the +// M0-TEST-01 CI (runs 24/25): only handles this process itself +// inherited from its parent (its kernel-assigned fd0/1/2) are delivered. +// A scenario child therefore cannot be given a capture pipe it inherits; +// its stdout must BE this process's stdout (plain inheritance), which the +// harness (the CTest check script's execute_process) captures. +// +// So mode 2 runs in two phases: +// phase 1 (this function, per run): emit `@@DETCHK-RUN--BEGIN@@`, +// spawn the scenario child WITHOUT redirecting its stdout (it +// inherits this process's stdout), wait for it, emit +// `@@DETCHK-RUN--END @@`. The child's ticks land between +// the markers in the harness's capture of this process's stdout. +// phase 2 (--compare-combined): the check script hands the combined +// stream back to detcheck, which splits it at the markers, re-runs +// the stream contract on each run, and compares. +// +// This function does not read the ticks. It returns the child's exit +// code, or -1 when the spawn itself failed (error set). +int spawnScenarioWithMarkers(const std::string& exe, + const std::vector& args, + const std::string& label, std::string& error) { + // The BEGIN marker must reach the capture BEFORE the child is born, so + // the child's ticks (on the same stream) cannot precede it. + std::printf("@@DETCHK-RUN-%s-BEGIN@@\n", label.c_str()); + std::fflush(stdout); + int result = -1; +#ifdef _WIN32 + { + const std::wstring exeW = toWide(exe); + const DWORD attrs = GetFileAttributesW(exeW.c_str()); + if (attrs == INVALID_FILE_ATTRIBUTES) { + error = "scenario executable not found (lastError=" + + std::to_string(GetLastError()) + "): " + exe; + } else { + std::wstring cmd = exeW; + for (const std::string& a : args) cmd += L" " + quoteArg(a); + // Zeroed STARTUPINFO (no dwFlags): the child inherits this process's + // standard handles (the harness's capture). No self-created handle + // is involved - the kind the CI Windows runner never delivers (see + // above). A zeroed STARTUPINFO is passed rather than NULL: on the + // CI Windows runner the NULL form makes CreateProcessW fail while + // the zeroed form spawns fine (measured in the M0-TEST-01 CI, run + // 34732057356) - do not "simplify" this to NULL. + STARTUPINFOW si{}; + si.cb = sizeof si; + PROCESS_INFORMATION pi{}; + if (!CreateProcessW(exeW.c_str(), cmd.data(), nullptr, nullptr, TRUE, + 0, nullptr, nullptr, &si, &pi)) { + error = "CreateProcessW failed (lastError=" + + std::to_string(GetLastError()) + "): " + exe; + } else { + // Wait for termination BEFORE reading the exit code + // (GetExitCodeProcess on a live process returns STILL_ACTIVE). + if (WaitForSingleObject(pi.hProcess, INFINITE) != WAIT_OBJECT_0) { + error = "WaitForSingleObject(process) failed"; + } else { + DWORD raw = 0; + GetExitCodeProcess(pi.hProcess, &raw); + result = static_cast(raw); + } + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + } } } - CloseHandle(readH); - DWORD code = 0; - GetExitCodeProcess(pi.hProcess, &code); - CloseHandle(pi.hThread); - CloseHandle(pi.hProcess); - r.exitCode = static_cast(code); - finishPending(r.lines, pending, r); - if (r.exitCode != 0 && r.error.empty()) { - r.error = "scenario process exited with code " + std::to_string(code); +#else + { + if (::access(exe.c_str(), X_OK) != 0) { + error = "scenario executable not found: " + exe; + } else { + const pid_t pid = fork(); + if (pid < 0) { + error = "fork() failed"; + } else if (pid == 0) { + // Child: NO stdout redirect - it inherits this process's stdout + // (the harness's capture), so its ticks land between the parent's + // markers. + std::vector argv; + argv.push_back(const_cast(exe.c_str())); + for (const std::string& a : args) + argv.push_back(const_cast(a.c_str())); + argv.push_back(nullptr); + execv(exe.c_str(), argv.data()); + _exit(127); // execv failed: the path is wrong or not executable + } else { + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + error = "waitpid() failed"; + } else if (WIFEXITED(status)) { + result = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + // Conventional "exit code" for a signalled child; main() reports + // it as a scenario process failure (exit 2), same as the old + // single-shot path. + result = 128 + WTERMSIG(status); + } + } + } } - r.ok = r.error.empty(); - return r; +#endif + // The END marker carries the child's exit code (or -1 when the spawn + // itself failed, in which case the check script stops at phase 1; the + // marker keeps the captured stream parseable either way). + std::printf("@@DETCHK-RUN-%s-END %d@@\n", label.c_str(), result); + std::fflush(stdout); + return result; } -#else // POSIX (Linux, macOS) - -RunResult runScenario(const std::string& exe, - const std::vector& args) { - RunResult r; - int pipefd[2]; - if (pipe(pipefd) != 0) { - r.error = "pipe() failed"; - return r; - } - const pid_t pid = fork(); - if (pid < 0) { - r.error = "fork() failed"; - ::close(pipefd[0]); - ::close(pipefd[1]); - return r; - } - if (pid == 0) { - // Child: stdout goes to the pipe; stderr is inherited, so a scenario - // crash report still reaches the CI log. - if (::dup2(pipefd[1], 1) < 0) _exit(127); - ::close(pipefd[0]); - ::close(pipefd[1]); - std::vector argv; - argv.push_back(const_cast(exe.c_str())); - for (const std::string& a : args) argv.push_back(const_cast(a.c_str())); - argv.push_back(nullptr); - execv(exe.c_str(), argv.data()); - _exit(127); // execv failed: the path is wrong or not executable - } - ::close(pipefd[1]); +// Read a whole file, bounded to maxBytes (bounded work, CORE-003). +bool readFileBounded(const std::string& path, std::string& out, + std::size_t maxBytes) { + std::FILE* f = nullptr; +#ifdef _WIN32 + if (fopen_s(&f, path.c_str(), "rb") != 0) return false; +#else + f = std::fopen(path.c_str(), "rb"); + if (!f) return false; +#endif + out.clear(); char buf[65536]; - std::string pending; for (;;) { - const ssize_t n = ::read(pipefd[0], buf, sizeof buf); - if (n < 0) { - if (errno == EINTR) continue; - r.error = "read() of the scenario stdout failed"; - break; - } - if (n == 0) break; // EOF: the scenario finished - if (!appendChunk(r.lines, pending, buf, static_cast(n), - kMaxTicks)) { - r.error = "scenario output is unbounded (line or tick count exceeds " - "the contract)"; - break; + const std::size_t n = std::fread(buf, 1, sizeof buf, f); + if (n == 0) break; + if (out.size() + n > maxBytes) { + std::fclose(f); + return false; } + out.append(buf, n); } - ::close(pipefd[0]); - int status = 0; - if (waitpid(pid, &status, 0) < 0) { - r.error = "waitpid() failed"; - } - if (WIFEXITED(status)) { - r.exitCode = WEXITSTATUS(status); - } else if (WIFSIGNALED(status)) { - r.exitCode = 128 + WTERMSIG(status); - if (r.error.empty()) { - r.error = "scenario process was killed by signal " + - std::to_string(WTERMSIG(status)); + std::fclose(f); + return true; +} + +// Extract one run's tick lines from the combined stream: the lines +// strictly between `@@DETCHK-RUN--BEGIN@@` and the +// `@@DETCHK-RUN--END @@` line. Returns false when either marker +// is missing (a truncated capture is a contract failure, CORE-008). +bool extractRunStream(const std::vector& lines, + const std::string& label, + std::vector& out) { + const std::string begin = "@@DETCHK-RUN-" + label + "-BEGIN@@"; + const std::string endPrefix = "@@DETCHK-RUN-" + label + "-END "; + bool inStream = false; + for (const std::string& ln : lines) { + if (ln == begin) { + inStream = true; + continue; + } + if (inStream) { + if (ln.rfind(endPrefix, 0) == 0) return true; + out.push_back(ln); } } - finishPending(r.lines, pending, r); - if (r.exitCode != 0 && r.error.empty()) { - r.error = "scenario process exited with code " + std::to_string(r.exitCode); - } - r.ok = r.error.empty(); - return r; + return false; } -#endif // _WIN32 - // --- The built-in synthetic scenario (M0 stand-in) ------------------------- struct SyntheticBody { @@ -587,27 +638,35 @@ CompareResult compareStreams(const std::vector& a, return c; } -std::string basenameOf(std::string_view path) { - const auto pos = path.find_last_of("/\\"); - return pos == std::string_view::npos ? std::string(path) - : std::string(path.substr(pos + 1)); -} - void report(std::string_view scenarioName, const CompareResult& c, std::size_t ticks, std::string_view labelA, std::string_view labelB) { + auto emit = [](const char* line) { std::printf("%s\n", line); }; if (c.ok) { - std::printf("detcheck scenario=%s result=OK ticks=%llu\n", - std::string(scenarioName).c_str(), - static_cast(ticks)); - std::printf(" run-a: %s\n", std::string(labelA).c_str()); - std::printf(" run-b: %s\n", std::string(labelB).c_str()); + char buf[256]; + std::snprintf(buf, sizeof buf, + "detcheck scenario=%s result=OK ticks=%llu", + std::string(scenarioName).c_str(), + static_cast(ticks)); + emit(buf); + std::snprintf(buf, sizeof buf, " run-a: %s", + std::string(labelA).c_str()); + emit(buf); + std::snprintf(buf, sizeof buf, " run-b: %s", + std::string(labelB).c_str()); + emit(buf); } else { - std::printf("detcheck scenario=%s result=DIVERGED first_diff_tick=%llu\n", - std::string(scenarioName).c_str(), - static_cast(c.firstDiffTick)); - std::printf(" run-a: %s\n", c.lineA.c_str()); - std::printf(" run-b: %s\n", c.lineB.c_str()); + char buf[256]; + std::snprintf(buf, sizeof buf, + "detcheck scenario=%s result=DIVERGED " + "first_diff_tick=%llu", + std::string(scenarioName).c_str(), + static_cast(c.firstDiffTick)); + emit(buf); + std::snprintf(buf, sizeof buf, " run-a: %s", c.lineA.c_str()); + emit(buf); + std::snprintf(buf, sizeof buf, " run-b: %s", c.lineB.c_str()); + emit(buf); } } @@ -617,6 +676,8 @@ struct Args { std::string scenario; // mode 1 std::string runA; // mode 2 std::string runB; // mode 2 + bool cmpCombined = false; // mode 3 + std::string cmpCombinedFile; // mode 3 combined stream file bool ticksSet = false; int ticks = kDefaultTicks; bool seedSet = false; @@ -631,12 +692,23 @@ void printUsage(std::FILE* out) { "[--seed=HEX|DEC]\n" " laige-detcheck --run-a= --run-b=" " [-- scenario-args...]\n" + " laige-detcheck --compare-combined=\n" " laige-detcheck --help\n" "\n" " --scenario built-in scenario: synthetic | " "synthetic-perturbed\n" " --run-a/--run-b two builds of the same scenario (two build\n" - " configurations)\n" + " configurations). Two-stage capture: this\n" + " invocation SPAWNS both binaries (each child's\n" + " stdout is inherited from this process, so its\n" + " ticks flow to this process's stdout, delimited\n" + " by @@DETCHK-RUN-A/B-BEGIN/END@@ markers) and\n" + " does NOT compare; the caller hands the\n" + " combined stream to --compare-combined.\n" + " --compare-combined read a combined stream (markers + both tick\n" + " streams), validate both runs against the\n" + " stream contract, compare, and report (phase 2\n" + " of --run-a/--run-b)\n" " --ticks built-in scenario tick count (1..%d, " "default %d)\n" " --seed built-in scenario seed (0xHEX or decimal)\n" @@ -644,7 +716,10 @@ void printUsage(std::FILE* out) { " are passed to both scenario binaries\n" "\n" "Exit codes: 0 = match, 1 = divergence, 2 = error (usage, unknown\n" - "scenario, scenario failure, contract violation).\n", + "scenario, spawn failure, scenario failure, contract violation).\n" + "--run-a/--run-b exits 0 when both scenario processes ran to\n" + "completion and 2 on any spawn or scenario failure (the comparison\n" + "happens in the --compare-combined phase).\n", kMaxTicks, kDefaultTicks); } @@ -674,6 +749,10 @@ bool parseArgs(int argc, char** argv, Args& a) { } else if (key == "--run-b") { if (!hasValue) return false; a.runB = value; + } else if (key == "--compare-combined") { + if (!hasValue) return false; + a.cmpCombined = true; + a.cmpCombinedFile = value; } else if (key == "--ticks") { if (!hasValue || value.empty()) return false; std::uint64_t v = 0; @@ -715,10 +794,12 @@ int main(int argc, char** argv) { } const bool mode1 = !a.scenario.empty(); const bool mode2 = !a.runA.empty() || !a.runB.empty(); - if (mode1 && mode2) { + const bool mode3 = a.cmpCombined; + const int modeCount = (mode1 ? 1 : 0) + (mode2 ? 1 : 0) + (mode3 ? 1 : 0); + if (modeCount != 1) { std::fprintf(stderr, - "laige-detcheck: --scenario and --run-a/--run-b are " - "mutually exclusive\n"); + "laige-detcheck: exactly one of --scenario, " + "--run-a/--run-b, or --compare-combined is required\n"); printUsage(stderr); return 2; } @@ -736,10 +817,6 @@ int main(int argc, char** argv) { printUsage(stderr); return 2; } - if (!mode1 && !mode2) { - printUsage(stderr); - return 2; - } if (mode2 && (a.runA.empty() || a.runB.empty())) { std::fprintf(stderr, "laige-detcheck: both --run-a and --run-b are required\n"); @@ -754,38 +831,97 @@ int main(int argc, char** argv) { return 2; } - // --- Mode 2: two scenario binaries (two build configurations) ---------- - if (mode2) { - const RunResult resA = runScenario(a.runA, a.positionals); - if (!resA.ok) { - std::fprintf(stderr, "laige-detcheck: scenario run-a: %s\n", - resA.error.c_str()); + // --- Mode 3: compare a combined stream (phase 2) ------------------------ + if (mode3) { + // The combined stream holds, per run: a BEGIN marker line, the tick + // lines, and an END marker line. Its size is bounded by the contract + // (two runs of kMaxTicks lines of kMaxLineBytes, plus the markers); + // reading more is a contract failure, not an allocation (CORE-003). + const std::size_t maxBytes = + (2 * static_cast(kMaxTicks) + 8) * (kMaxLineBytes + 1); + std::string content; + if (!readFileBounded(a.cmpCombinedFile, content, maxBytes)) { + std::fprintf(stderr, + "laige-detcheck: cannot read combined stream file: %s\n", + a.cmpCombinedFile.c_str()); return 2; } - const std::string errA = validateStream(resA.lines, kMaxTicks); - if (!errA.empty()) { - std::fprintf(stderr, "laige-detcheck: scenario run-a: %s\n", - errA.c_str()); + std::vector lines; + std::string pending; + RunResult bounded; // carries the unbounded-output error if any + const bool okA = appendChunk(lines, pending, content.data(), + content.size(), 2 * kMaxTicks + 8); + if (okA) finishPending(lines, pending, bounded); + if (!okA || !bounded.error.empty()) { + std::fprintf(stderr, + "laige-detcheck: combined stream is unbounded (line or " + "tick count exceeds the contract)\n"); return 2; } - const RunResult resB = runScenario(a.runB, a.positionals); - if (!resB.ok) { - std::fprintf(stderr, "laige-detcheck: scenario run-b: %s\n", - resB.error.c_str()); + std::vector linesA, linesB; + if (!extractRunStream(lines, "A", linesA)) { + std::fprintf(stderr, + "laige-detcheck: combined stream is missing the " + "run-A markers (@@DETCHK-RUN-A-BEGIN/END@@)\n"); return 2; } - const std::string errB = validateStream(resB.lines, kMaxTicks); + if (!extractRunStream(lines, "B", linesB)) { + std::fprintf(stderr, + "laige-detcheck: combined stream is missing the " + "run-B markers (@@DETCHK-RUN-B-BEGIN/END@@)\n"); + return 2; + } + const std::string errA = validateStream(linesA, kMaxTicks); + if (!errA.empty()) { + std::fprintf(stderr, "laige-detcheck: scenario run-a: %s\n", errA.c_str()); + return 2; + } + const std::string errB = validateStream(linesB, kMaxTicks); if (!errB.empty()) { - std::fprintf(stderr, "laige-detcheck: scenario run-b: %s\n", - errB.c_str()); + std::fprintf(stderr, "laige-detcheck: scenario run-b: %s\n", errB.c_str()); return 2; } - const CompareResult c = compareStreams(resA.lines, resB.lines); - report(basenameOf(a.runA) + " vs " + basenameOf(a.runB), c, - resA.lines.size(), a.runA, a.runB); + const CompareResult c = compareStreams(linesA, linesB); + report("combined", c, linesA.size(), "run-a", "run-b"); return c.ok ? 0 : 1; } + // --- Mode 2: two scenario binaries (phase 1 of two-stage capture) ------ + // Each scenario child inherits THIS process's stdout (no capture pipe - + // see spawnScenarioWithMarkers), so its ticks land in the harness's + // capture of this process's stdout, between the BEGIN/END markers. The + // comparison happens in phase 2 (--compare-combined), invoked by the + // check script over the combined stream. + if (mode2) { + std::string errA; + const int codeA = + spawnScenarioWithMarkers(a.runA, a.positionals, "A", errA); + if (codeA < 0) { + std::fprintf(stderr, "laige-detcheck: scenario run-a: %s\n", errA.c_str()); + return 2; + } + if (codeA != 0) { + std::fprintf(stderr, "laige-detcheck: scenario run-a: scenario process exited with " + "code %d (command: %s)\n", + codeA, a.runA.c_str()); + return 2; + } + std::string errB; + const int codeB = + spawnScenarioWithMarkers(a.runB, a.positionals, "B", errB); + if (codeB < 0) { + std::fprintf(stderr, "laige-detcheck: scenario run-b: %s\n", errB.c_str()); + return 2; + } + if (codeB != 0) { + std::fprintf(stderr, "laige-detcheck: scenario run-b: scenario process exited with " + "code %d (command: %s)\n", + codeB, a.runB.c_str()); + return 2; + } + return 0; + } + // --- Mode 1: built-in scenario, two in-process runs -------------------- const RunSpec specA{ std::string(a.scenario) + "[seed=" + formatSeed(a.seed) +