From 8a2f93f68f507a4cc578b1e78dd0912c60bdfddb Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:58:17 -0800 Subject: [PATCH 1/5] Record fresh M5 BactoTraits finalization and guarded release verification --- .../bactotraits-identity-20261115.md | 4 + .../m5-bactotraits-release-20261116.json | 297 +++++++++++++++ .../m5-bactotraits-release-20261116.md | 134 +++++++ .../adhoc/m5_release_archive_stat_20261116.py | 48 +++ scripts/adhoc/m5_release_chain_20261116.py | 345 ++++++++++++++++++ scripts/adhoc/m5_release_checks_20261116.py | 68 ++++ .../adhoc/m5_release_environment_20261116.py | 208 +++++++++++ .../adhoc/m5_release_gate_setup_20261116.py | 169 +++++++++ scripts/adhoc/m5_release_lineage_20261116.py | 211 +++++++++++ .../adhoc/m5_release_preservation_20261116.py | 57 +++ scripts/adhoc/m5_release_results_20261116.py | 119 ++++++ scripts/adhoc/m5_release_state_20261116.py | 121 ++++++ tests/test_m5_release_chain.py | 95 +++++ tests/test_m5_release_progress.py | 36 ++ 14 files changed, 1912 insertions(+) create mode 100644 docs/validation/m5-bactotraits-release-20261116.json create mode 100644 docs/validation/m5-bactotraits-release-20261116.md create mode 100644 scripts/adhoc/m5_release_archive_stat_20261116.py create mode 100644 scripts/adhoc/m5_release_chain_20261116.py create mode 100644 scripts/adhoc/m5_release_checks_20261116.py create mode 100644 scripts/adhoc/m5_release_environment_20261116.py create mode 100644 scripts/adhoc/m5_release_gate_setup_20261116.py create mode 100644 scripts/adhoc/m5_release_lineage_20261116.py create mode 100644 scripts/adhoc/m5_release_preservation_20261116.py create mode 100644 scripts/adhoc/m5_release_results_20261116.py create mode 100644 scripts/adhoc/m5_release_state_20261116.py create mode 100644 tests/test_m5_release_chain.py create mode 100644 tests/test_m5_release_progress.py diff --git a/docs/validation/bactotraits-identity-20261115.md b/docs/validation/bactotraits-identity-20261115.md index c433ded..a8981ec 100644 --- a/docs/validation/bactotraits-identity-20261115.md +++ b/docs/validation/bactotraits-identity-20261115.md @@ -1,5 +1,9 @@ # BactoTraits mixed-strain source investigation +Later execution status is recorded in the +[M5 release verification follow-up](m5-bactotraits-release-20261116.md). +The investigation below preserves its original pre-merge, pre-finalization scope. + This investigation follows [#130](https://github.com/CultureBotAI/EdgeMech/issues/130) for the exact fresh-05 occurrence `urn:kg-microbe:occurrence:v1:1e9dec218a07f7e480add8ff273e3a9285f79d57746ca74041702839207a8136`: diff --git a/docs/validation/m5-bactotraits-release-20261116.json b/docs/validation/m5-bactotraits-release-20261116.json new file mode 100644 index 0000000..48ebcba --- /dev/null +++ b/docs/validation/m5-bactotraits-release-20261116.json @@ -0,0 +1,297 @@ +{ + "profile": "m5-bactotraits-release-verification/v1", + "observed_at": "2026-11-16T20:43:58.815480+00:00", + "export_helper_sha256": "96f3d2bf325ca2889700089a2d3276a86339ff9fd70bc8b8a2eb28adc6eb744a", + "fresh_baseline_candidate_pair_verified": true, + "bounded_source_finalization_verified": true, + "merged_graph_verified": false, + "published_release_verified": false, + "source_row": 8521, + "bacdive_id": 17851, + "scientific_claim_resolved": false, + "neighboring_rows_quarantined": false, + "results": { + "baseline-m5-20261116-r1": { + "original_receipt": "/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116/data/validation-bactotraits-130/baseline-m5-20261116-r1/receipt.json", + "receipt": { + "sha256": "8d2a780d27670106456450fb4c8d1063424570b7bf9277387c8abcb09a389eb0", + "bytes": 36761 + }, + "producer_revision": "ef27b97a0b450767da54235c6105b5250c178adc", + "edges": 21, + "nodes": 22, + "quarantine_rows": 0, + "finalization_run": false, + "artifacts": { + "authority/ncbitaxon-label.json": { + "bytes": 955, + "sha256": "277127387d0107c00ae57bafd5c51a1ef8e5399e31bc491ffa6e132fb85e063f" + }, + "cache/pystow/README.md": { + "bytes": 1566, + "sha256": "8d471e43c6d74b56b35252feb86132ce8f32caea360865cf4ec48a4c5fe0ffef" + }, + "input-plan.json": { + "bytes": 4248, + "sha256": "f6566b3f337b34eeab0da90c66cb59ac3260c84dc5848fd7b18ea30a953e14fa" + }, + "lookup/bacdive.tsv": { + "bytes": 1419, + "sha256": "e238208d460a47c437e6ca4da3e3aea262270d8d30ee006c201f227347f4647d" + }, + "mappings/custom_curies.yaml": { + "bytes": 34718, + "sha256": "5b201563b8042eb51a6984f56354be2b3420deae0d029ce707c961ec6eeab784" + }, + "mappings/metpo_alias_mappings.tsv": { + "bytes": 19165, + "sha256": "15c4aa67644fd0aac233dbc909cda0b8bbfcbc50bb998fc8bf3e06a7b19ace06" + }, + "mappings/prefixmap.json": { + "bytes": 2033, + "sha256": "a11f11fc43db2d3d0ffd74aca08fbc1da08033074b355ef2e0a1d6d547c29762" + }, + "raw/BactoTraits_databaseV2_Jun2022.csv": { + "bytes": 2074, + "sha256": "e5c10497510e580d1cabb4f76b51c90b9fa3472547d25461aa5d834ac69ef9dd" + }, + "templates/metpo-properties.tsv": { + "bytes": 21929, + "sha256": "8ef8929f3f8d3ed61ba41f062548a3d4d2e058932c522b7ddb3a01d1e06cef63" + }, + "templates/metpo_sheet.tsv": { + "bytes": 81520, + "sha256": "ec9700569bf5e83d39fe4c29f525c06d3c396adfad2ce0bf33072b8b1a1b1409" + }, + "tmp/bactotraits/bactotraits.tsv": { + "bytes": 1590, + "sha256": "078c43494f96aadca7ae8c1c5c2d63c3438fcb1c5dfe8198b4fae1cf7dc786f6" + }, + "tmp/bactotraits/bactotraits_mapping.tsv": { + "bytes": 125, + "sha256": "c889f33b711569ee00c60a96fb7bf29e7efeaa9f2011dd7b9388e347c638a006" + }, + "tmp/bactotraits/bactotraits_mapping.tsv.complete": { + "bytes": 64, + "sha256": "62ef0d8f6c56253c12c1218cf1650fd87297ceeb3dd218affd4e4e5d77a1bdd7" + }, + "transform.stderr.txt": { + "bytes": 855, + "sha256": "19641439822a224c9e250a772efe7ee7a4bfbb6c687a26ddba5eda551436b1cb" + }, + "transform.stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "transformed/bactotraits/edges.tsv": { + "bytes": 2396, + "sha256": "666438931337cc512cfeac82a451bbfad1da6e54daa69d730b199ac5373b43a0" + }, + "transformed/bactotraits/nodes.tsv": { + "bytes": 1897, + "sha256": "e21c1fcba073d3885493e4819017d4773d3fe02a48680be5e9c9e297ed19202e" + } + } + }, + "candidate-m5-20261116-r1": { + "original_receipt": "/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116/data/validation-bactotraits-130/candidate-m5-20261116-r1/receipt.json", + "receipt": { + "sha256": "03446ab15dbe56bf277d25c810b5b0a612a73fe5850d73447d112ab74e611477", + "bytes": 32000 + }, + "producer_revision": "2d94ed094167d94402f21400a9ad7dc4f7f20438", + "edges": 0, + "nodes": 0, + "quarantine_rows": 1, + "finalization_run": false, + "artifacts": { + "authority/ncbitaxon-label.json": { + "bytes": 955, + "sha256": "277127387d0107c00ae57bafd5c51a1ef8e5399e31bc491ffa6e132fb85e063f" + }, + "cache/pystow/README.md": { + "bytes": 1566, + "sha256": "8d471e43c6d74b56b35252feb86132ce8f32caea360865cf4ec48a4c5fe0ffef" + }, + "input-plan.json": { + "bytes": 4245, + "sha256": "28cec7d826dd75872a034b80f6c6b224488fc7ea5d0b291c05ca7029c52c534e" + }, + "lookup/bacdive.tsv": { + "bytes": 1419, + "sha256": "e238208d460a47c437e6ca4da3e3aea262270d8d30ee006c201f227347f4647d" + }, + "mappings/custom_curies.yaml": { + "bytes": 34718, + "sha256": "5b201563b8042eb51a6984f56354be2b3420deae0d029ce707c961ec6eeab784" + }, + "mappings/metpo_alias_mappings.tsv": { + "bytes": 19165, + "sha256": "15c4aa67644fd0aac233dbc909cda0b8bbfcbc50bb998fc8bf3e06a7b19ace06" + }, + "mappings/prefixmap.json": { + "bytes": 2033, + "sha256": "a11f11fc43db2d3d0ffd74aca08fbc1da08033074b355ef2e0a1d6d547c29762" + }, + "raw/BactoTraits_databaseV2_Jun2022.csv": { + "bytes": 2074, + "sha256": "e5c10497510e580d1cabb4f76b51c90b9fa3472547d25461aa5d834ac69ef9dd" + }, + "templates/metpo-properties.tsv": { + "bytes": 21929, + "sha256": "8ef8929f3f8d3ed61ba41f062548a3d4d2e058932c522b7ddb3a01d1e06cef63" + }, + "templates/metpo_sheet.tsv": { + "bytes": 81520, + "sha256": "ec9700569bf5e83d39fe4c29f525c06d3c396adfad2ce0bf33072b8b1a1b1409" + }, + "tmp/bactotraits/bactotraits.tsv": { + "bytes": 1168, + "sha256": "dd171b985cad4887a7b72e8ea653d7938fb708bd49a507eeb58c4fc7d6277bf5" + }, + "tmp/bactotraits/bactotraits_mapping.tsv": { + "bytes": 125, + "sha256": "c889f33b711569ee00c60a96fb7bf29e7efeaa9f2011dd7b9388e347c638a006" + }, + "tmp/bactotraits/bactotraits_mapping.tsv.complete": { + "bytes": 64, + "sha256": "62ef0d8f6c56253c12c1218cf1650fd87297ceeb3dd218affd4e4e5d77a1bdd7" + }, + "transform.stderr.txt": { + "bytes": 855, + "sha256": "19641439822a224c9e250a772efe7ee7a4bfbb6c687a26ddba5eda551436b1cb" + }, + "transform.stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "transformed/bactotraits/bactotraits_identity_quarantine.tsv": { + "bytes": 3761, + "sha256": "28d62147d49735406a35c721db345497c7e044f69f6d723caa0367f996158918" + }, + "transformed/bactotraits/edges.tsv": { + "bytes": 86, + "sha256": "8d142a6d6931b88959df0577cfad52cb545edb2ba48222f11a34a05bc6336d46" + }, + "transformed/bactotraits/nodes.tsv": { + "bytes": 73, + "sha256": "0604cdfbcc9fdc122da6154ea79a7c434e12b501f91650f57297164edd2cd3d9" + } + } + }, + "candidate-finalized-m5-20261116-r1": { + "original_receipt": "/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116/data/validation-bactotraits-130/candidate-finalized-m5-20261116-r1/receipt.json", + "receipt": { + "sha256": "0c832333ac93018e95e81d2ba53e9e587b58c4838840859ba823f8f7fd441996", + "bytes": 50984 + }, + "producer_revision": "2d94ed094167d94402f21400a9ad7dc4f7f20438", + "edges": 0, + "nodes": 0, + "quarantine_rows": 1, + "finalization_run": true, + "artifacts": { + "authority/ncbitaxon-label.json": { + "bytes": 955, + "sha256": "277127387d0107c00ae57bafd5c51a1ef8e5399e31bc491ffa6e132fb85e063f" + }, + "cache/pystow/README.md": { + "bytes": 1566, + "sha256": "8d471e43c6d74b56b35252feb86132ce8f32caea360865cf4ec48a4c5fe0ffef" + }, + "execution.stderr.txt": { + "bytes": 855, + "sha256": "19641439822a224c9e250a772efe7ee7a4bfbb6c687a26ddba5eda551436b1cb" + }, + "execution.stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "input-plan.json": { + "bytes": 41480, + "sha256": "bb5f896abd27db61ecbfbde63718b8cff8ebd688d62143a949077579ff14bb56" + }, + "lookup/bacdive.tsv": { + "bytes": 1419, + "sha256": "e238208d460a47c437e6ca4da3e3aea262270d8d30ee006c201f227347f4647d" + }, + "mappings/custom_curies.yaml": { + "bytes": 34718, + "sha256": "5b201563b8042eb51a6984f56354be2b3420deae0d029ce707c961ec6eeab784" + }, + "mappings/metpo_alias_mappings.tsv": { + "bytes": 19165, + "sha256": "15c4aa67644fd0aac233dbc909cda0b8bbfcbc50bb998fc8bf3e06a7b19ace06" + }, + "mappings/prefixmap.json": { + "bytes": 2033, + "sha256": "a11f11fc43db2d3d0ffd74aca08fbc1da08033074b355ef2e0a1d6d547c29762" + }, + "merge.json": { + "bytes": 1118, + "sha256": "4e9e9164952bf1339bb2189d7c220e4877d9d3f7297436ca0479369a5fc71bc2" + }, + "raw/BactoTraits_databaseV2_Jun2022.csv": { + "bytes": 2074, + "sha256": "e5c10497510e580d1cabb4f76b51c90b9fa3472547d25461aa5d834ac69ef9dd" + }, + "templates/metpo-properties.tsv": { + "bytes": 21929, + "sha256": "8ef8929f3f8d3ed61ba41f062548a3d4d2e058932c522b7ddb3a01d1e06cef63" + }, + "templates/metpo_sheet.tsv": { + "bytes": 81520, + "sha256": "ec9700569bf5e83d39fe4c29f525c06d3c396adfad2ce0bf33072b8b1a1b1409" + }, + "tmp/bactotraits/bactotraits.tsv": { + "bytes": 1168, + "sha256": "dd171b985cad4887a7b72e8ea653d7938fb708bd49a507eeb58c4fc7d6277bf5" + }, + "tmp/bactotraits/bactotraits_mapping.tsv": { + "bytes": 125, + "sha256": "c889f33b711569ee00c60a96fb7bf29e7efeaa9f2011dd7b9388e347c638a006" + }, + "tmp/bactotraits/bactotraits_mapping.tsv.complete": { + "bytes": 64, + "sha256": "62ef0d8f6c56253c12c1218cf1650fd87297ceeb3dd218affd4e4e5d77a1bdd7" + }, + "transformed/bactotraits/bactotraits_identity_quarantine.tsv": { + "bytes": 3761, + "sha256": "28d62147d49735406a35c721db345497c7e044f69f6d723caa0367f996158918" + }, + "transformed/bactotraits/edges.tsv": { + "bytes": 99, + "sha256": "1a1f91d2da59ddf2aad7f854f52b7e2b23aaba455918bdd7382980bf6e25079f" + }, + "transformed/bactotraits/go_reference_resolution.tsv": { + "bytes": 204, + "sha256": "082792210b81208e4bfab7db9686e1d64a7bc98366f9bc0b7a4a302d1e8cfacc" + }, + "transformed/bactotraits/nodes.tsv": { + "bytes": 73, + "sha256": "0604cdfbcc9fdc122da6154ea79a7c434e12b501f91650f57297164edd2cd3d9" + }, + "transformed/bactotraits/source_canonicalization.tsv": { + "bytes": 48, + "sha256": "4e6a2e08ad0ec3ed46d69cf49c5790c4c0bc253c6cb18d9f1866363bedbd4870" + }, + "transformed/bactotraits/source_finalization.json": { + "bytes": 10036, + "sha256": "fb0ebb33062aa6060d20d2580f9000137f39fe4260526c7ed6f7ada473ce20c6" + }, + "transformed/bactotraits/source_fingerprint.json": { + "bytes": 2086, + "sha256": "987b8cb6db6403a9b3dc2a8140abb4a1a0e053b7ef8a1564743dfab03f43d9e6" + }, + "transformed/bactotraits/source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "transformed/bactotraits/source_reference_resolution.tsv": { + "bytes": 102, + "sha256": "727041d59de466a4823b0c919310b7efcea1d8695f53b0b09e2e5192656798c8" + } + } + } + } +} diff --git a/docs/validation/m5-bactotraits-release-20261116.md b/docs/validation/m5-bactotraits-release-20261116.md new file mode 100644 index 0000000..82ad60b --- /dev/null +++ b/docs/validation/m5-bactotraits-release-20261116.md @@ -0,0 +1,134 @@ +# M5 BactoTraits graph-release verification — 2026-11-16 + +The new M5 pair verifies the exact-row correction through the actual producer, +and a separate fresh candidate verifies that its complete quarantine audit and +zero graph rows survive production source finalization. Strict merge refuses +the incomplete upstream chain. No corrected merged graph or published release +is established by this evidence. + +| Check | Actual result | +| --- | --- | +| Fresh baseline, `ef27b97a0b450767da54235c6105b5250c178adc` | 21 edges, 22 nodes | +| Fresh corrected candidate, `2d94ed094167d94402f21400a9ad7dc4f7f20438` | 0 edges, 0 nodes, 1 complete quarantine audit | +| Separate fresh candidate through `finalize(fresh_run=True)` | 0 edges, 0 nodes, identical quarantine audit | +| Strict `load_and_merge(processes=1)` | Refused missing current BacDive whole-source finalization; no merged output | +| Current raw reviewed-input preflight | Passed against original M5 raw root | +| Current published corrected release | Unavailable; dependency tracked in [#143](https://github.com/CultureBotAI/EdgeMech/issues/143) | + +The producer merge is [KG-Microbe PR #1867](https://github.com/Knowledge-Graph-Hub/kg-microbe/pull/1867), +reviewed head `a86c63e778ebf8a646abb9d4d9e0718dfd49f5a0`. The earlier +[identity investigation](bactotraits-identity-20261115.md) is a historical +pre-finalization record. Its old receipts and every nested cache file remain +unchanged; none was used in place of the new baseline. + +## Current publication and preserved graph + +The live GitHub API inspection found 14 releases, with latest tag +[2025-03-07](https://github.com/Knowledge-Graph-Hub/kg-microbe/releases/tag/2025-03-07), +published `2025-03-07T23:03:41Z`. No November 16 corrected release was present. +The retained canonical M5 `merged-kg.tar.gz` has SHA256 +`3d529dd0024843f9278ecfbfc8703413061ef9c4c7474805ecec03aa8cfdfd4c`. +Its embedded historical manifest records generation `2026-10-18T11:56:34+00:00`, +producer `bcfa6bf5b-dirty`, Python 3.10.11, 17,927,641 edges and 3,440,986 nodes. +Those counts are manifest claims, not a new graph-wide validation. + +[Upstream #887](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/887) +remains open and the current release workflow retains its retired artifact URL. +A new URL probe failed DNS; this execution did not reproduce the issue's +historical HTTP404. Publication requires a fresh complete dependency chain, +canonical merge and lineage verification, a working artifact publication source, +separate human approval, and verification of the published asset bytes. + +Independent review found that the first archive stat observation rounded its +nanosecond timestamp by 24 ns through JavaScript Number serialization. The original +observation is preserved. A new Python-written observation records exact +`st_mtime_ns=1792324632633850376` and confirms the unchanged archive SHA256. +[#147](https://github.com/CultureBotAI/EdgeMech/issues/147) tracks this corrected +metadata recording; it is not evidence of archive mutation. + +## New execution and provenance + +Both new replay runs used the unchanged +`scripts/adhoc/replay_bactotraits_identity_130_20261115.py`, SHA256 +`060d7e4dbb6189051f989a4a9efbf55336408e4224ecba5a4975e710538ea90e`, +with `--plan` before each `--run` and a new `--baseline-receipt`. They used one +unit at a time, exclusive output stages, identical shared inputs and the same +absolute `.venv310/bin/python` executable. Python is 3.10.11, built on M5 on +October 10 with Clang 21.0.0; old Python version strings were not rewritten. + +The original CSV SHA256 is +`18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b`. +The complete decoded row 8521/BacDive17851 SHA256 is +`345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a`. +The receipts bind the original and staged CSV, actual BacDive lookup, METPO +tables, local alias and prefix mappings, producer Python source files and policy. +NCBITaxon DB metadata and actual queried statements are bound; the DB and OWL +release agree at `2026-07-12`. This is not a whole-database hash. + +The first Poetry install failed DNS. The isolated Python 3.10 environment was +then installed offline from all 346 applicable main/dev lock packages using +`--no-index --no-deps --require-hashes`. Installed versions and `pip check` pass. +335 wheels match lock hashes directly; 11 cached built wheels have separate +wheel hashes and co-located source archives matching the lock. The latter is +source/archive association, not a reproducible-build attestation. Lock SHA256: +`a9e8336e172faef6449ed79a2dde8df7932a5cfb9ba368e7a93a7c7ab83d7f6f`. + +| Receipt | SHA256 | +| --- | --- | +| New baseline | `8d2a780d27670106456450fb4c8d1063424570b7bf9277387c8abcb09a389eb0` | +| New pre-finalization candidate | `03446ab15dbe56bf277d25c810b5b0a612a73fe5850d73447d112ab74e611477` | +| New finalized candidate | `0c832333ac93018e95e81d2ba53e9e587b58c4838840859ba823f8f7fd441996` | + +The separate finalization helper runs the actual producer and finalizer on the +same live object; it does not reopen old outputs and claim a fresh run. It pins +shared policy files before/after use, the schema trio, finalization code, +configuration, preceding receipts and resulting files. The complete quarantine +audit remains SHA256 +`28d62147d49735406a35c721db345497c7e044f69f6d723caa0367f996158918`. +Finalized empty edge TSV SHA256 is +`1a1f91d2da59ddf2aad7f854f52b7e2b23aaba455918bdd7382980bf6e25079f`; +the finalizer adds canonical columns without adding assertions. + +All three new stages were copied byte-for-byte beneath the required invocation +output root `.temp/m5-worker-run-20261116/results-r1/`; original paths in receipts +remain unchanged. The [compact result](m5-bactotraits-release-20261116.json) +records every receipt-bound output hash. Complete receipts, plans, stdout/stderr, +publication responses, environment inventory and historical preservation checks +remain private under that invocation root. + +## Lineage and scope + +The full original CSV has 19,456 data records and exactly one matching BacDive +17851/conflicting-accession occurrence. The lookup has 36,596 records, with one +mapping to NCBITaxon:1123071. The producer rebuilds its mapping and quarantines +the complete reviewed row before its pruned copy and graph writes. The retained +October copies were inspected read-only; stale completion records cannot admit +them through the strict current merge path. + +The equivalent historical Madin assertion has independent source provenance: +raw row 50,223, DSM18772, `motility=no`, `data_source=gold`, `ref_id=13521`; +the transformed edge has `infores:madin_etal`. It is not evidence of leakage +from this BactoTraits row. The BacDive lookup separately records motility=no, +although the inspected historical BacDive edge file has no matching assertion. +Neither independent source was newly scientifically reassessed here. + +The lineage helper streamed eight explicit input paths, including ignored data, +and hashed them before/after. Its source search used `rg --no-ignore --hidden`. +This is not absence evidence across arbitrary copies or uninspected graphs. +No neighboring row was quarantined, no replacement organism was inferred, and +the correction does not establish trait falsity or graph-wide validity. + +## Independent review and checks + +Read-only adversarial review independently recomputed all new stage artifact +hashes, checked complete audit JSON against the original row, compared actual +runtime versions, and verified all 37 historical files. It found no defect in +the bounded producer/finalizer/admission evidence. Two evidence bookkeeping +findings were filed and fixed: [#146](https://github.com/CultureBotAI/EdgeMech/issues/146) +for concurrent progress updates and [#147](https://github.com/CultureBotAI/EdgeMech/issues/147) +for stat precision. A second read-only review verified both fixes. + +The finalization helper's six focused tests passed. The progress regression +passes and checks both command retention and metadata when launchers finish +out of order. Final repository check results and the production dependency +canary will be appended after execution; this document does not imply they passed. diff --git a/scripts/adhoc/m5_release_archive_stat_20261116.py b/scripts/adhoc/m5_release_archive_stat_20261116.py new file mode 100644 index 0000000..e2959a3 --- /dev/null +++ b/scripts/adhoc/m5_release_archive_stat_20261116.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Preserve exact archive stat integers after review found JS precision loss. + +2026-11-16: this read-only observation supplements, never rewrites, the original +publication receipt. A nanosecond rounding difference is not a byte mutation. +""" + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +ARCHIVE = Path( + "/Users/marcin/Documents/VIMSS/ontology/KG-Hub/KG-Microbe/kg-microbe/" + "data/merged/merged-kg.tar.gz" +) + + +def main(): + """Hash the preserved archive and serialize exact stat values directly in Python.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + args = parser.parse_args() + before = ARCHIVE.stat() + digest = hashlib.sha256() + with ARCHIVE.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + after = ARCHIVE.stat() + fields = ("st_size", "st_mtime_ns", "st_ctime_ns", "st_dev", "st_ino") + assert all(getattr(before, key) == getattr(after, key) for key in fields) + result = { + "path": str(ARCHIVE), "sha256": digest.hexdigest(), + "observed_at": datetime.now(timezone.utc).isoformat(), + "stat": {key: getattr(after, key) for key in fields}, + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "note": "Original publication stat lost 24ns in JS integer rounding; archive bytes match.", + "published_release_verified": False, + } + assert result["sha256"] == "3d529dd0024843f9278ecfbfc8703413061ef9c4c7474805ecec03aa8cfdfd4c" + with args.output.open("x") as stream: + stream.write(json.dumps(result, indent=2) + "\n") + print(result["sha256"]) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_chain_20261116.py b/scripts/adhoc/m5_release_chain_20261116.py new file mode 100644 index 0000000..d35ded5 --- /dev/null +++ b/scripts/adhoc/m5_release_chain_20261116.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""2026-11-16: preserve real M5 production units at the canonical KG output base. + +The reviewed KG unit launcher supplies named input staging, pinned code/config, +private caches, one actual CLI source and raw side-effect inventory. This wrapper +changes only output placement to the task checkout's initially absent canonical +source directory, so BacDive's fixed upstream paths select the actual new output. +It captures every file in that source tree and checks pre-existing upstream trees +remain unchanged. No completion markers are invented, copied or relabeled. +Separate source units share the output base but own disjoint source directories. +Original M5 inputs and the earlier RO canary stages are read-only. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SHARED_LAUNCHER = "scripts/adhoc/run_bactotraits_production_unit_20261116.py" +UPSTREAM = {"ontologies": (), "bacdive": ("ontologies",), "bactotraits": ("ontologies", "bacdive")} +GO_RECEIPT_SHA256 = "dd96bdb5d482e75458b545a658ddd4d1117e53e61f38d84b5f27cabc17d90001" +JAVA_PROOFS = { + "java-runtime-supplement-r3.json": "b697bb006e0754712d28cd0a3bf47628b3f11cd24506ebeab51058d98a175ac1", + "java-runtime-implementation-r3.json": "ec76081f31d0e639082cb3eb359f82f2c138fd31e8538f2c7282b4dfd14a1451", +} +ANCILLARY = ( + "kg_microbe/transform_utils/ontologies/xrefs", + "kg_microbe/transform_utils/ontologies/trees", + "kg_microbe/transform_utils/bacdive/tmp", + "kg_microbe/transform_utils/bactotraits/tmp", +) + + +def load_launcher(root): + """Import the separately reviewed helper without invoking a producer or changing KG code.""" + path = root / SHARED_LAUNCHER + spec = importlib.util.spec_from_file_location("m5_kg_production_unit", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def inventory(directory, launcher): + """Stream every member of one explicit owned subtree, including ignored and hidden files.""" + if not directory.exists(): + return {} + return { + str(path.relative_to(directory)): launcher.identity(launcher.safe_path(path)) + for path in sorted(directory.rglob("*")) + if path.is_file() + } + + +def java_provenance(environment, launcher): + """Preserve the full JVM version output, including any banner before the actual version.""" + command = shutil.which("java", path=environment.get("PATH")) + launcher.require(command is not None, "Java is unavailable") + result = subprocess.run( # noqa: S603 -- PATH-selected Java with fixed read-only version arguments. + [command, "-version"], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + selected_home = environment.get("JAVA_HOME") + if not selected_home and Path("/usr/libexec/java_home").is_file(): + home_result = subprocess.run( # noqa: S603 -- fixed macOS JDK locator, read-only. + ["/usr/libexec/java_home"], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if home_result.returncode == 0: + selected_home = home_result.stdout.strip() + resolved = Path(command).resolve() + actual = Path(selected_home) / "bin/java" if selected_home else resolved + return { + "command": [command, "-version"], + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "path": command, + "resolved_path": str(resolved), + "launcher_identity": launcher.identity(resolved), + "selected_java_home": selected_home, + "selected_jdk_java_path": str(actual.resolve()) if actual.is_file() else None, + "selected_jdk_java_identity": launcher.identity(actual) if actual.is_file() else None, + } + + +def bind_prior_authorities(plan, args, launcher): + """Require the actually rebuilt GO cache and inspected JDK, not just whichever bytes now exist.""" + pins = plan["implementation_and_policy_inputs"] + receipt_path = ( + args.kg_root / "data/validation-bactotraits-130/production-ro-canary-r3/receipt.json" + ) + receipt_identity = launcher.identity(receipt_path) + launcher.require( + receipt_identity["sha256"] == GO_RECEIPT_SHA256, "Reviewed RO/GO receipt changed" + ) + pins[str(receipt_path)] = receipt_identity + receipt = json.loads(receipt_path.read_text()) + go = {} + for name in ("go.db", "go.db.toolchain.json"): + path = args.kg_root / "data/raw" / name + expected = receipt["task_raw_after"][name] + launcher.require(launcher.identity(path) == expected, f"Fresh r3 GO cache changed: {name}") + pins[str(path)] = expected + go[name] = {"path": str(path), **expected} + plan["retained_go_cache"] = { + "receipt": {"path": str(receipt_path), **receipt_identity}, + "members": go, + } + proofs = {} + for name, expected_sha in JAVA_PROOFS.items(): + path = args.output_root / name + observed = launcher.identity(path) + launcher.require(observed["sha256"] == expected_sha, "Reviewed Java supplement changed") + pins[str(path)] = observed + proofs[name] = {"path": str(path), **observed} + implementation = json.loads( + (args.output_root / "java-runtime-implementation-r3.json").read_text() + ) + for role in ("actual_runtime_launcher", "actual_runtime_vm"): + native = implementation[role] + path = Path(native["path"]) + expected = {key: native[key] for key in ("bytes", "sha256")} + launcher.require( + str(path.resolve()) == native["resolved"] and launcher.identity(path) == expected, + "Inspected Java implementation changed", + ) + pins[str(path)] = expected + actual_java = implementation["actual_runtime_launcher"]["resolved"] + launcher.require( + plan["java"]["selected_jdk_java_path"] == actual_java, "Different Java runtime selected" + ) + # Freeze the actual JDK proved by -XshowSettings in the supplement, avoiding + # any later macOS /Library/Java/Home selector change during the long unit. + plan["environment"]["JAVA_HOME"] = str(Path(actual_java).parent.parent) + plan["java_supplements"] = proofs + + +def prepare(args, launcher): + """Pin this wrapper and canonical placement without opening any output for writing.""" + launcher.require( + args.kg_root.resolve() == launcher.ROOT, "Selected KG checkout differs from helper root" + ) + plan = launcher.prepare(args) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "NUMEXPR_MAX_THREADS", + "VECLIB_MAXIMUM_THREADS", + "RAYON_NUM_THREADS", + "BLIS_NUM_THREADS", + ): + plan["environment"][name] = "1" + for name in ("ROBOT_JAVA_ARGS", "_JAVA_OPTIONS"): + plan["environment"][name] += " -XX:ActiveProcessorCount=1" + output = launcher.safe_path(args.kg_root / "data/transformed") + source = launcher.safe_path(output / args.source) + launcher.require( + not source.exists(), f"Canonical source output already exists: {source}; preserve it" + ) + prior = {} + for name in UPSTREAM[args.source]: + directory = output / name + for member in ("source_finalization.json", "source_fingerprint.json"): + launcher.require( + (directory / member).is_file(), + f"Completed upstream evidence missing: {directory / member}", + ) + prior[name] = inventory(directory, launcher) + plan["command"][plan["command"].index("-o") + 1] = str(output) + plan["canonical_output_base"] = str(output) + plan["canonical_source_directory"] = str(source) + plan["upstream_before"] = prior + plan["ancillary_before"] = { + name: inventory(launcher.safe_path(args.kg_root / name), launcher) for name in ANCILLARY + } + fresh_ancillary = { + "bacdive": ( + "bacdive.tsv", + "bacdive_physiology_metabolism.tsv", + "bacdive_name_tax_classification.tsv", + ), + "bactotraits": ("bactotraits_mapping.tsv", "bactotraits.tsv"), + } + for name in fresh_ancillary.get(args.source, ()): + path = args.kg_root / "kg_microbe/transform_utils" / args.source / "tmp" / name + launcher.require(not path.exists(), f"Existing ancillary output must be preserved: {path}") + legacy = args.kg_root / "kg_microbe/transform_utils/bacdive/metabolite_mapping.json" + plan["implementation_and_policy_inputs"][str(legacy)] = launcher.identity(legacy) + if args.source == "bactotraits": + launcher.require( + args.bacdive_receipt is not None, "BactoTraits requires the fresh BacDive unit receipt" + ) + previous = json.loads(args.bacdive_receipt.read_text()) + launcher.require( + previous["returncode"] == 0 and previous["plan"]["source"] == "bacdive", + "BacDive unit did not complete", + ) + expected = previous["ancillary_after"]["kg_microbe/transform_utils/bacdive/tmp"][ + "bacdive.tsv" + ] + lookup = args.kg_root / "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv" + launcher.require( + launcher.identity(lookup) == expected, + "Fresh BacDive lookup differs from its actual unit", + ) + plan["implementation_and_policy_inputs"][str(lookup)] = expected + plan["implementation_and_policy_inputs"][str(args.bacdive_receipt)] = launcher.identity( + args.bacdive_receipt + ) + plan["fresh_bacdive_lookup"] = { + "path": str(lookup), + **expected, + "receipt": str(args.bacdive_receipt), + } + wrapper = Path(__file__).resolve() + plan["implementation_and_policy_inputs"][str(wrapper)] = launcher.identity(wrapper) + plan["wrapper"] = {"path": str(wrapper), **launcher.identity(wrapper)} + output_root = launcher.safe_path(args.output_root) + launcher.require(output_root.is_dir(), "Worker output root must already exist") + record = output_root / f"chain-{args.unit}.json" + launcher.require(not record.exists(), "Chain receipt already exists; preserve it") + plan["chain_receipt"] = str(record) + plan["java"] = java_provenance({**os.environ, **plan["environment"]}, launcher) + bind_prior_authorities(plan, args, launcher) + plan["java_provenance_issue"] = "https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1872" + return plan + + +def verify_members(directory, record, launcher): + """Independently compare persisted finalizer member hashes, without inventing a success marker.""" + for section in ("members", "audit_members"): + launcher.require( + isinstance(record.get(section), dict) and bool(record[section]), f"Missing {section}" + ) + for name, expected in record[section].items(): + launcher.require(Path(name).name == name, "Escaping finalizer member name") + launcher.require( + launcher.identity(directory / name) == expected, f"Finalizer member changed: {name}" + ) + + +def run(plan, launcher): + """Run one canonical source, preserve its complete output tree and check upstream nonmutation.""" + source = Path(plan["canonical_source_directory"]) + launcher.require(not source.exists(), "Canonical output appeared after planning") + code = launcher.run(plan) + outputs = inventory(source, launcher) + ancillary = {name: inventory(launcher.ROOT / name, launcher) for name in ANCILLARY} + for directory, members in plan["ancillary_before"].items(): + launcher.require( + all(ancillary[directory].get(name) == expected for name, expected in members.items()), + f"Pre-existing ancillary input changed: {directory}", + ) + upstream_after = { + name: inventory(Path(plan["canonical_output_base"]) / name, launcher) + for name in plan["upstream_before"] + } + launcher.require( + upstream_after == plan["upstream_before"], "Previously completed upstream output changed" + ) + finalized = None + marker = None + if code == 0: + record_path = source / "source_finalization.json" + marker_path = source / "source_fingerprint.json" + launcher.require( + record_path.is_file() and marker_path.is_file(), + "CLI success lacks actual completion evidence", + ) + finalized = json.loads(record_path.read_text()) + marker = json.loads(marker_path.read_text()) + verify_members(source, finalized, launcher) + receipt = { + "profile": "m5-canonical-release-source-unit/v1", + "completed_at": datetime.now(timezone.utc).isoformat(), + "returncode": code, + "plan": plan, + "canonical_source_artifacts": outputs, + "ancillary_after": ancillary, + "ancillary_preexisting_unchanged": True, + "upstream_unchanged": True, + "finalization_record": finalized, + "producer_fingerprint": marker, + "unit_receipt": { + "path": str(Path(plan["stage"]) / "receipt.json"), + **launcher.identity(Path(plan["stage"]) / "receipt.json"), + }, + "merged_graph_verified": False, + "published_release_verified": False, + "scope": "One real source through the canonical CLI output base; independent review precedes the next unit.", + } + launcher.write_new(Path(plan["chain_receipt"]), receipt) + print( + json.dumps( + { + "chain_receipt": plan["chain_receipt"], + "returncode": code, + "source": str(source), + "artifact_count": len(outputs), + } + ) + ) + return code + + +def main(): + """Require a read-only plan or one explicit canonical-layout producer unit.""" + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--plan", action="store_true") + action.add_argument("--run", action="store_true") + parser.add_argument("--kg-root", type=Path, required=True) + parser.add_argument("--input-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--source", choices=tuple(UPSTREAM), required=True) + parser.add_argument("--unit", required=True) + parser.add_argument("--raw-input", action="append", required=True) + parser.add_argument("--bacdive-receipt", type=Path) + args = parser.parse_args() + launcher = load_launcher(args.kg_root) + plan = prepare(args, launcher) + if args.plan: + print(json.dumps({"action": "plan", "writes": 0, **plan}, sort_keys=True, indent=2)) + else: + raise SystemExit(run(plan, launcher)) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_checks_20261116.py b/scripts/adhoc/m5_release_checks_20261116.py new file mode 100644 index 0000000..f0f29cb --- /dev/null +++ b/scripts/adhoc/m5_release_checks_20261116.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Run one recorded M5 KG check through the prepared offline environment. + +Provenance: consumes m5_release_gate_setup_20261116.py's environment record. +This launcher records operational execution only. The repository's unchanged +run_pr_gates.py remains responsible for final-candidate and full-suite gates. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + + +def write(path: Path, record: dict) -> None: + """Atomically update this invocation's live receipt.""" + pending = path.with_suffix(".pending") + pending.write_text(json.dumps(record, indent=2) + "\n") + pending.replace(path) + + +def main() -> int: + """Run a single explicitly supplied command with durable logs.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--environment-record", required=True, type=Path) + parser.add_argument("--kg-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("Supply a command after --") + args.output.mkdir(parents=True, exist_ok=False) + specification = json.loads(args.environment_record.read_text()) + environment = dict(os.environ) + for key in list(environment): + if key in specification["unset"] or any(key.startswith(p) for p in specification["unset_prefixes"]): + environment.pop(key) + environment.update(specification["set"]) + record = {"started_at": datetime.now(timezone.utc).isoformat(), "cwd": str(args.kg_root), + "command": command, "environment_record": str(args.environment_record), + "environment_record_sha256": hashlib.sha256(args.environment_record.read_bytes()).hexdigest(), + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "status": "running"} + receipt = args.output / "receipt.json" + log_path = args.output / "command.log" + started = time.monotonic() + with log_path.open("x") as log: + process = subprocess.Popen(command, cwd=args.kg_root, env=environment, stdout=log, stderr=subprocess.STDOUT) + record["pid"] = process.pid + write(receipt, record) + print(json.dumps({"pid": process.pid, "receipt": str(receipt), "log": str(log_path)}), flush=True) + code = process.wait() + record.update(status="complete" if code == 0 else "failed", exit_code=code, + completed_at=datetime.now(timezone.utc).isoformat(), seconds=round(time.monotonic() - started, 3), + log_sha256=hashlib.sha256(log_path.read_bytes()).hexdigest()) + write(receipt, record) + print(json.dumps(record, indent=2)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_environment_20261116.py b/scripts/adhoc/m5_release_environment_20261116.py new file mode 100644 index 0000000..bc37e1f --- /dev/null +++ b/scripts/adhoc/m5_release_environment_20261116.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Inventory and install the M5 KG lock from its read-only Poetry artifact cache. + +Provenance: the November 16 release worker has a fresh Python 3.10 KG venv, +but online Poetry installation failed DNS. This helper selects main/dev lock +records using the TARGET interpreter's markers and wheel tags. Published +wheels must match poetry.lock SHA256. A locally built cached wheel is accepted +only with an explicit flag, a hash-matched co-located locked source archive, +and a separately recorded wheel hash; co-location is not a build attestation. +Run --plan first. Installation uses only the recorded local wheel paths, +--no-index, --no-deps and --require-hashes, followed by pip check. + +The planner needs Python 3.11+ with packaging, independently of the target +Python 3.10-3.12 environment. It neither downloads nor edits the shared cache. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tomllib +from datetime import datetime, timezone +from pathlib import Path + +from packaging.markers import Marker +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name, parse_wheel_filename + + +def sha256(path: Path) -> str: + """Hash bytes without loading large wheels at once.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def target_runtime(python: Path) -> dict: + """Ask the target's pip-vendored packaging for its actual platform.""" + program = ( + "import json,sys; from pip._vendor.packaging.markers import default_environment; " + "from pip._vendor.packaging.tags import sys_tags; " + "print(json.dumps({'executable':sys.executable,'version':sys.version," + "'markers':default_environment(),'tags':[str(t) for t in sys_tags()]}))" + ) + result = subprocess.run([str(python), "-c", program], check=True, capture_output=True, text=True) + runtime = json.loads(result.stdout) + version = runtime["markers"]["python_version"] + if version not in {"3.10", "3.11", "3.12"}: + raise ValueError(f"Unsupported KG target Python: {version}") + runtime["resolved_executable"] = str(python.resolve()) + runtime["executable_sha256"] = sha256(python.resolve()) + return runtime + + +def selected(package: dict, groups: set[str], environment: dict) -> bool: + """Respect per-group lock markers as well as Python version constraints.""" + active_groups = set(package["groups"]) & groups + if not active_groups: + return False + constraint = package.get("python-versions", "*") + if constraint != "*" and not SpecifierSet(constraint).contains(environment["python_full_version"]): + return False + markers = package.get("markers") + if isinstance(markers, dict): + return any(group not in markers or Marker(markers[group]).evaluate(environment) for group in active_groups) + return not markers or Marker(markers).evaluate(environment) + + +def write_json(path: Path, value: dict) -> None: + """Keep each new record exclusive rather than overwriting old receipts.""" + with path.open("x") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def plan(args: argparse.Namespace) -> int: + """Inventory compatible cache bytes without installing anything.""" + args.output.mkdir(parents=True, exist_ok=False) + runtime = target_runtime(args.python) + lock_path = args.kg_root / "poetry.lock" + lock = tomllib.loads(lock_path.read_text()) + groups = set(args.groups.split(",")) + packages = [p for p in lock["package"] if selected(p, groups, runtime["markers"])] + cache_files = sorted(path for path in args.cache.rglob("*") if path.is_file()) + wheel_index: dict[tuple[str, str], list[Path]] = {} + for path in cache_files: + if path.suffix != ".whl": + continue + name, version, _, tags = parse_wheel_filename(path.name) + if set(map(str, tags)) & set(runtime["tags"]): + wheel_index.setdefault((canonicalize_name(name), str(version)), []).append(path) + records = [] + missing = [] + for package in packages: + name, version = package["name"], package["version"] + locked_files = {entry["file"]: entry["hash"].removeprefix("sha256:") for entry in package["files"]} + candidates = wheel_index.get((canonicalize_name(name), version), []) + choices = [] + for path in candidates: + digest = sha256(path) + record = {"name": name, "version": version, "path": str(path), "sha256": digest} + if locked_files.get(path.name) == digest: + record["status"] = "lock_matched_wheel" + choices.append(record) + continue + sources = [] + for source_name, source_digest in locked_files.items(): + source = path.parent / source_name + if source.suffix != ".whl" and source.is_file() and sha256(source) == source_digest: + sources.append({"path": str(source), "sha256": source_digest}) + if sources and args.allow_cached_built_wheels: + record.update(status="cached_built_wheel", locked_sources=sources, + provenance_limit="Co-located pinned source; cached build not independently attested.") + choices.append(record) + if choices: + choices.sort(key=lambda record: (record["status"] != "lock_matched_wheel", record["path"])) + records.append(choices[0]) + else: + missing.append({"name": name, "version": version, "compatible_cached_wheels": list(map(str, candidates)), + "locked_filenames": list(locked_files)}) + names = [canonicalize_name(record["name"]) for record in records] + if len(names) != len(set(names)): + raise ValueError("Multiple selected lock versions for a package; refuse ambiguous installation") + requirements = "".join(f"{Path(r['path']).as_uri()} --hash=sha256:{r['sha256']}\n" for r in records) + requirements_path = args.output / "requirements.txt" + with requirements_path.open("x") as handle: + handle.write(requirements) + inventory = { + "created_at": datetime.now(timezone.utc).isoformat(), "mode": "offline_inventory", + "planner_python": sys.version, "target": runtime, "groups": sorted(groups), + "kg_root": str(args.kg_root), "lock_path": str(lock_path), "lock_sha256": sha256(lock_path), + "helper_sha256": sha256(Path(__file__)), "cache": str(args.cache), + "cache_files_enumerated": len(cache_files), "selected_lock_packages": len(packages), + "requirements_sha256": sha256(requirements_path), "packages": records, "missing": missing, + "allow_cached_built_wheels": args.allow_cached_built_wheels, + } + write_json(args.output / "inventory.json", inventory) + print(json.dumps({"output": str(args.output), "selected": len(packages), "available": len(records), + "cached_built": [r["name"] for r in records if r["status"] == "cached_built_wheel"], + "missing": [{"name": p["name"], "version": p["version"]} for p in missing]}, indent=2)) + return 2 if missing else 0 + + +def install(args: argparse.Namespace) -> int: + """Install only a complete, unchanged, previously inspected inventory.""" + inventory_path = args.output / "inventory.json" + inventory = json.loads(inventory_path.read_text()) + if inventory["missing"]: + raise ValueError("Inventory incomplete; installation is blocked") + runtime = target_runtime(args.python) + if runtime != inventory["target"]: + raise ValueError("Target interpreter changed after planning") + if sha256(Path(inventory["lock_path"])) != inventory["lock_sha256"]: + raise ValueError("Lock changed after planning") + requirements = args.output / "requirements.txt" + if sha256(requirements) != inventory["requirements_sha256"]: + raise ValueError("Requirements changed after planning") + for record in inventory["packages"]: + if sha256(Path(record["path"])) != record["sha256"]: + raise ValueError(f"Wheel changed after planning: {record['name']}") + for source in record.get("locked_sources", []): + if sha256(Path(source["path"])) != source["sha256"]: + raise ValueError(f"Locked source changed: {record['name']}") + scratch = args.output / "pip-scratch" + scratch.mkdir(exist_ok=False) + environment = dict(os.environ, TMPDIR=str(scratch)) + command = [str(args.python), "-m", "pip", "--isolated", "--disable-pip-version-check", "install", + "--no-index", "--no-cache-dir", "--no-deps", "--require-hashes", "-r", str(requirements)] + with (args.output / "install.log").open("x") as log: + process = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, env=environment) + check = subprocess.run([str(args.python), "-m", "pip", "--isolated", "check"], capture_output=True, text=True) + installed = subprocess.run([str(args.python), "-m", "pip", "--isolated", "list", "--format=json"], + check=True, capture_output=True, text=True) + receipt = {"completed_at": datetime.now(timezone.utc).isoformat(), "command": command, + "inventory_sha256": sha256(inventory_path), "returncode": process.returncode, + "pip_check_returncode": check.returncode, "pip_check": check.stdout + check.stderr, + "installed": json.loads(installed.stdout), "install_log_sha256": sha256(args.output / "install.log")} + write_json(args.output / "install-receipt.json", receipt) + print(json.dumps({key: value for key, value in receipt.items() if key != "installed"}, indent=2)) + return process.returncode or check.returncode + + +def main() -> int: + """Keep planning and mutation explicit and separate.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kg-root", required=True, type=Path) + parser.add_argument("--python", required=True, type=Path) + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--groups", default="main,dev") + parser.add_argument("--allow-cached-built-wheels", action="store_true") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", action="store_true") + mode.add_argument("--install", action="store_true") + args = parser.parse_args() + for name in ("kg_root", "python", "cache", "output"): + setattr(args, name, getattr(args, name).absolute()) + return plan(args) if args.plan else install(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_gate_setup_20261116.py b/scripts/adhoc/m5_release_gate_setup_20261116.py new file mode 100644 index 0000000..54726cd --- /dev/null +++ b/scripts/adhoc/m5_release_gate_setup_20261116.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Prepare isolated offline KG PR gates from retained M5 package bytes. + +Provenance: use the November 16 environment inventory for all KG lock wheels. +The two unpinned tox-only tools are reconstructed from the canonical checkout's +installed pure-Python distributions, verifying each retained file against its +installed RECORD. Reconstructed hashes are local artifact hashes, not claimed +PyPI wheel hashes. Original environments and caches are read-only. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import io +import json +import os +import shutil +import subprocess +import zipfile +from datetime import datetime, timezone +from pathlib import Path + + +def digest(path: Path) -> str: + """Hash a local file without changing it.""" + with path.open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def repack(source: Path, wheelhouse: Path) -> dict: + """Reconstruct a pure-Python wheel from RECORD-verified installed files.""" + package = source.parent + wheel_text = (source / "WHEEL").read_text() + if "Root-Is-Purelib: true" not in wheel_text or "Tag: py3-none-any" not in wheel_text: + raise ValueError(f"Unsupported retained tool wheel: {source.name}") + stem = source.name.removesuffix(".dist-info") + destination = wheelhouse / f"{stem}-py3-none-any.whl" + retained = [] + excluded = [] + for relative, recorded_hash, size in csv.reader((source / "RECORD").read_text().splitlines()): + path = Path(relative) + if ".." in path.parts or path.suffix == ".pyc" or path.name in {"RECORD", "INSTALLER", "REQUESTED"}: + excluded.append(relative) + continue + if path.is_absolute() or not recorded_hash.startswith("sha256="): + raise ValueError(f"Unverifiable retained file: {relative}") + resolved = (package / path).resolve() + if not resolved.is_relative_to(package.resolve()): + raise ValueError(f"Retained file escapes site-packages: {relative}") + content = resolved.read_bytes() + actual = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + if recorded_hash != "sha256=" + actual or int(size) != len(content): + raise ValueError(f"Installed RECORD mismatch: {relative}") + retained.append((relative, content, recorded_hash)) + record_path = f"{source.name}/RECORD" + record = io.StringIO(newline="") + writer = csv.writer(record, lineterminator="\n") + for name, content, hash_value in sorted(retained): + writer.writerow([name, hash_value, len(content)]) + writer.writerow([record_path, "", ""]) + retained.append((record_path, record.getvalue().encode(), "")) + with zipfile.ZipFile(destination, "x", compression=zipfile.ZIP_DEFLATED) as wheel: + for name, content, _ in sorted(retained): + member = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + member.compress_type = zipfile.ZIP_DEFLATED + member.external_attr = 0o644 << 16 + wheel.writestr(member, content) + return {"path": str(destination), "sha256": digest(destination), "source_dist_info": str(source), + "source_record_sha256": digest(source / "RECORD"), "verified_source_files": len(retained) - 1, + "excluded_installer_files": excluded, "provenance": "Reconstructed from installed RECORD-verified files"} + + +def setup(args: argparse.Namespace) -> int: + """Prepare an exclusive wheelhouse, environment record and read-only probes.""" + args.output.mkdir(parents=True, exist_ok=False) + setup_root = args.kg_root / "data/validation/pr-gates/m5-setup" + setup_root.mkdir(parents=True, exist_ok=False) + wheelhouse = setup_root / "wheelhouse" + wheelhouse.mkdir() + inventory = json.loads(args.inventory.read_text()) + if inventory["missing"]: + raise ValueError("KG lock inventory is incomplete") + copied = [] + for item in inventory["packages"]: + source = Path(item["path"]) + if digest(source) != item["sha256"]: + raise ValueError(f"Cached wheel changed: {source.name}") + destination = wheelhouse / source.name + if destination.exists(): + raise FileExistsError(destination) + shutil.copyfile(source, destination) + if digest(destination) != item["sha256"]: + raise ValueError(f"Copied wheel mismatch: {source.name}") + copied.append({"path": str(destination), "sha256": item["sha256"], "source": str(source)}) + canonical = args.canonical_root / ".tox" + repacked = [repack(canonical / "codespell/lib/python3.10/site-packages/codespell-2.4.3.dist-info", wheelhouse), + repack(canonical / "docstr-coverage/lib/python3.10/site-packages/docstr_coverage-2.3.2.dist-info", wheelhouse)] + constraints = setup_root / "constraints.txt" + constraints.write_text("".join(f"{item['name']}=={item['version']}\n" for item in inventory["packages"]) + + "codespell==2.4.3\ndocstr-coverage==2.3.2\n") + # The final runner intentionally strips TOX* selectors. A private .tox link + # keeps its standard unmodified command's physical output under data/. + tox_link = args.kg_root / ".tox" + if os.path.lexists(tox_link): + raise FileExistsError(f"Preserve pre-existing tox output: {tox_link}") + tox_output = setup_root / "tox" + tox_output.mkdir() + tox_link.symlink_to(tox_output, target_is_directory=True) + for name in ("tmp", "pip-cache", "virtualenv", "poetry-cache", "xdg-cache", "matplotlib"): + (setup_root / name).mkdir() + environment = { + "VIRTUAL_ENV": str(args.kg_root / ".venv310"), + "PATH": str(args.kg_root / ".venv310/bin") + ":/opt/homebrew/bin:/usr/bin:/bin", + "PIP_NO_INDEX": "1", "PIP_FIND_LINKS": str(wheelhouse), "PIP_CONSTRAINT": str(constraints), + "PIP_CACHE_DIR": str(setup_root / "pip-cache"), "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "VIRTUALENV_OVERRIDE_APP_DATA": str(setup_root / "virtualenv"), + "VIRTUALENV_NO_PERIODIC_UPDATE": "1", "VIRTUALENV_DOWNLOAD": "0", + "POETRY_CACHE_DIR": str(setup_root / "poetry-cache"), "TMPDIR": str(setup_root / "tmp"), + "XDG_CACHE_HOME": str(setup_root / "xdg-cache"), "MPLCONFIGDIR": str(setup_root / "matplotlib"), + } + unset = ["CONDA_DEFAULT_ENV", "CONDA_PREFIX", "PYTEST_ADDOPTS", "POETRY_VIRTUALENVS_IN_PROJECT"] + effective = dict(os.environ) + for key in list(effective): + if key in unset or key.startswith("TOX"): + effective.pop(key) + effective.update(environment) + with (args.output / "environment.json").open("x") as handle: + json.dump({"set": environment, "unset": unset, "unset_prefixes": ["TOX"]}, handle, indent=2) + handle.write("\n") + commands = [ + ["poetry", "run", "python", "-c", "import sys; print(sys.version); print(sys.executable)"], + ["poetry", "env", "info", "--path"], + ["poetry", "run", "tox", "--version"], + ["poetry", "run", "tox", "c", "-e", "codespell", "-k", "pass_env"], + ["poetry", "run", "python", "scripts/run_pr_gates.py", "m5-final-pending", "--dry-run"], + ] + results = [] + for index, command in enumerate(commands): + result = subprocess.run(command, cwd=args.kg_root, env=effective, capture_output=True, text=True) + log = args.output / f"probe-{index + 1}.log" + with log.open("x") as handle: + handle.write(result.stdout + result.stderr) + results.append({"command": command, "returncode": result.returncode, "log": str(log), "sha256": digest(log)}) + receipt = {"completed_at": datetime.now(timezone.utc).isoformat(), "helper_sha256": digest(Path(__file__)), + "inventory_sha256": digest(args.inventory), "setup_root": str(setup_root), + "copied_wheels": copied, "reconstructed_tool_wheels": repacked, + "constraints_sha256": digest(constraints), "probes": results, + "scope": "Environment preparation only; no source formatting or full test suite executed."} + with (args.output / "setup-receipt.json").open("x") as handle: + json.dump(receipt, handle, indent=2) + handle.write("\n") + print(json.dumps({"setup_root": str(setup_root), "copied_wheels": len(copied), + "reconstructed_tools": repacked, "probes": results}, indent=2)) + return int(any(result["returncode"] for result in results)) + + +def main() -> int: + """Parse explicit task-only setup paths.""" + parser = argparse.ArgumentParser(description=__doc__) + for name in ("kg-root", "canonical-root", "inventory", "output"): + parser.add_argument("--" + name, required=True, type=Path) + return setup(parser.parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_lineage_20261116.py b/scripts/adhoc/m5_release_lineage_20261116.py new file mode 100644 index 0000000..87b3d3d --- /dev/null +++ b/scripts/adhoc/m5_release_lineage_20261116.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Inspect bounded retained source lineage for the M5 BactoTraits correction. + +2026-11-16: read only the original CSV, the declared BacDive lookup, the old +BactoTraits intermediates and three explicitly selected source edge files. +Also inspect the distinct Madin source row for an equivalent historical edge. +Keep occurrence ordinals and hashes. This does not establish graph-wide absence, +current release freshness, trait falsity or a replacement organism identity. +The selected files are inspected directly, including ignored data; no discovery +walk or remote request is performed. Results use an exclusive output directory. +""" + +import argparse +import csv +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path + +CANONICAL = Path("/Users/marcin/Documents/VIMSS/ontology/KG-Hub/KG-Microbe/kg-microbe") +CODE = Path("/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116") +SOURCE_SHA = "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" +ROW_SHA = "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a" +SUBJECT = "NCBITaxon:1123071" +OBJECT = "METPO:1000703" + + +def sha256(path): + """Hash a selected regular file without retaining its contents in memory.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def identity(path): + """Record content and filesystem identity for one explicitly selected input.""" + before = path.stat() + result = { + "path": str(path), + "bytes": before.st_size, + "mtime_ns": before.st_mtime_ns, + "inode": before.st_ino, + "sha256": sha256(path), + } + after = path.stat() + assert (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) == ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ), f"Input changed while hashing: {path}" + return result + + +def rows(path, *, raw=False, delimiter="\t"): + """Stream exact CSV/TSV records with data-row ordinals, including duplicates.""" + with path.open(encoding="ISO-8859-1" if raw else "utf-8", newline="") as stream: + reader = csv.reader(stream, delimiter=";" if raw else delimiter) + if raw: + next(reader) + next(reader) + header = next(reader) + assert len(header) == len(set(header)), f"Duplicate header: {path}" + for ordinal, values in enumerate(reader, 1): + assert len(values) == len(header), f"Malformed row {ordinal}: {path}" + yield ordinal, dict(zip(header, values, strict=True)) + + +def scan(path, selector, *, raw=False, delimiter="\t"): + """Retain matching complete records and verify source identity after reading.""" + pin = identity(path) + matches = [] + count = 0 + for count, row in rows(path, raw=raw, delimiter=delimiter): + if selector(row): + matches.append({"source_data_row": count, "row": row}) + assert identity(path) == pin, f"Input changed during scan: {path}" + return { + "input": pin, + "data_rows": count, + "matches": matches, + "matched_occurrences": len(matches), + } + + +def inspect(root): + """Trace the reviewed selector, accession occurrences and retained joins.""" + raw = scan( + root / "data/raw/BactoTraits_databaseV2_Jun2022.csv", + lambda row: ( + row["Bacdive_ID"] == "17851" + or "19353" in row["culture collection codes"] + or "14586" in row["culture collection codes"] + ), + raw=True, + ) + assert raw["input"]["sha256"] == SOURCE_SHA, "Original BactoTraits bytes changed" + selected = [m for m in raw["matches"] if m["row"]["Bacdive_ID"] == "17851"] + assert len(selected) == 1 and selected[0]["source_data_row"] == 8521 + raw_row = selected[0]["row"] + row_digest = hashlib.sha256( + json.dumps(raw_row, ensure_ascii=False, indent=2, sort_keys=True).encode() + ).hexdigest() + assert row_digest == ROW_SHA, "Reviewed complete original row changed" + lookup = scan( + root / "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv", + lambda row: row["bacdive_id"] == "bacdive:17851" or row["ncbitaxon_id"] == SUBJECT, + ) + intermediates = {} + for filename in ("bactotraits.tsv", "bactotraits_mapping.tsv"): + intermediates[filename] = scan( + root / "kg_microbe/transform_utils/bactotraits/tmp" / filename, + lambda row: ( + row.get("Bacdive_ID") in ("17851", "bacdive:17851") + or row.get("ncbitaxon_id") == SUBJECT + ), + ) + products = {} + for source in ("bactotraits", "bacdive", "madin_etal"): + item = scan( + root / "data/transformed" / source / "edges.tsv", + lambda row: row.get("subject") == SUBJECT and row.get("object") == OBJECT, + ) + products[source] = item + madin = scan( + root / "data/raw/madin_etal.csv", lambda row: row["tax_id"] == "1123071", delimiter="," + ) + implementations = { + name: identity(CODE / name) + for name in ( + "kg_microbe/transform_utils/bactotraits/bactotraits.py", + "kg_microbe/transform_utils/bactotraits/identity_quarantine.py", + "kg_microbe/transform_utils/madin_etal/madin_etal.py", + "mappings/canonical/bactotraits_identity_quarantine.json", + "merge.yaml", + ) + } + return { + "profile": "m5-bactotraits-retained-lineage/v1", + "completed_at": datetime.now(timezone.utc).isoformat(), + "helper_sha256": sha256(Path(__file__)), + "raw_source": raw, + "selected_complete_row_sha256": row_digest, + "bacdive_lookup": lookup, + "historical_intermediates": intermediates, + "historical_source_graph_occurrences": products, + "independent_madin_raw_occurrences": madin, + "inspected_implementation_and_config": implementations, + "scope": "Selected original CSV, lookup, two old intermediates, three old source edge files and Madin raw CSV; ignored files included by direct path.", + "graph_wide_absence_verified": False, + "current_source_freshness_verified": False, + "published_release_verified": False, + "scientific_trait_truth_resolved": False, + "neighboring_rows_quarantined": False, + } + + +def main(): + """Plan or execute one bounded source lineage inspection.""" + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_mutually_exclusive_group(required=True) + actions.add_argument("--plan", action="store_true") + actions.add_argument("--run", action="store_true") + parser.add_argument("--input-root", type=Path, default=CANONICAL) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + parser.error("Output already exists; select a fresh exclusive directory") + csv.field_size_limit(16 * 1024 * 1024) + if args.plan: + print( + json.dumps( + { + "action": "plan", + "input_root": str(args.input_root), + "output": str(args.output), + "network_calls": 0, + "source_files": 8, + "concurrency": 1, + "writes": 0, + } + ) + ) + return + os.umask(0o077) + args.output.mkdir(parents=True, mode=0o700) + result = inspect(args.input_root) + receipt = args.output / "receipt.json" + with receipt.open("x") as stream: + json.dump(result, stream, indent=2) + stream.write("\n") + print( + json.dumps( + { + "receipt": str(receipt), + "sha256": sha256(receipt), + "matches_by_source": { + key: value["matched_occurrences"] + for key, value in result["historical_source_graph_occurrences"].items() + }, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_preservation_20261116.py b/scripts/adhoc/m5_release_preservation_20261116.py new file mode 100644 index 0000000..9b50a6d --- /dev/null +++ b/scripts/adhoc/m5_release_preservation_20261116.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Hash every nested artifact in the two explicitly retained v3 replay stages. + +2026-11-16: read-only preservation check for M5 release verification. This +walk is confined to the named evidence roots and includes hidden cache files. +""" + +import argparse +import hashlib +import json +from pathlib import Path + +RETAINED = Path( + "/Users/marcin/agent-work/EdgeMech/retained-pr135-pr1867-20261116/" + "kg-candidate/data/validation-bactotraits-130" +) + + +def inventory(): + """Verify all receipt-bound bytes and inventory every nested regular file.""" + result = {} + for name in ("baseline-v3-20261116", "candidate-v3-20261116"): + root = RETAINED / name + files = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Unexpected symlink: {path}") + if path.is_file(): + raw = path.read_bytes() + files[str(path.relative_to(root))] = { + "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw) + } + receipt = json.loads((root / "receipt.json").read_text()) + for relative, expected in receipt["artifacts"].items(): + if files[relative] != expected: + raise ValueError(f"Retained receipt artifact differs: {name}/{relative}") + result[name] = files + return result + + +def main(): + """Save a new manifest or compare all current retained files with an old one.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compare", type=Path) + args = parser.parse_args() + result = inventory() + if args.compare and json.loads(args.compare.read_text()) != result: + raise ValueError("Retained evidence inventory changed") + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + stream.write(json.dumps(result, indent=2) + "\n") + print(json.dumps({"files": sum(map(len, result.values())), "unchanged": bool(args.compare)})) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_results_20261116.py b/scripts/adhoc/m5_release_results_20261116.py new file mode 100644 index 0000000..4570b22 --- /dev/null +++ b/scripts/adhoc/m5_release_results_20261116.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Export verified M5 row/finalizer evidence beneath this invocation's output root. + +2026-11-16: preserve original receipt paths and all nested files unchanged. The +compact result is suitable for Git; complete private receipts remain local. +""" + +import argparse +import hashlib +import json +import shutil +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +KG = ROOT.parent / "m5-kg-bactotraits-release-20261116" +STAGES = ( + "baseline-m5-20261116-r1", + "candidate-m5-20261116-r1", + "candidate-finalized-m5-20261116-r1", +) + + +def identity(path): + """Identify one bounded evidence file using exact persisted bytes.""" + raw = path.read_bytes() + return {"sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw)} + + +def verify(stage): + """Reject any changed or escaping receipt-bound artifact before export.""" + receipt = json.loads((stage / "receipt.json").read_text()) + for name, expected in receipt["artifacts"].items(): + path = (stage / name).resolve() + if not path.is_relative_to(stage.resolve()) or identity(path) != expected: + raise ValueError(f"Receipt artifact differs: {path}") + return receipt + + +def main(): + """Copy only the named new stages to an exclusive result root and verify readback.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--plan", action="store_true") + action.add_argument("--run", action="store_true") + args = parser.parse_args() + output = args.output.absolute() + required = ROOT / ".temp/m5-worker-run-20261116" + if not output.is_relative_to(required) or output.exists(): + raise ValueError("Use a fresh directory beneath this invocation's required output root") + results = {} + receipts = {} + for name in STAGES: + stage = KG / "data/validation-bactotraits-130" / name + receipt = verify(stage) + receipts[name] = receipt + results[name] = { + "original_receipt": str(stage / "receipt.json"), + "receipt": identity(stage / "receipt.json"), + "producer_revision": receipt["git_revision"], + "edges": receipt.get("all_edge_count", receipt.get("finalized_edge_count")), + "nodes": receipt.get("node_count", receipt.get("finalized_node_count")), + "quarantine_rows": receipt["quarantine_rows"], + "finalization_run": receipt["finalization_run"], + "artifacts": receipt["artifacts"], + } + baseline, candidate, finalized = [receipts[name] for name in STAGES] + if not ( + baseline["mode"] == "baseline" + and baseline["git_revision"] == "ef27b97a0b450767da54235c6105b5250c178adc" + and (baseline["all_edge_count"], baseline["node_count"]) == (21, 22) + and candidate["mode"] == "candidate" + and (candidate["all_edge_count"], candidate["node_count"], candidate["quarantine_rows"]) == (0, 0, 1) + and (finalized["finalized_edge_count"], finalized["finalized_node_count"], finalized["quarantine_rows"]) == (0, 0, 1) + and finalized["finalization_run"] is True + and finalized["merged_graph_verified"] is False + and finalized["published_release_verified"] is False + and baseline["shared_inputs"] == candidate["shared_inputs"] == finalized["shared_inputs"] + and candidate["baseline_receipt"] == finalized["baseline_receipt"] + and candidate["baseline_receipt"]["sha256"] == results[STAGES[0]]["receipt"]["sha256"] + and finalized["candidate_receipt"]["sha256"] == results[STAGES[1]]["receipt"]["sha256"] + ): + raise ValueError("Fresh pair/finalization semantics or receipt links differ") + summary = { + "profile": "m5-bactotraits-release-verification/v1", + "observed_at": datetime.now(timezone.utc).isoformat(), + "export_helper_sha256": identity(Path(__file__))["sha256"], + "fresh_baseline_candidate_pair_verified": True, + "bounded_source_finalization_verified": True, + "merged_graph_verified": False, + "published_release_verified": False, + "source_row": 8521, + "bacdive_id": 17851, + "scientific_claim_resolved": False, + "neighboring_rows_quarantined": False, + "results": results, + } + if args.plan: + print(json.dumps({"ready": True, "output": str(output), "stages": list(results)})) + return + output.mkdir(mode=0o700) + for name in STAGES: + source = KG / "data/validation-bactotraits-130" / name + destination = output / name + shutil.copytree(source, destination) + verify(destination) + for path in source.rglob("*"): + if path.is_symlink(): + raise ValueError(f"Unexpected symlink in a new stage: {path}") + if path.is_file() and identity(path) != identity(destination / path.relative_to(source)): + raise ValueError(f"Export differs: {path}") + with (output / "results.json").open("x") as stream: + stream.write(json.dumps(summary, indent=2) + "\n") + print(json.dumps({"results": str(output / "results.json"), **identity(output / "results.json")})) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_state_20261116.py b/scripts/adhoc/m5_release_state_20261116.py new file mode 100644 index 0000000..7c79148 --- /dev/null +++ b/scripts/adhoc/m5_release_state_20261116.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Record the authorized M5 release track's commands and durable progress. + +2026-11-16: all logs and state remain beneath the isolated evidence checkout. +This launcher runs one command at a time without shell interpretation; receipts +from the producer and finalizer remain the scientific/provenance evidence. +""" + +import argparse +import datetime +import fcntl +import hashlib +import json +import os +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUTPUT = ROOT / ".temp/m5-worker-run-20261116" + + +def _save(state): + """Atomically replace progress and retain individual execution records.""" + state["commands"] = sorted( + (json.loads(path.read_text()) for path in (OUTPUT / "commands").glob("*.json")), + key=lambda entry: entry["started_at"], + ) + state["updated_at"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + for path in (ROOT / ".temp/m5-release-progress.json", OUTPUT / "progress.json"): + temporary = path.with_suffix(f".{os.getpid()}.pending") + temporary.write_text(json.dumps(state, indent=2) + "\n") + temporary.replace(path) + lines = ["# M5 release verification", "", f"Phase: {state['phase']}", + f"Updated: {state['updated_at']}", f"Launcher PID: {state['pid']}", "", + "Published release verified: false", ""] + for entry in state.get("commands", []): + lines.append(f"- {entry['label']}: {entry['status']}; PID {entry.get('child_pid')}; " + f"exit {entry.get('exit_code')}; log {entry['log']}") + lines.extend(["", "Blockers:", *[f"- {item}" for item in state.get("blockers", [])]]) + if state.get("process_observation"): + observed = state["process_observation"] + lines.extend(["", f"External process: {observed['status']}", + f"Evidence: {observed['path']}", f"SHA256: {observed['sha256']}"]) + for path in (ROOT / ".temp/M5_RELEASE_PROGRESS.md", OUTPUT / "M5_RELEASE_PROGRESS.md"): + temporary = path.with_suffix(f".{os.getpid()}.pending") + temporary.write_text("\n".join(lines) + "\n") + temporary.replace(path) + + +def save(updates): + """Apply explicit metadata changes to current state while holding the aggregate lock.""" + with (OUTPUT / "progress.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + progress = ROOT / ".temp/m5-release-progress.json" + state = json.loads(progress.read_text()) if progress.exists() else {} + state.update(updates) + _save(state) + return state + + +def save_command(path, entry): + """Atomically publish one independent launcher's current execution record.""" + temporary = path.with_suffix(f".{os.getpid()}.pending") + temporary.write_text(json.dumps(entry, indent=2) + "\n") + temporary.replace(path) + + +def main(): + """Run a bounded task command, preserving its argv, PID, output and status.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--phase", required=True) + parser.add_argument("--label") + parser.add_argument("--cwd", type=Path, default=ROOT) + parser.add_argument("--blocker", action="append") + parser.add_argument("--results", type=Path) + parser.add_argument("--observe-process", type=Path) + parser.add_argument("--process-status") + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + os.umask(0o077) + OUTPUT.mkdir(parents=True, exist_ok=True) + state = {"phase": args.phase, "pid": os.getpid()} + if args.blocker: + state["blockers"] = args.blocker + if args.results: + state["results"] = json.loads(args.results.read_text()) + if args.observe_process: + if not args.process_status: + parser.error("--process-status required for an external process observation") + raw = args.observe_process.read_bytes() + state["process_observation"] = { + "path": str(args.observe_process.absolute()), "sha256": hashlib.sha256(raw).hexdigest(), + "status": args.process_status, "record": json.loads(raw), + } + state = save(state) + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + return 0 + if not args.label: + parser.error("--label required for command") + logs = OUTPUT / "commands" + logs.mkdir(exist_ok=True) + logfile = logs / (args.label + ".log") + command_record = logs / (args.label + ".json") + entry = {"label": args.label, "argv": command, "cwd": str(args.cwd), + "log": str(logfile), "status": "running", "started_at": state["updated_at"]} + with logfile.open("x") as log: + child = subprocess.Popen(command, cwd=args.cwd, stdout=log, stderr=subprocess.STDOUT) + entry["child_pid"] = child.pid + save_command(command_record, entry) + save({}) + code = child.wait() + entry.update(exit_code=code, status="passed" if code == 0 else "failed") + save_command(command_record, entry) + save({}) + print(json.dumps(entry)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_m5_release_chain.py b/tests/test_m5_release_chain.py new file mode 100644 index 0000000..dab81d0 --- /dev/null +++ b/tests/test_m5_release_chain.py @@ -0,0 +1,95 @@ +"""Keep canonical release units exclusive and reject scoped or tampered completion evidence.""" + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + +HELPER = Path(__file__).resolve().parents[1] / "scripts/adhoc/m5_release_chain_20261116.py" + + +@pytest.fixture +def chain(tmp_path): + """Use an isolated output root and restore process permissions after direct launcher calls.""" + spec = importlib.util.spec_from_file_location("m5_release_chain", HELPER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def require(condition, message): + if not condition: + raise ValueError(message) + + def identity(path): + raw = path.read_bytes() + return {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} + + root = tmp_path.resolve() + launcher = SimpleNamespace(ROOT=root, require=require, identity=identity, safe_path=lambda p: p) + previous = os.umask(0o077) + os.umask(previous) + try: + yield module, launcher, root + finally: + os.umask(previous) + + +def test_existing_canonical_source_is_refused_before_producer(chain): + """Even an empty source directory belongs to prior work and cannot be reused implicitly.""" + module, launcher, root = chain + existing = root / "data/transformed/ontologies" + existing.mkdir(parents=True) + sentinel = existing / "retained.tsv" + sentinel.write_text("previous source evidence\n") + launcher.prepare = lambda args: {"environment": {"ROBOT_JAVA_ARGS": "", "_JAVA_OPTIONS": ""}} + args = SimpleNamespace(kg_root=root, source="ontologies") + with pytest.raises(ValueError, match="Canonical source output already exists"): + module.prepare(args, launcher) + assert sentinel.read_text() == "previous source evidence\n" + + +@pytest.mark.parametrize("damage", ["scoped_only", "changed_member"]) +def test_cli_success_cannot_certify_incomplete_or_changed_source(chain, damage): + """A zero exit code with a scoped record or edited TSV never supplies whole-source evidence.""" + module, launcher, root = chain + source = root / "data/transformed/ontologies" + receipt_path = root / "chain.json" + plan = { + "canonical_output_base": str(source.parent), + "canonical_source_directory": str(source), + "upstream_before": {}, + "ancillary_before": {name: {} for name in module.ANCILLARY}, + "chain_receipt": str(receipt_path), + } + + def produced(_): + source.mkdir(parents=True) + nodes = source / "ro_nodes.tsv" + nodes.write_text("id\tcategory\n") + audit = source / "audit.tsv" + audit.write_text("record\n") + report = { + "members": {nodes.name: launcher.identity(nodes)}, + "audit_members": {audit.name: launcher.identity(audit)}, + } + filename = ( + "ro_source_finalization.json" if damage == "scoped_only" else "source_finalization.json" + ) + (source / filename).write_text(json.dumps(report)) + (source / "source_fingerprint.json").write_text('{"version": 3}\n') + if damage == "changed_member": + nodes.write_text("id\tcategory\nchanged:1\tbiolink:NamedThing\n") + return 0 + + launcher.run = produced + message = ( + "CLI success lacks actual completion evidence" + if damage == "scoped_only" + else "Finalizer member changed" + ) + with pytest.raises(ValueError, match=message): + module.run(plan, launcher) + assert not receipt_path.exists() diff --git a/tests/test_m5_release_progress.py b/tests/test_m5_release_progress.py new file mode 100644 index 0000000..5712a03 --- /dev/null +++ b/tests/test_m5_release_progress.py @@ -0,0 +1,36 @@ +"""Retain command evidence when independent M5 launchers finish out of order.""" + +import importlib.util +import json +from pathlib import Path + + +def test_stale_launcher_state_cannot_erase_another_command(tmp_path): + """Rebuild aggregate progress from atomic records, including overlapping commands.""" + helper = Path(__file__).parents[1] / "scripts/adhoc/m5_release_state_20261116.py" + spec = importlib.util.spec_from_file_location("release_progress", helper) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.ROOT = tmp_path + module.OUTPUT = tmp_path / ".temp/m5-worker-run-20261116" + logs = module.OUTPUT / "commands" + logs.mkdir(parents=True) + baseline = {"label": "baseline", "status": "running", "log": "baseline.log", + "started_at": "2026-11-16T20:00:00Z"} + preflight = {"label": "preflight", "status": "passed", "log": "preflight.log", + "started_at": "2026-11-16T20:01:00Z"} + stale_state = {"phase": "baseline", "pid": 1, "commands": [baseline]} + module.save_command(logs / "baseline.json", baseline) + module.save(stale_state) + module.save_command(logs / "preflight.json", preflight) + module.save({"phase": "preflight", "pid": 2, "blockers": ["current dependency"], + "results": {"preflight_passed": True}}) + baseline["status"] = "passed" + module.save_command(logs / "baseline.json", baseline) + module.save({}) + result = json.loads((module.OUTPUT / "progress.json").read_text()) + assert result["commands"] == [baseline, preflight] + assert result["phase"] == "preflight" + assert result["blockers"] == ["current dependency"] + assert result["results"] == {"preflight_passed": True} + assert "preflight: passed" in (module.ROOT / ".temp/M5_RELEASE_PROGRESS.md").read_text() From 1149730f5964c91a8335579af403fcbb34b9333e Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:19:35 -0800 Subject: [PATCH 2/5] Verify strict M5 merge inputs and harden gate evidence boundaries --- .../m5-bactotraits-release-20261116.md | 59 +- .../adhoc/m5_release_gate_retry_20261116.py | 61 ++ scripts/adhoc/m5_release_merge_20261116.py | 524 ++++++++++++++++++ scripts/adhoc/m5_release_state_20261116.py | 45 ++ tests/test_m5_release_gate_retry.py | 44 ++ tests/test_m5_release_merge.py | 284 ++++++++++ tests/test_m5_release_progress.py | 92 +++ 7 files changed, 1105 insertions(+), 4 deletions(-) create mode 100644 scripts/adhoc/m5_release_gate_retry_20261116.py create mode 100644 scripts/adhoc/m5_release_merge_20261116.py create mode 100644 tests/test_m5_release_gate_retry.py create mode 100644 tests/test_m5_release_merge.py diff --git a/docs/validation/m5-bactotraits-release-20261116.md b/docs/validation/m5-bactotraits-release-20261116.md index 82ad60b..468d4c4 100644 --- a/docs/validation/m5-bactotraits-release-20261116.md +++ b/docs/validation/m5-bactotraits-release-20261116.md @@ -118,6 +118,35 @@ This is not absence evidence across arbitrary copies or uninspected graphs. No neighboring row was quarantined, no replacement organism was inferred, and the correction does not establish trait falsity or graph-wide validity. +## Actual canonical dependency chain + +The ordinary CLI whole-ontology unit completed on committed KG candidate +`b631cacf5bca2f5f993e85bacb96578919cc7d92`, whose production implementation is +unchanged from the corrected merged code. It wrote 1,355,041 nodes and 1,729,253 +edges across 14 graph pairs. Independent review counted those rows and verified +97 persisted artifacts and 526 named input, implementation and authority paths. +The actual whole-source finalization and producer fingerprint both completed; +the earlier scoped RO canary was not substituted for whole-source completion. +The complete chain receipt SHA256 is +`c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e`. + +The next BacDive unit uses the same canonical output placement, launcher, +Python environment and single-worker/thread limits. Its plan pins six original +inputs totaling 14.875 GB, including the actual 14.074 GB NCBITaxon DB, with +188.416 GB free before staging. BacDive and full BactoTraits completion, followed +by the ordinary strict BactoTraits-only merge, remain pending in this checkpoint. +No completion marker is synthesized and no freshness check is disabled. + +The strict merge helper preserves canonical semantics and records explicit +path-only configuration changes: absolute source paths and exclusive output +paths, including statistics. This prevents the canonical cwd-relative stats +filename from overwriting a task-checkout artifact +([#160](https://github.com/CultureBotAI/EdgeMech/issues/160)). It requires all three +complete source receipts and verifies the complete quarantined row, retained +intermediate joins and streamed merged outputs. Its 24 focused tests also pass +independently. A BactoTraits-only candidate cannot establish full-snapshot +publication. + ## Independent review and checks Read-only adversarial review independently recomputed all new stage artifact @@ -128,7 +157,29 @@ findings were filed and fixed: [#146](https://github.com/CultureBotAI/EdgeMech/i for concurrent progress updates and [#147](https://github.com/CultureBotAI/EdgeMech/issues/147) for stat precision. A second read-only review verified both fixes. -The finalization helper's six focused tests passed. The progress regression -passes and checks both command retention and metadata when launchers finish -out of order. Final repository check results and the production dependency -canary will be appended after execution; this document does not imply they passed. +The finalization helper's six focused tests and production-launcher four focused +tests passed. EdgeMech's final full suite passed 777 tests after the progress +boundary fixes. Required EdgeMech lint and offline wheel/source package builds +also passed. Independent review reran the 23 helper-boundary checks and verified +the actual retained five-case retry receipt against its persisted log bytes. + +The first KG standard gate passed its cheap stages and 9,999 tests but failed +four scratch-context assertions and one output-root teardown guard. A versioned +environment moves scratch outside Git into durable `~/agent-work`; all five +affected checks pass without changing assertions or guards +([#161](https://github.com/CultureBotAI/EdgeMech/issues/161)). A second gate exposed +pre-existing fixture dependence on newly produced canonical taxonomy data +([KG #1873](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1873)). A +pristine verification checkout at exactly the same committed candidate passes +that real test canary; the unchanged standard full gate there remains pending. +The populated-checkout failure and every earlier log remain preserved. + +Read-only review found two new helper boundary cases: exact temporary-root paths +were accepted, and imported terminal receipts did not require consistent exit +status and log hashes. Both are fixed with 23 focused tests +([#162](https://github.com/CultureBotAI/EdgeMech/issues/162)); actual prior test +receipts remain valid. The earlier executed helper bytes are retained by hash. +The existing Java toolchain recorder can mistake `_JAVA_OPTIONS` text for the +version ([KG #1872](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1872)); +actual Java version and binary/library hashes were captured separately without +rewriting the old record or claiming this provenance-only field gates admission. diff --git a/scripts/adhoc/m5_release_gate_retry_20261116.py b/scripts/adhoc/m5_release_gate_retry_20261116.py new file mode 100644 index 0000000..9f01541 --- /dev/null +++ b/scripts/adhoc/m5_release_gate_retry_20261116.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Record an environment-only retry after M5 gate scratch-context failures. + +2026-11-16 provenance: the first standard gate put pytest scratch inside KG, +changing path normalization and Git discovery. Use durable scratch outside any +checkout; preserve the old record and all failures. No gate, assertions, source, +tox selectors, or filesystem guards are changed. Required results stay in OUT. +""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + + +def durable_scratch(path): + """Reject exact temporary roots and every descendant after resolving aliases (#162).""" + scratch = path.resolve(strict=True) + forbidden = (Path("/tmp").resolve(), Path("/private/tmp"), Path("/private/var/folders")) + if not scratch.is_dir() or any(scratch == root or scratch.is_relative_to(root) for root in forbidden): + raise ValueError("Use a durable existing scratch directory") + return scratch + + +def main(): + """Write a versioned environment record after verifying the scratch boundary.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--previous", required=True, type=Path) + parser.add_argument("--scratch-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + scratch = durable_scratch(args.scratch_root) + except (ValueError, OSError) as error: + parser.error(str(error)) + probe = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=scratch, + text=True, capture_output=True, check=False) + if probe.returncode != 128 or "not a git repository" not in probe.stderr: + parser.error("Scratch must be outside all Git checkouts") + before = args.previous.read_bytes() + environment = json.loads(before) + previous_tmp = environment["set"]["TMPDIR"] + environment["set"]["TMPDIR"] = str(scratch) + args.output.mkdir(parents=True, exist_ok=False) + destination = args.output / "environment.json" + destination.write_text(json.dumps(environment, indent=2) + "\n") + receipt = {"previous": str(args.previous), "previous_sha256": hashlib.sha256(before).hexdigest(), + "environment": str(destination), + "environment_sha256": hashlib.sha256(destination.read_bytes()).hexdigest(), + "delta": {"set.TMPDIR": {"before": previous_tmp, "after": str(scratch)}}, + "git_probe": {"returncode": probe.returncode, "stderr": probe.stderr.strip()}, + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "scope": "Environment-only preparation; no test or gate executed."} + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_merge_20261116.py b/scripts/adhoc/m5_release_merge_20261116.py new file mode 100644 index 0000000..0ed611c --- /dev/null +++ b/scripts/adhoc/m5_release_merge_20261116.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Verify one strict M5 BactoTraits merge after all canonical producers complete. + +2026-11-16 provenance: this is the ordinary KG merge CLI, selecting only the +canonical bactotraits source and one process. No finalization/freshness bypass +or low-level merge is used. All three actual source receipts and their complete +persisted trees are required. The canonical configuration retains its semantics; +source filenames become absolute and output-only filenames move to this unit's +exclusive directory, including stats (EdgeMech #160). Historical outputs remain +read-only. Plan before run; neither a plan nor this bounded candidate is a +full-snapshot release, publication, or scientific graph-wide validation. +""" + +from __future__ import annotations + +import argparse +import copy +import csv +import hashlib +import importlib.metadata +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +SOURCES = ("ontologies", "bacdive", "bactotraits") +SOURCE_SHA = "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" +ROW_SHA = "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a" +SPO = ("NCBITaxon:1123071", "biolink:has_phenotype", "METPO:1000703") +AUDIT = "bactotraits_identity_quarantine.tsv" +THREADS = ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS", + "NUMEXPR_MAX_THREADS", "VECLIB_MAXIMUM_THREADS", "RAYON_NUM_THREADS", "BLIS_NUM_THREADS") + + +def require(condition, message): + """Refuse incomplete or changed evidence explicitly, including under python -O.""" + if not condition: + raise ValueError(message) + + +def safe(path): + """Require a literal path without parent traversal or symlinked components.""" + path = Path(path).absolute() + require(".." not in path.parts, "Parent traversal is not permitted") + require(not any(item.is_symlink() for item in (path, *path.parents)), f"Symlinked path: {path}") + return path + + +def identity(path): + """Stream exact bytes and reject changes during hashing.""" + path = safe(path) + require(path.is_file(), f"Required file missing: {path}") + before = path.stat() + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + after = path.stat() + fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + require(all(getattr(before, key) == getattr(after, key) for key in fields), f"File changed while reading: {path}") + return {"bytes": before.st_size, "sha256": digest.hexdigest()} + + +def snapshot(directory): + """Inventory a named producer tree, including nested ignored files and rejecting links.""" + directory = safe(directory) + require(directory.is_dir(), f"Required producer directory missing: {directory}") + result = {} + for path in sorted(directory.rglob("*")): + require(not path.is_symlink(), f"Symlink in producer tree: {path}") + if path.is_file(): + result[str(path.relative_to(directory))] = identity(path) + return result + + +def read_json(path): + """Read bounded receipts without accepting duplicate JSON keys.""" + def pairs(items): + result = {} + for key, value in items: + require(key not in result, f"Duplicate JSON key: {key}") + result[key] = value + return result + + return json.loads(safe(path).read_text(), object_pairs_hook=pairs) + + +def write_new(path, value): + """Preserve every earlier receipt using exclusive creation.""" + with path.open("x") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + + +def bind(pins, path, expected=None): + """Bind an input once and reject contradictory prior byte identities.""" + path = safe(path) + observed = identity(path) + if expected is not None: + require(observed == expected, f"Receipt-bound file changed: {path}") + require(str(path) not in pins or pins[str(path)] == observed, f"Conflicting input identity: {path}") + pins[str(path)] = observed + return observed + + +def verify_chain(kg_root, receipt_paths): + """Require real whole-source completion and an unchanged recursive upstream chain.""" + receipts, pins = {}, {} + for source in SOURCES: + path = safe(receipt_paths[source]) + bind(pins, path) + receipt = read_json(path) + require(receipt.get("profile") == "m5-canonical-release-source-unit/v1", "Unsupported source receipt") + require(receipt.get("returncode") == 0, f"{source}: producer did not complete") + plan = receipt["plan"] + require(plan.get("source") == source, f"{source}: wrong producer receipt") + directory = kg_root / "data/transformed" / source + require(plan.get("canonical_source_directory") == str(directory), f"{source}: noncanonical output") + require(receipt.get("upstream_unchanged") is True and receipt.get("ancillary_preexisting_unchanged") is True, + f"{source}: preservation checks are incomplete") + recorded = receipt["canonical_source_artifacts"] + require(snapshot(directory) == recorded, f"{source}: persisted source tree changed") + for member in ("source_finalization.json", "source_fingerprint.json"): + require(member in recorded, f"{source}: whole-source completion missing") + finalization = read_json(directory / "source_finalization.json") + marker = read_json(directory / "source_fingerprint.json") + require(finalization == receipt.get("finalization_record") and finalization.get("source") == source, + f"{source}: finalization receipt mismatch") + require(marker == receipt.get("producer_fingerprint") and marker.get("version") == 3, + f"{source}: producer fingerprint mismatch") + for section in ("members", "audit_members"): + require(bool(finalization.get(section)), f"{source}: missing finalization {section}") + for name, expected in finalization[section].items(): + require(Path(name).name == name and name in recorded, "Invalid finalization member") + require(recorded[name] == expected, f"{source}: finalization member differs: {name}") + if source == "ontologies": + require("ncbitaxon_nodes.tsv" in finalization["members"] + and "chebi_nodes.tsv" in finalization["members"], "Scoped ontology output cannot qualify") + unit_ref = receipt["unit_receipt"] + bind(pins, Path(unit_ref["path"]), {key: unit_ref[key] for key in ("bytes", "sha256")}) + unit = read_json(Path(unit_ref["path"])) + require(unit.get("returncode") == 0 and unit.get("source") == source, "Underlying CLI unit incomplete") + require(unit.get("git_revision") == plan.get("git_revision"), "Producer revision mismatch") + for name, expected in plan["implementation_and_policy_inputs"].items(): + bind(pins, kg_root / name, expected) + for original in plan["originals"]: + bind(pins, Path(original["source"]), {key: original[key] for key in ("bytes", "sha256")}) + for upstream in SOURCES[:SOURCES.index(source)]: + require(plan["upstream_before"].get(upstream) == receipts[upstream]["canonical_source_artifacts"], + f"{source}: upstream {upstream} is not the completed receipt") + receipts[source] = receipt + latest = read_json(Path(receipts["bactotraits"]["unit_receipt"]["path"])) + for name, expected in latest["task_raw_after"].items(): + require(Path(name).name == name, "Invalid raw snapshot member") + bind(pins, kg_root / "data/raw" / name, expected) + for directory, files in receipts["bactotraits"]["ancillary_after"].items(): + for name, expected in files.items(): + bind(pins, kg_root / directory / name, expected) + return receipts, pins + + +def relocated_configuration(canonical, kg_root, output): + """Change only explicit path placement; preserve source options and graph semantics.""" + require(isinstance(canonical, dict), "Invalid canonical merge configuration") + require(canonical.get("configuration", {}).get("allow_unfinalized_sources", False) is False, + "Unfinalized source opt-out is forbidden") + result = copy.deepcopy(canonical) + deltas = [] + + def replace(container, key, value, path): + previous = container[key] + container[key] = value + deltas.append({"path": path, "before": previous, "after": value}) + + replace(result["configuration"], "output_directory", str(output), ["configuration", "output_directory"]) + sources = result["merged_graph"]["source"] + require("bactotraits" in sources, "Canonical BactoTraits source missing") + for name, source in sources.items(): + inputs = source["input"] + filenames = inputs["filename"] + require(isinstance(filenames, list) and all(isinstance(value, str) for value in filenames), + "Canonical source filenames must be a list") + resolved = [] + for filename in filenames: + path = Path(filename) + require(not path.is_absolute() and ".." not in path.parts, "Noncanonical source path") + resolved.append(str(safe(kg_root / path))) + replace(inputs, "filename", resolved, ["merged_graph", "source", name, "input", "filename"]) + groups = [(["merged_graph", "operations"], result["merged_graph"].get("operations", []))] + groups += [(["merged_graph", "source", name, "operations"], source.get("operations", [])) + for name, source in sources.items()] + targets = set() + for prefix, operations in groups: + for index, operation in enumerate(operations): + args = operation.get("args", {}) + if "filename" not in args: + continue + require(operation.get("name") == "kgx.graph_operations.summarize_graph.generate_graph_stats", + "Unreviewed output-producing merge operation") + filename = Path(args["filename"]) + require(filename.name == str(filename), "Noncanonical operation output filename") + target = str(output / filename.name) + require(target not in targets, "Operation output paths collide") + targets.add(target) + replace(args, "filename", target, [*prefix, index, "args", "filename"]) + # Reversing every declared path-only delta must reconstruct the canonical object. + recovered = copy.deepcopy(result) + for delta in reversed(deltas): + node = recovered + for key in delta["path"][:-1]: + node = node[key] + node[delta["path"][-1]] = delta["before"] + require(recovered == canonical, "Merge semantics changed beyond recorded path placement") + return result, deltas + + +def literal_rows(path, delimiter="\t", encoding="utf-8", skip=0, quoting=csv.QUOTE_MINIMAL): + """Stream well-formed literal records and preserve their data-row ordinal.""" + with path.open(encoding=encoding, newline="") as stream: + reader = csv.reader(stream, delimiter=delimiter, quoting=quoting) + for _ in range(skip): + next(reader) + header = next(reader) + require(len(header) == len(set(header)), f"Duplicate table headers: {path}") + for ordinal, values in enumerate(reader, 1): + require(len(values) == len(header), f"Malformed row {ordinal}: {path}") + yield ordinal, dict(zip(header, values)) + + +def verify_quarantine(kg_root): + """Replay the complete quarantined row, fresh joins and finalized source edge stream.""" + raw = kg_root / "data/raw/BactoTraits_databaseV2_Jun2022.csv" + require(identity(raw)["sha256"] == SOURCE_SHA, "BactoTraits raw source changed") + selected = [(ordinal, row) for ordinal, row in literal_rows(raw, ";", "ISO-8859-1", 2) + if row.get("Bacdive_ID") == "17851"] + require(len(selected) == 1 and selected[0][0] == 8521, "Reviewed raw occurrence is not unique row8521") + row = selected[0][1] + row_sha = hashlib.sha256(json.dumps(row, ensure_ascii=False, indent=2, sort_keys=True).encode()).hexdigest() + require(row_sha == ROW_SHA, "Complete reviewed source row changed") + audit = kg_root / "data/transformed/bactotraits" / AUDIT + audits = list(literal_rows(audit)) + require(len(audits) == 1, "Expected exactly one complete quarantine audit occurrence") + record = audits[0][1] + expected = {"source_data_row": "8521", "reviewed_source_data_row": "8521", "bacdive_id": "17851", + "ncbitaxon_id": SPO[0], "source_sha256": SOURCE_SHA, "reviewed_source_sha256": SOURCE_SHA, + "row_sha256": ROW_SHA, "disposition": "quarantine_mixed_strain_row"} + require(all(record.get(key) == value for key, value in expected.items()), "Quarantine audit identity mismatch") + require(json.loads(record["raw_row_json"]) == row, "Quarantine audit omits or changes original row fields") + policy = kg_root / "mappings/canonical/bactotraits_identity_quarantine.json" + require(record.get("policy_sha256") == identity(policy)["sha256"], "Quarantine policy identity mismatch") + decisions = read_json(policy).get("decisions", []) + require(len(decisions) == 1 and decisions[0].get("bacdive_id") == "17851", + "Policy must retain only the independently reviewed row") + decision = decisions[0] + for field in ("rule_id", "full_name", "culture_collection_codes", "rationale"): + require(record.get(field) == decision.get(field), f"Quarantine audit policy field mismatch: {field}") + require(record.get("full_name") == row.get("Full_name") + and record.get("culture_collection_codes") == row.get("culture collection codes"), + "Quarantine audit identity columns differ from complete original row") + require(record.get("evidence_urls") == "|".join(decision.get("evidence_urls", [])), + "Quarantine audit evidence URLs differ from policy") + require(bool(record.get("rule_id")) and bool(record.get("evidence_urls")) and bool(record.get("rationale")), + "Quarantine audit lacks decision evidence") + joins = {} + for source, filename, key in (("bacdive", "bacdive.tsv", "bacdive_id"), + ("bactotraits", "bactotraits_mapping.tsv", "Bacdive_ID"), + ("bactotraits", "bactotraits.tsv", "Bacdive_ID")): + path = kg_root / "kg_microbe/transform_utils" / source / "tmp" / filename + rows = [{"source_data_row": ordinal, "row": values} for ordinal, values in literal_rows(path) + if values.get(key) in ("17851", "bacdive:17851") or values.get("ncbitaxon_id") == SPO[0]] + target = [item for item in rows if item["row"].get(key) in ("17851", "bacdive:17851")] + if filename == "bactotraits.tsv": + require(not target, "Quarantined source row leaked into retained BactoTraits intermediate") + else: + require(len(target) == 1 and target[0]["row"].get("ncbitaxon_id") == SPO[0], + "Fresh source lookup does not preserve the reviewed exact identity") + joins[f"{source}/{filename}"] = {"identity": identity(path), "matching_occurrences": rows} + with (kg_root / "data/transformed/bactotraits/edges.tsv").open("rb") as stream: + edges = inspect_tsv(stream, edges=True) + require(edges["target_spo_occurrences"] == 0, + "Target SPO remains in BactoTraits-only source output; independent lineage review is required") + return {"audit": {"path": str(audit), **identity(audit), "rows": 1, "record": record}, + "source_row_sha256": row_sha, "fresh_joins": joins, "source_edges": edges} + + +def inspect_tsv(stream, *, edges=False): + """Hash literal persisted TSV bytes and count targeted provenance while streaming.""" + digest, size, lines, tail = hashlib.sha256(), 0, 0, b"" + + def decoded(): + nonlocal size, lines, tail + for line in stream: + digest.update(line) + size += len(line) + lines += line.count(b"\n") + tail = line[-1:] + yield line.decode("utf-8") + + reader = csv.reader(decoded(), delimiter="\t", quoting=csv.QUOTE_NONE) + header = next(reader, None) + require(header and len(header) == len(set(header)), "Missing or duplicate persisted TSV header") + needed = ("subject", "predicate", "object", "primary_knowledge_source") if edges else () + require(set(needed).issubset(header), "Persisted edge provenance columns missing") + count, target_count, bactotraits_count, examples = 0, 0, 0, [] + for count, values in enumerate(reader, 1): + require(len(values) == len(header), "Malformed persisted TSV row") + if edges: + row = dict(zip(header, values)) + if tuple(row[key] for key in ("subject", "predicate", "object")) == SPO: + target_count += 1 + if "infores:bactotraits" in row["primary_knowledge_source"].split("|"): + bactotraits_count += 1 + if len(examples) < 10: + examples.append({"source_data_row": count, "row": row}) + result = {"bytes": size, "sha256": digest.hexdigest(), "rows": count} + require(count == max(0, lines + int(bool(tail) and tail != b"\n") - 1), "Physical/parsed TSV row counts differ") + if edges: + result.update(target_spo_occurrences=target_count, bactotraits_target_occurrences=bactotraits_count, + target_examples=examples) + return result + + +def inspect_archive(path, config_sha, revision, python_version): + """Verify the actual compressed graph members and embedded strict-merge provenance.""" + archive_identity = identity(path) + seen, members, manifest = set(), {}, None + with tarfile.open(path, "r|gz") as archive: + for member in archive: + require(member.isfile() and Path(member.name).name == member.name and member.name not in seen, + "Unsafe or duplicate graph archive member") + seen.add(member.name) + stream = archive.extractfile(member) + if member.name == "manifest.json": + require(member.size <= 1024 * 1024, "Unexpectedly large graph manifest") + manifest = json.loads(stream.read()) + else: + require(member.name in {"merged-kg_nodes.tsv", "merged-kg_edges.tsv", "merged-kg_reference_resolution.tsv"}, + "Unexpected graph archive member") + members[member.name] = inspect_tsv(stream, edges=member.name == "merged-kg_edges.tsv") + require(manifest is not None and manifest.get("manifest_version") == 1, "Graph manifest missing or unsupported") + expected_names = {"merged-kg_nodes.tsv", "merged-kg_edges.tsv", "merged-kg_reference_resolution.tsv"} + require(set(members) == set(manifest.get("members", {})) == expected_names, "Graph manifest member set differs") + for name, observed in members.items(): + require({key: observed[key] for key in ("bytes", "sha256", "rows")} == manifest["members"][name], + f"Persisted graph member differs from manifest: {name}") + provenance = manifest.get("provenance", {}) + require(provenance.get("source_finalization") == {"required": True, "diagnostic_opt_out": False}, + "Archive lacks strict finalization provenance") + require(provenance.get("merge_config", {}).get("sha256") == config_sha, "Archive config identity mismatch") + require(provenance.get("commit") == revision, "Archive producer revision is dirty or changed") + require(provenance.get("python") == python_version, "Archive interpreter changed") + require(members["merged-kg_edges.tsv"]["target_spo_occurrences"] == 0, + "Target SPO remains in BactoTraits-only merge; independent lineage review is required") + require(identity(path) == archive_identity, "Graph archive changed during inspection") + return {"archive": {"path": str(path), **archive_identity}, "members": members, "manifest": manifest} + + +def runtime(): + """Bind the actual supported interpreter and complete installed distribution versions.""" + require((3, 10) <= sys.version_info[:2] <= (3, 12), "Run merge helper with pinned KG Python3.10-3.12") + executable = Path(sys.executable).absolute() + return {"python": sys.version, "executable": str(executable), "resolved": str(executable.resolve()), + "binary": identity(executable.resolve()), + "packages": {re.sub(r"[-_.]+", "-", dist.metadata["Name"].lower()): dist.version + for dist in importlib.metadata.distributions()}} + + +def prepare(args): + """Build an immutable read-only plan requiring all completed upstream receipts.""" + kg_root, output_root, output = safe(args.kg_root), safe(args.output_root), safe(args.output) + require(output.parent == output_root and output_root.is_dir(), "Candidate must be directly beneath worker output root") + require(not output.exists(), "Candidate output already exists; preserve it") + receipts, pins = verify_chain(kg_root, {source: getattr(args, source + "_receipt") for source in SOURCES}) + revision = subprocess.check_output(["git", "-C", str(kg_root), "rev-parse", "HEAD"], text=True).strip() + short_revision = subprocess.check_output(["git", "-C", str(kg_root), "rev-parse", "--short", "HEAD"], text=True).strip() + status = subprocess.check_output(["git", "-C", str(kg_root), "status", "--porcelain", "--untracked-files=no"], text=True) + require(not status, "KG tracked candidate is not clean") + require(all(receipt["plan"]["git_revision"] == revision for receipt in receipts.values()), "Source producer commits differ") + actual_runtime = runtime() + for name in ("inventory.json", "install-receipt.json", "installed-version-verification.json"): + bind(pins, output_root / "environment" / name) + installed = read_json(output_root / "environment/inventory.json") + expected_packages = {re.sub(r"[-_.]+", "-", item["name"].lower()): item["version"] + for item in installed["packages"]} + require(not installed["missing"] and actual_runtime["packages"] == expected_packages, + "Actual merge packages differ from pinned offline installation") + prior = receipts["bactotraits"]["plan"] + require(actual_runtime["python"] == prior["python"] + and actual_runtime["executable"] == prior["interpreter"]["executable"], "Merge interpreter differs from producer") + canonical_path = kg_root / "merge.yaml" + bind(pins, canonical_path) + bind(pins, Path(__file__).resolve()) + configuration, deltas = relocated_configuration(yaml.safe_load(canonical_path.read_text()), kg_root, output) + config_text = yaml.safe_dump(configuration, sort_keys=False) + quarantine = verify_quarantine(kg_root) + require(AUDIT in receipts["bactotraits"]["finalization_record"]["audit_members"], + "Quarantine audit is not bound by finalization") + lineage_path = safe(args.lineage_receipt) + bind(pins, lineage_path) + lineage = read_json(lineage_path) + require(lineage.get("profile") == "m5-bactotraits-retained-lineage/v1", "Wrong historical lineage receipt") + historical_madin = lineage["historical_source_graph_occurrences"]["madin_etal"] + require(historical_madin["matched_occurrences"] == 1 + and historical_madin["matches"][0]["row"]["primary_knowledge_source"] == "infores:madin_etal", + "Independent historical Madin provenance missing") + for item in (historical_madin["input"], lineage["independent_madin_raw_occurrences"]["input"]): + bind(pins, Path(item["path"]), {key: item[key] for key in ("bytes", "sha256")}) + environment = dict(prior["environment"]) + old_stage = prior["stage"] + environment = {key: value.replace(old_stage, str(output)) for key, value in environment.items()} + for key in THREADS: + environment[key] = "1" + environment["PYTHONPATH"] = str(kg_root) + environment["VIRTUAL_ENV"] = str(Path(actual_runtime["executable"]).parent.parent) + protected = {} + for delta in deltas: + if delta["path"][-1] == "filename" and "operations" in delta["path"]: + path = kg_root / delta["before"] + protected[str(path)] = identity(path) if path.exists() else None + if path.exists(): + bind(pins, path) + binding = {"kg_root": str(kg_root), "output_root": str(output_root), "output": str(output), + "git_revision": revision, "short_revision": short_revision, "runtime": actual_runtime, + "configuration": configuration, "configuration_text": config_text, "path_only_deltas": deltas, + "input_pins": pins, "source_trees": {source: receipt["canonical_source_artifacts"] for source, receipt in receipts.items()}, + "quarantine": quarantine, "historical_independent_madin": historical_madin, + "historical_independent_madin_raw": lineage["independent_madin_raw_occurrences"], + "environment": environment, "protected_original_outputs": protected, + "command": [actual_runtime["executable"], "-B", "-m", "kg_microbe.run", "merge", "-y", + str(output / "merge.yaml"), "-p", "1", "-s", "bactotraits"], "processes": 1} + return {"profile": "m5-strict-bactotraits-merge-plan/v1", "action": "plan", "writes": 0, + "binding": binding, "binding_sha256": hashlib.sha256(json.dumps(binding, sort_keys=True).encode()).hexdigest(), + "free_disk_bytes": shutil.disk_usage(output_root).free} + + +def verify_after(binding): + """Reject any input, producer tree, runtime or protected-output change after the CLI.""" + for name, expected in binding["input_pins"].items(): + require(identity(Path(name)) == expected, f"Merge input changed during execution: {name}") + for source, expected in binding["source_trees"].items(): + require(snapshot(Path(binding["kg_root"]) / "data/transformed" / source) == expected, "Producer output changed during merge") + require(runtime() == binding["runtime"], "Merge runtime changed") + for name, expected in binding["protected_original_outputs"].items(): + path = Path(name) + require((identity(path) if path.exists() else None) == expected, "Tracked or existing statistics were overwritten") + + +def run(plan, plan_file): + """Execute the actual strict CLI once and verify persisted archive bytes before success.""" + approved_plan = read_json(plan_file) + require(approved_plan.get("profile") == plan["profile"] + and approved_plan.get("binding_sha256") == plan["binding_sha256"] + and approved_plan.get("binding") == plan["binding"], "Inputs or plan changed since --plan") + binding = plan["binding"] + output = Path(binding["output"]) + require(not output.exists(), "Candidate output appeared after planning") + os.umask(0o077) + output.mkdir(mode=0o700) + (output / "tmp").mkdir(mode=0o700) + write_new(output / "plan.json", plan) + config = output / "merge.yaml" + with config.open("x") as stream: + stream.write(binding["configuration_text"]) + environment = {key: value for key, value in os.environ.items() + if key not in ("CONDA_DEFAULT_ENV", "CONDA_PREFIX") and not key.startswith("TOX")} + environment.update(binding["environment"]) + result = {"profile": "m5-strict-bactotraits-merge/v1", "started_at": datetime.now(timezone.utc).isoformat(), + "plan_sha256": identity(plan_file)["sha256"], "binding_sha256": plan["binding_sha256"], + "candidate_merged_graph_verified": False, "full_snapshot_release_verified": False, + "published_release_verified": False, "scientific_trait_truth_resolved": False, + "graph_wide_validation_claimed": False} + try: + with (output / "stdout.log").open("x") as stdout, (output / "stderr.log").open("x") as stderr: + child = subprocess.Popen(binding["command"], cwd=binding["kg_root"], env=environment, stdout=stdout, stderr=stderr) + write_new(output / "process.json", {"pid": child.pid, "parent_pid": os.getpid(), "command": binding["command"]}) + result["returncode"] = child.wait() + verify_after(binding) + require(result["returncode"] == 0, "Strict KG merge CLI did not complete; preserve logs") + result["persisted_graph"] = inspect_archive(output / "merged-kg.tar.gz", identity(config)["sha256"], + binding["short_revision"], binding["runtime"]["python"].split()[0]) + result["quarantine_after"] = verify_quarantine(Path(binding["kg_root"])) + require(result["quarantine_after"] == binding["quarantine"], "Quarantine/lineage changed during merge") + result.update(status="complete", candidate_merged_graph_verified=True, inputs_and_upstreams_unchanged=True) + except Exception as error: + result.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + result["completed_at"] = datetime.now(timezone.utc).isoformat() + result["output_artifacts"] = {str(path.relative_to(output)): identity(path) + for path in sorted(output.rglob("*")) if path.is_file()} + write_new(output / "receipt.json", result) + print(json.dumps({"receipt": str(output / "receipt.json"), "status": result["status"], + "candidate_merged_graph_verified": result["candidate_merged_graph_verified"]})) + + +def main(): + """Keep planning and actual merge invocation distinct and explicit.""" + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", action="store_true") + mode.add_argument("--run", action="store_true") + for name in ("kg-root", "output-root", "output", "ontologies-receipt", "bacdive-receipt", "bactotraits-receipt", "lineage-receipt"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--plan-file", type=Path) + args = parser.parse_args() + if args.run and args.plan_file is None: + parser.error("--run requires the newly recorded --plan-file") + csv.field_size_limit(16 * 1024 * 1024) + plan = prepare(args) + if args.plan: + print(json.dumps(plan, indent=2, sort_keys=True)) + else: + run(plan, args.plan_file) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_state_20261116.py b/scripts/adhoc/m5_release_state_20261116.py index 7c79148..341991f 100644 --- a/scripts/adhoc/m5_release_state_20261116.py +++ b/scripts/adhoc/m5_release_state_20261116.py @@ -65,6 +65,40 @@ def save_command(path, entry): temporary.replace(path) +def import_check_record(path, label): + """Validate the launcher's terminal evidence before publishing imported progress (#162).""" + raw = path.read_bytes() + receipt = json.loads(raw) + status = receipt.get("status") + if status not in {"running", "complete", "failed"}: + raise ValueError("Unsupported imported check status") + terminal_fields = {"exit_code", "completed_at", "seconds", "log_sha256"} + logfile = path.parent / "command.log" + if not logfile.is_file(): + raise ValueError("Imported check log is missing") + if status == "running": + if terminal_fields & receipt.keys(): + raise ValueError("Running check receipt contains terminal fields") + else: + code = receipt.get("exit_code") + if type(code) is not int or (status == "complete") != (code == 0): + raise ValueError("Imported terminal status contradicts exit code") + if not isinstance(receipt.get("completed_at"), str) or not receipt["completed_at"]: + raise ValueError("Imported terminal check lacks completion time") + expected = receipt.get("log_sha256") + digest = hashlib.sha256() + with logfile.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + if expected != digest.hexdigest(): + raise ValueError("Imported completed check log hash differs or is missing") + return {"label": label, "argv": receipt["command"], "cwd": receipt["cwd"], "log": str(logfile), + "status": "passed" if status == "complete" else status, + "child_pid": receipt["pid"], "started_at": receipt["started_at"], + "exit_code": receipt.get("exit_code"), "source_receipt": str(path), + "source_receipt_sha256": hashlib.sha256(raw).hexdigest()} + + def main(): """Run a bounded task command, preserving its argv, PID, output and status.""" parser = argparse.ArgumentParser(description=__doc__) @@ -75,11 +109,22 @@ def main(): parser.add_argument("--results", type=Path) parser.add_argument("--observe-process", type=Path) parser.add_argument("--process-status") + parser.add_argument("--import-check", type=Path) parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args() os.umask(0o077) OUTPUT.mkdir(parents=True, exist_ok=True) state = {"phase": args.phase, "pid": os.getpid()} + if args.import_check: + if not args.label: + parser.error("--label required for --import-check") + try: + entry = import_check_record(args.import_check, args.label) + except (ValueError, OSError, KeyError, TypeError) as error: + parser.error(str(error)) + logs = OUTPUT / "commands" + logs.mkdir(exist_ok=True) + save_command(logs / (args.label + ".json"), entry) if args.blocker: state["blockers"] = args.blocker if args.results: diff --git a/tests/test_m5_release_gate_retry.py b/tests/test_m5_release_gate_retry.py new file mode 100644 index 0000000..ed3e78d --- /dev/null +++ b/tests/test_m5_release_gate_retry.py @@ -0,0 +1,44 @@ +"""Keep durable gate scratch outside ephemeral filesystem roots, including aliases.""" + +import importlib.util +from pathlib import Path + +import pytest + + +@pytest.fixture +def retry(): + """Load only the retry boundary helper, without creating an environment record.""" + path = Path(__file__).parents[1] / "scripts/adhoc/m5_release_gate_retry_20261116.py" + spec = importlib.util.spec_from_file_location("m5_gate_retry", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("name", ["/tmp", "/private/tmp", "/private/var/folders"]) +def test_exact_existing_ephemeral_roots_are_refused_without_writing_them(retry, name): + """The root itself must be rejected, not just strings containing its trailing slash.""" + path = Path(name) + if not path.exists(): + pytest.skip("Platform does not have this temporary root") + with pytest.raises(ValueError, match="durable"): + retry.durable_scratch(path) + + +def test_durable_named_alias_to_temporary_root_cannot_pass(retry, tmp_path): + """A durable-looking name does not change the resolved storage lifetime.""" + alias = tmp_path / "durable-gate-scratch" + alias.symlink_to(Path("/tmp"), target_is_directory=True) + with pytest.raises(ValueError, match="durable"): + retry.durable_scratch(alias) + + +def test_actual_durable_directory_remains_usable(retry): + """The boundary repair accepts task-owned durable scratch and does not require /tmp.""" + # Read only an existing directory; pytest itself may place tmp_path in /tmp. + scratch = Path(__file__).resolve().parent + ephemeral = (Path("/tmp").resolve(), Path("/private/tmp"), Path("/private/var/folders")) + if any(scratch.is_relative_to(root) for root in ephemeral): + pytest.skip("Checkout itself is in ephemeral storage") + assert retry.durable_scratch(scratch) == scratch diff --git a/tests/test_m5_release_merge.py b/tests/test_m5_release_merge.py new file mode 100644 index 0000000..b817bcf --- /dev/null +++ b/tests/test_m5_release_merge.py @@ -0,0 +1,284 @@ +"""Reject stale/scoped sources, unsafe config placement and laundered candidate archives.""" + +import copy +import csv +import hashlib +import importlib.util +import io +import json +import tarfile +from pathlib import Path + +import pytest + +HELPER = Path(__file__).resolve().parents[1] / "scripts/adhoc/m5_release_merge_20261116.py" + + +@pytest.fixture +def merge(): + """Load the standalone helper without importing KG's graph pipeline.""" + spec = importlib.util.spec_from_file_location("m5_release_merge", HELPER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_only_declared_path_placement_changes_and_existing_stats_remain(merge, tmp_path): + """Preserve every source/semantic option and never write tracked statistics during planning.""" + kg = tmp_path / "kg" + kg.mkdir() + stats = kg / "merged_graph_stats.yaml" + stats.write_bytes(b"retained historical statistics\n") + canonical = { + "configuration": {"output_directory": "data/merged", "checkpoint": False, + "preserve": ["primary_knowledge_source"]}, + "merged_graph": { + "source": {"bactotraits": {"name": "BactoTraits", "input": {"format": "tsv", "filename": [ + "data/transformed/bactotraits/nodes.tsv", "data/transformed/bactotraits/edges.tsv"]}}, + "madin_etal": {"input": {"format": "tsv", "filename": ["data/transformed/madin_etal/edges.tsv"]}}}, + "operations": [{"name": "kgx.graph_operations.summarize_graph.generate_graph_stats", "args": { + "filename": "merged_graph_stats.yaml", "edge_facet_properties": ["source"]}}], + "destination": {"merged-kg-tsv": {"format": "tsv", "compression": "tar.gz", "filename": "merged-kg"}}, + }, + } + retained = copy.deepcopy(canonical) + output = tmp_path / "exclusive-candidate" + configured, deltas = merge.relocated_configuration(canonical, kg, output) + assert canonical == retained + assert configured["merged_graph"]["destination"] == canonical["merged_graph"]["destination"] + assert configured["configuration"]["preserve"] == ["primary_knowledge_source"] + assert configured["merged_graph"]["operations"][0]["args"]["filename"] == str(output / stats.name) + assert configured["merged_graph"]["operations"][0]["args"]["edge_facet_properties"] == ["source"] + assert configured["merged_graph"]["source"]["bactotraits"]["input"]["filename"] == [ + str(kg / "data/transformed/bactotraits/nodes.tsv"), str(kg / "data/transformed/bactotraits/edges.tsv")] + assert len(deltas) == 4 + assert stats.read_bytes() == b"retained historical statistics\n" + assert not output.exists() + + +@pytest.mark.parametrize("diagnostic", [True, "false", 0, None]) +def test_diagnostic_or_ambiguous_finalization_flags_never_plan(merge, tmp_path, diagnostic): + """Only the exact false boolean or absent canonical setting can pass.""" + with pytest.raises(ValueError, match="opt-out"): + merge.relocated_configuration({"configuration": {"allow_unfinalized_sources": diagnostic}}, + tmp_path, tmp_path / "out") + + +def source_chain(merge, root): + """Create small whole-source contracts whose byte identities can be independently changed.""" + receipts, paths, snapshots = {}, {}, {} + units = root / "units" + units.mkdir() + for source in merge.SOURCES: + directory = root / "data/transformed" / source + directory.mkdir(parents=True) + names = ("ncbitaxon_nodes.tsv", "chebi_nodes.tsv") if source == "ontologies" else ("nodes.tsv", "edges.tsv") + for name in names: + (directory / name).write_text("id\tcategory\ntest:1\tbiolink:NamedThing\n") + (directory / "audit.tsv").write_text("check\tstatus\nsource\tpass\n") + finalization = {"source": source, "members": {name: merge.identity(directory / name) for name in names}, + "audit_members": {"audit.tsv": merge.identity(directory / "audit.tsv")}} + marker = {"version": 3, "source": source} + (directory / "source_finalization.json").write_text(json.dumps(finalization)) + (directory / "source_fingerprint.json").write_text(json.dumps(marker)) + unit_path = units / (source + ".json") + unit_path.write_text(json.dumps({"returncode": 0, "source": source, "git_revision": "producer-pin", + "task_raw_after": {}})) + record = {"profile": "m5-canonical-release-source-unit/v1", "returncode": 0, + "upstream_unchanged": True, "ancillary_preexisting_unchanged": True, + "plan": {"source": source, "canonical_source_directory": str(directory), + "git_revision": "producer-pin", "implementation_and_policy_inputs": {}, "originals": [], + "upstream_before": copy.deepcopy(snapshots)}, + "canonical_source_artifacts": merge.snapshot(directory), "finalization_record": finalization, + "producer_fingerprint": marker, "ancillary_after": {}, + "unit_receipt": {"path": str(unit_path), **merge.identity(unit_path)}} + path = root / (source + "-chain.json") + path.write_text(json.dumps(record)) + receipts[source], paths[source] = record, path + snapshots[source] = record["canonical_source_artifacts"] + return receipts, paths + + +@pytest.mark.parametrize("damage", ["scope", "source_bytes", "upstream", "failed_unit", "failed_chain"]) +def test_source_chain_must_bind_complete_actual_upstreams(merge, tmp_path, damage): + """A clean-looking latest receipt cannot launder an incomplete or changed earlier producer.""" + root = tmp_path / "kg" + root.mkdir() + records, paths = source_chain(merge, root) + merge.verify_chain(root, paths) + source = root / "data/transformed/ontologies" + if damage == "scope": + (source / "source_finalization.json").rename(source / "ro_source_finalization.json") + records["ontologies"]["canonical_source_artifacts"] = merge.snapshot(source) + paths["ontologies"].write_text(json.dumps(records["ontologies"])) + elif damage == "source_bytes": + (source / "ncbitaxon_nodes.tsv").write_text("tampered retained output\n") + elif damage == "upstream": + records["bacdive"]["plan"]["upstream_before"]["ontologies"] = {} + paths["bacdive"].write_text(json.dumps(records["bacdive"])) + elif damage == "failed_unit": + unit_path = Path(records["bactotraits"]["unit_receipt"]["path"]) + unit = json.loads(unit_path.read_text()) + unit["returncode"] = 1 + unit_path.write_text(json.dumps(unit)) + records["bactotraits"]["unit_receipt"].update(merge.identity(unit_path)) + paths["bactotraits"].write_text(json.dumps(records["bactotraits"])) + else: + records["ontologies"]["returncode"] = 1 + paths["ontologies"].write_text(json.dumps(records["ontologies"])) + with pytest.raises(ValueError): + merge.verify_chain(root, paths) + + +def graph_archive(path, *, target=False, provider="infores:bactotraits", diagnostic=False, tamper=False, duplicate=False): + """Write compact transport bytes with independently constructed manifest hashes.""" + obj = "METPO:1000703" if target else "METPO:9999999" + members = { + "merged-kg_nodes.tsv": b"id\tcategory\nNCBITaxon:1123071\tbiolink:OrganismTaxon\n", + "merged-kg_edges.tsv": ("subject\tpredicate\tobject\tprimary_knowledge_source\n" + f"NCBITaxon:1123071\tbiolink:has_phenotype\t{obj}\t{provider}\n").encode(), + "merged-kg_reference_resolution.tsv": b"check\tstatus\nendpoint_closure\tpass\n", + } + manifest = {"manifest_version": 1, "members": { + name: {"sha256": hashlib.sha256(value).hexdigest(), "bytes": len(value), "rows": value.count(b"\n") - 1} + for name, value in members.items()}, + "provenance": {"commit": "b631cacf", "python": "3.10.11", "merge_config": {"sha256": "a" * 64}, + "source_finalization": {"required": not diagnostic, "diagnostic_opt_out": diagnostic}}} + if tamper: + members["merged-kg_nodes.tsv"] += b"tampered:1\tbiolink:NamedThing\n" + with tarfile.open(path, "w:gz") as archive: + for name, payload in [*members.items(), ("manifest.json", json.dumps(manifest).encode())]: + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + if duplicate: + info = tarfile.TarInfo("merged-kg_edges.tsv") + info.size = len(members[info.name]) + archive.addfile(info, io.BytesIO(members[info.name])) + + +def test_actual_archive_counts_hashes_and_strict_provenance_are_verified(merge, tmp_path): + """A valid compressed candidate returns independently streamed member counts.""" + archive = tmp_path / "candidate.tar.gz" + graph_archive(archive) + result = merge.inspect_archive(archive, "a" * 64, "b631cacf", "3.10.11") + assert result["members"]["merged-kg_edges.tsv"]["rows"] == 1 + assert result["members"]["merged-kg_edges.tsv"]["target_spo_occurrences"] == 0 + assert result["archive"]["sha256"] == hashlib.sha256(archive.read_bytes()).hexdigest() + + +@pytest.mark.parametrize("options", [ + {"target": True}, {"target": True, "provider": ""}, {"diagnostic": True}, {"tamper": True}, {"duplicate": True}, +]) +def test_valid_exit_or_rehashed_manifest_cannot_hide_candidate_failure(merge, tmp_path, options): + """Reject surviving target assertions, provenance loss, diagnostics and archive tampering.""" + archive = tmp_path / "candidate.tar.gz" + graph_archive(archive, **options) + with pytest.raises(ValueError): + merge.inspect_archive(archive, "a" * 64, "b631cacf", "3.10.11") + + +def test_changed_plan_blocks_before_output_creation_or_cli(merge, tmp_path, monkeypatch): + """A stale plan cannot start any candidate process or reserve its output.""" + output = tmp_path / "must-stay-absent" + planned = {"profile": "m5-strict-bactotraits-merge-plan/v1", "binding": {"output": str(output)}, + "binding_sha256": "prior"} + plan_file = tmp_path / "old-plan.json" + plan_file.write_text(json.dumps(planned)) + current = {**planned, "binding_sha256": "changed-inputs"} + monkeypatch.setattr(merge.subprocess, "Popen", lambda *args, **kwargs: pytest.fail("CLI started with stale plan")) + with pytest.raises(ValueError, match="since --plan"): + merge.run(current, plan_file) + assert not output.exists() + + +def test_changed_protected_stats_are_a_required_failure(merge, tmp_path): + """Even an absent historical stats path is protected from new merge side effects.""" + stats = tmp_path / "merged_graph_stats.yaml" + binding = {"input_pins": {}, "source_trees": {}, "runtime": {}, "protected_original_outputs": {str(stats): None}} + stats.write_text("unexpected stats written in KG checkout\n") + with pytest.MonkeyPatch.context() as patch: + patch.setattr(merge, "runtime", lambda: {}) + with pytest.raises(ValueError, match="statistics were overwritten"): + merge.verify_after(binding) + + +def quarantine_tree(merge, root, monkeypatch): + """Use an explicitly synthetic CSV hash while preserving exact-row and complete-field contracts.""" + raw = root / "data/raw/BactoTraits_databaseV2_Jun2022.csv" + raw.parent.mkdir(parents=True) + row = {"strain": "fixture", "Bacdive_ID": "17851", "Full_name": "mixed identity fixture", + "culture collection codes": "DSM 19353, JCM 14586", "uninterpreted measurement": "0.5"} + with raw.open("w", newline="", encoding="ISO-8859-1") as stream: + writer = csv.writer(stream, delimiter=";") + writer.writerow(["fixture metadata1"]) + writer.writerow(["fixture metadata2"]) + writer.writerow(row) + for _ in range(8520): + writer.writerow(["neighbor", "7", "independent neighbor", "DSM 7", "1.0"]) + writer.writerow(row.values()) + monkeypatch.setattr(merge, "SOURCE_SHA", hashlib.sha256(raw.read_bytes()).hexdigest()) + monkeypatch.setattr(merge, "ROW_SHA", hashlib.sha256(json.dumps(row, ensure_ascii=False, indent=2, + sort_keys=True).encode()).hexdigest()) + policy = root / "mappings/canonical/bactotraits_identity_quarantine.json" + policy.parent.mkdir(parents=True) + decision = {"bacdive_id": "17851", "rule_id": "fixture-mixed-row", "full_name": row["Full_name"], + "culture_collection_codes": row["culture collection codes"], "rationale": "Fixture identity conflict", + "evidence_urls": ["https://example.test/fixture"]} + policy.write_text(json.dumps({"decisions": [decision]})) + source = root / "data/transformed/bactotraits" + source.mkdir(parents=True) + record = {**decision, "source_data_row": "8521", "reviewed_source_data_row": "8521", "ncbitaxon_id": merge.SPO[0], + "source_sha256": merge.SOURCE_SHA, "reviewed_source_sha256": merge.SOURCE_SHA, "row_sha256": merge.ROW_SHA, + "disposition": "quarantine_mixed_strain_row", "raw_row_json": json.dumps(row), + "policy_sha256": hashlib.sha256(policy.read_bytes()).hexdigest(), + "evidence_urls": "https://example.test/fixture"} + audit = source / merge.AUDIT + + def write_audit(records): + with audit.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(record), delimiter="\t") + writer.writeheader() + writer.writerows(records) + + write_audit([record]) + for folder, filename, key, bid, taxon in ( + ("bacdive", "bacdive.tsv", "bacdive_id", "bacdive:17851", merge.SPO[0]), + ("bactotraits", "bactotraits_mapping.tsv", "Bacdive_ID", "bacdive:17851", merge.SPO[0]), + ("bactotraits", "bactotraits.tsv", "Bacdive_ID", "bacdive:7", "NCBITaxon:7"), + ): + path = root / "kg_microbe/transform_utils" / folder / "tmp" / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{key}\tncbitaxon_id\n{bid}\t{taxon}\n") + (source / "edges.tsv").write_text("subject\tpredicate\tobject\tprimary_knowledge_source\n" + "NCBITaxon:7\tbiolink:has_phenotype\tMETPO:1000703\tinfores:bactotraits\n") + return record, write_audit + + +@pytest.mark.parametrize("damage", ["duplicate", "omitted_field", "source_hash", "identity_column", "retained_copy", "spo_leak"]) +def test_complete_audit_and_retained_joins_are_required(merge, tmp_path, monkeypatch, damage): + """A correct ordinal/hash label alone cannot hide row truncation or a leaking retained join.""" + record, write_audit = quarantine_tree(merge, tmp_path, monkeypatch) + result = merge.verify_quarantine(tmp_path) + assert result["audit"]["rows"] == 1 + assert result["source_edges"]["rows"] == 1 # Neighboring row remains present. + if damage == "duplicate": + write_audit([record, record]) + elif damage == "omitted_field": + raw_row = json.loads(record["raw_row_json"]) + del raw_row["uninterpreted measurement"] + write_audit([{**record, "raw_row_json": json.dumps(raw_row)}]) + elif damage == "source_hash": + write_audit([{**record, "source_sha256": "0" * 64}]) + elif damage == "identity_column": + write_audit([{**record, "full_name": "silent replacement identity"}]) + elif damage == "retained_copy": + path = tmp_path / "kg_microbe/transform_utils/bactotraits/tmp/bactotraits.tsv" + with path.open("a") as stream: + stream.write(f"bacdive:17851\t{merge.SPO[0]}\n") + else: + path = tmp_path / "data/transformed/bactotraits/edges.tsv" + with path.open("a") as stream: + stream.write("\t".join((*merge.SPO, "infores:bactotraits")) + "\n") + with pytest.raises(ValueError): + merge.verify_quarantine(tmp_path) diff --git a/tests/test_m5_release_progress.py b/tests/test_m5_release_progress.py index 5712a03..bcd0144 100644 --- a/tests/test_m5_release_progress.py +++ b/tests/test_m5_release_progress.py @@ -1,9 +1,12 @@ """Retain command evidence when independent M5 launchers finish out of order.""" +import hashlib import importlib.util import json from pathlib import Path +import pytest + def test_stale_launcher_state_cannot_erase_another_command(tmp_path): """Rebuild aggregate progress from atomic records, including overlapping commands.""" @@ -34,3 +37,92 @@ def test_stale_launcher_state_cannot_erase_another_command(tmp_path): assert result["blockers"] == ["current dependency"] assert result["results"] == {"preflight_passed": True} assert "preflight: passed" in (module.ROOT / ".temp/M5_RELEASE_PROGRESS.md").read_text() + + +@pytest.fixture +def progress_module(tmp_path): + """Keep imported check validation separate from actual release progress.""" + helper = Path(__file__).parents[1] / "scripts/adhoc/m5_release_state_20261116.py" + spec = importlib.util.spec_from_file_location("release_progress_import", helper) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.ROOT = tmp_path + module.OUTPUT = tmp_path / ".temp/m5-worker-run-20261116" + return module + + +def check_receipt(tmp_path, status="complete", code=0): + """Write authentic-looking operational evidence with a byte-bound test log.""" + log = tmp_path / "command.log" + log.write_bytes(b"5 passed in 2.94s\n") + receipt = {"status": status, "command": ["pytest", "five-selected-cases"], "cwd": str(tmp_path), + "pid": 123, "started_at": "2026-11-16T21:14:40+00:00"} + if status != "running": + receipt.update(exit_code=code, completed_at="2026-11-16T21:14:44+00:00", seconds=4.0, + log_sha256=hashlib.sha256(log.read_bytes()).hexdigest()) + path = tmp_path / "receipt.json" + path.write_text(json.dumps(receipt)) + return path, receipt + + +@pytest.mark.parametrize(("status", "code", "expected"), [("complete", 0, "passed"), ("failed", 1, "failed"), + ("failed", -9, "failed"), ("running", None, "running")]) +def test_import_preserves_valid_observed_terminal_or_running_status(progress_module, tmp_path, status, code, expected): + """An active process stays running; a signal failure cannot become a passing result.""" + path, _ = check_receipt(tmp_path, status, code) + entry = progress_module.import_check_record(path, "actual-check") + assert entry["status"] == expected + assert entry["exit_code"] == code + assert entry["source_receipt_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.mark.parametrize(("status", "code"), [("complete", 1), ("failed", 0), ("complete", None), + ("complete", False), ("failed", True)]) +def test_contradictory_terminal_evidence_is_refused(progress_module, tmp_path, status, code): + """Boolean, missing and conflicting exit codes never justify terminal progress status.""" + path, _ = check_receipt(tmp_path, status, code) + with pytest.raises(ValueError, match="contradicts exit code"): + progress_module.import_check_record(path, "invalid-check") + + +@pytest.mark.parametrize("field", ["exit_code", "completed_at", "seconds", "log_sha256"]) +def test_running_receipt_cannot_carry_terminal_evidence(progress_module, tmp_path, field): + """A running label cannot conceal any already completed or contradictory field.""" + path, receipt = check_receipt(tmp_path, "running") + receipt[field] = None + path.write_text(json.dumps(receipt)) + with pytest.raises(ValueError, match="contains terminal fields"): + progress_module.import_check_record(path, "invalid-running") + + +@pytest.mark.parametrize("damage", ["changed", "missing_hash", "missing_log"]) +def test_terminal_import_requires_the_actual_receipt_bound_log(progress_module, tmp_path, damage): + """A claimed passing receipt cannot import changed or absent result evidence.""" + path, receipt = check_receipt(tmp_path) + log = tmp_path / "command.log" + if damage == "changed": + log.write_bytes(b"5 failed\n") + elif damage == "missing_hash": + receipt.pop("log_sha256") + path.write_text(json.dumps(receipt)) + else: + log.unlink() + with pytest.raises(ValueError, match="log"): + progress_module.import_check_record(path, "invalid-log") + + +def test_failed_import_cannot_replace_existing_command_progress(progress_module, tmp_path, monkeypatch): + """Validate before updating either the imported command row or aggregate state.""" + path, _ = check_receipt(tmp_path, "complete", 1) + commands = progress_module.OUTPUT / "commands" + commands.mkdir(parents=True) + existing = commands / "retained.json" + retained = b'{"label":"retained","status":"running"}\n' + existing.write_bytes(retained) + monkeypatch.setattr("sys.argv", ["m5_release_state", "--phase", "check", "--label", "retained", + "--import-check", str(path)]) + with pytest.raises(SystemExit) as error: + progress_module.main() + assert error.value.code == 2 + assert existing.read_bytes() == retained + assert not (tmp_path / ".temp/m5-release-progress.json").exists() From 047159aa6d04794511efdf78a317114562e5b004 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:23:46 -0800 Subject: [PATCH 3/5] Record exact remaining M5 full-release producer dependencies --- .../m5-bactotraits-release-20261116.md | 12 ++ ...lease_publication_dependencies_20261116.py | 157 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 scripts/adhoc/m5_release_publication_dependencies_20261116.py diff --git a/docs/validation/m5-bactotraits-release-20261116.md b/docs/validation/m5-bactotraits-release-20261116.md index 468d4c4..7a1d078 100644 --- a/docs/validation/m5-bactotraits-release-20261116.md +++ b/docs/validation/m5-bactotraits-release-20261116.md @@ -39,6 +39,18 @@ historical HTTP404. Publication requires a fresh complete dependency chain, canonical merge and lineage verification, a working artifact publication source, separate human approval, and verification of the published asset bytes. +The second GitHub inspection at `2026-11-16T21:20:12Z` again found 14 releases, +with newest asset modification on August 26. The canonical configuration has +24 source keys from 15 producers. After the planned ontologies/BacDive/BactoTraits +chain, 12 additional whole-source completions remain: gold, gtdb, lpsn, lpsn_api, +madin_etal, mediadive, metatraits, metatraits_gtdb, microbedecoder, +ontologies_stubs, prego (habitat only), and rhea_mappings. Independent review +confirmed their completion records were absent using exact paths including +ignored files. This establishes an unmet full-release dependency; it does not +by itself establish that rebuilding them is infeasible. The global ontology +raw-input preflight does not certify those sources' input availability or +freshness. + Independent review found that the first archive stat observation rounded its nanosecond timestamp by 24 ns through JavaScript Number serialization. The original observation is preserved. A new Python-written observation records exact diff --git a/scripts/adhoc/m5_release_publication_dependencies_20261116.py b/scripts/adhoc/m5_release_publication_dependencies_20261116.py new file mode 100644 index 0000000..e549294 --- /dev/null +++ b/scripts/adhoc/m5_release_publication_dependencies_20261116.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Read the exact canonical full-release dependency gap without running producers. + +2026-11-16 provenance: parse only merge.yaml, the lazy registry/constants and +its selected producer class definitions. Inspect explicitly configured graph +paths and whole-source completion records directly, including ignored files. +Read release-preflight policy pins without hashing or downloading bulk inputs. +Presence is not freshness or completion validation. A future successful +ontologies/BacDive/BactoTraits chain still covers only part of canonical merge. +""" + +import argparse +import ast +import hashlib +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import yaml + + +def identity(path): + """Bind small implementation/configuration files read for this audit.""" + payload = path.read_bytes() + return {"path": str(path), "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest()} + + +def inspect(root): + """Derive source requirements from maintained code rather than a remembered source list.""" + pins = {} + + def parse(relative): + path = root / relative + pins[relative] = identity(path) + return ast.parse(path.read_text()) + + constants = {} + for node in parse("kg_microbe/transform_utils/constants.py").body: + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + try: + constants[node.targets[0].id] = ast.literal_eval(node.value) + except (ValueError, TypeError): + pass + registry = {} + for node in parse("kg_microbe/transform.py").body: + if isinstance(node, ast.Assign) and any(isinstance(key, ast.Name) and key.id == "DATA_SOURCES" for key in node.targets): + for key, value in zip(node.value.keys, node.value.values): + name = constants[key.id] if isinstance(key, ast.Name) else ast.literal_eval(key) + registry[name] = ast.literal_eval(value.args[0]) + dependencies = {} + + def declaration(source): + if source in dependencies: + return dependencies[source] + module_name, class_name = registry[source].rsplit(".", 1) + relative = module_name.replace(".", "/") + ".py" + module = parse(relative) + cls = next(node for node in module.body if isinstance(node, ast.ClassDef) and node.name == class_name) + declarations = [node for node in cls.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "TRANSFORM_INPUTS" for target in node.targets)] + + def evaluate(expression): + if isinstance(expression, ast.Name): + return constants[expression.id] + if isinstance(expression, ast.Attribute) and expression.attr == "TRANSFORM_INPUTS": + inherited = next(key for key, value in registry.items() if value.endswith("." + expression.value.id)) + return declaration(inherited)["upstreams"] + if isinstance(expression, (ast.Tuple, ast.List)): + result = [] + for value in expression.elts: + result.extend(evaluate(value.value) if isinstance(value, ast.Starred) else [evaluate(value)]) + return result + return ast.literal_eval(expression) + + if declarations: + node = declarations[0] + result = {"implementation": relative, "line": node.lineno, "upstreams": evaluate(node.value)} + else: + parse("kg_microbe/transform_utils/transform.py") + result = {"implementation": relative, "line": cls.lineno, "upstreams": [], + "inherited_empty_default": "kg_microbe/transform_utils/transform.py:Transform.TRANSFORM_INPUTS"} + dependencies[source] = result + return result + + config_path = root / "merge.yaml" + pins["merge.yaml"] = identity(config_path) + config = yaml.safe_load(config_path.read_text()) + sources, producers = {}, {} + for name, source in config["merged_graph"]["source"].items(): + files = source["input"]["filename"] + directories = {str(Path(value).parent) for value in files} + if len(directories) != 1: + raise ValueError(f"Source spans unexpected producer directories: {name}") + relative = directories.pop() + directory_name = Path(relative).name + producer = "prego" if directory_name == "prego_habitat" else directory_name + declaration(producer) + sources[name] = {"producer": producer, "directory": relative, + "configured_inputs": [{"path": value, "present": (root / value).is_file()} for value in files]} + if producer not in producers: + records = {} + for filename in ("source_finalization.json", "source_fingerprint.json"): + path = root / relative / filename + records[filename] = {"present": path.is_file()} + if path.is_file(): + records[filename].update(identity(path)) + data = json.loads(path.read_text()) + records[filename]["record_version"] = data.get("version") + records[filename]["record_source"] = data.get("source") + producers[producer] = {"directory": relative, "canonical_source_keys": [], + "dependency_declaration": dependencies[producer], "completion_records": records, + "freshness_verified": False} + producers[producer]["canonical_source_keys"].append(name) + # This module exposes small read-only pin collectors; do not run bulk preflight. + sys.path.insert(0, str(root)) + from kg_microbe.utils import release_preflight + + for name, _ in (*release_preflight.PINNING_POLICIES, *release_preflight.DERIVED_PINS): + pins[name] = identity(root / name) + for name in ("kg_microbe/utils/release_preflight.py", "kg_microbe/merge_utils/source_freshness.py", + "kg_microbe/merge_utils/merge_kg.py", ".github/workflows/release.yml"): + pins[name] = identity(root / name) + planned = {"ontologies", "bacdive", "bactotraits"} + return {"profile": "m5-full-release-dependency-gap/v1", "observed_at": datetime.now(timezone.utc).isoformat(), + "kg_root": str(root), "producer_commit": subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip(), + "helper": identity(Path(__file__).resolve()), "implementation_config_policy_pins": pins, + "canonical_source_key_count": len(sources), "canonical_producer_count": len(producers), + "canonical_sources": sources, "producers": producers, + "planned_chain": sorted(planned), + "remaining_producers_after_successful_planned_chain": sorted(set(producers) - planned), + "remaining_source_keys_after_successful_planned_chain": [key for key, value in sources.items() if value["producer"] not in planned], + "reviewed_release_raw_pins": release_preflight.reviewed_release_pins(root), + "derived_raw_pins": release_preflight.reviewed_release_pins(root, policies=release_preflight.DERIVED_PINS), + "additional_preflight_requirement": "NCIT SQLite authority validation; NCIT is used by whole ontologies, ontologies_stubs and MediaDive. Existing UPA JSON must match its derived pin.", + "scope": "Exact configured paths were checked directly, including ignored files. Presence only; no current freshness, graph acceptance or publication validation. No graph scans, producer execution or downloads."} + + +def main(): + """Keep each audit in a fresh exclusive output directory.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kg-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = inspect(args.kg_root.resolve(strict=True)) + args.output.mkdir(parents=True, exist_ok=False) + with (args.output / "dependency-gap.json").open("x") as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({"output": str(args.output), "canonical_source_keys": result["canonical_source_key_count"], + "producers": result["canonical_producer_count"], + "remaining_after_planned_chain": result["remaining_producers_after_successful_planned_chain"]}, indent=2)) + + +if __name__ == "__main__": + main() From 2d2d2fa5b06b8b33faae1898d85ae577edc76731 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:56:00 -0800 Subject: [PATCH 4/5] Record verified canonical M5 BactoTraits source chain --- .../m5-bactotraits-production-20261116.json | 486 ++++++++++++++++++ .../m5-bactotraits-release-20261116.md | 52 +- .../adhoc/summarize_m5_production_20261116.py | 353 +++++++++++++ 3 files changed, 885 insertions(+), 6 deletions(-) create mode 100644 docs/validation/m5-bactotraits-production-20261116.json create mode 100644 scripts/adhoc/summarize_m5_production_20261116.py diff --git a/docs/validation/m5-bactotraits-production-20261116.json b/docs/validation/m5-bactotraits-production-20261116.json new file mode 100644 index 0000000..5379d34 --- /dev/null +++ b/docs/validation/m5-bactotraits-production-20261116.json @@ -0,0 +1,486 @@ +{ + "inputs": { + "current_go_cache": { + "fresh_build_receipt": { + "bytes": 75201, + "path": "data/validation-bactotraits-130/production-ro-canary-r3/receipt.json", + "sha256": "dd96bdb5d482e75458b545a658ddd4d1117e53e61f38d84b5f27cabc17d90001" + }, + "java_history_note": "Original toolchain record is preserved; separately pinned supplements identify the actual JVM.", + "members": { + "go.db": { + "bytes": 403886080, + "path": "data/raw/go.db", + "sha256": "61312e19177a3280ff1d186129dccdb1db63bf2cbad4faef088c7ace4cf16d4d" + }, + "go.db.toolchain.json": { + "bytes": 922, + "path": "data/raw/go.db.toolchain.json", + "sha256": "c7141519a33b8629b16cb8b90d7f1b17d88e9d3f7086eafc75d3e2036a434d57" + } + } + }, + "fresh_bacdive_lookup": { + "bytes": 39069347, + "path": "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv", + "sha256": "b01032d4bc41b3342ab12e9aa68901551e83cb05bfa2a9b1b1aa4d2ab1479904", + "source_receipt": "chain-canonical-bacdive-r1.json" + }, + "original_full_csv": { + "bytes": 8973561, + "original_retained_identity_matches_staged": true, + "path": "data/raw/BactoTraits_databaseV2_Jun2022.csv", + "sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" + }, + "retained_ncbitaxon_db": { + "authority_version": "2026-07-12", + "build_history": "Retained database has no historical toolchain record; no original builder is inferred.", + "bytes": 14073847808, + "matching_owl_gzip_version": "2026-07-12", + "original_retained_identity_matches_staged": true, + "path": "data/raw/ncbitaxon.db", + "sha256": "8474ba4f4c341b8578ac10d5661b315be8a3c6f0687b4d2e7e61b55e34faea34", + "toolchain_record_present": false + }, + "retained_ncbitaxon_owl_gzip": { + "bytes": 71846044, + "original_retained_identity_matches_staged": true, + "path": "data/raw/ncbitaxon.owl.gz", + "sha256": "65be93c022e45b0fe6d6d311bf0a18ba08cc29f7de5e0207ee06823ec65f5671" + } + }, + "merged_graph_verified": false, + "observed_at": "2026-11-16T21:53:36.263324+00:00", + "profile": "m5-canonical-production-summary/v1", + "published_release_verified": false, + "quarantine_audit": { + "bacdive_id": "17851", + "bytes": 3764, + "complete_raw_row_field_count": 105, + "disposition": "quarantine_mixed_strain_row", + "exact_copy_path": "full-bactotraits-audit/bactotraits_identity_quarantine.tsv", + "ncbitaxon_id": "NCBITaxon:1123071", + "original_path": "data/transformed/bactotraits/bactotraits_identity_quarantine.tsv", + "policy_sha256": "b26d514f1a1fa97d6d5d5a44498fc8bf1f98fa7e2ad3883d4fdfeb6789523844", + "reviewed_source_data_row": "8521", + "reviewed_source_sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b", + "row_count": 1, + "row_sha256": "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a", + "rule_id": "bactotraits-17851-mixed-strain", + "sha256": "ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160", + "source_data_row": "8521", + "source_sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" + }, + "runtime": { + "authority_guards": { + "KG_CHEBI_VERSION_CHECK": "strict", + "KG_GO_VERSION_CHECK": "strict", + "KG_NCBITAXON_VERSION_CHECK": "strict", + "KG_SEMSQL_BUILD": "on" + }, + "canonical_wrapper": { + "bytes": 15032, + "path": "scripts/adhoc/m5_release_chain_20261116.py", + "sha256": "affc4be45c6444634220680cdac9dd941a3b2a36e1e2b40dd151208c0a1158ca" + }, + "interpreter_binary_observed_during_summary": { + "bytes": 33816, + "name": "python3.10", + "sha256": "dd20469f55259b6d0aafbbd2a2b250a3645dfcbd1c70923cbb478e07537c951e" + }, + "java": { + "implementation": { + "actual_runtime_launcher": { + "bytes": 70432, + "file": "java", + "sha256": "c8586bb2233ed876cbf5a4e556389b2569e13b4413fc595aeafa530a6ba9ee6b" + }, + "actual_runtime_vm": { + "bytes": 20062816, + "file": "libjvm.dylib", + "sha256": "983be5b9be8d8c90442c4e33bf789db0c2ba9bca022ace978dffd5c1b4c9c100" + } + }, + "original_go_record_java_metadata_issue": "https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1872", + "supplement_files": { + "java-runtime-implementation-r3.json": { + "bytes": 1786, + "sha256": "ec76081f31d0e639082cb3eb359f82f2c138fd31e8538f2c7282b4dfd14a1451" + }, + "java-runtime-supplement-r3.json": { + "bytes": 2223, + "sha256": "b697bb006e0754712d28cd0a3bf47628b3f11cd24506ebeab51058d98a175ac1" + } + }, + "version_output": "openjdk version \"27\" 2026-09-15\nOpenJDK Runtime Environment Temurin-27+35 (build 27+35)\nOpenJDK 64-Bit Server VM Temurin-27+35 (build 27+35, mixed mode, sharing)" + }, + "kg_launcher": { + "bytes": 11117, + "path": "scripts/adhoc/run_bactotraits_production_unit_20261116.py", + "sha256": "2a079d57f95ce857d737c7fe22eabb71944a7456f29ba37107e009b33927bcb3" + }, + "native_thread_caps": { + "BLIS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "NUMEXPR_MAX_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "RAYON_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": "1" + }, + "packages": { + "PyYAML": "6.0.3", + "kgx": "2.7.0", + "oaklib": "0.7.4", + "pandas": "2.3.3", + "semsql": "0.4.0" + }, + "poetry_lock": { + "bytes": 706221, + "sha256": "a9e8336e172faef6449ed79a2dde8df7932a5cfb9ba368e7a93a7c7ab83d7f6f" + }, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "python": "3.10.11 (main, Oct 10 2026, 16:38:29) [Clang 21.0.0 (clang-2100.3.34.2)]", + "source_processes": 1 + }, + "scope": "Three actual source producers; no merge or published-release success is asserted.", + "sources": { + "bacdive": { + "chain_receipt": { + "bytes": 89180901, + "path": "chain-canonical-bacdive-r1.json", + "sha256": "6742881bc5743bdf8ed12023638634ad628f482377ad8f0ad0799936585103bd" + }, + "completed_at": "2026-11-16T21:41:50.812680+00:00", + "configuration_digest": "0a81d9b04db86753ae8760ca855fa9a46d1636e08aae174a6714edb3b7fc236c", + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "counts": { + "edges": 4454892, + "nodes": 298502 + }, + "existing_raw_inputs_unchanged": true, + "finalizer_code_fingerprint": "f4586479a17fc1e8ae7c7749b18780a7823c160abfde359d97076466d5d1d722", + "finalizer_graph_and_audit_members": { + "clinical_isolation_category_dispositions.tsv": { + "bytes": 8316125, + "sha256": "2a30533412b36fdfba70bb3d9b1b909a0f6cde0e0b8749032e0e183062f99df3" + }, + "ec_substrate_corrections.tsv": { + "bytes": 5628, + "sha256": "05b16360ec55536a1880e4c046ab1914207f0946a937a3a438376a07c6d512db" + }, + "edges.tsv": { + "bytes": 729983149, + "sha256": "c4e024e1074d0708132eae3ab822175bab7138184880c26c0b645888f28e7a95" + }, + "go_reference_resolution.tsv": { + "bytes": 11809, + "sha256": "ed92b622a4c301bab97580348d495b5ce46a9fe2e283f91a341838b941bfb812" + }, + "isolation_category_identity_dispositions.tsv": { + "bytes": 78073408, + "sha256": "6ad2f49af2526851bd408efa1e69079248418356cd7a4c0e76966f5bf02fa94d" + }, + "ncit_isolation_category_dispositions.tsv": { + "bytes": 30491, + "sha256": "f05f5bae3cfb21b20c8659c94540b72ec5988e910faa1a1fbbc6ad33c13faa16" + }, + "nodes.tsv": { + "bytes": 40014221, + "sha256": "444b7be456e3af5c9e19423616135110214e4de7f543cbbe0f3977a630ee96d9" + }, + "source_canonicalization.tsv": { + "bytes": 1149, + "sha256": "cbe429560d5c20c35126b32883c7fc5ae1b8bcb1c44b1da1443b1734c5a161cd" + }, + "source_finalization.json": { + "bytes": 84796227, + "sha256": "368e62a41633f45d3581600a535e38a9d7d6b36c0bae713729e68de07954a7b5" + }, + "source_fingerprint.json": { + "bytes": 3376, + "sha256": "e065fc15fb93666afd2decb37cd45ed9de560e1a0db85978aa1facb5ae9632b6" + }, + "source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "source_reference_resolution.tsv": { + "bytes": 1110744, + "sha256": "3218bd82c4b99f6f4d401299b6aca3c5507afe1c4b10393113d67493ff8d9555" + }, + "taxonomy_assertion_dispositions.tsv": { + "bytes": 27838410, + "sha256": "e2289bbbcff0afe42fb4edfd352c8c889b2c03b9f6a57d1feaed5e1707f2e9d0" + } + }, + "inner_receipt": { + "bytes": 99263, + "path": "data/validation-bactotraits-130/canonical-bacdive-r1/receipt.json", + "sha256": "4daca3c36bf310ea158ef4a57f8b1b30488733915ba6fcbc7ae429d4ffb32c9b" + }, + "original_inputs_unchanged_at_execution_end": true, + "preexisting_ancillary_unchanged": true, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "source_directory": "data/transformed/bacdive", + "started_at": "2026-11-16T21:18:31.276450+00:00", + "upstream_unchanged": true + }, + "bactotraits": { + "chain_receipt": { + "bytes": 116138, + "path": "chain-canonical-bactotraits-r1.json", + "sha256": "36968cb32f7c10ec883160503e369bf1ad19ecfa3b8203395e327f24db93a445" + }, + "completed_at": "2026-11-16T21:46:09.913751+00:00", + "configuration_digest": "c8918a56d98964bdb47272b6f09fe48379ff8a77ebb6d91a431aaff267ced6df", + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "counts": { + "edges": 87788, + "nodes": 9757 + }, + "existing_raw_inputs_unchanged": true, + "finalizer_code_fingerprint": "f4586479a17fc1e8ae7c7749b18780a7823c160abfde359d97076466d5d1d722", + "finalizer_graph_and_audit_members": { + "bactotraits_identity_quarantine.tsv": { + "bytes": 3764, + "sha256": "ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160" + }, + "edges.tsv": { + "bytes": 9864659, + "sha256": "9331f45f9b3e049e7498779777b769f2675eb339ab91f7eee15cbf8a689d07db" + }, + "go_reference_resolution.tsv": { + "bytes": 204, + "sha256": "082792210b81208e4bfab7db9686e1d64a7bc98366f9bc0b7a4a302d1e8cfacc" + }, + "nodes.tsv": { + "bytes": 894226, + "sha256": "64898938613a6b242d20e513d57e351fecf72a77d14ca3e301d63a1962367b64" + }, + "source_canonicalization.tsv": { + "bytes": 48, + "sha256": "4e6a2e08ad0ec3ed46d69cf49c5790c4c0bc253c6cb18d9f1866363bedbd4870" + }, + "source_finalization.json": { + "bytes": 10419, + "sha256": "4261b67c32a05e6f7bc7fc2cd1cbef49ec8fb975eeb47b4de55cae5dfc8ec6c6" + }, + "source_fingerprint.json": { + "bytes": 2020, + "sha256": "6b19adc93c888ce9a6881d528a5bc5c4a9fcce91d9bef79401426132f2723968" + }, + "source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "source_reference_resolution.tsv": { + "bytes": 312981, + "sha256": "4939410635da7ec5fe52e13bf0a8cc4934739cc0dd6507fcdd7bc098591b0d97" + } + }, + "inner_receipt": { + "bytes": 102453, + "path": "data/validation-bactotraits-130/canonical-bactotraits-r1/receipt.json", + "sha256": "dd87233b672b4c9c9d830425c7ee8ce19bbc66ed32e349fbed4e1eb05687ef48" + }, + "original_inputs_unchanged_at_execution_end": true, + "preexisting_ancillary_unchanged": true, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "source_directory": "data/transformed/bactotraits", + "started_at": "2026-11-16T21:45:41.395528+00:00", + "upstream_unchanged": true + }, + "ontologies": { + "chain_receipt": { + "bytes": 156174, + "path": "chain-canonical-ontologies-r1.json", + "sha256": "c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e" + }, + "completed_at": "2026-11-16T21:15:47.530538+00:00", + "configuration_digest": "5418f48bc234704ad984abaf79a26481a4a5161a1b7e9e9561a80571d7db6670", + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "counts": { + "edges": 1729253, + "nodes": 1355041 + }, + "existing_raw_inputs_unchanged": true, + "finalizer_code_fingerprint": "f4586479a17fc1e8ae7c7749b18780a7823c160abfde359d97076466d5d1d722", + "finalizer_graph_and_audit_members": { + "chebi_edges.tsv": { + "bytes": 46268462, + "sha256": "9a74dd1a3d92a5f82fd7361c58d1ba971abde1f8ebb5c8f2ed0c77990544e1e1" + }, + "chebi_nodes.tsv": { + "bytes": 66294335, + "sha256": "02024ca0a29cafaac16b43a06a786edd0368a8a646b75ab1002b6239c1219f60" + }, + "ec_edges.tsv": { + "bytes": 1260032, + "sha256": "c86152c532704fb368c7c5ce98ed0371b8570bc1fd5bb51aa4ada938de3b0d19" + }, + "ec_nodes.tsv": { + "bytes": 2065473, + "sha256": "84418ecbd7f46a4d08e63c90d4c5f2a6bb1f3ba88b0b564d8e02053c7a1a0ca1" + }, + "envo_edges.tsv": { + "bytes": 1215217, + "sha256": "86296283b8956be14f87d541ac71a10092ea1001dd68c7e15d894bd876cbdc8a" + }, + "envo_nodes.tsv": { + "bytes": 1292759, + "sha256": "ff195ed17339df155c90e3af7677f6f6c6a02a0a4b8222f86d7665c68641db0a" + }, + "foodon_edges.tsv": { + "bytes": 6566822, + "sha256": "cf0014be93a186021667d59a6c69e59ad1ed5fcfafee620ccde105a18f2eca2f" + }, + "foodon_kernel_quarantine.tsv": { + "bytes": 94917, + "sha256": "6b7feb1493a629c9b5dcaabe7f2662663f42316c634f0ce384e5c045c6535137" + }, + "foodon_nodes.tsv": { + "bytes": 7066327, + "sha256": "5337050e9527b4c80b59d7f4afb1a543579503bcb676de9834d0658c31a67e5d" + }, + "go_edges.tsv": { + "bytes": 7735534, + "sha256": "dfa175a58b2317e112b3d920aee93eab4046bb76af2a3b03b3e4f28b3a180125" + }, + "go_nodes.tsv": { + "bytes": 15147094, + "sha256": "23df80dbbbc80cf7a6ac411eff2265b9c0b1cc5c15793ec8495ca74fb59d6ac3" + }, + "go_reference_resolution.tsv": { + "bytes": 49595009, + "sha256": "e0ea9d2401f6b6b50a7864a7e6e92da01335373774df38cc08de125b27bba926" + }, + "hp_edges.tsv": { + "bytes": 2584221, + "sha256": "0adb692f275158d3d1d35eff6b0239315dfb7f52d320ecc7fbf7a1eb8b4a4c88" + }, + "hp_nodes.tsv": { + "bytes": 5094035, + "sha256": "2eedfe6bb7c1bed2a5d96a7af9837e10c155cf608ff9002d2321ce0d20e9768c" + }, + "metpo_edges.tsv": { + "bytes": 34308, + "sha256": "57dfb68fa09db5e7b9284470df4c717b9b3f9830e151a731cbed5a1c91abf8cb" + }, + "metpo_nodes.tsv": { + "bytes": 65883, + "sha256": "6483bee2d730fb55ab58830aa1a87106fbf2531192a4018324a8579169f48266" + }, + "mondo_edges.tsv": { + "bytes": 12721442, + "sha256": "fac90452121124151fef1705a443fed42b0dedc6e1cd3522d7b7a51be5be6519" + }, + "mondo_nodes.tsv": { + "bytes": 17176022, + "sha256": "2bd3b13e6131d6eb780a3f248858ea61672f1109678d4d05afc2ee164b6b86d0" + }, + "native_relation_dispositions.tsv": { + "bytes": 1125605, + "sha256": "d25dc3e19cbd1a18162ae04b48685f7e7c8760dd405fbd83c86e909338de51a9" + }, + "ncbitaxon_edges.tsv": { + "bytes": 116390818, + "sha256": "4b9597974319bb030fd07682f00755404d26ee6297ece30f99feb88915048e95" + }, + "ncbitaxon_nodes.tsv": { + "bytes": 104657178, + "sha256": "c7c9bd85b3871a8d323a833a164b4bab3e2ef9798a3456e55a218476dad56580" + }, + "ontology_metadata_exclusions.tsv": { + "bytes": 818850, + "sha256": "472c6fda2dc56544fe0103e59d099dc62ffc56d62cd7145d486d4a2d34f512d1" + }, + "ontology_parent_dispositions.tsv": { + "bytes": 15455868, + "sha256": "3d59e84b8e6d4939d75ae7f3b811763b778555424a9b618298b56de84394aedf" + }, + "pato_edges.tsv": { + "bytes": 2064720, + "sha256": "bc682e3966169f40b7cbf78a52844f9638609c021d89ade77817b34f724490e6" + }, + "pato_nodes.tsv": { + "bytes": 2341468, + "sha256": "45734a73037daa50601d4262c704129b7488f88ed93d7d56bbc54037573747c8" + }, + "ro_edges.tsv": { + "bytes": 9271, + "sha256": "015699c43eede656b82759b30ae68ee873b59dd910cf62306fe54bc78772b9ca" + }, + "ro_nodes.tsv": { + "bytes": 177483, + "sha256": "94a72f26e170a9fb28e00712ecc1f09551273f18dcbb0ff749dcc5bbc365efe9" + }, + "source_canonicalization.tsv": { + "bytes": 25524472, + "sha256": "c3e9b35325164fd3d015c59e3536f67fc98454bed2a071ab8220a99c6214ce29" + }, + "source_finalization.json": { + "bytes": 46411, + "sha256": "f6d80648081b1f18fcff51eaf05209eeeae8d5e10d9b4b369b18653f10fe16b2" + }, + "source_fingerprint.json": { + "bytes": 4385, + "sha256": "76f4f151a24a864daa4e9ed0edb6ee94449154d35977c7ab95299faf1a2e24db" + }, + "source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "source_reference_resolution.tsv": { + "bytes": 279610, + "sha256": "37734fa7d77e5833b074ac365b6debce50f1ac98dab1420b091522d4ca7a5dce" + }, + "taxrank_edges.tsv": { + "bytes": 9228, + "sha256": "0a6d62353510003c438a82d66eb4fafd5c0be9e034ae771686512334fa81ff11" + }, + "taxrank_nodes.tsv": { + "bytes": 9046, + "sha256": "b4ded9e2479674558e8c3100fbd07fcc27b6690ea46feb79fb159640c873973e" + }, + "uberon_edges.tsv": { + "bytes": 8350913, + "sha256": "5d75961bf657df579ab02496c9c03205db4dd54aa2999a2118d28dd64b35302f" + }, + "uberon_nodes.tsv": { + "bytes": 8122603, + "sha256": "23f96f9a513b081b0ef24a22d9f4c7d662489bfbac014679fe4492196f04586a" + }, + "upa_edges.tsv": { + "bytes": 307754, + "sha256": "189f856fd89c32be0880f65abfa0283a8f968dcf568d26ddcf3c4501db69a87d" + }, + "upa_nodes.tsv": { + "bytes": 398234, + "sha256": "4cb2b06ffe73eed9a66e6c1f29e4afd3010eb9a667cbdef412e488d3e30c9fd2" + }, + "upa_route_review.tsv": { + "bytes": 11779832, + "sha256": "3614e6abc27b1b66e3f1dd81db6ca9214860063bce73abe446c16e5dc12df9f2" + } + }, + "inner_receipt": { + "bytes": 89899, + "path": "data/validation-bactotraits-130/canonical-ontologies-r1/receipt.json", + "sha256": "f6169dcdfc7822cbc4de7174a59a516ee4b1acf1e15af886da6f4add0246631f" + }, + "original_inputs_unchanged_at_execution_end": true, + "preexisting_ancillary_unchanged": true, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "source_directory": "data/transformed/ontologies", + "started_at": "2026-11-16T21:00:05.307047+00:00", + "upstream_unchanged": true + } + }, + "summary_helper": { + "bytes": 15495, + "path": "summarize_m5_production_20261116.py", + "sha256": "74d2e5d18d3d68c2cb79d7439bfcc991252e8ae8e9d7405887b8ddd8d00a9435" + } +} diff --git a/docs/validation/m5-bactotraits-release-20261116.md b/docs/validation/m5-bactotraits-release-20261116.md index 7a1d078..7a5ef98 100644 --- a/docs/validation/m5-bactotraits-release-20261116.md +++ b/docs/validation/m5-bactotraits-release-20261116.md @@ -142,12 +142,38 @@ the earlier scoped RO canary was not substituted for whole-source completion. The complete chain receipt SHA256 is `c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e`. -The next BacDive unit uses the same canonical output placement, launcher, +The BacDive unit used the same canonical output placement, launcher, Python environment and single-worker/thread limits. Its plan pins six original inputs totaling 14.875 GB, including the actual 14.074 GB NCBITaxon DB, with -188.416 GB free before staging. BacDive and full BactoTraits completion, followed -by the ordinary strict BactoTraits-only merge, remain pending in this checkpoint. -No completion marker is synthesized and no freshness check is disabled. +188.416 GB free before staging. It completed with 298,502 nodes and 4,454,892 +edges, nine finalization audits and an actual producer fingerprint. The full +private chain receipt SHA256 is +`6742881bc5743bdf8ed12023638634ad628f482377ad8f0ad0799936585103bd`; +the wrapper confirmed existing raw inputs, implementation and upstream outputs +were unchanged. The retained NCBITaxon database matched the authority version +but had no historical toolchain record; the ordinary run records that limitation +and no historical record was invented. Independent review verified all 16 source +artifacts, 97 upstream artifacts, 521 named paths and exact graph counts. The +target SPO has zero occurrences in the new BacDive graph; its fresh lookup has +one BacDive17851 to NCBITaxon:1123071 mapping at data row 22,928. + +The full BactoTraits producer then completed with 9,757 nodes and 87,788 edges, +using the original full CSV and that new lookup. Independent review verified +all nine source artifacts, 113 upstream artifacts and 530 named paths. Exactly +one audit preserves all 105 fields of original row 8521, with the expected +complete-row and full-CSV hashes. Its SHA256 is +`ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160`. +The fresh lookup and mapping each retain one identity entry for traceability; +the pruned producer cache has none, and the finalized source has zero target +SPO occurrences. The full BactoTraits chain receipt SHA256 is +`36968cb32f7c10ec883160503e369bf1ad19ecfa3b8203395e327f24db93a445`. + +The [compact production record](m5-bactotraits-production-20261116.json) binds +all three completed sources, full authority bytes, runtime, configurations, +implementation and output hashes without embedding native histories. Its +source-only scope does not assert merged-graph or publication success. The +strict merged candidate is currently running in this checkpoint. No completion +marker was synthesized and no freshness check was disabled. The strict merge helper preserves canonical semantics and records explicit path-only configuration changes: absolute source paths and exclusive output @@ -159,6 +185,14 @@ intermediate joins and streamed merged outputs. Its 24 focused tests also pass independently. A BactoTraits-only candidate cannot establish full-snapshot publication. +The first strict merge plan refused an inherited EdgeMech `PYTHONPATH` because +it exposed an extra distribution. The failed plan was preserved; a new plan +used the explicit KG path and the same native thread limits as the producers, +and matched all 346 pinned distributions. No package or runtime guard changed +([#163](https://github.com/CultureBotAI/EdgeMech/issues/163)). Independent review +then verified 457 input pins and all 26 path-only configuration deltas before +the ordinary single-process CLI was started. + ## Independent review and checks Read-only adversarial review independently recomputed all new stage artifact @@ -183,8 +217,14 @@ affected checks pass without changing assertions or guards pre-existing fixture dependence on newly produced canonical taxonomy data ([KG #1873](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1873)). A pristine verification checkout at exactly the same committed candidate passes -that real test canary; the unchanged standard full gate there remains pending. -The populated-checkout failure and every earlier log remain preserved. +that real test canary and all four unchanged standard gate stages, with 9,968 +passed and 56 skipped. The populated-checkout run completed with 9,974 passed, +33 failed and 17 skipped; all 33 failures have the same fixture-cache mismatch, +and the affected fixture cases pass in the pristine checkout. The pristine +checkout has additional ordinary local-data skips; this is not a claim that +every optional integration test ran there. No assertions, selectors or +producer guards were changed. The known fixture defect, populated-checkout +failure and every earlier log remain preserved. Read-only review found two new helper boundary cases: exact temporary-root paths were accepted, and imported terminal receipts did not require consistent exit diff --git a/scripts/adhoc/summarize_m5_production_20261116.py b/scripts/adhoc/summarize_m5_production_20261116.py new file mode 100644 index 0000000..ed6ebb4 --- /dev/null +++ b/scripts/adhoc/summarize_m5_production_20261116.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""2026-11-16: summarize completed M5 native producers without bulk contracts. + +Reads the three immutable canonical source receipts and checks their selected +files against the saved hashes. The compact report is a projection, not a new +producer/finalization receipt. NCBITaxon version queries use SQLite mode=ro and +the exact ontology-node version predicate used by KG ontology_utils. No build, +merge, network request, freshness override, or old-receipt edit is performed. +The complete BactoTraits audit is copied byte-for-byte to a fresh output path. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import json +import re +import sqlite3 +import sys +from contextlib import closing +from datetime import datetime, timezone +from pathlib import Path + +SOURCES = ("ontologies", "bacdive", "bactotraits") +# Exact receipts observed after the separate native executions and their reviews. +CHAIN_SHA256 = { + "ontologies": "c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e", + "bacdive": "6742881bc5743bdf8ed12023638634ad628f482377ad8f0ad0799936585103bd", + "bactotraits": "36968cb32f7c10ec883160503e369bf1ad19ecfa3b8203395e327f24db93a445", +} +LAUNCHER = "scripts/adhoc/run_bactotraits_production_unit_20261116.py" +AUDIT = "bactotraits_identity_quarantine.tsv" +CAPS = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "NUMEXPR_MAX_THREADS", + "VECLIB_MAXIMUM_THREADS", + "RAYON_NUM_THREADS", + "BLIS_NUM_THREADS", +) + + +def require(condition, message): + """Refuse a compact success summary if its source evidence has drifted.""" + if not condition: + raise ValueError(message) + + +def identity(path): + """Stream exact bytes; never load a database into memory.""" + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + size += len(block) + digest.update(block) + return {"bytes": size, "sha256": digest.hexdigest()} + + +def checked(path, expected): + """Return a saved identity only after checking the actual file.""" + require(identity(path) == expected, f"Evidence changed: {path}") + return dict(expected) + + +def read_record(path): + """Read one already-completed evidence record without printing its payload.""" + with path.open() as stream: + return json.load(stream) + + +def collect(args): + """Build a bounded projection of recorded execution and current selected members.""" + summary_path = args.output_root / "canonical-production-summary.json" + audit_copy = args.output_root / "full-bactotraits-audit" / AUDIT + require(args.output_root.is_dir(), "Worker output root must already exist") + require( + not summary_path.exists() and not audit_copy.parent.exists(), "Preserve existing results" + ) + result = { + "profile": "m5-canonical-production-summary/v1", + "observed_at": datetime.now(timezone.utc).isoformat(), + "summary_helper": {"path": Path(__file__).name, **identity(Path(__file__))}, + "scope": "Three actual source producers; no merge or published-release success is asserted.", + "merged_graph_verified": False, + "published_release_verified": False, + "sources": {}, + } + original_inputs = {} + runtime = None + for source in SOURCES: + unit_name = f"canonical-{source}-r1" + chain_path = args.output_root / f"chain-{unit_name}.json" + chain_identity = identity(chain_path) + require(chain_identity["sha256"] == CHAIN_SHA256[source], "Reviewed chain receipt changed") + chain = read_record(chain_path) + plan = chain["plan"] + source_dir = args.kg_root / "data/transformed" / source + unit_path = args.kg_root / "data/validation-bactotraits-130" / unit_name / "receipt.json" + require(chain["returncode"] == 0 and plan["source"] == source, "Producer did not complete") + require(plan["canonical_source_directory"] == str(source_dir), "Unexpected source path") + require(chain["unit_receipt"]["path"] == str(unit_path), "Unexpected unit receipt path") + expected_unit = {key: chain["unit_receipt"][key] for key in ("bytes", "sha256")} + checked(unit_path, expected_unit) + unit = read_record(unit_path) + require(unit["returncode"] == 0, "Native CLI did not complete") + require(not unit["task_raw_changed_files"], "A pre-existing raw input changed") + require( + chain["upstream_unchanged"] and chain["ancillary_preexisting_unchanged"], "Input drift" + ) + require(plan["processes"] == 1, "Unexpected source concurrency") + require(all(plan["environment"][name] == "1" for name in CAPS), "Native cap drift") + recorded_runtime = { + "producer_commit": plan["git_revision"], + "python": plan["python"], + "packages": plan["packages"], + "poetry_lock": checked( + args.kg_root / "poetry.lock", + plan["implementation_and_policy_inputs"]["poetry.lock"], + ), + "kg_launcher": { + "path": LAUNCHER, + **checked( + args.kg_root / LAUNCHER, plan["implementation_and_policy_inputs"][LAUNCHER] + ), + }, + "canonical_wrapper": { + "path": "scripts/adhoc/" + Path(plan["wrapper"]["path"]).name, + **checked( + Path(plan["wrapper"]["path"]), + {key: plan["wrapper"][key] for key in ("bytes", "sha256")}, + ), + }, + "source_processes": 1, + "native_thread_caps": {name: plan["environment"][name] for name in CAPS}, + "authority_guards": { + name: plan["environment"][name] + for name in ( + "KG_SEMSQL_BUILD", + "KG_GO_VERSION_CHECK", + "KG_NCBITAXON_VERSION_CHECK", + "KG_CHEBI_VERSION_CHECK", + ) + }, + } + java_proofs = { + name: checked(Path(item["path"]), {key: item[key] for key in ("bytes", "sha256")}) + for name, item in plan["java_supplements"].items() + } + java_native = read_record(args.output_root / "java-runtime-implementation-r3.json") + java_implementation = {} + for role in ("actual_runtime_launcher", "actual_runtime_vm"): + item = java_native[role] + java_implementation[role] = { + "file": Path(item["path"]).name, + **checked(Path(item["path"]), {key: item[key] for key in ("bytes", "sha256")}), + } + recorded_runtime["java"] = { + "version_output": "\n".join( + line + for line in plan["java"]["stderr"].splitlines() + if not line.startswith("Picked up _JAVA_OPTIONS:") + ), + "implementation": java_implementation, + "supplement_files": java_proofs, + "original_go_record_java_metadata_issue": plan["java_provenance_issue"], + } + require( + plan["python"] == sys.version, "Summary interpreter differs from recorded execution" + ) + require( + plan["interpreter"]["executable"] == sys.executable, "Different interpreter selected" + ) + if runtime is None: + runtime = recorded_runtime + require(runtime == recorded_runtime, "Source runtime or implementation differs") + selected = { + "source_finalization.json": chain["canonical_source_artifacts"][ + "source_finalization.json" + ], + "source_fingerprint.json": chain["canonical_source_artifacts"][ + "source_fingerprint.json" + ], + **chain["finalization_record"]["members"], + **chain["finalization_record"]["audit_members"], + } + for name, expected in selected.items(): + require(Path(name).name == name, "Unexpected nested finalizer member") + checked(source_dir / name, expected) + result["sources"][source] = { + "producer_commit": plan["git_revision"], + "started_at": unit["started_at"], + "completed_at": unit["completed_at"], + "counts": chain["finalization_record"]["summaries"]["rows"], + "chain_receipt": {"path": chain_path.name, **chain_identity}, + "inner_receipt": {"path": str(unit_path.relative_to(args.kg_root)), **expected_unit}, + "source_directory": str(source_dir.relative_to(args.kg_root)), + "finalizer_graph_and_audit_members": selected, + "finalizer_code_fingerprint": chain["finalization_record"]["finalizer_code"], + "configuration_digest": hashlib.sha256( + json.dumps(plan["environment"], sort_keys=True, separators=(",", ":")).encode() + ).hexdigest(), + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "original_inputs_unchanged_at_execution_end": True, + "existing_raw_inputs_unchanged": True, + "upstream_unchanged": chain["upstream_unchanged"], + "preexisting_ancillary_unchanged": chain["ancillary_preexisting_unchanged"], + } + original_inputs.update({Path(item["source"]).name: item for item in plan["originals"]}) + if source == "bactotraits": + fresh_lookup = plan["fresh_bacdive_lookup"] + result["runtime"] = runtime + result["runtime"]["interpreter_binary_observed_during_summary"] = { + "name": Path(sys.executable).resolve().name, + **identity(Path(sys.executable).resolve()), + } + inputs = {} + for role, name in ( + ("original_full_csv", "BactoTraits_databaseV2_Jun2022.csv"), + ("retained_ncbitaxon_db", "ncbitaxon.db"), + ("retained_ncbitaxon_owl_gzip", "ncbitaxon.owl.gz"), + ): + recorded = original_inputs[name] + expected = {key: recorded[key] for key in ("bytes", "sha256")} + inputs[role] = { + "path": "data/raw/" + name, + **checked(args.kg_root / "data/raw" / name, expected), + "original_retained_identity_matches_staged": True, + } + lookup_path = args.kg_root / "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv" + inputs["fresh_bacdive_lookup"] = { + "path": str(lookup_path.relative_to(args.kg_root)), + **checked(lookup_path, {key: fresh_lookup[key] for key in ("bytes", "sha256")}), + "source_receipt": Path(fresh_lookup["receipt"]).name, + } + go_cache = plan["retained_go_cache"] + go_members = {} + for name, item in go_cache["members"].items(): + path = args.kg_root / "data/raw" / name + require(item["path"] == str(path), "Unexpected GO cache location") + go_members[name] = { + "path": str(path.relative_to(args.kg_root)), + **checked(path, {key: item[key] for key in ("bytes", "sha256")}), + } + go_receipt = go_cache["receipt"] + go_receipt_path = Path(go_receipt["path"]) + inputs["current_go_cache"] = { + "members": go_members, + "fresh_build_receipt": { + "path": str(go_receipt_path.relative_to(args.kg_root)), + **checked(go_receipt_path, {key: go_receipt[key] for key in ("bytes", "sha256")}), + }, + "java_history_note": "Original toolchain record is preserved; separately pinned supplements identify the actual JVM.", + } + db = args.kg_root / "data/raw/ncbitaxon.db" + with closing(sqlite3.connect(db.as_uri() + "?mode=ro", uri=True)) as connection: + version = connection.execute( + "SELECT value FROM statements WHERE predicate = 'owl:versionInfo' " + "AND subject LIKE '%ncbitaxon%' AND value IS NOT NULL LIMIT 1" + ).fetchone() + require(version is not None, "Missing NCBITaxon version stamp") + db_version = re.search(r"\d{4}-\d{2}-\d{2}", str(version[0])) + with gzip.open(args.kg_root / "data/raw/ncbitaxon.owl.gz", "rt", errors="replace") as stream: + head = stream.read(2_000_000) + owl_version = re.search(r"versionIRI[^>]{0,160}?(\d{4}-\d{2}-\d{2})", head) + require( + db_version and owl_version and db_version[0] == owl_version[1], "Authority version drift" + ) + inputs["retained_ncbitaxon_db"].update( + { + "authority_version": db_version[0], + "matching_owl_gzip_version": owl_version[1], + "toolchain_record_present": db.with_name(db.name + ".toolchain.json").exists(), + "build_history": "Retained database has no historical toolchain record; no original builder is inferred.", + } + ) + result["inputs"] = inputs + source_audit = args.kg_root / "data/transformed/bactotraits" / AUDIT + with source_audit.open(newline="") as stream: + audits = list(csv.DictReader(stream, delimiter="\t")) + require(len(audits) == 1, "Unexpected quarantine audit count") + row = audits[0] + require(row["source_data_row"] == row["reviewed_source_data_row"] == "8521", "Wrong row") + require(row["bacdive_id"] == "17851", "Wrong BacDive identity") + result["quarantine_audit"] = { + key: row[key] + for key in ( + "rule_id", + "source_data_row", + "reviewed_source_data_row", + "bacdive_id", + "ncbitaxon_id", + "disposition", + "row_sha256", + "source_sha256", + "reviewed_source_sha256", + "policy_sha256", + ) + } + result["quarantine_audit"].update( + { + "row_count": 1, + "complete_raw_row_field_count": len(json.loads(row["raw_row_json"])), + "original_path": str(source_audit.relative_to(args.kg_root)), + "exact_copy_path": str(audit_copy.relative_to(args.output_root)), + **identity(source_audit), + } + ) + return result, summary_path, source_audit, audit_copy + + +def main(): + """Keep the plan read-only and both new output paths exclusive.""" + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--plan", action="store_true") + action.add_argument("--run", action="store_true") + parser.add_argument("--kg-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + result, summary_path, source_audit, audit_copy = collect(args) + if args.plan: + print( + json.dumps( + { + "action": "plan", + "writes": 0, + "planned_outputs": [ + summary_path.name, + str(audit_copy.relative_to(args.output_root)), + ], + **result, + }, + indent=2, + ) + ) + return + audit_copy.parent.mkdir(mode=0o700) + with audit_copy.open("xb") as stream: + stream.write(source_audit.read_bytes()) + require(identity(audit_copy) == identity(source_audit), "Audit copy changed") + with summary_path.open("x") as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({"summary": summary_path.name, **identity(summary_path)})) + + +if __name__ == "__main__": + main() From 33af8e00bce25767967d6ae58660c796635fc045 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:05:38 -0800 Subject: [PATCH 5/5] Verify strict M5 BactoTraits merged candidate and publication gap --- ...bactotraits-merged-candidate-20261116.json | 93 +++++++++++++++++++ .../m5-bactotraits-release-20261116.md | 65 ++++++++++--- 2 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 docs/validation/m5-bactotraits-merged-candidate-20261116.json diff --git a/docs/validation/m5-bactotraits-merged-candidate-20261116.json b/docs/validation/m5-bactotraits-merged-candidate-20261116.json new file mode 100644 index 0000000..2d9809c --- /dev/null +++ b/docs/validation/m5-bactotraits-merged-candidate-20261116.json @@ -0,0 +1,93 @@ +{ + "profile": "m5-bactotraits-merged-candidate-summary/v1", + "scope": "Actual strict BactoTraits-only merged candidate; other canonical sources were not selected.", + "candidate_merged_graph_verified": true, + "full_snapshot_release_verified": false, + "published_release_verified": false, + "scientific_trait_truth_resolved": false, + "graph_wide_validation_claimed": false, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "corrected_merged_code": "2d94ed094167d94402f21400a9ad7dc4f7f20438", + "python": "3.10.11", + "source_selection": ["bactotraits"], + "processes": 1, + "started_at": "2026-11-16T21:52:35.127010+00:00", + "completed_at": "2026-11-16T22:01:06.333825+00:00", + "returncode": 0, + "source_finalization": {"required": true, "diagnostic_opt_out": false}, + "inputs_and_upstreams_unchanged": true, + "production_summary": { + "path": "m5-bactotraits-production-20261116.json", + "bytes": 19889, + "sha256": "d1cf3d011bc1d5153d8e13e5422f861f8e5e3cf210f28b27dcc16be9d04b80ac" + }, + "private_plan": { + "path": ".temp/m5-worker-run-20261116/strict-merge-plan-r2.json", + "bytes": 196433, + "sha256": "570e72b056f4b0f604d78035b6ec944bc9c95c63719ccf9e3fed3ff9fc792a16", + "input_pins": 457, + "path_only_configuration_deltas": 26, + "installed_distributions": 346 + }, + "private_receipt": { + "path": ".temp/m5-worker-run-20261116/merged-candidate-r1/receipt.json", + "bytes": 15817, + "sha256": "111a1d66ae90ae0136212c06f379f15fee447ebdc9db3f70e4fd4e1d806b03be" + }, + "archive": { + "path": ".temp/m5-worker-run-20261116/merged-candidate-r1/merged-kg.tar.gz", + "bytes": 447980, + "sha256": "d217563f5d29ca2bc7ac729253acf5f83d23519d02a22ed91a5cc974ccf49976" + }, + "archive_members": { + "merged-kg_edges.tsv": { + "bytes": 9864659, + "rows": 87788, + "sha256": "e99d3bba38cd366a0b84d4c8a7058581e7293b9004a9474fa20e02a658c0d6b2", + "target_spo_occurrences": 0, + "bactotraits_target_occurrences": 0 + }, + "merged-kg_nodes.tsv": { + "bytes": 894226, + "rows": 9757, + "sha256": "64898938613a6b242d20e513d57e351fecf72a77d14ca3e301d63a1962367b64" + }, + "merged-kg_reference_resolution.tsv": { + "bytes": 140, + "rows": 1, + "sha256": "c6f0f8652fb373cf058fe3ffd2dd4a3a0a8b06016c8f23d17c18ae451b804685" + } + }, + "merge_configuration": { + "sha256": "c4c3bb66b6d84ea60441d2727fdda644d8c64bfceb9c760e14f9c5e20e1bee87", + "canonical_sha256": "926c238e577931db06f6674ec96bdf0d85d185a7bcc5028ed6f5bde293d5d1d2", + "change_scope": "Absolute source paths and exclusive output paths, including statistics; canonical graph semantics preserved." + }, + "repository_schema": { + "version": "4.4.2", + "digest": "2d1649c4b3ee554abaab3a24fd74d23b0a34637ec3e3d4de58afc6c530c59f3f" + }, + "quarantine": { + "source_data_row": 8521, + "bacdive_id": 17851, + "audit_rows": 1, + "complete_raw_fields": 105, + "audit_sha256": "ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160", + "complete_row_sha256": "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a", + "full_source_sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b", + "lookup_identity_occurrences": 1, + "mapping_identity_occurrences": 1, + "pruned_row_occurrences": 0 + }, + "target_spo": ["NCBITaxon:1123071", "biolink:has_phenotype", "METPO:1000703"], + "historical_independent_madin_occurrences": 1, + "historical_madin_was_not_newly_scientifically_reassessed": true, + "independent_archive_review": { + "status": "passed", + "path": ".temp/m5-worker-run-20261116/adversarial-review/review-r16.json", + "sha256": "bf397abd283fc5103731be4642662a8c729916cf484a66d8067e2c8fd4c9b14b", + "verified_output_artifacts": 13, + "verified_input_pins": 457, + "all_edges_primary_knowledge_source": "infores:bactotraits" + } +} diff --git a/docs/validation/m5-bactotraits-release-20261116.md b/docs/validation/m5-bactotraits-release-20261116.md index 7a5ef98..83e930b 100644 --- a/docs/validation/m5-bactotraits-release-20261116.md +++ b/docs/validation/m5-bactotraits-release-20261116.md @@ -1,17 +1,21 @@ # M5 BactoTraits graph-release verification — 2026-11-16 -The new M5 pair verifies the exact-row correction through the actual producer, -and a separate fresh candidate verifies that its complete quarantine audit and -zero graph rows survive production source finalization. Strict merge refuses -the incomplete upstream chain. No corrected merged graph or published release -is established by this evidence. +The exact-row correction is verified through a new M5 baseline/candidate pair, +the complete current ontologies → BacDive → BactoTraits producer chain, and an +actual strict BactoTraits-only merged candidate. The persisted candidate has +9,757 nodes and 87,788 edges, with zero occurrences of the quarantined target +SPO. Its complete original-row audit remains intact. This verifies a bounded +candidate; a complete canonical snapshot and corrected published release remain +unverified. | Check | Actual result | | --- | --- | | Fresh baseline, `ef27b97a0b450767da54235c6105b5250c178adc` | 21 edges, 22 nodes | | Fresh corrected candidate, `2d94ed094167d94402f21400a9ad7dc4f7f20438` | 0 edges, 0 nodes, 1 complete quarantine audit | | Separate fresh candidate through `finalize(fresh_run=True)` | 0 edges, 0 nodes, identical quarantine audit | -| Strict `load_and_merge(processes=1)` | Refused missing current BacDive whole-source finalization; no merged output | +| Earlier bounded `load_and_merge(processes=1)` attempt | Correctly refused its missing upstream completion; old receipt unchanged | +| Full current BactoTraits source | 87,788 edges, 9,757 nodes, 1 complete 105-field quarantine audit | +| Ordinary strict BactoTraits-only merge | 87,788 edges, 9,757 nodes; target SPO count 0; persisted archive verified | | Current raw reviewed-input preflight | Passed against original M5 raw root | | Current published corrected release | Unavailable; dependency tracked in [#143](https://github.com/CultureBotAI/EdgeMech/issues/143) | @@ -39,10 +43,12 @@ historical HTTP404. Publication requires a fresh complete dependency chain, canonical merge and lineage verification, a working artifact publication source, separate human approval, and verification of the published asset bytes. -The second GitHub inspection at `2026-11-16T21:20:12Z` again found 14 releases, -with newest asset modification on August 26. The canonical configuration has -24 source keys from 15 producers. After the planned ontologies/BacDive/BactoTraits -chain, 12 additional whole-source completions remain: gold, gtdb, lpsn, lpsn_api, +The complete GitHub inspection at `2026-11-16T21:20:12Z` again found 14 releases, +with newest asset modification on August 26. The final latest-release/issue +refresh at `2026-11-16T22:01:40Z` still showed latest `2025-03-07` and #887 open. +The canonical configuration has 24 source keys from 15 producers. After the +completed ontologies/BacDive/BactoTraits chain, 12 additional whole-source +completions remain: gold, gtdb, lpsn, lpsn_api, madin_etal, mediadive, metatraits, metatraits_gtdb, microbedecoder, ontologies_stubs, prego (habitat only), and rhea_mappings. Independent review confirmed their completion records were absent using exact paths including @@ -172,8 +178,9 @@ The [compact production record](m5-bactotraits-production-20261116.json) binds all three completed sources, full authority bytes, runtime, configurations, implementation and output hashes without embedding native histories. Its source-only scope does not assert merged-graph or publication success. The -strict merged candidate is currently running in this checkpoint. No completion -marker was synthesized and no freshness check was disabled. +separate [merged-candidate record](m5-bactotraits-merged-candidate-20261116.json) +binds the actual merge result. No completion marker was synthesized and no +freshness check was disabled. The strict merge helper preserves canonical semantics and records explicit path-only configuration changes: absolute source paths and exclusive output @@ -193,6 +200,23 @@ and matched all 346 pinned distributions. No package or runtime guard changed then verified 457 input pins and all 26 path-only configuration deltas before the ordinary single-process CLI was started. +The strict CLI and persisted verification completed successfully at +`2026-11-16T22:01:06Z`. Archive SHA256 is +`d217563f5d29ca2bc7ac729253acf5f83d23519d02a22ed91a5cc974ccf49976` +(447,980 bytes); the complete merge receipt SHA256 is +`111a1d66ae90ae0136212c06f379f15fee447ebdc9db3f70e4fd4e1d806b03be`. +Streaming readback verified 9,757 node rows, 87,788 edge rows, all member hashes, +the reference-resolution member and embedded producer/configuration/schema +provenance. The manifest records Python 3.10.11, producer `b631cacf5`, required +source finalization and no diagnostic opt-out. Input/source trees, original +statistics and the complete quarantine audit stayed unchanged. The candidate +contains zero target SPO rows and zero BactoTraits-provenance target rows. +Independent final review verified all 13 output artifacts, all three source +trees, all 457 input pins after execution, protected original statistics, +the actual interpreter and all 346 distributions. Every merged edge has +`infores:bactotraits` as its primary knowledge source. Its review receipt SHA256 +is `bf397abd283fc5103731be4642662a8c729916cf484a66d8067e2c8fd4c9b14b`. + ## Independent review and checks Read-only adversarial review independently recomputed all new stage artifact @@ -203,6 +227,23 @@ findings were filed and fixed: [#146](https://github.com/CultureBotAI/EdgeMech/i for concurrent progress updates and [#147](https://github.com/CultureBotAI/EdgeMech/issues/147) for stat precision. A second read-only review verified both fixes. +Production-launcher gaps in shared configuration pins +([KG #1869](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1869)) and +canonical raw side-effect recording +([KG #1870](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1870)) were +filed and fixed before actual source units ran. The original canonical archive, +all 37 retained v3 files (including nested hidden caches), and the clean `ef27b97` +baseline passed final independent preservation review. The original archive's +exact Python stat fields also remain unchanged. All failed attempts and older +receipts remain preserved. + +After merge completion, the retained 37-file inventory and original archive +were checked again: byte hashes and exact archive stat fields still match the +initial observations. The complete result is ready for review in +[EdgeMech PR #151](https://github.com/CultureBotAI/EdgeMech/pull/151) and companion +[KG-Microbe PR #1871](https://github.com/Knowledge-Graph-Hub/kg-microbe/pull/1871). +No new PR was merged and no release was published. + The finalization helper's six focused tests and production-launcher four focused tests passed. EdgeMech's final full suite passed 777 tests after the progress boundary fixes. Required EdgeMech lint and offline wheel/source package builds