diff --git a/docs/validation/bactotraits-identity-20261115.md b/docs/validation/bactotraits-identity-20261115.md index c433ded..a8981ec 100644 --- a/docs/validation/bactotraits-identity-20261115.md +++ b/docs/validation/bactotraits-identity-20261115.md @@ -1,5 +1,9 @@ # BactoTraits mixed-strain source investigation +Later execution status is recorded in the +[M5 release verification follow-up](m5-bactotraits-release-20261116.md). +The investigation below preserves its original pre-merge, pre-finalization scope. + This investigation follows [#130](https://github.com/CultureBotAI/EdgeMech/issues/130) for the exact fresh-05 occurrence `urn:kg-microbe:occurrence:v1:1e9dec218a07f7e480add8ff273e3a9285f79d57746ca74041702839207a8136`: diff --git a/docs/validation/m5-bactotraits-merged-candidate-20261116.json b/docs/validation/m5-bactotraits-merged-candidate-20261116.json new file mode 100644 index 0000000..2d9809c --- /dev/null +++ b/docs/validation/m5-bactotraits-merged-candidate-20261116.json @@ -0,0 +1,93 @@ +{ + "profile": "m5-bactotraits-merged-candidate-summary/v1", + "scope": "Actual strict BactoTraits-only merged candidate; other canonical sources were not selected.", + "candidate_merged_graph_verified": true, + "full_snapshot_release_verified": false, + "published_release_verified": false, + "scientific_trait_truth_resolved": false, + "graph_wide_validation_claimed": false, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "corrected_merged_code": "2d94ed094167d94402f21400a9ad7dc4f7f20438", + "python": "3.10.11", + "source_selection": ["bactotraits"], + "processes": 1, + "started_at": "2026-11-16T21:52:35.127010+00:00", + "completed_at": "2026-11-16T22:01:06.333825+00:00", + "returncode": 0, + "source_finalization": {"required": true, "diagnostic_opt_out": false}, + "inputs_and_upstreams_unchanged": true, + "production_summary": { + "path": "m5-bactotraits-production-20261116.json", + "bytes": 19889, + "sha256": "d1cf3d011bc1d5153d8e13e5422f861f8e5e3cf210f28b27dcc16be9d04b80ac" + }, + "private_plan": { + "path": ".temp/m5-worker-run-20261116/strict-merge-plan-r2.json", + "bytes": 196433, + "sha256": "570e72b056f4b0f604d78035b6ec944bc9c95c63719ccf9e3fed3ff9fc792a16", + "input_pins": 457, + "path_only_configuration_deltas": 26, + "installed_distributions": 346 + }, + "private_receipt": { + "path": ".temp/m5-worker-run-20261116/merged-candidate-r1/receipt.json", + "bytes": 15817, + "sha256": "111a1d66ae90ae0136212c06f379f15fee447ebdc9db3f70e4fd4e1d806b03be" + }, + "archive": { + "path": ".temp/m5-worker-run-20261116/merged-candidate-r1/merged-kg.tar.gz", + "bytes": 447980, + "sha256": "d217563f5d29ca2bc7ac729253acf5f83d23519d02a22ed91a5cc974ccf49976" + }, + "archive_members": { + "merged-kg_edges.tsv": { + "bytes": 9864659, + "rows": 87788, + "sha256": "e99d3bba38cd366a0b84d4c8a7058581e7293b9004a9474fa20e02a658c0d6b2", + "target_spo_occurrences": 0, + "bactotraits_target_occurrences": 0 + }, + "merged-kg_nodes.tsv": { + "bytes": 894226, + "rows": 9757, + "sha256": "64898938613a6b242d20e513d57e351fecf72a77d14ca3e301d63a1962367b64" + }, + "merged-kg_reference_resolution.tsv": { + "bytes": 140, + "rows": 1, + "sha256": "c6f0f8652fb373cf058fe3ffd2dd4a3a0a8b06016c8f23d17c18ae451b804685" + } + }, + "merge_configuration": { + "sha256": "c4c3bb66b6d84ea60441d2727fdda644d8c64bfceb9c760e14f9c5e20e1bee87", + "canonical_sha256": "926c238e577931db06f6674ec96bdf0d85d185a7bcc5028ed6f5bde293d5d1d2", + "change_scope": "Absolute source paths and exclusive output paths, including statistics; canonical graph semantics preserved." + }, + "repository_schema": { + "version": "4.4.2", + "digest": "2d1649c4b3ee554abaab3a24fd74d23b0a34637ec3e3d4de58afc6c530c59f3f" + }, + "quarantine": { + "source_data_row": 8521, + "bacdive_id": 17851, + "audit_rows": 1, + "complete_raw_fields": 105, + "audit_sha256": "ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160", + "complete_row_sha256": "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a", + "full_source_sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b", + "lookup_identity_occurrences": 1, + "mapping_identity_occurrences": 1, + "pruned_row_occurrences": 0 + }, + "target_spo": ["NCBITaxon:1123071", "biolink:has_phenotype", "METPO:1000703"], + "historical_independent_madin_occurrences": 1, + "historical_madin_was_not_newly_scientifically_reassessed": true, + "independent_archive_review": { + "status": "passed", + "path": ".temp/m5-worker-run-20261116/adversarial-review/review-r16.json", + "sha256": "bf397abd283fc5103731be4642662a8c729916cf484a66d8067e2c8fd4c9b14b", + "verified_output_artifacts": 13, + "verified_input_pins": 457, + "all_edges_primary_knowledge_source": "infores:bactotraits" + } +} diff --git a/docs/validation/m5-bactotraits-production-20261116.json b/docs/validation/m5-bactotraits-production-20261116.json new file mode 100644 index 0000000..5379d34 --- /dev/null +++ b/docs/validation/m5-bactotraits-production-20261116.json @@ -0,0 +1,486 @@ +{ + "inputs": { + "current_go_cache": { + "fresh_build_receipt": { + "bytes": 75201, + "path": "data/validation-bactotraits-130/production-ro-canary-r3/receipt.json", + "sha256": "dd96bdb5d482e75458b545a658ddd4d1117e53e61f38d84b5f27cabc17d90001" + }, + "java_history_note": "Original toolchain record is preserved; separately pinned supplements identify the actual JVM.", + "members": { + "go.db": { + "bytes": 403886080, + "path": "data/raw/go.db", + "sha256": "61312e19177a3280ff1d186129dccdb1db63bf2cbad4faef088c7ace4cf16d4d" + }, + "go.db.toolchain.json": { + "bytes": 922, + "path": "data/raw/go.db.toolchain.json", + "sha256": "c7141519a33b8629b16cb8b90d7f1b17d88e9d3f7086eafc75d3e2036a434d57" + } + } + }, + "fresh_bacdive_lookup": { + "bytes": 39069347, + "path": "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv", + "sha256": "b01032d4bc41b3342ab12e9aa68901551e83cb05bfa2a9b1b1aa4d2ab1479904", + "source_receipt": "chain-canonical-bacdive-r1.json" + }, + "original_full_csv": { + "bytes": 8973561, + "original_retained_identity_matches_staged": true, + "path": "data/raw/BactoTraits_databaseV2_Jun2022.csv", + "sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" + }, + "retained_ncbitaxon_db": { + "authority_version": "2026-07-12", + "build_history": "Retained database has no historical toolchain record; no original builder is inferred.", + "bytes": 14073847808, + "matching_owl_gzip_version": "2026-07-12", + "original_retained_identity_matches_staged": true, + "path": "data/raw/ncbitaxon.db", + "sha256": "8474ba4f4c341b8578ac10d5661b315be8a3c6f0687b4d2e7e61b55e34faea34", + "toolchain_record_present": false + }, + "retained_ncbitaxon_owl_gzip": { + "bytes": 71846044, + "original_retained_identity_matches_staged": true, + "path": "data/raw/ncbitaxon.owl.gz", + "sha256": "65be93c022e45b0fe6d6d311bf0a18ba08cc29f7de5e0207ee06823ec65f5671" + } + }, + "merged_graph_verified": false, + "observed_at": "2026-11-16T21:53:36.263324+00:00", + "profile": "m5-canonical-production-summary/v1", + "published_release_verified": false, + "quarantine_audit": { + "bacdive_id": "17851", + "bytes": 3764, + "complete_raw_row_field_count": 105, + "disposition": "quarantine_mixed_strain_row", + "exact_copy_path": "full-bactotraits-audit/bactotraits_identity_quarantine.tsv", + "ncbitaxon_id": "NCBITaxon:1123071", + "original_path": "data/transformed/bactotraits/bactotraits_identity_quarantine.tsv", + "policy_sha256": "b26d514f1a1fa97d6d5d5a44498fc8bf1f98fa7e2ad3883d4fdfeb6789523844", + "reviewed_source_data_row": "8521", + "reviewed_source_sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b", + "row_count": 1, + "row_sha256": "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a", + "rule_id": "bactotraits-17851-mixed-strain", + "sha256": "ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160", + "source_data_row": "8521", + "source_sha256": "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" + }, + "runtime": { + "authority_guards": { + "KG_CHEBI_VERSION_CHECK": "strict", + "KG_GO_VERSION_CHECK": "strict", + "KG_NCBITAXON_VERSION_CHECK": "strict", + "KG_SEMSQL_BUILD": "on" + }, + "canonical_wrapper": { + "bytes": 15032, + "path": "scripts/adhoc/m5_release_chain_20261116.py", + "sha256": "affc4be45c6444634220680cdac9dd941a3b2a36e1e2b40dd151208c0a1158ca" + }, + "interpreter_binary_observed_during_summary": { + "bytes": 33816, + "name": "python3.10", + "sha256": "dd20469f55259b6d0aafbbd2a2b250a3645dfcbd1c70923cbb478e07537c951e" + }, + "java": { + "implementation": { + "actual_runtime_launcher": { + "bytes": 70432, + "file": "java", + "sha256": "c8586bb2233ed876cbf5a4e556389b2569e13b4413fc595aeafa530a6ba9ee6b" + }, + "actual_runtime_vm": { + "bytes": 20062816, + "file": "libjvm.dylib", + "sha256": "983be5b9be8d8c90442c4e33bf789db0c2ba9bca022ace978dffd5c1b4c9c100" + } + }, + "original_go_record_java_metadata_issue": "https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1872", + "supplement_files": { + "java-runtime-implementation-r3.json": { + "bytes": 1786, + "sha256": "ec76081f31d0e639082cb3eb359f82f2c138fd31e8538f2c7282b4dfd14a1451" + }, + "java-runtime-supplement-r3.json": { + "bytes": 2223, + "sha256": "b697bb006e0754712d28cd0a3bf47628b3f11cd24506ebeab51058d98a175ac1" + } + }, + "version_output": "openjdk version \"27\" 2026-09-15\nOpenJDK Runtime Environment Temurin-27+35 (build 27+35)\nOpenJDK 64-Bit Server VM Temurin-27+35 (build 27+35, mixed mode, sharing)" + }, + "kg_launcher": { + "bytes": 11117, + "path": "scripts/adhoc/run_bactotraits_production_unit_20261116.py", + "sha256": "2a079d57f95ce857d737c7fe22eabb71944a7456f29ba37107e009b33927bcb3" + }, + "native_thread_caps": { + "BLIS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "NUMEXPR_MAX_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "RAYON_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": "1" + }, + "packages": { + "PyYAML": "6.0.3", + "kgx": "2.7.0", + "oaklib": "0.7.4", + "pandas": "2.3.3", + "semsql": "0.4.0" + }, + "poetry_lock": { + "bytes": 706221, + "sha256": "a9e8336e172faef6449ed79a2dde8df7932a5cfb9ba368e7a93a7c7ab83d7f6f" + }, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "python": "3.10.11 (main, Oct 10 2026, 16:38:29) [Clang 21.0.0 (clang-2100.3.34.2)]", + "source_processes": 1 + }, + "scope": "Three actual source producers; no merge or published-release success is asserted.", + "sources": { + "bacdive": { + "chain_receipt": { + "bytes": 89180901, + "path": "chain-canonical-bacdive-r1.json", + "sha256": "6742881bc5743bdf8ed12023638634ad628f482377ad8f0ad0799936585103bd" + }, + "completed_at": "2026-11-16T21:41:50.812680+00:00", + "configuration_digest": "0a81d9b04db86753ae8760ca855fa9a46d1636e08aae174a6714edb3b7fc236c", + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "counts": { + "edges": 4454892, + "nodes": 298502 + }, + "existing_raw_inputs_unchanged": true, + "finalizer_code_fingerprint": "f4586479a17fc1e8ae7c7749b18780a7823c160abfde359d97076466d5d1d722", + "finalizer_graph_and_audit_members": { + "clinical_isolation_category_dispositions.tsv": { + "bytes": 8316125, + "sha256": "2a30533412b36fdfba70bb3d9b1b909a0f6cde0e0b8749032e0e183062f99df3" + }, + "ec_substrate_corrections.tsv": { + "bytes": 5628, + "sha256": "05b16360ec55536a1880e4c046ab1914207f0946a937a3a438376a07c6d512db" + }, + "edges.tsv": { + "bytes": 729983149, + "sha256": "c4e024e1074d0708132eae3ab822175bab7138184880c26c0b645888f28e7a95" + }, + "go_reference_resolution.tsv": { + "bytes": 11809, + "sha256": "ed92b622a4c301bab97580348d495b5ce46a9fe2e283f91a341838b941bfb812" + }, + "isolation_category_identity_dispositions.tsv": { + "bytes": 78073408, + "sha256": "6ad2f49af2526851bd408efa1e69079248418356cd7a4c0e76966f5bf02fa94d" + }, + "ncit_isolation_category_dispositions.tsv": { + "bytes": 30491, + "sha256": "f05f5bae3cfb21b20c8659c94540b72ec5988e910faa1a1fbbc6ad33c13faa16" + }, + "nodes.tsv": { + "bytes": 40014221, + "sha256": "444b7be456e3af5c9e19423616135110214e4de7f543cbbe0f3977a630ee96d9" + }, + "source_canonicalization.tsv": { + "bytes": 1149, + "sha256": "cbe429560d5c20c35126b32883c7fc5ae1b8bcb1c44b1da1443b1734c5a161cd" + }, + "source_finalization.json": { + "bytes": 84796227, + "sha256": "368e62a41633f45d3581600a535e38a9d7d6b36c0bae713729e68de07954a7b5" + }, + "source_fingerprint.json": { + "bytes": 3376, + "sha256": "e065fc15fb93666afd2decb37cd45ed9de560e1a0db85978aa1facb5ae9632b6" + }, + "source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "source_reference_resolution.tsv": { + "bytes": 1110744, + "sha256": "3218bd82c4b99f6f4d401299b6aca3c5507afe1c4b10393113d67493ff8d9555" + }, + "taxonomy_assertion_dispositions.tsv": { + "bytes": 27838410, + "sha256": "e2289bbbcff0afe42fb4edfd352c8c889b2c03b9f6a57d1feaed5e1707f2e9d0" + } + }, + "inner_receipt": { + "bytes": 99263, + "path": "data/validation-bactotraits-130/canonical-bacdive-r1/receipt.json", + "sha256": "4daca3c36bf310ea158ef4a57f8b1b30488733915ba6fcbc7ae429d4ffb32c9b" + }, + "original_inputs_unchanged_at_execution_end": true, + "preexisting_ancillary_unchanged": true, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "source_directory": "data/transformed/bacdive", + "started_at": "2026-11-16T21:18:31.276450+00:00", + "upstream_unchanged": true + }, + "bactotraits": { + "chain_receipt": { + "bytes": 116138, + "path": "chain-canonical-bactotraits-r1.json", + "sha256": "36968cb32f7c10ec883160503e369bf1ad19ecfa3b8203395e327f24db93a445" + }, + "completed_at": "2026-11-16T21:46:09.913751+00:00", + "configuration_digest": "c8918a56d98964bdb47272b6f09fe48379ff8a77ebb6d91a431aaff267ced6df", + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "counts": { + "edges": 87788, + "nodes": 9757 + }, + "existing_raw_inputs_unchanged": true, + "finalizer_code_fingerprint": "f4586479a17fc1e8ae7c7749b18780a7823c160abfde359d97076466d5d1d722", + "finalizer_graph_and_audit_members": { + "bactotraits_identity_quarantine.tsv": { + "bytes": 3764, + "sha256": "ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160" + }, + "edges.tsv": { + "bytes": 9864659, + "sha256": "9331f45f9b3e049e7498779777b769f2675eb339ab91f7eee15cbf8a689d07db" + }, + "go_reference_resolution.tsv": { + "bytes": 204, + "sha256": "082792210b81208e4bfab7db9686e1d64a7bc98366f9bc0b7a4a302d1e8cfacc" + }, + "nodes.tsv": { + "bytes": 894226, + "sha256": "64898938613a6b242d20e513d57e351fecf72a77d14ca3e301d63a1962367b64" + }, + "source_canonicalization.tsv": { + "bytes": 48, + "sha256": "4e6a2e08ad0ec3ed46d69cf49c5790c4c0bc253c6cb18d9f1866363bedbd4870" + }, + "source_finalization.json": { + "bytes": 10419, + "sha256": "4261b67c32a05e6f7bc7fc2cd1cbef49ec8fb975eeb47b4de55cae5dfc8ec6c6" + }, + "source_fingerprint.json": { + "bytes": 2020, + "sha256": "6b19adc93c888ce9a6881d528a5bc5c4a9fcce91d9bef79401426132f2723968" + }, + "source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "source_reference_resolution.tsv": { + "bytes": 312981, + "sha256": "4939410635da7ec5fe52e13bf0a8cc4934739cc0dd6507fcdd7bc098591b0d97" + } + }, + "inner_receipt": { + "bytes": 102453, + "path": "data/validation-bactotraits-130/canonical-bactotraits-r1/receipt.json", + "sha256": "dd87233b672b4c9c9d830425c7ee8ce19bbc66ed32e349fbed4e1eb05687ef48" + }, + "original_inputs_unchanged_at_execution_end": true, + "preexisting_ancillary_unchanged": true, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "source_directory": "data/transformed/bactotraits", + "started_at": "2026-11-16T21:45:41.395528+00:00", + "upstream_unchanged": true + }, + "ontologies": { + "chain_receipt": { + "bytes": 156174, + "path": "chain-canonical-ontologies-r1.json", + "sha256": "c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e" + }, + "completed_at": "2026-11-16T21:15:47.530538+00:00", + "configuration_digest": "5418f48bc234704ad984abaf79a26481a4a5161a1b7e9e9561a80571d7db6670", + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "counts": { + "edges": 1729253, + "nodes": 1355041 + }, + "existing_raw_inputs_unchanged": true, + "finalizer_code_fingerprint": "f4586479a17fc1e8ae7c7749b18780a7823c160abfde359d97076466d5d1d722", + "finalizer_graph_and_audit_members": { + "chebi_edges.tsv": { + "bytes": 46268462, + "sha256": "9a74dd1a3d92a5f82fd7361c58d1ba971abde1f8ebb5c8f2ed0c77990544e1e1" + }, + "chebi_nodes.tsv": { + "bytes": 66294335, + "sha256": "02024ca0a29cafaac16b43a06a786edd0368a8a646b75ab1002b6239c1219f60" + }, + "ec_edges.tsv": { + "bytes": 1260032, + "sha256": "c86152c532704fb368c7c5ce98ed0371b8570bc1fd5bb51aa4ada938de3b0d19" + }, + "ec_nodes.tsv": { + "bytes": 2065473, + "sha256": "84418ecbd7f46a4d08e63c90d4c5f2a6bb1f3ba88b0b564d8e02053c7a1a0ca1" + }, + "envo_edges.tsv": { + "bytes": 1215217, + "sha256": "86296283b8956be14f87d541ac71a10092ea1001dd68c7e15d894bd876cbdc8a" + }, + "envo_nodes.tsv": { + "bytes": 1292759, + "sha256": "ff195ed17339df155c90e3af7677f6f6c6a02a0a4b8222f86d7665c68641db0a" + }, + "foodon_edges.tsv": { + "bytes": 6566822, + "sha256": "cf0014be93a186021667d59a6c69e59ad1ed5fcfafee620ccde105a18f2eca2f" + }, + "foodon_kernel_quarantine.tsv": { + "bytes": 94917, + "sha256": "6b7feb1493a629c9b5dcaabe7f2662663f42316c634f0ce384e5c045c6535137" + }, + "foodon_nodes.tsv": { + "bytes": 7066327, + "sha256": "5337050e9527b4c80b59d7f4afb1a543579503bcb676de9834d0658c31a67e5d" + }, + "go_edges.tsv": { + "bytes": 7735534, + "sha256": "dfa175a58b2317e112b3d920aee93eab4046bb76af2a3b03b3e4f28b3a180125" + }, + "go_nodes.tsv": { + "bytes": 15147094, + "sha256": "23df80dbbbc80cf7a6ac411eff2265b9c0b1cc5c15793ec8495ca74fb59d6ac3" + }, + "go_reference_resolution.tsv": { + "bytes": 49595009, + "sha256": "e0ea9d2401f6b6b50a7864a7e6e92da01335373774df38cc08de125b27bba926" + }, + "hp_edges.tsv": { + "bytes": 2584221, + "sha256": "0adb692f275158d3d1d35eff6b0239315dfb7f52d320ecc7fbf7a1eb8b4a4c88" + }, + "hp_nodes.tsv": { + "bytes": 5094035, + "sha256": "2eedfe6bb7c1bed2a5d96a7af9837e10c155cf608ff9002d2321ce0d20e9768c" + }, + "metpo_edges.tsv": { + "bytes": 34308, + "sha256": "57dfb68fa09db5e7b9284470df4c717b9b3f9830e151a731cbed5a1c91abf8cb" + }, + "metpo_nodes.tsv": { + "bytes": 65883, + "sha256": "6483bee2d730fb55ab58830aa1a87106fbf2531192a4018324a8579169f48266" + }, + "mondo_edges.tsv": { + "bytes": 12721442, + "sha256": "fac90452121124151fef1705a443fed42b0dedc6e1cd3522d7b7a51be5be6519" + }, + "mondo_nodes.tsv": { + "bytes": 17176022, + "sha256": "2bd3b13e6131d6eb780a3f248858ea61672f1109678d4d05afc2ee164b6b86d0" + }, + "native_relation_dispositions.tsv": { + "bytes": 1125605, + "sha256": "d25dc3e19cbd1a18162ae04b48685f7e7c8760dd405fbd83c86e909338de51a9" + }, + "ncbitaxon_edges.tsv": { + "bytes": 116390818, + "sha256": "4b9597974319bb030fd07682f00755404d26ee6297ece30f99feb88915048e95" + }, + "ncbitaxon_nodes.tsv": { + "bytes": 104657178, + "sha256": "c7c9bd85b3871a8d323a833a164b4bab3e2ef9798a3456e55a218476dad56580" + }, + "ontology_metadata_exclusions.tsv": { + "bytes": 818850, + "sha256": "472c6fda2dc56544fe0103e59d099dc62ffc56d62cd7145d486d4a2d34f512d1" + }, + "ontology_parent_dispositions.tsv": { + "bytes": 15455868, + "sha256": "3d59e84b8e6d4939d75ae7f3b811763b778555424a9b618298b56de84394aedf" + }, + "pato_edges.tsv": { + "bytes": 2064720, + "sha256": "bc682e3966169f40b7cbf78a52844f9638609c021d89ade77817b34f724490e6" + }, + "pato_nodes.tsv": { + "bytes": 2341468, + "sha256": "45734a73037daa50601d4262c704129b7488f88ed93d7d56bbc54037573747c8" + }, + "ro_edges.tsv": { + "bytes": 9271, + "sha256": "015699c43eede656b82759b30ae68ee873b59dd910cf62306fe54bc78772b9ca" + }, + "ro_nodes.tsv": { + "bytes": 177483, + "sha256": "94a72f26e170a9fb28e00712ecc1f09551273f18dcbb0ff749dcc5bbc365efe9" + }, + "source_canonicalization.tsv": { + "bytes": 25524472, + "sha256": "c3e9b35325164fd3d015c59e3536f67fc98454bed2a071ab8220a99c6214ce29" + }, + "source_finalization.json": { + "bytes": 46411, + "sha256": "f6d80648081b1f18fcff51eaf05209eeeae8d5e10d9b4b369b18653f10fe16b2" + }, + "source_fingerprint.json": { + "bytes": 4385, + "sha256": "76f4f151a24a864daa4e9ed0edb6ee94449154d35977c7ab95299faf1a2e24db" + }, + "source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "source_reference_resolution.tsv": { + "bytes": 279610, + "sha256": "37734fa7d77e5833b074ac365b6debce50f1ac98dab1420b091522d4ca7a5dce" + }, + "taxrank_edges.tsv": { + "bytes": 9228, + "sha256": "0a6d62353510003c438a82d66eb4fafd5c0be9e034ae771686512334fa81ff11" + }, + "taxrank_nodes.tsv": { + "bytes": 9046, + "sha256": "b4ded9e2479674558e8c3100fbd07fcc27b6690ea46feb79fb159640c873973e" + }, + "uberon_edges.tsv": { + "bytes": 8350913, + "sha256": "5d75961bf657df579ab02496c9c03205db4dd54aa2999a2118d28dd64b35302f" + }, + "uberon_nodes.tsv": { + "bytes": 8122603, + "sha256": "23f96f9a513b081b0ef24a22d9f4c7d662489bfbac014679fe4492196f04586a" + }, + "upa_edges.tsv": { + "bytes": 307754, + "sha256": "189f856fd89c32be0880f65abfa0283a8f968dcf568d26ddcf3c4501db69a87d" + }, + "upa_nodes.tsv": { + "bytes": 398234, + "sha256": "4cb2b06ffe73eed9a66e6c1f29e4afd3010eb9a667cbdef412e488d3e30c9fd2" + }, + "upa_route_review.tsv": { + "bytes": 11779832, + "sha256": "3614e6abc27b1b66e3f1dd81db6ca9214860063bce73abe446c16e5dc12df9f2" + } + }, + "inner_receipt": { + "bytes": 89899, + "path": "data/validation-bactotraits-130/canonical-ontologies-r1/receipt.json", + "sha256": "f6169dcdfc7822cbc4de7174a59a516ee4b1acf1e15af886da6f4add0246631f" + }, + "original_inputs_unchanged_at_execution_end": true, + "preexisting_ancillary_unchanged": true, + "producer_commit": "b631cacf5bca2f5f993e85bacb96578919cc7d92", + "source_directory": "data/transformed/ontologies", + "started_at": "2026-11-16T21:00:05.307047+00:00", + "upstream_unchanged": true + } + }, + "summary_helper": { + "bytes": 15495, + "path": "summarize_m5_production_20261116.py", + "sha256": "74d2e5d18d3d68c2cb79d7439bfcc991252e8ae8e9d7405887b8ddd8d00a9435" + } +} diff --git a/docs/validation/m5-bactotraits-release-20261116.json b/docs/validation/m5-bactotraits-release-20261116.json new file mode 100644 index 0000000..48ebcba --- /dev/null +++ b/docs/validation/m5-bactotraits-release-20261116.json @@ -0,0 +1,297 @@ +{ + "profile": "m5-bactotraits-release-verification/v1", + "observed_at": "2026-11-16T20:43:58.815480+00:00", + "export_helper_sha256": "96f3d2bf325ca2889700089a2d3276a86339ff9fd70bc8b8a2eb28adc6eb744a", + "fresh_baseline_candidate_pair_verified": true, + "bounded_source_finalization_verified": true, + "merged_graph_verified": false, + "published_release_verified": false, + "source_row": 8521, + "bacdive_id": 17851, + "scientific_claim_resolved": false, + "neighboring_rows_quarantined": false, + "results": { + "baseline-m5-20261116-r1": { + "original_receipt": "/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116/data/validation-bactotraits-130/baseline-m5-20261116-r1/receipt.json", + "receipt": { + "sha256": "8d2a780d27670106456450fb4c8d1063424570b7bf9277387c8abcb09a389eb0", + "bytes": 36761 + }, + "producer_revision": "ef27b97a0b450767da54235c6105b5250c178adc", + "edges": 21, + "nodes": 22, + "quarantine_rows": 0, + "finalization_run": false, + "artifacts": { + "authority/ncbitaxon-label.json": { + "bytes": 955, + "sha256": "277127387d0107c00ae57bafd5c51a1ef8e5399e31bc491ffa6e132fb85e063f" + }, + "cache/pystow/README.md": { + "bytes": 1566, + "sha256": "8d471e43c6d74b56b35252feb86132ce8f32caea360865cf4ec48a4c5fe0ffef" + }, + "input-plan.json": { + "bytes": 4248, + "sha256": "f6566b3f337b34eeab0da90c66cb59ac3260c84dc5848fd7b18ea30a953e14fa" + }, + "lookup/bacdive.tsv": { + "bytes": 1419, + "sha256": "e238208d460a47c437e6ca4da3e3aea262270d8d30ee006c201f227347f4647d" + }, + "mappings/custom_curies.yaml": { + "bytes": 34718, + "sha256": "5b201563b8042eb51a6984f56354be2b3420deae0d029ce707c961ec6eeab784" + }, + "mappings/metpo_alias_mappings.tsv": { + "bytes": 19165, + "sha256": "15c4aa67644fd0aac233dbc909cda0b8bbfcbc50bb998fc8bf3e06a7b19ace06" + }, + "mappings/prefixmap.json": { + "bytes": 2033, + "sha256": "a11f11fc43db2d3d0ffd74aca08fbc1da08033074b355ef2e0a1d6d547c29762" + }, + "raw/BactoTraits_databaseV2_Jun2022.csv": { + "bytes": 2074, + "sha256": "e5c10497510e580d1cabb4f76b51c90b9fa3472547d25461aa5d834ac69ef9dd" + }, + "templates/metpo-properties.tsv": { + "bytes": 21929, + "sha256": "8ef8929f3f8d3ed61ba41f062548a3d4d2e058932c522b7ddb3a01d1e06cef63" + }, + "templates/metpo_sheet.tsv": { + "bytes": 81520, + "sha256": "ec9700569bf5e83d39fe4c29f525c06d3c396adfad2ce0bf33072b8b1a1b1409" + }, + "tmp/bactotraits/bactotraits.tsv": { + "bytes": 1590, + "sha256": "078c43494f96aadca7ae8c1c5c2d63c3438fcb1c5dfe8198b4fae1cf7dc786f6" + }, + "tmp/bactotraits/bactotraits_mapping.tsv": { + "bytes": 125, + "sha256": "c889f33b711569ee00c60a96fb7bf29e7efeaa9f2011dd7b9388e347c638a006" + }, + "tmp/bactotraits/bactotraits_mapping.tsv.complete": { + "bytes": 64, + "sha256": "62ef0d8f6c56253c12c1218cf1650fd87297ceeb3dd218affd4e4e5d77a1bdd7" + }, + "transform.stderr.txt": { + "bytes": 855, + "sha256": "19641439822a224c9e250a772efe7ee7a4bfbb6c687a26ddba5eda551436b1cb" + }, + "transform.stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "transformed/bactotraits/edges.tsv": { + "bytes": 2396, + "sha256": "666438931337cc512cfeac82a451bbfad1da6e54daa69d730b199ac5373b43a0" + }, + "transformed/bactotraits/nodes.tsv": { + "bytes": 1897, + "sha256": "e21c1fcba073d3885493e4819017d4773d3fe02a48680be5e9c9e297ed19202e" + } + } + }, + "candidate-m5-20261116-r1": { + "original_receipt": "/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116/data/validation-bactotraits-130/candidate-m5-20261116-r1/receipt.json", + "receipt": { + "sha256": "03446ab15dbe56bf277d25c810b5b0a612a73fe5850d73447d112ab74e611477", + "bytes": 32000 + }, + "producer_revision": "2d94ed094167d94402f21400a9ad7dc4f7f20438", + "edges": 0, + "nodes": 0, + "quarantine_rows": 1, + "finalization_run": false, + "artifacts": { + "authority/ncbitaxon-label.json": { + "bytes": 955, + "sha256": "277127387d0107c00ae57bafd5c51a1ef8e5399e31bc491ffa6e132fb85e063f" + }, + "cache/pystow/README.md": { + "bytes": 1566, + "sha256": "8d471e43c6d74b56b35252feb86132ce8f32caea360865cf4ec48a4c5fe0ffef" + }, + "input-plan.json": { + "bytes": 4245, + "sha256": "28cec7d826dd75872a034b80f6c6b224488fc7ea5d0b291c05ca7029c52c534e" + }, + "lookup/bacdive.tsv": { + "bytes": 1419, + "sha256": "e238208d460a47c437e6ca4da3e3aea262270d8d30ee006c201f227347f4647d" + }, + "mappings/custom_curies.yaml": { + "bytes": 34718, + "sha256": "5b201563b8042eb51a6984f56354be2b3420deae0d029ce707c961ec6eeab784" + }, + "mappings/metpo_alias_mappings.tsv": { + "bytes": 19165, + "sha256": "15c4aa67644fd0aac233dbc909cda0b8bbfcbc50bb998fc8bf3e06a7b19ace06" + }, + "mappings/prefixmap.json": { + "bytes": 2033, + "sha256": "a11f11fc43db2d3d0ffd74aca08fbc1da08033074b355ef2e0a1d6d547c29762" + }, + "raw/BactoTraits_databaseV2_Jun2022.csv": { + "bytes": 2074, + "sha256": "e5c10497510e580d1cabb4f76b51c90b9fa3472547d25461aa5d834ac69ef9dd" + }, + "templates/metpo-properties.tsv": { + "bytes": 21929, + "sha256": "8ef8929f3f8d3ed61ba41f062548a3d4d2e058932c522b7ddb3a01d1e06cef63" + }, + "templates/metpo_sheet.tsv": { + "bytes": 81520, + "sha256": "ec9700569bf5e83d39fe4c29f525c06d3c396adfad2ce0bf33072b8b1a1b1409" + }, + "tmp/bactotraits/bactotraits.tsv": { + "bytes": 1168, + "sha256": "dd171b985cad4887a7b72e8ea653d7938fb708bd49a507eeb58c4fc7d6277bf5" + }, + "tmp/bactotraits/bactotraits_mapping.tsv": { + "bytes": 125, + "sha256": "c889f33b711569ee00c60a96fb7bf29e7efeaa9f2011dd7b9388e347c638a006" + }, + "tmp/bactotraits/bactotraits_mapping.tsv.complete": { + "bytes": 64, + "sha256": "62ef0d8f6c56253c12c1218cf1650fd87297ceeb3dd218affd4e4e5d77a1bdd7" + }, + "transform.stderr.txt": { + "bytes": 855, + "sha256": "19641439822a224c9e250a772efe7ee7a4bfbb6c687a26ddba5eda551436b1cb" + }, + "transform.stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "transformed/bactotraits/bactotraits_identity_quarantine.tsv": { + "bytes": 3761, + "sha256": "28d62147d49735406a35c721db345497c7e044f69f6d723caa0367f996158918" + }, + "transformed/bactotraits/edges.tsv": { + "bytes": 86, + "sha256": "8d142a6d6931b88959df0577cfad52cb545edb2ba48222f11a34a05bc6336d46" + }, + "transformed/bactotraits/nodes.tsv": { + "bytes": 73, + "sha256": "0604cdfbcc9fdc122da6154ea79a7c434e12b501f91650f57297164edd2cd3d9" + } + } + }, + "candidate-finalized-m5-20261116-r1": { + "original_receipt": "/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116/data/validation-bactotraits-130/candidate-finalized-m5-20261116-r1/receipt.json", + "receipt": { + "sha256": "0c832333ac93018e95e81d2ba53e9e587b58c4838840859ba823f8f7fd441996", + "bytes": 50984 + }, + "producer_revision": "2d94ed094167d94402f21400a9ad7dc4f7f20438", + "edges": 0, + "nodes": 0, + "quarantine_rows": 1, + "finalization_run": true, + "artifacts": { + "authority/ncbitaxon-label.json": { + "bytes": 955, + "sha256": "277127387d0107c00ae57bafd5c51a1ef8e5399e31bc491ffa6e132fb85e063f" + }, + "cache/pystow/README.md": { + "bytes": 1566, + "sha256": "8d471e43c6d74b56b35252feb86132ce8f32caea360865cf4ec48a4c5fe0ffef" + }, + "execution.stderr.txt": { + "bytes": 855, + "sha256": "19641439822a224c9e250a772efe7ee7a4bfbb6c687a26ddba5eda551436b1cb" + }, + "execution.stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "input-plan.json": { + "bytes": 41480, + "sha256": "bb5f896abd27db61ecbfbde63718b8cff8ebd688d62143a949077579ff14bb56" + }, + "lookup/bacdive.tsv": { + "bytes": 1419, + "sha256": "e238208d460a47c437e6ca4da3e3aea262270d8d30ee006c201f227347f4647d" + }, + "mappings/custom_curies.yaml": { + "bytes": 34718, + "sha256": "5b201563b8042eb51a6984f56354be2b3420deae0d029ce707c961ec6eeab784" + }, + "mappings/metpo_alias_mappings.tsv": { + "bytes": 19165, + "sha256": "15c4aa67644fd0aac233dbc909cda0b8bbfcbc50bb998fc8bf3e06a7b19ace06" + }, + "mappings/prefixmap.json": { + "bytes": 2033, + "sha256": "a11f11fc43db2d3d0ffd74aca08fbc1da08033074b355ef2e0a1d6d547c29762" + }, + "merge.json": { + "bytes": 1118, + "sha256": "4e9e9164952bf1339bb2189d7c220e4877d9d3f7297436ca0479369a5fc71bc2" + }, + "raw/BactoTraits_databaseV2_Jun2022.csv": { + "bytes": 2074, + "sha256": "e5c10497510e580d1cabb4f76b51c90b9fa3472547d25461aa5d834ac69ef9dd" + }, + "templates/metpo-properties.tsv": { + "bytes": 21929, + "sha256": "8ef8929f3f8d3ed61ba41f062548a3d4d2e058932c522b7ddb3a01d1e06cef63" + }, + "templates/metpo_sheet.tsv": { + "bytes": 81520, + "sha256": "ec9700569bf5e83d39fe4c29f525c06d3c396adfad2ce0bf33072b8b1a1b1409" + }, + "tmp/bactotraits/bactotraits.tsv": { + "bytes": 1168, + "sha256": "dd171b985cad4887a7b72e8ea653d7938fb708bd49a507eeb58c4fc7d6277bf5" + }, + "tmp/bactotraits/bactotraits_mapping.tsv": { + "bytes": 125, + "sha256": "c889f33b711569ee00c60a96fb7bf29e7efeaa9f2011dd7b9388e347c638a006" + }, + "tmp/bactotraits/bactotraits_mapping.tsv.complete": { + "bytes": 64, + "sha256": "62ef0d8f6c56253c12c1218cf1650fd87297ceeb3dd218affd4e4e5d77a1bdd7" + }, + "transformed/bactotraits/bactotraits_identity_quarantine.tsv": { + "bytes": 3761, + "sha256": "28d62147d49735406a35c721db345497c7e044f69f6d723caa0367f996158918" + }, + "transformed/bactotraits/edges.tsv": { + "bytes": 99, + "sha256": "1a1f91d2da59ddf2aad7f854f52b7e2b23aaba455918bdd7382980bf6e25079f" + }, + "transformed/bactotraits/go_reference_resolution.tsv": { + "bytes": 204, + "sha256": "082792210b81208e4bfab7db9686e1d64a7bc98366f9bc0b7a4a302d1e8cfacc" + }, + "transformed/bactotraits/nodes.tsv": { + "bytes": 73, + "sha256": "0604cdfbcc9fdc122da6154ea79a7c434e12b501f91650f57297164edd2cd3d9" + }, + "transformed/bactotraits/source_canonicalization.tsv": { + "bytes": 48, + "sha256": "4e6a2e08ad0ec3ed46d69cf49c5790c4c0bc253c6cb18d9f1866363bedbd4870" + }, + "transformed/bactotraits/source_finalization.json": { + "bytes": 10036, + "sha256": "fb0ebb33062aa6060d20d2580f9000137f39fe4260526c7ed6f7ada473ce20c6" + }, + "transformed/bactotraits/source_fingerprint.json": { + "bytes": 2086, + "sha256": "987b8cb6db6403a9b3dc2a8140abb4a1a0e053b7ef8a1564743dfab03f43d9e6" + }, + "transformed/bactotraits/source_metadata_exclusions.tsv": { + "bytes": 62, + "sha256": "f0bfb347a870f737dd13e48920e6eda693bc99db3f081a4326bc76efdd538564" + }, + "transformed/bactotraits/source_reference_resolution.tsv": { + "bytes": 102, + "sha256": "727041d59de466a4823b0c919310b7efcea1d8695f53b0b09e2e5192656798c8" + } + } + } + } +} diff --git a/docs/validation/m5-bactotraits-release-20261116.md b/docs/validation/m5-bactotraits-release-20261116.md new file mode 100644 index 0000000..83e930b --- /dev/null +++ b/docs/validation/m5-bactotraits-release-20261116.md @@ -0,0 +1,278 @@ +# M5 BactoTraits graph-release verification — 2026-11-16 + +The exact-row correction is verified through a new M5 baseline/candidate pair, +the complete current ontologies → BacDive → BactoTraits producer chain, and an +actual strict BactoTraits-only merged candidate. The persisted candidate has +9,757 nodes and 87,788 edges, with zero occurrences of the quarantined target +SPO. Its complete original-row audit remains intact. This verifies a bounded +candidate; a complete canonical snapshot and corrected published release remain +unverified. + +| Check | Actual result | +| --- | --- | +| Fresh baseline, `ef27b97a0b450767da54235c6105b5250c178adc` | 21 edges, 22 nodes | +| Fresh corrected candidate, `2d94ed094167d94402f21400a9ad7dc4f7f20438` | 0 edges, 0 nodes, 1 complete quarantine audit | +| Separate fresh candidate through `finalize(fresh_run=True)` | 0 edges, 0 nodes, identical quarantine audit | +| Earlier bounded `load_and_merge(processes=1)` attempt | Correctly refused its missing upstream completion; old receipt unchanged | +| Full current BactoTraits source | 87,788 edges, 9,757 nodes, 1 complete 105-field quarantine audit | +| Ordinary strict BactoTraits-only merge | 87,788 edges, 9,757 nodes; target SPO count 0; persisted archive verified | +| Current raw reviewed-input preflight | Passed against original M5 raw root | +| Current published corrected release | Unavailable; dependency tracked in [#143](https://github.com/CultureBotAI/EdgeMech/issues/143) | + +The producer merge is [KG-Microbe PR #1867](https://github.com/Knowledge-Graph-Hub/kg-microbe/pull/1867), +reviewed head `a86c63e778ebf8a646abb9d4d9e0718dfd49f5a0`. The earlier +[identity investigation](bactotraits-identity-20261115.md) is a historical +pre-finalization record. Its old receipts and every nested cache file remain +unchanged; none was used in place of the new baseline. + +## Current publication and preserved graph + +The live GitHub API inspection found 14 releases, with latest tag +[2025-03-07](https://github.com/Knowledge-Graph-Hub/kg-microbe/releases/tag/2025-03-07), +published `2025-03-07T23:03:41Z`. No November 16 corrected release was present. +The retained canonical M5 `merged-kg.tar.gz` has SHA256 +`3d529dd0024843f9278ecfbfc8703413061ef9c4c7474805ecec03aa8cfdfd4c`. +Its embedded historical manifest records generation `2026-10-18T11:56:34+00:00`, +producer `bcfa6bf5b-dirty`, Python 3.10.11, 17,927,641 edges and 3,440,986 nodes. +Those counts are manifest claims, not a new graph-wide validation. + +[Upstream #887](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/887) +remains open and the current release workflow retains its retired artifact URL. +A new URL probe failed DNS; this execution did not reproduce the issue's +historical HTTP404. Publication requires a fresh complete dependency chain, +canonical merge and lineage verification, a working artifact publication source, +separate human approval, and verification of the published asset bytes. + +The complete GitHub inspection at `2026-11-16T21:20:12Z` again found 14 releases, +with newest asset modification on August 26. The final latest-release/issue +refresh at `2026-11-16T22:01:40Z` still showed latest `2025-03-07` and #887 open. +The canonical configuration has 24 source keys from 15 producers. After the +completed ontologies/BacDive/BactoTraits chain, 12 additional whole-source +completions remain: gold, gtdb, lpsn, lpsn_api, +madin_etal, mediadive, metatraits, metatraits_gtdb, microbedecoder, +ontologies_stubs, prego (habitat only), and rhea_mappings. Independent review +confirmed their completion records were absent using exact paths including +ignored files. This establishes an unmet full-release dependency; it does not +by itself establish that rebuilding them is infeasible. The global ontology +raw-input preflight does not certify those sources' input availability or +freshness. + +Independent review found that the first archive stat observation rounded its +nanosecond timestamp by 24 ns through JavaScript Number serialization. The original +observation is preserved. A new Python-written observation records exact +`st_mtime_ns=1792324632633850376` and confirms the unchanged archive SHA256. +[#147](https://github.com/CultureBotAI/EdgeMech/issues/147) tracks this corrected +metadata recording; it is not evidence of archive mutation. + +## New execution and provenance + +Both new replay runs used the unchanged +`scripts/adhoc/replay_bactotraits_identity_130_20261115.py`, SHA256 +`060d7e4dbb6189051f989a4a9efbf55336408e4224ecba5a4975e710538ea90e`, +with `--plan` before each `--run` and a new `--baseline-receipt`. They used one +unit at a time, exclusive output stages, identical shared inputs and the same +absolute `.venv310/bin/python` executable. Python is 3.10.11, built on M5 on +October 10 with Clang 21.0.0; old Python version strings were not rewritten. + +The original CSV SHA256 is +`18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b`. +The complete decoded row 8521/BacDive17851 SHA256 is +`345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a`. +The receipts bind the original and staged CSV, actual BacDive lookup, METPO +tables, local alias and prefix mappings, producer Python source files and policy. +NCBITaxon DB metadata and actual queried statements are bound; the DB and OWL +release agree at `2026-07-12`. This is not a whole-database hash. + +The first Poetry install failed DNS. The isolated Python 3.10 environment was +then installed offline from all 346 applicable main/dev lock packages using +`--no-index --no-deps --require-hashes`. Installed versions and `pip check` pass. +335 wheels match lock hashes directly; 11 cached built wheels have separate +wheel hashes and co-located source archives matching the lock. The latter is +source/archive association, not a reproducible-build attestation. Lock SHA256: +`a9e8336e172faef6449ed79a2dde8df7932a5cfb9ba368e7a93a7c7ab83d7f6f`. + +| Receipt | SHA256 | +| --- | --- | +| New baseline | `8d2a780d27670106456450fb4c8d1063424570b7bf9277387c8abcb09a389eb0` | +| New pre-finalization candidate | `03446ab15dbe56bf277d25c810b5b0a612a73fe5850d73447d112ab74e611477` | +| New finalized candidate | `0c832333ac93018e95e81d2ba53e9e587b58c4838840859ba823f8f7fd441996` | + +The separate finalization helper runs the actual producer and finalizer on the +same live object; it does not reopen old outputs and claim a fresh run. It pins +shared policy files before/after use, the schema trio, finalization code, +configuration, preceding receipts and resulting files. The complete quarantine +audit remains SHA256 +`28d62147d49735406a35c721db345497c7e044f69f6d723caa0367f996158918`. +Finalized empty edge TSV SHA256 is +`1a1f91d2da59ddf2aad7f854f52b7e2b23aaba455918bdd7382980bf6e25079f`; +the finalizer adds canonical columns without adding assertions. + +All three new stages were copied byte-for-byte beneath the required invocation +output root `.temp/m5-worker-run-20261116/results-r1/`; original paths in receipts +remain unchanged. The [compact result](m5-bactotraits-release-20261116.json) +records every receipt-bound output hash. Complete receipts, plans, stdout/stderr, +publication responses, environment inventory and historical preservation checks +remain private under that invocation root. + +## Lineage and scope + +The full original CSV has 19,456 data records and exactly one matching BacDive +17851/conflicting-accession occurrence. The lookup has 36,596 records, with one +mapping to NCBITaxon:1123071. The producer rebuilds its mapping and quarantines +the complete reviewed row before its pruned copy and graph writes. The retained +October copies were inspected read-only; stale completion records cannot admit +them through the strict current merge path. + +The equivalent historical Madin assertion has independent source provenance: +raw row 50,223, DSM18772, `motility=no`, `data_source=gold`, `ref_id=13521`; +the transformed edge has `infores:madin_etal`. It is not evidence of leakage +from this BactoTraits row. The BacDive lookup separately records motility=no, +although the inspected historical BacDive edge file has no matching assertion. +Neither independent source was newly scientifically reassessed here. + +The lineage helper streamed eight explicit input paths, including ignored data, +and hashed them before/after. Its source search used `rg --no-ignore --hidden`. +This is not absence evidence across arbitrary copies or uninspected graphs. +No neighboring row was quarantined, no replacement organism was inferred, and +the correction does not establish trait falsity or graph-wide validity. + +## Actual canonical dependency chain + +The ordinary CLI whole-ontology unit completed on committed KG candidate +`b631cacf5bca2f5f993e85bacb96578919cc7d92`, whose production implementation is +unchanged from the corrected merged code. It wrote 1,355,041 nodes and 1,729,253 +edges across 14 graph pairs. Independent review counted those rows and verified +97 persisted artifacts and 526 named input, implementation and authority paths. +The actual whole-source finalization and producer fingerprint both completed; +the earlier scoped RO canary was not substituted for whole-source completion. +The complete chain receipt SHA256 is +`c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e`. + +The BacDive unit used the same canonical output placement, launcher, +Python environment and single-worker/thread limits. Its plan pins six original +inputs totaling 14.875 GB, including the actual 14.074 GB NCBITaxon DB, with +188.416 GB free before staging. It completed with 298,502 nodes and 4,454,892 +edges, nine finalization audits and an actual producer fingerprint. The full +private chain receipt SHA256 is +`6742881bc5743bdf8ed12023638634ad628f482377ad8f0ad0799936585103bd`; +the wrapper confirmed existing raw inputs, implementation and upstream outputs +were unchanged. The retained NCBITaxon database matched the authority version +but had no historical toolchain record; the ordinary run records that limitation +and no historical record was invented. Independent review verified all 16 source +artifacts, 97 upstream artifacts, 521 named paths and exact graph counts. The +target SPO has zero occurrences in the new BacDive graph; its fresh lookup has +one BacDive17851 to NCBITaxon:1123071 mapping at data row 22,928. + +The full BactoTraits producer then completed with 9,757 nodes and 87,788 edges, +using the original full CSV and that new lookup. Independent review verified +all nine source artifacts, 113 upstream artifacts and 530 named paths. Exactly +one audit preserves all 105 fields of original row 8521, with the expected +complete-row and full-CSV hashes. Its SHA256 is +`ca287b5941761b5ba857a142d2896cefb285039a0fa5638d23075e5e26297160`. +The fresh lookup and mapping each retain one identity entry for traceability; +the pruned producer cache has none, and the finalized source has zero target +SPO occurrences. The full BactoTraits chain receipt SHA256 is +`36968cb32f7c10ec883160503e369bf1ad19ecfa3b8203395e327f24db93a445`. + +The [compact production record](m5-bactotraits-production-20261116.json) binds +all three completed sources, full authority bytes, runtime, configurations, +implementation and output hashes without embedding native histories. Its +source-only scope does not assert merged-graph or publication success. The +separate [merged-candidate record](m5-bactotraits-merged-candidate-20261116.json) +binds the actual merge result. No completion marker was synthesized and no +freshness check was disabled. + +The strict merge helper preserves canonical semantics and records explicit +path-only configuration changes: absolute source paths and exclusive output +paths, including statistics. This prevents the canonical cwd-relative stats +filename from overwriting a task-checkout artifact +([#160](https://github.com/CultureBotAI/EdgeMech/issues/160)). It requires all three +complete source receipts and verifies the complete quarantined row, retained +intermediate joins and streamed merged outputs. Its 24 focused tests also pass +independently. A BactoTraits-only candidate cannot establish full-snapshot +publication. + +The first strict merge plan refused an inherited EdgeMech `PYTHONPATH` because +it exposed an extra distribution. The failed plan was preserved; a new plan +used the explicit KG path and the same native thread limits as the producers, +and matched all 346 pinned distributions. No package or runtime guard changed +([#163](https://github.com/CultureBotAI/EdgeMech/issues/163)). Independent review +then verified 457 input pins and all 26 path-only configuration deltas before +the ordinary single-process CLI was started. + +The strict CLI and persisted verification completed successfully at +`2026-11-16T22:01:06Z`. Archive SHA256 is +`d217563f5d29ca2bc7ac729253acf5f83d23519d02a22ed91a5cc974ccf49976` +(447,980 bytes); the complete merge receipt SHA256 is +`111a1d66ae90ae0136212c06f379f15fee447ebdc9db3f70e4fd4e1d806b03be`. +Streaming readback verified 9,757 node rows, 87,788 edge rows, all member hashes, +the reference-resolution member and embedded producer/configuration/schema +provenance. The manifest records Python 3.10.11, producer `b631cacf5`, required +source finalization and no diagnostic opt-out. Input/source trees, original +statistics and the complete quarantine audit stayed unchanged. The candidate +contains zero target SPO rows and zero BactoTraits-provenance target rows. +Independent final review verified all 13 output artifacts, all three source +trees, all 457 input pins after execution, protected original statistics, +the actual interpreter and all 346 distributions. Every merged edge has +`infores:bactotraits` as its primary knowledge source. Its review receipt SHA256 +is `bf397abd283fc5103731be4642662a8c729916cf484a66d8067e2c8fd4c9b14b`. + +## Independent review and checks + +Read-only adversarial review independently recomputed all new stage artifact +hashes, checked complete audit JSON against the original row, compared actual +runtime versions, and verified all 37 historical files. It found no defect in +the bounded producer/finalizer/admission evidence. Two evidence bookkeeping +findings were filed and fixed: [#146](https://github.com/CultureBotAI/EdgeMech/issues/146) +for concurrent progress updates and [#147](https://github.com/CultureBotAI/EdgeMech/issues/147) +for stat precision. A second read-only review verified both fixes. + +Production-launcher gaps in shared configuration pins +([KG #1869](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1869)) and +canonical raw side-effect recording +([KG #1870](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1870)) were +filed and fixed before actual source units ran. The original canonical archive, +all 37 retained v3 files (including nested hidden caches), and the clean `ef27b97` +baseline passed final independent preservation review. The original archive's +exact Python stat fields also remain unchanged. All failed attempts and older +receipts remain preserved. + +After merge completion, the retained 37-file inventory and original archive +were checked again: byte hashes and exact archive stat fields still match the +initial observations. The complete result is ready for review in +[EdgeMech PR #151](https://github.com/CultureBotAI/EdgeMech/pull/151) and companion +[KG-Microbe PR #1871](https://github.com/Knowledge-Graph-Hub/kg-microbe/pull/1871). +No new PR was merged and no release was published. + +The finalization helper's six focused tests and production-launcher four focused +tests passed. EdgeMech's final full suite passed 777 tests after the progress +boundary fixes. Required EdgeMech lint and offline wheel/source package builds +also passed. Independent review reran the 23 helper-boundary checks and verified +the actual retained five-case retry receipt against its persisted log bytes. + +The first KG standard gate passed its cheap stages and 9,999 tests but failed +four scratch-context assertions and one output-root teardown guard. A versioned +environment moves scratch outside Git into durable `~/agent-work`; all five +affected checks pass without changing assertions or guards +([#161](https://github.com/CultureBotAI/EdgeMech/issues/161)). A second gate exposed +pre-existing fixture dependence on newly produced canonical taxonomy data +([KG #1873](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1873)). A +pristine verification checkout at exactly the same committed candidate passes +that real test canary and all four unchanged standard gate stages, with 9,968 +passed and 56 skipped. The populated-checkout run completed with 9,974 passed, +33 failed and 17 skipped; all 33 failures have the same fixture-cache mismatch, +and the affected fixture cases pass in the pristine checkout. The pristine +checkout has additional ordinary local-data skips; this is not a claim that +every optional integration test ran there. No assertions, selectors or +producer guards were changed. The known fixture defect, populated-checkout +failure and every earlier log remain preserved. + +Read-only review found two new helper boundary cases: exact temporary-root paths +were accepted, and imported terminal receipts did not require consistent exit +status and log hashes. Both are fixed with 23 focused tests +([#162](https://github.com/CultureBotAI/EdgeMech/issues/162)); actual prior test +receipts remain valid. The earlier executed helper bytes are retained by hash. +The existing Java toolchain recorder can mistake `_JAVA_OPTIONS` text for the +version ([KG #1872](https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1872)); +actual Java version and binary/library hashes were captured separately without +rewriting the old record or claiming this provenance-only field gates admission. diff --git a/scripts/adhoc/m5_release_archive_stat_20261116.py b/scripts/adhoc/m5_release_archive_stat_20261116.py new file mode 100644 index 0000000..e2959a3 --- /dev/null +++ b/scripts/adhoc/m5_release_archive_stat_20261116.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Preserve exact archive stat integers after review found JS precision loss. + +2026-11-16: this read-only observation supplements, never rewrites, the original +publication receipt. A nanosecond rounding difference is not a byte mutation. +""" + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +ARCHIVE = Path( + "/Users/marcin/Documents/VIMSS/ontology/KG-Hub/KG-Microbe/kg-microbe/" + "data/merged/merged-kg.tar.gz" +) + + +def main(): + """Hash the preserved archive and serialize exact stat values directly in Python.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + args = parser.parse_args() + before = ARCHIVE.stat() + digest = hashlib.sha256() + with ARCHIVE.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + after = ARCHIVE.stat() + fields = ("st_size", "st_mtime_ns", "st_ctime_ns", "st_dev", "st_ino") + assert all(getattr(before, key) == getattr(after, key) for key in fields) + result = { + "path": str(ARCHIVE), "sha256": digest.hexdigest(), + "observed_at": datetime.now(timezone.utc).isoformat(), + "stat": {key: getattr(after, key) for key in fields}, + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "note": "Original publication stat lost 24ns in JS integer rounding; archive bytes match.", + "published_release_verified": False, + } + assert result["sha256"] == "3d529dd0024843f9278ecfbfc8703413061ef9c4c7474805ecec03aa8cfdfd4c" + with args.output.open("x") as stream: + stream.write(json.dumps(result, indent=2) + "\n") + print(result["sha256"]) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_chain_20261116.py b/scripts/adhoc/m5_release_chain_20261116.py new file mode 100644 index 0000000..d35ded5 --- /dev/null +++ b/scripts/adhoc/m5_release_chain_20261116.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""2026-11-16: preserve real M5 production units at the canonical KG output base. + +The reviewed KG unit launcher supplies named input staging, pinned code/config, +private caches, one actual CLI source and raw side-effect inventory. This wrapper +changes only output placement to the task checkout's initially absent canonical +source directory, so BacDive's fixed upstream paths select the actual new output. +It captures every file in that source tree and checks pre-existing upstream trees +remain unchanged. No completion markers are invented, copied or relabeled. +Separate source units share the output base but own disjoint source directories. +Original M5 inputs and the earlier RO canary stages are read-only. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SHARED_LAUNCHER = "scripts/adhoc/run_bactotraits_production_unit_20261116.py" +UPSTREAM = {"ontologies": (), "bacdive": ("ontologies",), "bactotraits": ("ontologies", "bacdive")} +GO_RECEIPT_SHA256 = "dd96bdb5d482e75458b545a658ddd4d1117e53e61f38d84b5f27cabc17d90001" +JAVA_PROOFS = { + "java-runtime-supplement-r3.json": "b697bb006e0754712d28cd0a3bf47628b3f11cd24506ebeab51058d98a175ac1", + "java-runtime-implementation-r3.json": "ec76081f31d0e639082cb3eb359f82f2c138fd31e8538f2c7282b4dfd14a1451", +} +ANCILLARY = ( + "kg_microbe/transform_utils/ontologies/xrefs", + "kg_microbe/transform_utils/ontologies/trees", + "kg_microbe/transform_utils/bacdive/tmp", + "kg_microbe/transform_utils/bactotraits/tmp", +) + + +def load_launcher(root): + """Import the separately reviewed helper without invoking a producer or changing KG code.""" + path = root / SHARED_LAUNCHER + spec = importlib.util.spec_from_file_location("m5_kg_production_unit", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def inventory(directory, launcher): + """Stream every member of one explicit owned subtree, including ignored and hidden files.""" + if not directory.exists(): + return {} + return { + str(path.relative_to(directory)): launcher.identity(launcher.safe_path(path)) + for path in sorted(directory.rglob("*")) + if path.is_file() + } + + +def java_provenance(environment, launcher): + """Preserve the full JVM version output, including any banner before the actual version.""" + command = shutil.which("java", path=environment.get("PATH")) + launcher.require(command is not None, "Java is unavailable") + result = subprocess.run( # noqa: S603 -- PATH-selected Java with fixed read-only version arguments. + [command, "-version"], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + selected_home = environment.get("JAVA_HOME") + if not selected_home and Path("/usr/libexec/java_home").is_file(): + home_result = subprocess.run( # noqa: S603 -- fixed macOS JDK locator, read-only. + ["/usr/libexec/java_home"], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if home_result.returncode == 0: + selected_home = home_result.stdout.strip() + resolved = Path(command).resolve() + actual = Path(selected_home) / "bin/java" if selected_home else resolved + return { + "command": [command, "-version"], + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "path": command, + "resolved_path": str(resolved), + "launcher_identity": launcher.identity(resolved), + "selected_java_home": selected_home, + "selected_jdk_java_path": str(actual.resolve()) if actual.is_file() else None, + "selected_jdk_java_identity": launcher.identity(actual) if actual.is_file() else None, + } + + +def bind_prior_authorities(plan, args, launcher): + """Require the actually rebuilt GO cache and inspected JDK, not just whichever bytes now exist.""" + pins = plan["implementation_and_policy_inputs"] + receipt_path = ( + args.kg_root / "data/validation-bactotraits-130/production-ro-canary-r3/receipt.json" + ) + receipt_identity = launcher.identity(receipt_path) + launcher.require( + receipt_identity["sha256"] == GO_RECEIPT_SHA256, "Reviewed RO/GO receipt changed" + ) + pins[str(receipt_path)] = receipt_identity + receipt = json.loads(receipt_path.read_text()) + go = {} + for name in ("go.db", "go.db.toolchain.json"): + path = args.kg_root / "data/raw" / name + expected = receipt["task_raw_after"][name] + launcher.require(launcher.identity(path) == expected, f"Fresh r3 GO cache changed: {name}") + pins[str(path)] = expected + go[name] = {"path": str(path), **expected} + plan["retained_go_cache"] = { + "receipt": {"path": str(receipt_path), **receipt_identity}, + "members": go, + } + proofs = {} + for name, expected_sha in JAVA_PROOFS.items(): + path = args.output_root / name + observed = launcher.identity(path) + launcher.require(observed["sha256"] == expected_sha, "Reviewed Java supplement changed") + pins[str(path)] = observed + proofs[name] = {"path": str(path), **observed} + implementation = json.loads( + (args.output_root / "java-runtime-implementation-r3.json").read_text() + ) + for role in ("actual_runtime_launcher", "actual_runtime_vm"): + native = implementation[role] + path = Path(native["path"]) + expected = {key: native[key] for key in ("bytes", "sha256")} + launcher.require( + str(path.resolve()) == native["resolved"] and launcher.identity(path) == expected, + "Inspected Java implementation changed", + ) + pins[str(path)] = expected + actual_java = implementation["actual_runtime_launcher"]["resolved"] + launcher.require( + plan["java"]["selected_jdk_java_path"] == actual_java, "Different Java runtime selected" + ) + # Freeze the actual JDK proved by -XshowSettings in the supplement, avoiding + # any later macOS /Library/Java/Home selector change during the long unit. + plan["environment"]["JAVA_HOME"] = str(Path(actual_java).parent.parent) + plan["java_supplements"] = proofs + + +def prepare(args, launcher): + """Pin this wrapper and canonical placement without opening any output for writing.""" + launcher.require( + args.kg_root.resolve() == launcher.ROOT, "Selected KG checkout differs from helper root" + ) + plan = launcher.prepare(args) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "NUMEXPR_MAX_THREADS", + "VECLIB_MAXIMUM_THREADS", + "RAYON_NUM_THREADS", + "BLIS_NUM_THREADS", + ): + plan["environment"][name] = "1" + for name in ("ROBOT_JAVA_ARGS", "_JAVA_OPTIONS"): + plan["environment"][name] += " -XX:ActiveProcessorCount=1" + output = launcher.safe_path(args.kg_root / "data/transformed") + source = launcher.safe_path(output / args.source) + launcher.require( + not source.exists(), f"Canonical source output already exists: {source}; preserve it" + ) + prior = {} + for name in UPSTREAM[args.source]: + directory = output / name + for member in ("source_finalization.json", "source_fingerprint.json"): + launcher.require( + (directory / member).is_file(), + f"Completed upstream evidence missing: {directory / member}", + ) + prior[name] = inventory(directory, launcher) + plan["command"][plan["command"].index("-o") + 1] = str(output) + plan["canonical_output_base"] = str(output) + plan["canonical_source_directory"] = str(source) + plan["upstream_before"] = prior + plan["ancillary_before"] = { + name: inventory(launcher.safe_path(args.kg_root / name), launcher) for name in ANCILLARY + } + fresh_ancillary = { + "bacdive": ( + "bacdive.tsv", + "bacdive_physiology_metabolism.tsv", + "bacdive_name_tax_classification.tsv", + ), + "bactotraits": ("bactotraits_mapping.tsv", "bactotraits.tsv"), + } + for name in fresh_ancillary.get(args.source, ()): + path = args.kg_root / "kg_microbe/transform_utils" / args.source / "tmp" / name + launcher.require(not path.exists(), f"Existing ancillary output must be preserved: {path}") + legacy = args.kg_root / "kg_microbe/transform_utils/bacdive/metabolite_mapping.json" + plan["implementation_and_policy_inputs"][str(legacy)] = launcher.identity(legacy) + if args.source == "bactotraits": + launcher.require( + args.bacdive_receipt is not None, "BactoTraits requires the fresh BacDive unit receipt" + ) + previous = json.loads(args.bacdive_receipt.read_text()) + launcher.require( + previous["returncode"] == 0 and previous["plan"]["source"] == "bacdive", + "BacDive unit did not complete", + ) + expected = previous["ancillary_after"]["kg_microbe/transform_utils/bacdive/tmp"][ + "bacdive.tsv" + ] + lookup = args.kg_root / "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv" + launcher.require( + launcher.identity(lookup) == expected, + "Fresh BacDive lookup differs from its actual unit", + ) + plan["implementation_and_policy_inputs"][str(lookup)] = expected + plan["implementation_and_policy_inputs"][str(args.bacdive_receipt)] = launcher.identity( + args.bacdive_receipt + ) + plan["fresh_bacdive_lookup"] = { + "path": str(lookup), + **expected, + "receipt": str(args.bacdive_receipt), + } + wrapper = Path(__file__).resolve() + plan["implementation_and_policy_inputs"][str(wrapper)] = launcher.identity(wrapper) + plan["wrapper"] = {"path": str(wrapper), **launcher.identity(wrapper)} + output_root = launcher.safe_path(args.output_root) + launcher.require(output_root.is_dir(), "Worker output root must already exist") + record = output_root / f"chain-{args.unit}.json" + launcher.require(not record.exists(), "Chain receipt already exists; preserve it") + plan["chain_receipt"] = str(record) + plan["java"] = java_provenance({**os.environ, **plan["environment"]}, launcher) + bind_prior_authorities(plan, args, launcher) + plan["java_provenance_issue"] = "https://github.com/Knowledge-Graph-Hub/kg-microbe/issues/1872" + return plan + + +def verify_members(directory, record, launcher): + """Independently compare persisted finalizer member hashes, without inventing a success marker.""" + for section in ("members", "audit_members"): + launcher.require( + isinstance(record.get(section), dict) and bool(record[section]), f"Missing {section}" + ) + for name, expected in record[section].items(): + launcher.require(Path(name).name == name, "Escaping finalizer member name") + launcher.require( + launcher.identity(directory / name) == expected, f"Finalizer member changed: {name}" + ) + + +def run(plan, launcher): + """Run one canonical source, preserve its complete output tree and check upstream nonmutation.""" + source = Path(plan["canonical_source_directory"]) + launcher.require(not source.exists(), "Canonical output appeared after planning") + code = launcher.run(plan) + outputs = inventory(source, launcher) + ancillary = {name: inventory(launcher.ROOT / name, launcher) for name in ANCILLARY} + for directory, members in plan["ancillary_before"].items(): + launcher.require( + all(ancillary[directory].get(name) == expected for name, expected in members.items()), + f"Pre-existing ancillary input changed: {directory}", + ) + upstream_after = { + name: inventory(Path(plan["canonical_output_base"]) / name, launcher) + for name in plan["upstream_before"] + } + launcher.require( + upstream_after == plan["upstream_before"], "Previously completed upstream output changed" + ) + finalized = None + marker = None + if code == 0: + record_path = source / "source_finalization.json" + marker_path = source / "source_fingerprint.json" + launcher.require( + record_path.is_file() and marker_path.is_file(), + "CLI success lacks actual completion evidence", + ) + finalized = json.loads(record_path.read_text()) + marker = json.loads(marker_path.read_text()) + verify_members(source, finalized, launcher) + receipt = { + "profile": "m5-canonical-release-source-unit/v1", + "completed_at": datetime.now(timezone.utc).isoformat(), + "returncode": code, + "plan": plan, + "canonical_source_artifacts": outputs, + "ancillary_after": ancillary, + "ancillary_preexisting_unchanged": True, + "upstream_unchanged": True, + "finalization_record": finalized, + "producer_fingerprint": marker, + "unit_receipt": { + "path": str(Path(plan["stage"]) / "receipt.json"), + **launcher.identity(Path(plan["stage"]) / "receipt.json"), + }, + "merged_graph_verified": False, + "published_release_verified": False, + "scope": "One real source through the canonical CLI output base; independent review precedes the next unit.", + } + launcher.write_new(Path(plan["chain_receipt"]), receipt) + print( + json.dumps( + { + "chain_receipt": plan["chain_receipt"], + "returncode": code, + "source": str(source), + "artifact_count": len(outputs), + } + ) + ) + return code + + +def main(): + """Require a read-only plan or one explicit canonical-layout producer unit.""" + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--plan", action="store_true") + action.add_argument("--run", action="store_true") + parser.add_argument("--kg-root", type=Path, required=True) + parser.add_argument("--input-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--source", choices=tuple(UPSTREAM), required=True) + parser.add_argument("--unit", required=True) + parser.add_argument("--raw-input", action="append", required=True) + parser.add_argument("--bacdive-receipt", type=Path) + args = parser.parse_args() + launcher = load_launcher(args.kg_root) + plan = prepare(args, launcher) + if args.plan: + print(json.dumps({"action": "plan", "writes": 0, **plan}, sort_keys=True, indent=2)) + else: + raise SystemExit(run(plan, launcher)) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_checks_20261116.py b/scripts/adhoc/m5_release_checks_20261116.py new file mode 100644 index 0000000..f0f29cb --- /dev/null +++ b/scripts/adhoc/m5_release_checks_20261116.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Run one recorded M5 KG check through the prepared offline environment. + +Provenance: consumes m5_release_gate_setup_20261116.py's environment record. +This launcher records operational execution only. The repository's unchanged +run_pr_gates.py remains responsible for final-candidate and full-suite gates. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + + +def write(path: Path, record: dict) -> None: + """Atomically update this invocation's live receipt.""" + pending = path.with_suffix(".pending") + pending.write_text(json.dumps(record, indent=2) + "\n") + pending.replace(path) + + +def main() -> int: + """Run a single explicitly supplied command with durable logs.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--environment-record", required=True, type=Path) + parser.add_argument("--kg-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("Supply a command after --") + args.output.mkdir(parents=True, exist_ok=False) + specification = json.loads(args.environment_record.read_text()) + environment = dict(os.environ) + for key in list(environment): + if key in specification["unset"] or any(key.startswith(p) for p in specification["unset_prefixes"]): + environment.pop(key) + environment.update(specification["set"]) + record = {"started_at": datetime.now(timezone.utc).isoformat(), "cwd": str(args.kg_root), + "command": command, "environment_record": str(args.environment_record), + "environment_record_sha256": hashlib.sha256(args.environment_record.read_bytes()).hexdigest(), + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "status": "running"} + receipt = args.output / "receipt.json" + log_path = args.output / "command.log" + started = time.monotonic() + with log_path.open("x") as log: + process = subprocess.Popen(command, cwd=args.kg_root, env=environment, stdout=log, stderr=subprocess.STDOUT) + record["pid"] = process.pid + write(receipt, record) + print(json.dumps({"pid": process.pid, "receipt": str(receipt), "log": str(log_path)}), flush=True) + code = process.wait() + record.update(status="complete" if code == 0 else "failed", exit_code=code, + completed_at=datetime.now(timezone.utc).isoformat(), seconds=round(time.monotonic() - started, 3), + log_sha256=hashlib.sha256(log_path.read_bytes()).hexdigest()) + write(receipt, record) + print(json.dumps(record, indent=2)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_environment_20261116.py b/scripts/adhoc/m5_release_environment_20261116.py new file mode 100644 index 0000000..bc37e1f --- /dev/null +++ b/scripts/adhoc/m5_release_environment_20261116.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Inventory and install the M5 KG lock from its read-only Poetry artifact cache. + +Provenance: the November 16 release worker has a fresh Python 3.10 KG venv, +but online Poetry installation failed DNS. This helper selects main/dev lock +records using the TARGET interpreter's markers and wheel tags. Published +wheels must match poetry.lock SHA256. A locally built cached wheel is accepted +only with an explicit flag, a hash-matched co-located locked source archive, +and a separately recorded wheel hash; co-location is not a build attestation. +Run --plan first. Installation uses only the recorded local wheel paths, +--no-index, --no-deps and --require-hashes, followed by pip check. + +The planner needs Python 3.11+ with packaging, independently of the target +Python 3.10-3.12 environment. It neither downloads nor edits the shared cache. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tomllib +from datetime import datetime, timezone +from pathlib import Path + +from packaging.markers import Marker +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name, parse_wheel_filename + + +def sha256(path: Path) -> str: + """Hash bytes without loading large wheels at once.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def target_runtime(python: Path) -> dict: + """Ask the target's pip-vendored packaging for its actual platform.""" + program = ( + "import json,sys; from pip._vendor.packaging.markers import default_environment; " + "from pip._vendor.packaging.tags import sys_tags; " + "print(json.dumps({'executable':sys.executable,'version':sys.version," + "'markers':default_environment(),'tags':[str(t) for t in sys_tags()]}))" + ) + result = subprocess.run([str(python), "-c", program], check=True, capture_output=True, text=True) + runtime = json.loads(result.stdout) + version = runtime["markers"]["python_version"] + if version not in {"3.10", "3.11", "3.12"}: + raise ValueError(f"Unsupported KG target Python: {version}") + runtime["resolved_executable"] = str(python.resolve()) + runtime["executable_sha256"] = sha256(python.resolve()) + return runtime + + +def selected(package: dict, groups: set[str], environment: dict) -> bool: + """Respect per-group lock markers as well as Python version constraints.""" + active_groups = set(package["groups"]) & groups + if not active_groups: + return False + constraint = package.get("python-versions", "*") + if constraint != "*" and not SpecifierSet(constraint).contains(environment["python_full_version"]): + return False + markers = package.get("markers") + if isinstance(markers, dict): + return any(group not in markers or Marker(markers[group]).evaluate(environment) for group in active_groups) + return not markers or Marker(markers).evaluate(environment) + + +def write_json(path: Path, value: dict) -> None: + """Keep each new record exclusive rather than overwriting old receipts.""" + with path.open("x") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def plan(args: argparse.Namespace) -> int: + """Inventory compatible cache bytes without installing anything.""" + args.output.mkdir(parents=True, exist_ok=False) + runtime = target_runtime(args.python) + lock_path = args.kg_root / "poetry.lock" + lock = tomllib.loads(lock_path.read_text()) + groups = set(args.groups.split(",")) + packages = [p for p in lock["package"] if selected(p, groups, runtime["markers"])] + cache_files = sorted(path for path in args.cache.rglob("*") if path.is_file()) + wheel_index: dict[tuple[str, str], list[Path]] = {} + for path in cache_files: + if path.suffix != ".whl": + continue + name, version, _, tags = parse_wheel_filename(path.name) + if set(map(str, tags)) & set(runtime["tags"]): + wheel_index.setdefault((canonicalize_name(name), str(version)), []).append(path) + records = [] + missing = [] + for package in packages: + name, version = package["name"], package["version"] + locked_files = {entry["file"]: entry["hash"].removeprefix("sha256:") for entry in package["files"]} + candidates = wheel_index.get((canonicalize_name(name), version), []) + choices = [] + for path in candidates: + digest = sha256(path) + record = {"name": name, "version": version, "path": str(path), "sha256": digest} + if locked_files.get(path.name) == digest: + record["status"] = "lock_matched_wheel" + choices.append(record) + continue + sources = [] + for source_name, source_digest in locked_files.items(): + source = path.parent / source_name + if source.suffix != ".whl" and source.is_file() and sha256(source) == source_digest: + sources.append({"path": str(source), "sha256": source_digest}) + if sources and args.allow_cached_built_wheels: + record.update(status="cached_built_wheel", locked_sources=sources, + provenance_limit="Co-located pinned source; cached build not independently attested.") + choices.append(record) + if choices: + choices.sort(key=lambda record: (record["status"] != "lock_matched_wheel", record["path"])) + records.append(choices[0]) + else: + missing.append({"name": name, "version": version, "compatible_cached_wheels": list(map(str, candidates)), + "locked_filenames": list(locked_files)}) + names = [canonicalize_name(record["name"]) for record in records] + if len(names) != len(set(names)): + raise ValueError("Multiple selected lock versions for a package; refuse ambiguous installation") + requirements = "".join(f"{Path(r['path']).as_uri()} --hash=sha256:{r['sha256']}\n" for r in records) + requirements_path = args.output / "requirements.txt" + with requirements_path.open("x") as handle: + handle.write(requirements) + inventory = { + "created_at": datetime.now(timezone.utc).isoformat(), "mode": "offline_inventory", + "planner_python": sys.version, "target": runtime, "groups": sorted(groups), + "kg_root": str(args.kg_root), "lock_path": str(lock_path), "lock_sha256": sha256(lock_path), + "helper_sha256": sha256(Path(__file__)), "cache": str(args.cache), + "cache_files_enumerated": len(cache_files), "selected_lock_packages": len(packages), + "requirements_sha256": sha256(requirements_path), "packages": records, "missing": missing, + "allow_cached_built_wheels": args.allow_cached_built_wheels, + } + write_json(args.output / "inventory.json", inventory) + print(json.dumps({"output": str(args.output), "selected": len(packages), "available": len(records), + "cached_built": [r["name"] for r in records if r["status"] == "cached_built_wheel"], + "missing": [{"name": p["name"], "version": p["version"]} for p in missing]}, indent=2)) + return 2 if missing else 0 + + +def install(args: argparse.Namespace) -> int: + """Install only a complete, unchanged, previously inspected inventory.""" + inventory_path = args.output / "inventory.json" + inventory = json.loads(inventory_path.read_text()) + if inventory["missing"]: + raise ValueError("Inventory incomplete; installation is blocked") + runtime = target_runtime(args.python) + if runtime != inventory["target"]: + raise ValueError("Target interpreter changed after planning") + if sha256(Path(inventory["lock_path"])) != inventory["lock_sha256"]: + raise ValueError("Lock changed after planning") + requirements = args.output / "requirements.txt" + if sha256(requirements) != inventory["requirements_sha256"]: + raise ValueError("Requirements changed after planning") + for record in inventory["packages"]: + if sha256(Path(record["path"])) != record["sha256"]: + raise ValueError(f"Wheel changed after planning: {record['name']}") + for source in record.get("locked_sources", []): + if sha256(Path(source["path"])) != source["sha256"]: + raise ValueError(f"Locked source changed: {record['name']}") + scratch = args.output / "pip-scratch" + scratch.mkdir(exist_ok=False) + environment = dict(os.environ, TMPDIR=str(scratch)) + command = [str(args.python), "-m", "pip", "--isolated", "--disable-pip-version-check", "install", + "--no-index", "--no-cache-dir", "--no-deps", "--require-hashes", "-r", str(requirements)] + with (args.output / "install.log").open("x") as log: + process = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, env=environment) + check = subprocess.run([str(args.python), "-m", "pip", "--isolated", "check"], capture_output=True, text=True) + installed = subprocess.run([str(args.python), "-m", "pip", "--isolated", "list", "--format=json"], + check=True, capture_output=True, text=True) + receipt = {"completed_at": datetime.now(timezone.utc).isoformat(), "command": command, + "inventory_sha256": sha256(inventory_path), "returncode": process.returncode, + "pip_check_returncode": check.returncode, "pip_check": check.stdout + check.stderr, + "installed": json.loads(installed.stdout), "install_log_sha256": sha256(args.output / "install.log")} + write_json(args.output / "install-receipt.json", receipt) + print(json.dumps({key: value for key, value in receipt.items() if key != "installed"}, indent=2)) + return process.returncode or check.returncode + + +def main() -> int: + """Keep planning and mutation explicit and separate.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kg-root", required=True, type=Path) + parser.add_argument("--python", required=True, type=Path) + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--groups", default="main,dev") + parser.add_argument("--allow-cached-built-wheels", action="store_true") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", action="store_true") + mode.add_argument("--install", action="store_true") + args = parser.parse_args() + for name in ("kg_root", "python", "cache", "output"): + setattr(args, name, getattr(args, name).absolute()) + return plan(args) if args.plan else install(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_gate_retry_20261116.py b/scripts/adhoc/m5_release_gate_retry_20261116.py new file mode 100644 index 0000000..9f01541 --- /dev/null +++ b/scripts/adhoc/m5_release_gate_retry_20261116.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Record an environment-only retry after M5 gate scratch-context failures. + +2026-11-16 provenance: the first standard gate put pytest scratch inside KG, +changing path normalization and Git discovery. Use durable scratch outside any +checkout; preserve the old record and all failures. No gate, assertions, source, +tox selectors, or filesystem guards are changed. Required results stay in OUT. +""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + + +def durable_scratch(path): + """Reject exact temporary roots and every descendant after resolving aliases (#162).""" + scratch = path.resolve(strict=True) + forbidden = (Path("/tmp").resolve(), Path("/private/tmp"), Path("/private/var/folders")) + if not scratch.is_dir() or any(scratch == root or scratch.is_relative_to(root) for root in forbidden): + raise ValueError("Use a durable existing scratch directory") + return scratch + + +def main(): + """Write a versioned environment record after verifying the scratch boundary.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--previous", required=True, type=Path) + parser.add_argument("--scratch-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + scratch = durable_scratch(args.scratch_root) + except (ValueError, OSError) as error: + parser.error(str(error)) + probe = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=scratch, + text=True, capture_output=True, check=False) + if probe.returncode != 128 or "not a git repository" not in probe.stderr: + parser.error("Scratch must be outside all Git checkouts") + before = args.previous.read_bytes() + environment = json.loads(before) + previous_tmp = environment["set"]["TMPDIR"] + environment["set"]["TMPDIR"] = str(scratch) + args.output.mkdir(parents=True, exist_ok=False) + destination = args.output / "environment.json" + destination.write_text(json.dumps(environment, indent=2) + "\n") + receipt = {"previous": str(args.previous), "previous_sha256": hashlib.sha256(before).hexdigest(), + "environment": str(destination), + "environment_sha256": hashlib.sha256(destination.read_bytes()).hexdigest(), + "delta": {"set.TMPDIR": {"before": previous_tmp, "after": str(scratch)}}, + "git_probe": {"returncode": probe.returncode, "stderr": probe.stderr.strip()}, + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "scope": "Environment-only preparation; no test or gate executed."} + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_gate_setup_20261116.py b/scripts/adhoc/m5_release_gate_setup_20261116.py new file mode 100644 index 0000000..54726cd --- /dev/null +++ b/scripts/adhoc/m5_release_gate_setup_20261116.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Prepare isolated offline KG PR gates from retained M5 package bytes. + +Provenance: use the November 16 environment inventory for all KG lock wheels. +The two unpinned tox-only tools are reconstructed from the canonical checkout's +installed pure-Python distributions, verifying each retained file against its +installed RECORD. Reconstructed hashes are local artifact hashes, not claimed +PyPI wheel hashes. Original environments and caches are read-only. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import io +import json +import os +import shutil +import subprocess +import zipfile +from datetime import datetime, timezone +from pathlib import Path + + +def digest(path: Path) -> str: + """Hash a local file without changing it.""" + with path.open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def repack(source: Path, wheelhouse: Path) -> dict: + """Reconstruct a pure-Python wheel from RECORD-verified installed files.""" + package = source.parent + wheel_text = (source / "WHEEL").read_text() + if "Root-Is-Purelib: true" not in wheel_text or "Tag: py3-none-any" not in wheel_text: + raise ValueError(f"Unsupported retained tool wheel: {source.name}") + stem = source.name.removesuffix(".dist-info") + destination = wheelhouse / f"{stem}-py3-none-any.whl" + retained = [] + excluded = [] + for relative, recorded_hash, size in csv.reader((source / "RECORD").read_text().splitlines()): + path = Path(relative) + if ".." in path.parts or path.suffix == ".pyc" or path.name in {"RECORD", "INSTALLER", "REQUESTED"}: + excluded.append(relative) + continue + if path.is_absolute() or not recorded_hash.startswith("sha256="): + raise ValueError(f"Unverifiable retained file: {relative}") + resolved = (package / path).resolve() + if not resolved.is_relative_to(package.resolve()): + raise ValueError(f"Retained file escapes site-packages: {relative}") + content = resolved.read_bytes() + actual = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + if recorded_hash != "sha256=" + actual or int(size) != len(content): + raise ValueError(f"Installed RECORD mismatch: {relative}") + retained.append((relative, content, recorded_hash)) + record_path = f"{source.name}/RECORD" + record = io.StringIO(newline="") + writer = csv.writer(record, lineterminator="\n") + for name, content, hash_value in sorted(retained): + writer.writerow([name, hash_value, len(content)]) + writer.writerow([record_path, "", ""]) + retained.append((record_path, record.getvalue().encode(), "")) + with zipfile.ZipFile(destination, "x", compression=zipfile.ZIP_DEFLATED) as wheel: + for name, content, _ in sorted(retained): + member = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + member.compress_type = zipfile.ZIP_DEFLATED + member.external_attr = 0o644 << 16 + wheel.writestr(member, content) + return {"path": str(destination), "sha256": digest(destination), "source_dist_info": str(source), + "source_record_sha256": digest(source / "RECORD"), "verified_source_files": len(retained) - 1, + "excluded_installer_files": excluded, "provenance": "Reconstructed from installed RECORD-verified files"} + + +def setup(args: argparse.Namespace) -> int: + """Prepare an exclusive wheelhouse, environment record and read-only probes.""" + args.output.mkdir(parents=True, exist_ok=False) + setup_root = args.kg_root / "data/validation/pr-gates/m5-setup" + setup_root.mkdir(parents=True, exist_ok=False) + wheelhouse = setup_root / "wheelhouse" + wheelhouse.mkdir() + inventory = json.loads(args.inventory.read_text()) + if inventory["missing"]: + raise ValueError("KG lock inventory is incomplete") + copied = [] + for item in inventory["packages"]: + source = Path(item["path"]) + if digest(source) != item["sha256"]: + raise ValueError(f"Cached wheel changed: {source.name}") + destination = wheelhouse / source.name + if destination.exists(): + raise FileExistsError(destination) + shutil.copyfile(source, destination) + if digest(destination) != item["sha256"]: + raise ValueError(f"Copied wheel mismatch: {source.name}") + copied.append({"path": str(destination), "sha256": item["sha256"], "source": str(source)}) + canonical = args.canonical_root / ".tox" + repacked = [repack(canonical / "codespell/lib/python3.10/site-packages/codespell-2.4.3.dist-info", wheelhouse), + repack(canonical / "docstr-coverage/lib/python3.10/site-packages/docstr_coverage-2.3.2.dist-info", wheelhouse)] + constraints = setup_root / "constraints.txt" + constraints.write_text("".join(f"{item['name']}=={item['version']}\n" for item in inventory["packages"]) + + "codespell==2.4.3\ndocstr-coverage==2.3.2\n") + # The final runner intentionally strips TOX* selectors. A private .tox link + # keeps its standard unmodified command's physical output under data/. + tox_link = args.kg_root / ".tox" + if os.path.lexists(tox_link): + raise FileExistsError(f"Preserve pre-existing tox output: {tox_link}") + tox_output = setup_root / "tox" + tox_output.mkdir() + tox_link.symlink_to(tox_output, target_is_directory=True) + for name in ("tmp", "pip-cache", "virtualenv", "poetry-cache", "xdg-cache", "matplotlib"): + (setup_root / name).mkdir() + environment = { + "VIRTUAL_ENV": str(args.kg_root / ".venv310"), + "PATH": str(args.kg_root / ".venv310/bin") + ":/opt/homebrew/bin:/usr/bin:/bin", + "PIP_NO_INDEX": "1", "PIP_FIND_LINKS": str(wheelhouse), "PIP_CONSTRAINT": str(constraints), + "PIP_CACHE_DIR": str(setup_root / "pip-cache"), "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "VIRTUALENV_OVERRIDE_APP_DATA": str(setup_root / "virtualenv"), + "VIRTUALENV_NO_PERIODIC_UPDATE": "1", "VIRTUALENV_DOWNLOAD": "0", + "POETRY_CACHE_DIR": str(setup_root / "poetry-cache"), "TMPDIR": str(setup_root / "tmp"), + "XDG_CACHE_HOME": str(setup_root / "xdg-cache"), "MPLCONFIGDIR": str(setup_root / "matplotlib"), + } + unset = ["CONDA_DEFAULT_ENV", "CONDA_PREFIX", "PYTEST_ADDOPTS", "POETRY_VIRTUALENVS_IN_PROJECT"] + effective = dict(os.environ) + for key in list(effective): + if key in unset or key.startswith("TOX"): + effective.pop(key) + effective.update(environment) + with (args.output / "environment.json").open("x") as handle: + json.dump({"set": environment, "unset": unset, "unset_prefixes": ["TOX"]}, handle, indent=2) + handle.write("\n") + commands = [ + ["poetry", "run", "python", "-c", "import sys; print(sys.version); print(sys.executable)"], + ["poetry", "env", "info", "--path"], + ["poetry", "run", "tox", "--version"], + ["poetry", "run", "tox", "c", "-e", "codespell", "-k", "pass_env"], + ["poetry", "run", "python", "scripts/run_pr_gates.py", "m5-final-pending", "--dry-run"], + ] + results = [] + for index, command in enumerate(commands): + result = subprocess.run(command, cwd=args.kg_root, env=effective, capture_output=True, text=True) + log = args.output / f"probe-{index + 1}.log" + with log.open("x") as handle: + handle.write(result.stdout + result.stderr) + results.append({"command": command, "returncode": result.returncode, "log": str(log), "sha256": digest(log)}) + receipt = {"completed_at": datetime.now(timezone.utc).isoformat(), "helper_sha256": digest(Path(__file__)), + "inventory_sha256": digest(args.inventory), "setup_root": str(setup_root), + "copied_wheels": copied, "reconstructed_tool_wheels": repacked, + "constraints_sha256": digest(constraints), "probes": results, + "scope": "Environment preparation only; no source formatting or full test suite executed."} + with (args.output / "setup-receipt.json").open("x") as handle: + json.dump(receipt, handle, indent=2) + handle.write("\n") + print(json.dumps({"setup_root": str(setup_root), "copied_wheels": len(copied), + "reconstructed_tools": repacked, "probes": results}, indent=2)) + return int(any(result["returncode"] for result in results)) + + +def main() -> int: + """Parse explicit task-only setup paths.""" + parser = argparse.ArgumentParser(description=__doc__) + for name in ("kg-root", "canonical-root", "inventory", "output"): + parser.add_argument("--" + name, required=True, type=Path) + return setup(parser.parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/m5_release_lineage_20261116.py b/scripts/adhoc/m5_release_lineage_20261116.py new file mode 100644 index 0000000..87b3d3d --- /dev/null +++ b/scripts/adhoc/m5_release_lineage_20261116.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Inspect bounded retained source lineage for the M5 BactoTraits correction. + +2026-11-16: read only the original CSV, the declared BacDive lookup, the old +BactoTraits intermediates and three explicitly selected source edge files. +Also inspect the distinct Madin source row for an equivalent historical edge. +Keep occurrence ordinals and hashes. This does not establish graph-wide absence, +current release freshness, trait falsity or a replacement organism identity. +The selected files are inspected directly, including ignored data; no discovery +walk or remote request is performed. Results use an exclusive output directory. +""" + +import argparse +import csv +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path + +CANONICAL = Path("/Users/marcin/Documents/VIMSS/ontology/KG-Hub/KG-Microbe/kg-microbe") +CODE = Path("/Users/marcin/agent-work/EdgeMech/m5-kg-bactotraits-release-20261116") +SOURCE_SHA = "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" +ROW_SHA = "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a" +SUBJECT = "NCBITaxon:1123071" +OBJECT = "METPO:1000703" + + +def sha256(path): + """Hash a selected regular file without retaining its contents in memory.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def identity(path): + """Record content and filesystem identity for one explicitly selected input.""" + before = path.stat() + result = { + "path": str(path), + "bytes": before.st_size, + "mtime_ns": before.st_mtime_ns, + "inode": before.st_ino, + "sha256": sha256(path), + } + after = path.stat() + assert (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) == ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ), f"Input changed while hashing: {path}" + return result + + +def rows(path, *, raw=False, delimiter="\t"): + """Stream exact CSV/TSV records with data-row ordinals, including duplicates.""" + with path.open(encoding="ISO-8859-1" if raw else "utf-8", newline="") as stream: + reader = csv.reader(stream, delimiter=";" if raw else delimiter) + if raw: + next(reader) + next(reader) + header = next(reader) + assert len(header) == len(set(header)), f"Duplicate header: {path}" + for ordinal, values in enumerate(reader, 1): + assert len(values) == len(header), f"Malformed row {ordinal}: {path}" + yield ordinal, dict(zip(header, values, strict=True)) + + +def scan(path, selector, *, raw=False, delimiter="\t"): + """Retain matching complete records and verify source identity after reading.""" + pin = identity(path) + matches = [] + count = 0 + for count, row in rows(path, raw=raw, delimiter=delimiter): + if selector(row): + matches.append({"source_data_row": count, "row": row}) + assert identity(path) == pin, f"Input changed during scan: {path}" + return { + "input": pin, + "data_rows": count, + "matches": matches, + "matched_occurrences": len(matches), + } + + +def inspect(root): + """Trace the reviewed selector, accession occurrences and retained joins.""" + raw = scan( + root / "data/raw/BactoTraits_databaseV2_Jun2022.csv", + lambda row: ( + row["Bacdive_ID"] == "17851" + or "19353" in row["culture collection codes"] + or "14586" in row["culture collection codes"] + ), + raw=True, + ) + assert raw["input"]["sha256"] == SOURCE_SHA, "Original BactoTraits bytes changed" + selected = [m for m in raw["matches"] if m["row"]["Bacdive_ID"] == "17851"] + assert len(selected) == 1 and selected[0]["source_data_row"] == 8521 + raw_row = selected[0]["row"] + row_digest = hashlib.sha256( + json.dumps(raw_row, ensure_ascii=False, indent=2, sort_keys=True).encode() + ).hexdigest() + assert row_digest == ROW_SHA, "Reviewed complete original row changed" + lookup = scan( + root / "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv", + lambda row: row["bacdive_id"] == "bacdive:17851" or row["ncbitaxon_id"] == SUBJECT, + ) + intermediates = {} + for filename in ("bactotraits.tsv", "bactotraits_mapping.tsv"): + intermediates[filename] = scan( + root / "kg_microbe/transform_utils/bactotraits/tmp" / filename, + lambda row: ( + row.get("Bacdive_ID") in ("17851", "bacdive:17851") + or row.get("ncbitaxon_id") == SUBJECT + ), + ) + products = {} + for source in ("bactotraits", "bacdive", "madin_etal"): + item = scan( + root / "data/transformed" / source / "edges.tsv", + lambda row: row.get("subject") == SUBJECT and row.get("object") == OBJECT, + ) + products[source] = item + madin = scan( + root / "data/raw/madin_etal.csv", lambda row: row["tax_id"] == "1123071", delimiter="," + ) + implementations = { + name: identity(CODE / name) + for name in ( + "kg_microbe/transform_utils/bactotraits/bactotraits.py", + "kg_microbe/transform_utils/bactotraits/identity_quarantine.py", + "kg_microbe/transform_utils/madin_etal/madin_etal.py", + "mappings/canonical/bactotraits_identity_quarantine.json", + "merge.yaml", + ) + } + return { + "profile": "m5-bactotraits-retained-lineage/v1", + "completed_at": datetime.now(timezone.utc).isoformat(), + "helper_sha256": sha256(Path(__file__)), + "raw_source": raw, + "selected_complete_row_sha256": row_digest, + "bacdive_lookup": lookup, + "historical_intermediates": intermediates, + "historical_source_graph_occurrences": products, + "independent_madin_raw_occurrences": madin, + "inspected_implementation_and_config": implementations, + "scope": "Selected original CSV, lookup, two old intermediates, three old source edge files and Madin raw CSV; ignored files included by direct path.", + "graph_wide_absence_verified": False, + "current_source_freshness_verified": False, + "published_release_verified": False, + "scientific_trait_truth_resolved": False, + "neighboring_rows_quarantined": False, + } + + +def main(): + """Plan or execute one bounded source lineage inspection.""" + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_mutually_exclusive_group(required=True) + actions.add_argument("--plan", action="store_true") + actions.add_argument("--run", action="store_true") + parser.add_argument("--input-root", type=Path, default=CANONICAL) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + parser.error("Output already exists; select a fresh exclusive directory") + csv.field_size_limit(16 * 1024 * 1024) + if args.plan: + print( + json.dumps( + { + "action": "plan", + "input_root": str(args.input_root), + "output": str(args.output), + "network_calls": 0, + "source_files": 8, + "concurrency": 1, + "writes": 0, + } + ) + ) + return + os.umask(0o077) + args.output.mkdir(parents=True, mode=0o700) + result = inspect(args.input_root) + receipt = args.output / "receipt.json" + with receipt.open("x") as stream: + json.dump(result, stream, indent=2) + stream.write("\n") + print( + json.dumps( + { + "receipt": str(receipt), + "sha256": sha256(receipt), + "matches_by_source": { + key: value["matched_occurrences"] + for key, value in result["historical_source_graph_occurrences"].items() + }, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_merge_20261116.py b/scripts/adhoc/m5_release_merge_20261116.py new file mode 100644 index 0000000..0ed611c --- /dev/null +++ b/scripts/adhoc/m5_release_merge_20261116.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Verify one strict M5 BactoTraits merge after all canonical producers complete. + +2026-11-16 provenance: this is the ordinary KG merge CLI, selecting only the +canonical bactotraits source and one process. No finalization/freshness bypass +or low-level merge is used. All three actual source receipts and their complete +persisted trees are required. The canonical configuration retains its semantics; +source filenames become absolute and output-only filenames move to this unit's +exclusive directory, including stats (EdgeMech #160). Historical outputs remain +read-only. Plan before run; neither a plan nor this bounded candidate is a +full-snapshot release, publication, or scientific graph-wide validation. +""" + +from __future__ import annotations + +import argparse +import copy +import csv +import hashlib +import importlib.metadata +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +SOURCES = ("ontologies", "bacdive", "bactotraits") +SOURCE_SHA = "18316d7a0b6c2bef6fcb96a7659c2e3f3734c14786f383c2a93fefe2ae226a4b" +ROW_SHA = "345fd2d00ee93334d975085716692dd4d4c530d5529f04c5b07af0766d5a512a" +SPO = ("NCBITaxon:1123071", "biolink:has_phenotype", "METPO:1000703") +AUDIT = "bactotraits_identity_quarantine.tsv" +THREADS = ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS", + "NUMEXPR_MAX_THREADS", "VECLIB_MAXIMUM_THREADS", "RAYON_NUM_THREADS", "BLIS_NUM_THREADS") + + +def require(condition, message): + """Refuse incomplete or changed evidence explicitly, including under python -O.""" + if not condition: + raise ValueError(message) + + +def safe(path): + """Require a literal path without parent traversal or symlinked components.""" + path = Path(path).absolute() + require(".." not in path.parts, "Parent traversal is not permitted") + require(not any(item.is_symlink() for item in (path, *path.parents)), f"Symlinked path: {path}") + return path + + +def identity(path): + """Stream exact bytes and reject changes during hashing.""" + path = safe(path) + require(path.is_file(), f"Required file missing: {path}") + before = path.stat() + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + after = path.stat() + fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + require(all(getattr(before, key) == getattr(after, key) for key in fields), f"File changed while reading: {path}") + return {"bytes": before.st_size, "sha256": digest.hexdigest()} + + +def snapshot(directory): + """Inventory a named producer tree, including nested ignored files and rejecting links.""" + directory = safe(directory) + require(directory.is_dir(), f"Required producer directory missing: {directory}") + result = {} + for path in sorted(directory.rglob("*")): + require(not path.is_symlink(), f"Symlink in producer tree: {path}") + if path.is_file(): + result[str(path.relative_to(directory))] = identity(path) + return result + + +def read_json(path): + """Read bounded receipts without accepting duplicate JSON keys.""" + def pairs(items): + result = {} + for key, value in items: + require(key not in result, f"Duplicate JSON key: {key}") + result[key] = value + return result + + return json.loads(safe(path).read_text(), object_pairs_hook=pairs) + + +def write_new(path, value): + """Preserve every earlier receipt using exclusive creation.""" + with path.open("x") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + + +def bind(pins, path, expected=None): + """Bind an input once and reject contradictory prior byte identities.""" + path = safe(path) + observed = identity(path) + if expected is not None: + require(observed == expected, f"Receipt-bound file changed: {path}") + require(str(path) not in pins or pins[str(path)] == observed, f"Conflicting input identity: {path}") + pins[str(path)] = observed + return observed + + +def verify_chain(kg_root, receipt_paths): + """Require real whole-source completion and an unchanged recursive upstream chain.""" + receipts, pins = {}, {} + for source in SOURCES: + path = safe(receipt_paths[source]) + bind(pins, path) + receipt = read_json(path) + require(receipt.get("profile") == "m5-canonical-release-source-unit/v1", "Unsupported source receipt") + require(receipt.get("returncode") == 0, f"{source}: producer did not complete") + plan = receipt["plan"] + require(plan.get("source") == source, f"{source}: wrong producer receipt") + directory = kg_root / "data/transformed" / source + require(plan.get("canonical_source_directory") == str(directory), f"{source}: noncanonical output") + require(receipt.get("upstream_unchanged") is True and receipt.get("ancillary_preexisting_unchanged") is True, + f"{source}: preservation checks are incomplete") + recorded = receipt["canonical_source_artifacts"] + require(snapshot(directory) == recorded, f"{source}: persisted source tree changed") + for member in ("source_finalization.json", "source_fingerprint.json"): + require(member in recorded, f"{source}: whole-source completion missing") + finalization = read_json(directory / "source_finalization.json") + marker = read_json(directory / "source_fingerprint.json") + require(finalization == receipt.get("finalization_record") and finalization.get("source") == source, + f"{source}: finalization receipt mismatch") + require(marker == receipt.get("producer_fingerprint") and marker.get("version") == 3, + f"{source}: producer fingerprint mismatch") + for section in ("members", "audit_members"): + require(bool(finalization.get(section)), f"{source}: missing finalization {section}") + for name, expected in finalization[section].items(): + require(Path(name).name == name and name in recorded, "Invalid finalization member") + require(recorded[name] == expected, f"{source}: finalization member differs: {name}") + if source == "ontologies": + require("ncbitaxon_nodes.tsv" in finalization["members"] + and "chebi_nodes.tsv" in finalization["members"], "Scoped ontology output cannot qualify") + unit_ref = receipt["unit_receipt"] + bind(pins, Path(unit_ref["path"]), {key: unit_ref[key] for key in ("bytes", "sha256")}) + unit = read_json(Path(unit_ref["path"])) + require(unit.get("returncode") == 0 and unit.get("source") == source, "Underlying CLI unit incomplete") + require(unit.get("git_revision") == plan.get("git_revision"), "Producer revision mismatch") + for name, expected in plan["implementation_and_policy_inputs"].items(): + bind(pins, kg_root / name, expected) + for original in plan["originals"]: + bind(pins, Path(original["source"]), {key: original[key] for key in ("bytes", "sha256")}) + for upstream in SOURCES[:SOURCES.index(source)]: + require(plan["upstream_before"].get(upstream) == receipts[upstream]["canonical_source_artifacts"], + f"{source}: upstream {upstream} is not the completed receipt") + receipts[source] = receipt + latest = read_json(Path(receipts["bactotraits"]["unit_receipt"]["path"])) + for name, expected in latest["task_raw_after"].items(): + require(Path(name).name == name, "Invalid raw snapshot member") + bind(pins, kg_root / "data/raw" / name, expected) + for directory, files in receipts["bactotraits"]["ancillary_after"].items(): + for name, expected in files.items(): + bind(pins, kg_root / directory / name, expected) + return receipts, pins + + +def relocated_configuration(canonical, kg_root, output): + """Change only explicit path placement; preserve source options and graph semantics.""" + require(isinstance(canonical, dict), "Invalid canonical merge configuration") + require(canonical.get("configuration", {}).get("allow_unfinalized_sources", False) is False, + "Unfinalized source opt-out is forbidden") + result = copy.deepcopy(canonical) + deltas = [] + + def replace(container, key, value, path): + previous = container[key] + container[key] = value + deltas.append({"path": path, "before": previous, "after": value}) + + replace(result["configuration"], "output_directory", str(output), ["configuration", "output_directory"]) + sources = result["merged_graph"]["source"] + require("bactotraits" in sources, "Canonical BactoTraits source missing") + for name, source in sources.items(): + inputs = source["input"] + filenames = inputs["filename"] + require(isinstance(filenames, list) and all(isinstance(value, str) for value in filenames), + "Canonical source filenames must be a list") + resolved = [] + for filename in filenames: + path = Path(filename) + require(not path.is_absolute() and ".." not in path.parts, "Noncanonical source path") + resolved.append(str(safe(kg_root / path))) + replace(inputs, "filename", resolved, ["merged_graph", "source", name, "input", "filename"]) + groups = [(["merged_graph", "operations"], result["merged_graph"].get("operations", []))] + groups += [(["merged_graph", "source", name, "operations"], source.get("operations", [])) + for name, source in sources.items()] + targets = set() + for prefix, operations in groups: + for index, operation in enumerate(operations): + args = operation.get("args", {}) + if "filename" not in args: + continue + require(operation.get("name") == "kgx.graph_operations.summarize_graph.generate_graph_stats", + "Unreviewed output-producing merge operation") + filename = Path(args["filename"]) + require(filename.name == str(filename), "Noncanonical operation output filename") + target = str(output / filename.name) + require(target not in targets, "Operation output paths collide") + targets.add(target) + replace(args, "filename", target, [*prefix, index, "args", "filename"]) + # Reversing every declared path-only delta must reconstruct the canonical object. + recovered = copy.deepcopy(result) + for delta in reversed(deltas): + node = recovered + for key in delta["path"][:-1]: + node = node[key] + node[delta["path"][-1]] = delta["before"] + require(recovered == canonical, "Merge semantics changed beyond recorded path placement") + return result, deltas + + +def literal_rows(path, delimiter="\t", encoding="utf-8", skip=0, quoting=csv.QUOTE_MINIMAL): + """Stream well-formed literal records and preserve their data-row ordinal.""" + with path.open(encoding=encoding, newline="") as stream: + reader = csv.reader(stream, delimiter=delimiter, quoting=quoting) + for _ in range(skip): + next(reader) + header = next(reader) + require(len(header) == len(set(header)), f"Duplicate table headers: {path}") + for ordinal, values in enumerate(reader, 1): + require(len(values) == len(header), f"Malformed row {ordinal}: {path}") + yield ordinal, dict(zip(header, values)) + + +def verify_quarantine(kg_root): + """Replay the complete quarantined row, fresh joins and finalized source edge stream.""" + raw = kg_root / "data/raw/BactoTraits_databaseV2_Jun2022.csv" + require(identity(raw)["sha256"] == SOURCE_SHA, "BactoTraits raw source changed") + selected = [(ordinal, row) for ordinal, row in literal_rows(raw, ";", "ISO-8859-1", 2) + if row.get("Bacdive_ID") == "17851"] + require(len(selected) == 1 and selected[0][0] == 8521, "Reviewed raw occurrence is not unique row8521") + row = selected[0][1] + row_sha = hashlib.sha256(json.dumps(row, ensure_ascii=False, indent=2, sort_keys=True).encode()).hexdigest() + require(row_sha == ROW_SHA, "Complete reviewed source row changed") + audit = kg_root / "data/transformed/bactotraits" / AUDIT + audits = list(literal_rows(audit)) + require(len(audits) == 1, "Expected exactly one complete quarantine audit occurrence") + record = audits[0][1] + expected = {"source_data_row": "8521", "reviewed_source_data_row": "8521", "bacdive_id": "17851", + "ncbitaxon_id": SPO[0], "source_sha256": SOURCE_SHA, "reviewed_source_sha256": SOURCE_SHA, + "row_sha256": ROW_SHA, "disposition": "quarantine_mixed_strain_row"} + require(all(record.get(key) == value for key, value in expected.items()), "Quarantine audit identity mismatch") + require(json.loads(record["raw_row_json"]) == row, "Quarantine audit omits or changes original row fields") + policy = kg_root / "mappings/canonical/bactotraits_identity_quarantine.json" + require(record.get("policy_sha256") == identity(policy)["sha256"], "Quarantine policy identity mismatch") + decisions = read_json(policy).get("decisions", []) + require(len(decisions) == 1 and decisions[0].get("bacdive_id") == "17851", + "Policy must retain only the independently reviewed row") + decision = decisions[0] + for field in ("rule_id", "full_name", "culture_collection_codes", "rationale"): + require(record.get(field) == decision.get(field), f"Quarantine audit policy field mismatch: {field}") + require(record.get("full_name") == row.get("Full_name") + and record.get("culture_collection_codes") == row.get("culture collection codes"), + "Quarantine audit identity columns differ from complete original row") + require(record.get("evidence_urls") == "|".join(decision.get("evidence_urls", [])), + "Quarantine audit evidence URLs differ from policy") + require(bool(record.get("rule_id")) and bool(record.get("evidence_urls")) and bool(record.get("rationale")), + "Quarantine audit lacks decision evidence") + joins = {} + for source, filename, key in (("bacdive", "bacdive.tsv", "bacdive_id"), + ("bactotraits", "bactotraits_mapping.tsv", "Bacdive_ID"), + ("bactotraits", "bactotraits.tsv", "Bacdive_ID")): + path = kg_root / "kg_microbe/transform_utils" / source / "tmp" / filename + rows = [{"source_data_row": ordinal, "row": values} for ordinal, values in literal_rows(path) + if values.get(key) in ("17851", "bacdive:17851") or values.get("ncbitaxon_id") == SPO[0]] + target = [item for item in rows if item["row"].get(key) in ("17851", "bacdive:17851")] + if filename == "bactotraits.tsv": + require(not target, "Quarantined source row leaked into retained BactoTraits intermediate") + else: + require(len(target) == 1 and target[0]["row"].get("ncbitaxon_id") == SPO[0], + "Fresh source lookup does not preserve the reviewed exact identity") + joins[f"{source}/{filename}"] = {"identity": identity(path), "matching_occurrences": rows} + with (kg_root / "data/transformed/bactotraits/edges.tsv").open("rb") as stream: + edges = inspect_tsv(stream, edges=True) + require(edges["target_spo_occurrences"] == 0, + "Target SPO remains in BactoTraits-only source output; independent lineage review is required") + return {"audit": {"path": str(audit), **identity(audit), "rows": 1, "record": record}, + "source_row_sha256": row_sha, "fresh_joins": joins, "source_edges": edges} + + +def inspect_tsv(stream, *, edges=False): + """Hash literal persisted TSV bytes and count targeted provenance while streaming.""" + digest, size, lines, tail = hashlib.sha256(), 0, 0, b"" + + def decoded(): + nonlocal size, lines, tail + for line in stream: + digest.update(line) + size += len(line) + lines += line.count(b"\n") + tail = line[-1:] + yield line.decode("utf-8") + + reader = csv.reader(decoded(), delimiter="\t", quoting=csv.QUOTE_NONE) + header = next(reader, None) + require(header and len(header) == len(set(header)), "Missing or duplicate persisted TSV header") + needed = ("subject", "predicate", "object", "primary_knowledge_source") if edges else () + require(set(needed).issubset(header), "Persisted edge provenance columns missing") + count, target_count, bactotraits_count, examples = 0, 0, 0, [] + for count, values in enumerate(reader, 1): + require(len(values) == len(header), "Malformed persisted TSV row") + if edges: + row = dict(zip(header, values)) + if tuple(row[key] for key in ("subject", "predicate", "object")) == SPO: + target_count += 1 + if "infores:bactotraits" in row["primary_knowledge_source"].split("|"): + bactotraits_count += 1 + if len(examples) < 10: + examples.append({"source_data_row": count, "row": row}) + result = {"bytes": size, "sha256": digest.hexdigest(), "rows": count} + require(count == max(0, lines + int(bool(tail) and tail != b"\n") - 1), "Physical/parsed TSV row counts differ") + if edges: + result.update(target_spo_occurrences=target_count, bactotraits_target_occurrences=bactotraits_count, + target_examples=examples) + return result + + +def inspect_archive(path, config_sha, revision, python_version): + """Verify the actual compressed graph members and embedded strict-merge provenance.""" + archive_identity = identity(path) + seen, members, manifest = set(), {}, None + with tarfile.open(path, "r|gz") as archive: + for member in archive: + require(member.isfile() and Path(member.name).name == member.name and member.name not in seen, + "Unsafe or duplicate graph archive member") + seen.add(member.name) + stream = archive.extractfile(member) + if member.name == "manifest.json": + require(member.size <= 1024 * 1024, "Unexpectedly large graph manifest") + manifest = json.loads(stream.read()) + else: + require(member.name in {"merged-kg_nodes.tsv", "merged-kg_edges.tsv", "merged-kg_reference_resolution.tsv"}, + "Unexpected graph archive member") + members[member.name] = inspect_tsv(stream, edges=member.name == "merged-kg_edges.tsv") + require(manifest is not None and manifest.get("manifest_version") == 1, "Graph manifest missing or unsupported") + expected_names = {"merged-kg_nodes.tsv", "merged-kg_edges.tsv", "merged-kg_reference_resolution.tsv"} + require(set(members) == set(manifest.get("members", {})) == expected_names, "Graph manifest member set differs") + for name, observed in members.items(): + require({key: observed[key] for key in ("bytes", "sha256", "rows")} == manifest["members"][name], + f"Persisted graph member differs from manifest: {name}") + provenance = manifest.get("provenance", {}) + require(provenance.get("source_finalization") == {"required": True, "diagnostic_opt_out": False}, + "Archive lacks strict finalization provenance") + require(provenance.get("merge_config", {}).get("sha256") == config_sha, "Archive config identity mismatch") + require(provenance.get("commit") == revision, "Archive producer revision is dirty or changed") + require(provenance.get("python") == python_version, "Archive interpreter changed") + require(members["merged-kg_edges.tsv"]["target_spo_occurrences"] == 0, + "Target SPO remains in BactoTraits-only merge; independent lineage review is required") + require(identity(path) == archive_identity, "Graph archive changed during inspection") + return {"archive": {"path": str(path), **archive_identity}, "members": members, "manifest": manifest} + + +def runtime(): + """Bind the actual supported interpreter and complete installed distribution versions.""" + require((3, 10) <= sys.version_info[:2] <= (3, 12), "Run merge helper with pinned KG Python3.10-3.12") + executable = Path(sys.executable).absolute() + return {"python": sys.version, "executable": str(executable), "resolved": str(executable.resolve()), + "binary": identity(executable.resolve()), + "packages": {re.sub(r"[-_.]+", "-", dist.metadata["Name"].lower()): dist.version + for dist in importlib.metadata.distributions()}} + + +def prepare(args): + """Build an immutable read-only plan requiring all completed upstream receipts.""" + kg_root, output_root, output = safe(args.kg_root), safe(args.output_root), safe(args.output) + require(output.parent == output_root and output_root.is_dir(), "Candidate must be directly beneath worker output root") + require(not output.exists(), "Candidate output already exists; preserve it") + receipts, pins = verify_chain(kg_root, {source: getattr(args, source + "_receipt") for source in SOURCES}) + revision = subprocess.check_output(["git", "-C", str(kg_root), "rev-parse", "HEAD"], text=True).strip() + short_revision = subprocess.check_output(["git", "-C", str(kg_root), "rev-parse", "--short", "HEAD"], text=True).strip() + status = subprocess.check_output(["git", "-C", str(kg_root), "status", "--porcelain", "--untracked-files=no"], text=True) + require(not status, "KG tracked candidate is not clean") + require(all(receipt["plan"]["git_revision"] == revision for receipt in receipts.values()), "Source producer commits differ") + actual_runtime = runtime() + for name in ("inventory.json", "install-receipt.json", "installed-version-verification.json"): + bind(pins, output_root / "environment" / name) + installed = read_json(output_root / "environment/inventory.json") + expected_packages = {re.sub(r"[-_.]+", "-", item["name"].lower()): item["version"] + for item in installed["packages"]} + require(not installed["missing"] and actual_runtime["packages"] == expected_packages, + "Actual merge packages differ from pinned offline installation") + prior = receipts["bactotraits"]["plan"] + require(actual_runtime["python"] == prior["python"] + and actual_runtime["executable"] == prior["interpreter"]["executable"], "Merge interpreter differs from producer") + canonical_path = kg_root / "merge.yaml" + bind(pins, canonical_path) + bind(pins, Path(__file__).resolve()) + configuration, deltas = relocated_configuration(yaml.safe_load(canonical_path.read_text()), kg_root, output) + config_text = yaml.safe_dump(configuration, sort_keys=False) + quarantine = verify_quarantine(kg_root) + require(AUDIT in receipts["bactotraits"]["finalization_record"]["audit_members"], + "Quarantine audit is not bound by finalization") + lineage_path = safe(args.lineage_receipt) + bind(pins, lineage_path) + lineage = read_json(lineage_path) + require(lineage.get("profile") == "m5-bactotraits-retained-lineage/v1", "Wrong historical lineage receipt") + historical_madin = lineage["historical_source_graph_occurrences"]["madin_etal"] + require(historical_madin["matched_occurrences"] == 1 + and historical_madin["matches"][0]["row"]["primary_knowledge_source"] == "infores:madin_etal", + "Independent historical Madin provenance missing") + for item in (historical_madin["input"], lineage["independent_madin_raw_occurrences"]["input"]): + bind(pins, Path(item["path"]), {key: item[key] for key in ("bytes", "sha256")}) + environment = dict(prior["environment"]) + old_stage = prior["stage"] + environment = {key: value.replace(old_stage, str(output)) for key, value in environment.items()} + for key in THREADS: + environment[key] = "1" + environment["PYTHONPATH"] = str(kg_root) + environment["VIRTUAL_ENV"] = str(Path(actual_runtime["executable"]).parent.parent) + protected = {} + for delta in deltas: + if delta["path"][-1] == "filename" and "operations" in delta["path"]: + path = kg_root / delta["before"] + protected[str(path)] = identity(path) if path.exists() else None + if path.exists(): + bind(pins, path) + binding = {"kg_root": str(kg_root), "output_root": str(output_root), "output": str(output), + "git_revision": revision, "short_revision": short_revision, "runtime": actual_runtime, + "configuration": configuration, "configuration_text": config_text, "path_only_deltas": deltas, + "input_pins": pins, "source_trees": {source: receipt["canonical_source_artifacts"] for source, receipt in receipts.items()}, + "quarantine": quarantine, "historical_independent_madin": historical_madin, + "historical_independent_madin_raw": lineage["independent_madin_raw_occurrences"], + "environment": environment, "protected_original_outputs": protected, + "command": [actual_runtime["executable"], "-B", "-m", "kg_microbe.run", "merge", "-y", + str(output / "merge.yaml"), "-p", "1", "-s", "bactotraits"], "processes": 1} + return {"profile": "m5-strict-bactotraits-merge-plan/v1", "action": "plan", "writes": 0, + "binding": binding, "binding_sha256": hashlib.sha256(json.dumps(binding, sort_keys=True).encode()).hexdigest(), + "free_disk_bytes": shutil.disk_usage(output_root).free} + + +def verify_after(binding): + """Reject any input, producer tree, runtime or protected-output change after the CLI.""" + for name, expected in binding["input_pins"].items(): + require(identity(Path(name)) == expected, f"Merge input changed during execution: {name}") + for source, expected in binding["source_trees"].items(): + require(snapshot(Path(binding["kg_root"]) / "data/transformed" / source) == expected, "Producer output changed during merge") + require(runtime() == binding["runtime"], "Merge runtime changed") + for name, expected in binding["protected_original_outputs"].items(): + path = Path(name) + require((identity(path) if path.exists() else None) == expected, "Tracked or existing statistics were overwritten") + + +def run(plan, plan_file): + """Execute the actual strict CLI once and verify persisted archive bytes before success.""" + approved_plan = read_json(plan_file) + require(approved_plan.get("profile") == plan["profile"] + and approved_plan.get("binding_sha256") == plan["binding_sha256"] + and approved_plan.get("binding") == plan["binding"], "Inputs or plan changed since --plan") + binding = plan["binding"] + output = Path(binding["output"]) + require(not output.exists(), "Candidate output appeared after planning") + os.umask(0o077) + output.mkdir(mode=0o700) + (output / "tmp").mkdir(mode=0o700) + write_new(output / "plan.json", plan) + config = output / "merge.yaml" + with config.open("x") as stream: + stream.write(binding["configuration_text"]) + environment = {key: value for key, value in os.environ.items() + if key not in ("CONDA_DEFAULT_ENV", "CONDA_PREFIX") and not key.startswith("TOX")} + environment.update(binding["environment"]) + result = {"profile": "m5-strict-bactotraits-merge/v1", "started_at": datetime.now(timezone.utc).isoformat(), + "plan_sha256": identity(plan_file)["sha256"], "binding_sha256": plan["binding_sha256"], + "candidate_merged_graph_verified": False, "full_snapshot_release_verified": False, + "published_release_verified": False, "scientific_trait_truth_resolved": False, + "graph_wide_validation_claimed": False} + try: + with (output / "stdout.log").open("x") as stdout, (output / "stderr.log").open("x") as stderr: + child = subprocess.Popen(binding["command"], cwd=binding["kg_root"], env=environment, stdout=stdout, stderr=stderr) + write_new(output / "process.json", {"pid": child.pid, "parent_pid": os.getpid(), "command": binding["command"]}) + result["returncode"] = child.wait() + verify_after(binding) + require(result["returncode"] == 0, "Strict KG merge CLI did not complete; preserve logs") + result["persisted_graph"] = inspect_archive(output / "merged-kg.tar.gz", identity(config)["sha256"], + binding["short_revision"], binding["runtime"]["python"].split()[0]) + result["quarantine_after"] = verify_quarantine(Path(binding["kg_root"])) + require(result["quarantine_after"] == binding["quarantine"], "Quarantine/lineage changed during merge") + result.update(status="complete", candidate_merged_graph_verified=True, inputs_and_upstreams_unchanged=True) + except Exception as error: + result.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + result["completed_at"] = datetime.now(timezone.utc).isoformat() + result["output_artifacts"] = {str(path.relative_to(output)): identity(path) + for path in sorted(output.rglob("*")) if path.is_file()} + write_new(output / "receipt.json", result) + print(json.dumps({"receipt": str(output / "receipt.json"), "status": result["status"], + "candidate_merged_graph_verified": result["candidate_merged_graph_verified"]})) + + +def main(): + """Keep planning and actual merge invocation distinct and explicit.""" + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", action="store_true") + mode.add_argument("--run", action="store_true") + for name in ("kg-root", "output-root", "output", "ontologies-receipt", "bacdive-receipt", "bactotraits-receipt", "lineage-receipt"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--plan-file", type=Path) + args = parser.parse_args() + if args.run and args.plan_file is None: + parser.error("--run requires the newly recorded --plan-file") + csv.field_size_limit(16 * 1024 * 1024) + plan = prepare(args) + if args.plan: + print(json.dumps(plan, indent=2, sort_keys=True)) + else: + run(plan, args.plan_file) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_preservation_20261116.py b/scripts/adhoc/m5_release_preservation_20261116.py new file mode 100644 index 0000000..9b50a6d --- /dev/null +++ b/scripts/adhoc/m5_release_preservation_20261116.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Hash every nested artifact in the two explicitly retained v3 replay stages. + +2026-11-16: read-only preservation check for M5 release verification. This +walk is confined to the named evidence roots and includes hidden cache files. +""" + +import argparse +import hashlib +import json +from pathlib import Path + +RETAINED = Path( + "/Users/marcin/agent-work/EdgeMech/retained-pr135-pr1867-20261116/" + "kg-candidate/data/validation-bactotraits-130" +) + + +def inventory(): + """Verify all receipt-bound bytes and inventory every nested regular file.""" + result = {} + for name in ("baseline-v3-20261116", "candidate-v3-20261116"): + root = RETAINED / name + files = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Unexpected symlink: {path}") + if path.is_file(): + raw = path.read_bytes() + files[str(path.relative_to(root))] = { + "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw) + } + receipt = json.loads((root / "receipt.json").read_text()) + for relative, expected in receipt["artifacts"].items(): + if files[relative] != expected: + raise ValueError(f"Retained receipt artifact differs: {name}/{relative}") + result[name] = files + return result + + +def main(): + """Save a new manifest or compare all current retained files with an old one.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compare", type=Path) + args = parser.parse_args() + result = inventory() + if args.compare and json.loads(args.compare.read_text()) != result: + raise ValueError("Retained evidence inventory changed") + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + stream.write(json.dumps(result, indent=2) + "\n") + print(json.dumps({"files": sum(map(len, result.values())), "unchanged": bool(args.compare)})) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_publication_dependencies_20261116.py b/scripts/adhoc/m5_release_publication_dependencies_20261116.py new file mode 100644 index 0000000..e549294 --- /dev/null +++ b/scripts/adhoc/m5_release_publication_dependencies_20261116.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Read the exact canonical full-release dependency gap without running producers. + +2026-11-16 provenance: parse only merge.yaml, the lazy registry/constants and +its selected producer class definitions. Inspect explicitly configured graph +paths and whole-source completion records directly, including ignored files. +Read release-preflight policy pins without hashing or downloading bulk inputs. +Presence is not freshness or completion validation. A future successful +ontologies/BacDive/BactoTraits chain still covers only part of canonical merge. +""" + +import argparse +import ast +import hashlib +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import yaml + + +def identity(path): + """Bind small implementation/configuration files read for this audit.""" + payload = path.read_bytes() + return {"path": str(path), "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest()} + + +def inspect(root): + """Derive source requirements from maintained code rather than a remembered source list.""" + pins = {} + + def parse(relative): + path = root / relative + pins[relative] = identity(path) + return ast.parse(path.read_text()) + + constants = {} + for node in parse("kg_microbe/transform_utils/constants.py").body: + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + try: + constants[node.targets[0].id] = ast.literal_eval(node.value) + except (ValueError, TypeError): + pass + registry = {} + for node in parse("kg_microbe/transform.py").body: + if isinstance(node, ast.Assign) and any(isinstance(key, ast.Name) and key.id == "DATA_SOURCES" for key in node.targets): + for key, value in zip(node.value.keys, node.value.values): + name = constants[key.id] if isinstance(key, ast.Name) else ast.literal_eval(key) + registry[name] = ast.literal_eval(value.args[0]) + dependencies = {} + + def declaration(source): + if source in dependencies: + return dependencies[source] + module_name, class_name = registry[source].rsplit(".", 1) + relative = module_name.replace(".", "/") + ".py" + module = parse(relative) + cls = next(node for node in module.body if isinstance(node, ast.ClassDef) and node.name == class_name) + declarations = [node for node in cls.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "TRANSFORM_INPUTS" for target in node.targets)] + + def evaluate(expression): + if isinstance(expression, ast.Name): + return constants[expression.id] + if isinstance(expression, ast.Attribute) and expression.attr == "TRANSFORM_INPUTS": + inherited = next(key for key, value in registry.items() if value.endswith("." + expression.value.id)) + return declaration(inherited)["upstreams"] + if isinstance(expression, (ast.Tuple, ast.List)): + result = [] + for value in expression.elts: + result.extend(evaluate(value.value) if isinstance(value, ast.Starred) else [evaluate(value)]) + return result + return ast.literal_eval(expression) + + if declarations: + node = declarations[0] + result = {"implementation": relative, "line": node.lineno, "upstreams": evaluate(node.value)} + else: + parse("kg_microbe/transform_utils/transform.py") + result = {"implementation": relative, "line": cls.lineno, "upstreams": [], + "inherited_empty_default": "kg_microbe/transform_utils/transform.py:Transform.TRANSFORM_INPUTS"} + dependencies[source] = result + return result + + config_path = root / "merge.yaml" + pins["merge.yaml"] = identity(config_path) + config = yaml.safe_load(config_path.read_text()) + sources, producers = {}, {} + for name, source in config["merged_graph"]["source"].items(): + files = source["input"]["filename"] + directories = {str(Path(value).parent) for value in files} + if len(directories) != 1: + raise ValueError(f"Source spans unexpected producer directories: {name}") + relative = directories.pop() + directory_name = Path(relative).name + producer = "prego" if directory_name == "prego_habitat" else directory_name + declaration(producer) + sources[name] = {"producer": producer, "directory": relative, + "configured_inputs": [{"path": value, "present": (root / value).is_file()} for value in files]} + if producer not in producers: + records = {} + for filename in ("source_finalization.json", "source_fingerprint.json"): + path = root / relative / filename + records[filename] = {"present": path.is_file()} + if path.is_file(): + records[filename].update(identity(path)) + data = json.loads(path.read_text()) + records[filename]["record_version"] = data.get("version") + records[filename]["record_source"] = data.get("source") + producers[producer] = {"directory": relative, "canonical_source_keys": [], + "dependency_declaration": dependencies[producer], "completion_records": records, + "freshness_verified": False} + producers[producer]["canonical_source_keys"].append(name) + # This module exposes small read-only pin collectors; do not run bulk preflight. + sys.path.insert(0, str(root)) + from kg_microbe.utils import release_preflight + + for name, _ in (*release_preflight.PINNING_POLICIES, *release_preflight.DERIVED_PINS): + pins[name] = identity(root / name) + for name in ("kg_microbe/utils/release_preflight.py", "kg_microbe/merge_utils/source_freshness.py", + "kg_microbe/merge_utils/merge_kg.py", ".github/workflows/release.yml"): + pins[name] = identity(root / name) + planned = {"ontologies", "bacdive", "bactotraits"} + return {"profile": "m5-full-release-dependency-gap/v1", "observed_at": datetime.now(timezone.utc).isoformat(), + "kg_root": str(root), "producer_commit": subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip(), + "helper": identity(Path(__file__).resolve()), "implementation_config_policy_pins": pins, + "canonical_source_key_count": len(sources), "canonical_producer_count": len(producers), + "canonical_sources": sources, "producers": producers, + "planned_chain": sorted(planned), + "remaining_producers_after_successful_planned_chain": sorted(set(producers) - planned), + "remaining_source_keys_after_successful_planned_chain": [key for key, value in sources.items() if value["producer"] not in planned], + "reviewed_release_raw_pins": release_preflight.reviewed_release_pins(root), + "derived_raw_pins": release_preflight.reviewed_release_pins(root, policies=release_preflight.DERIVED_PINS), + "additional_preflight_requirement": "NCIT SQLite authority validation; NCIT is used by whole ontologies, ontologies_stubs and MediaDive. Existing UPA JSON must match its derived pin.", + "scope": "Exact configured paths were checked directly, including ignored files. Presence only; no current freshness, graph acceptance or publication validation. No graph scans, producer execution or downloads."} + + +def main(): + """Keep each audit in a fresh exclusive output directory.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kg-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = inspect(args.kg_root.resolve(strict=True)) + args.output.mkdir(parents=True, exist_ok=False) + with (args.output / "dependency-gap.json").open("x") as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({"output": str(args.output), "canonical_source_keys": result["canonical_source_key_count"], + "producers": result["canonical_producer_count"], + "remaining_after_planned_chain": result["remaining_producers_after_successful_planned_chain"]}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_results_20261116.py b/scripts/adhoc/m5_release_results_20261116.py new file mode 100644 index 0000000..4570b22 --- /dev/null +++ b/scripts/adhoc/m5_release_results_20261116.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Export verified M5 row/finalizer evidence beneath this invocation's output root. + +2026-11-16: preserve original receipt paths and all nested files unchanged. The +compact result is suitable for Git; complete private receipts remain local. +""" + +import argparse +import hashlib +import json +import shutil +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +KG = ROOT.parent / "m5-kg-bactotraits-release-20261116" +STAGES = ( + "baseline-m5-20261116-r1", + "candidate-m5-20261116-r1", + "candidate-finalized-m5-20261116-r1", +) + + +def identity(path): + """Identify one bounded evidence file using exact persisted bytes.""" + raw = path.read_bytes() + return {"sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw)} + + +def verify(stage): + """Reject any changed or escaping receipt-bound artifact before export.""" + receipt = json.loads((stage / "receipt.json").read_text()) + for name, expected in receipt["artifacts"].items(): + path = (stage / name).resolve() + if not path.is_relative_to(stage.resolve()) or identity(path) != expected: + raise ValueError(f"Receipt artifact differs: {path}") + return receipt + + +def main(): + """Copy only the named new stages to an exclusive result root and verify readback.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--plan", action="store_true") + action.add_argument("--run", action="store_true") + args = parser.parse_args() + output = args.output.absolute() + required = ROOT / ".temp/m5-worker-run-20261116" + if not output.is_relative_to(required) or output.exists(): + raise ValueError("Use a fresh directory beneath this invocation's required output root") + results = {} + receipts = {} + for name in STAGES: + stage = KG / "data/validation-bactotraits-130" / name + receipt = verify(stage) + receipts[name] = receipt + results[name] = { + "original_receipt": str(stage / "receipt.json"), + "receipt": identity(stage / "receipt.json"), + "producer_revision": receipt["git_revision"], + "edges": receipt.get("all_edge_count", receipt.get("finalized_edge_count")), + "nodes": receipt.get("node_count", receipt.get("finalized_node_count")), + "quarantine_rows": receipt["quarantine_rows"], + "finalization_run": receipt["finalization_run"], + "artifacts": receipt["artifacts"], + } + baseline, candidate, finalized = [receipts[name] for name in STAGES] + if not ( + baseline["mode"] == "baseline" + and baseline["git_revision"] == "ef27b97a0b450767da54235c6105b5250c178adc" + and (baseline["all_edge_count"], baseline["node_count"]) == (21, 22) + and candidate["mode"] == "candidate" + and (candidate["all_edge_count"], candidate["node_count"], candidate["quarantine_rows"]) == (0, 0, 1) + and (finalized["finalized_edge_count"], finalized["finalized_node_count"], finalized["quarantine_rows"]) == (0, 0, 1) + and finalized["finalization_run"] is True + and finalized["merged_graph_verified"] is False + and finalized["published_release_verified"] is False + and baseline["shared_inputs"] == candidate["shared_inputs"] == finalized["shared_inputs"] + and candidate["baseline_receipt"] == finalized["baseline_receipt"] + and candidate["baseline_receipt"]["sha256"] == results[STAGES[0]]["receipt"]["sha256"] + and finalized["candidate_receipt"]["sha256"] == results[STAGES[1]]["receipt"]["sha256"] + ): + raise ValueError("Fresh pair/finalization semantics or receipt links differ") + summary = { + "profile": "m5-bactotraits-release-verification/v1", + "observed_at": datetime.now(timezone.utc).isoformat(), + "export_helper_sha256": identity(Path(__file__))["sha256"], + "fresh_baseline_candidate_pair_verified": True, + "bounded_source_finalization_verified": True, + "merged_graph_verified": False, + "published_release_verified": False, + "source_row": 8521, + "bacdive_id": 17851, + "scientific_claim_resolved": False, + "neighboring_rows_quarantined": False, + "results": results, + } + if args.plan: + print(json.dumps({"ready": True, "output": str(output), "stages": list(results)})) + return + output.mkdir(mode=0o700) + for name in STAGES: + source = KG / "data/validation-bactotraits-130" / name + destination = output / name + shutil.copytree(source, destination) + verify(destination) + for path in source.rglob("*"): + if path.is_symlink(): + raise ValueError(f"Unexpected symlink in a new stage: {path}") + if path.is_file() and identity(path) != identity(destination / path.relative_to(source)): + raise ValueError(f"Export differs: {path}") + with (output / "results.json").open("x") as stream: + stream.write(json.dumps(summary, indent=2) + "\n") + print(json.dumps({"results": str(output / "results.json"), **identity(output / "results.json")})) + + +if __name__ == "__main__": + main() diff --git a/scripts/adhoc/m5_release_state_20261116.py b/scripts/adhoc/m5_release_state_20261116.py new file mode 100644 index 0000000..341991f --- /dev/null +++ b/scripts/adhoc/m5_release_state_20261116.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Record the authorized M5 release track's commands and durable progress. + +2026-11-16: all logs and state remain beneath the isolated evidence checkout. +This launcher runs one command at a time without shell interpretation; receipts +from the producer and finalizer remain the scientific/provenance evidence. +""" + +import argparse +import datetime +import fcntl +import hashlib +import json +import os +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUTPUT = ROOT / ".temp/m5-worker-run-20261116" + + +def _save(state): + """Atomically replace progress and retain individual execution records.""" + state["commands"] = sorted( + (json.loads(path.read_text()) for path in (OUTPUT / "commands").glob("*.json")), + key=lambda entry: entry["started_at"], + ) + state["updated_at"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + for path in (ROOT / ".temp/m5-release-progress.json", OUTPUT / "progress.json"): + temporary = path.with_suffix(f".{os.getpid()}.pending") + temporary.write_text(json.dumps(state, indent=2) + "\n") + temporary.replace(path) + lines = ["# M5 release verification", "", f"Phase: {state['phase']}", + f"Updated: {state['updated_at']}", f"Launcher PID: {state['pid']}", "", + "Published release verified: false", ""] + for entry in state.get("commands", []): + lines.append(f"- {entry['label']}: {entry['status']}; PID {entry.get('child_pid')}; " + f"exit {entry.get('exit_code')}; log {entry['log']}") + lines.extend(["", "Blockers:", *[f"- {item}" for item in state.get("blockers", [])]]) + if state.get("process_observation"): + observed = state["process_observation"] + lines.extend(["", f"External process: {observed['status']}", + f"Evidence: {observed['path']}", f"SHA256: {observed['sha256']}"]) + for path in (ROOT / ".temp/M5_RELEASE_PROGRESS.md", OUTPUT / "M5_RELEASE_PROGRESS.md"): + temporary = path.with_suffix(f".{os.getpid()}.pending") + temporary.write_text("\n".join(lines) + "\n") + temporary.replace(path) + + +def save(updates): + """Apply explicit metadata changes to current state while holding the aggregate lock.""" + with (OUTPUT / "progress.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + progress = ROOT / ".temp/m5-release-progress.json" + state = json.loads(progress.read_text()) if progress.exists() else {} + state.update(updates) + _save(state) + return state + + +def save_command(path, entry): + """Atomically publish one independent launcher's current execution record.""" + temporary = path.with_suffix(f".{os.getpid()}.pending") + temporary.write_text(json.dumps(entry, indent=2) + "\n") + temporary.replace(path) + + +def import_check_record(path, label): + """Validate the launcher's terminal evidence before publishing imported progress (#162).""" + raw = path.read_bytes() + receipt = json.loads(raw) + status = receipt.get("status") + if status not in {"running", "complete", "failed"}: + raise ValueError("Unsupported imported check status") + terminal_fields = {"exit_code", "completed_at", "seconds", "log_sha256"} + logfile = path.parent / "command.log" + if not logfile.is_file(): + raise ValueError("Imported check log is missing") + if status == "running": + if terminal_fields & receipt.keys(): + raise ValueError("Running check receipt contains terminal fields") + else: + code = receipt.get("exit_code") + if type(code) is not int or (status == "complete") != (code == 0): + raise ValueError("Imported terminal status contradicts exit code") + if not isinstance(receipt.get("completed_at"), str) or not receipt["completed_at"]: + raise ValueError("Imported terminal check lacks completion time") + expected = receipt.get("log_sha256") + digest = hashlib.sha256() + with logfile.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + if expected != digest.hexdigest(): + raise ValueError("Imported completed check log hash differs or is missing") + return {"label": label, "argv": receipt["command"], "cwd": receipt["cwd"], "log": str(logfile), + "status": "passed" if status == "complete" else status, + "child_pid": receipt["pid"], "started_at": receipt["started_at"], + "exit_code": receipt.get("exit_code"), "source_receipt": str(path), + "source_receipt_sha256": hashlib.sha256(raw).hexdigest()} + + +def main(): + """Run a bounded task command, preserving its argv, PID, output and status.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--phase", required=True) + parser.add_argument("--label") + parser.add_argument("--cwd", type=Path, default=ROOT) + parser.add_argument("--blocker", action="append") + parser.add_argument("--results", type=Path) + parser.add_argument("--observe-process", type=Path) + parser.add_argument("--process-status") + parser.add_argument("--import-check", type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + os.umask(0o077) + OUTPUT.mkdir(parents=True, exist_ok=True) + state = {"phase": args.phase, "pid": os.getpid()} + if args.import_check: + if not args.label: + parser.error("--label required for --import-check") + try: + entry = import_check_record(args.import_check, args.label) + except (ValueError, OSError, KeyError, TypeError) as error: + parser.error(str(error)) + logs = OUTPUT / "commands" + logs.mkdir(exist_ok=True) + save_command(logs / (args.label + ".json"), entry) + if args.blocker: + state["blockers"] = args.blocker + if args.results: + state["results"] = json.loads(args.results.read_text()) + if args.observe_process: + if not args.process_status: + parser.error("--process-status required for an external process observation") + raw = args.observe_process.read_bytes() + state["process_observation"] = { + "path": str(args.observe_process.absolute()), "sha256": hashlib.sha256(raw).hexdigest(), + "status": args.process_status, "record": json.loads(raw), + } + state = save(state) + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + return 0 + if not args.label: + parser.error("--label required for command") + logs = OUTPUT / "commands" + logs.mkdir(exist_ok=True) + logfile = logs / (args.label + ".log") + command_record = logs / (args.label + ".json") + entry = {"label": args.label, "argv": command, "cwd": str(args.cwd), + "log": str(logfile), "status": "running", "started_at": state["updated_at"]} + with logfile.open("x") as log: + child = subprocess.Popen(command, cwd=args.cwd, stdout=log, stderr=subprocess.STDOUT) + entry["child_pid"] = child.pid + save_command(command_record, entry) + save({}) + code = child.wait() + entry.update(exit_code=code, status="passed" if code == 0 else "failed") + save_command(command_record, entry) + save({}) + print(json.dumps(entry)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/adhoc/summarize_m5_production_20261116.py b/scripts/adhoc/summarize_m5_production_20261116.py new file mode 100644 index 0000000..ed6ebb4 --- /dev/null +++ b/scripts/adhoc/summarize_m5_production_20261116.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""2026-11-16: summarize completed M5 native producers without bulk contracts. + +Reads the three immutable canonical source receipts and checks their selected +files against the saved hashes. The compact report is a projection, not a new +producer/finalization receipt. NCBITaxon version queries use SQLite mode=ro and +the exact ontology-node version predicate used by KG ontology_utils. No build, +merge, network request, freshness override, or old-receipt edit is performed. +The complete BactoTraits audit is copied byte-for-byte to a fresh output path. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import json +import re +import sqlite3 +import sys +from contextlib import closing +from datetime import datetime, timezone +from pathlib import Path + +SOURCES = ("ontologies", "bacdive", "bactotraits") +# Exact receipts observed after the separate native executions and their reviews. +CHAIN_SHA256 = { + "ontologies": "c5defa64ee549fbc4ef6978ca4cc1fbe441ee319d465eb1188a836cc6534119e", + "bacdive": "6742881bc5743bdf8ed12023638634ad628f482377ad8f0ad0799936585103bd", + "bactotraits": "36968cb32f7c10ec883160503e369bf1ad19ecfa3b8203395e327f24db93a445", +} +LAUNCHER = "scripts/adhoc/run_bactotraits_production_unit_20261116.py" +AUDIT = "bactotraits_identity_quarantine.tsv" +CAPS = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "NUMEXPR_MAX_THREADS", + "VECLIB_MAXIMUM_THREADS", + "RAYON_NUM_THREADS", + "BLIS_NUM_THREADS", +) + + +def require(condition, message): + """Refuse a compact success summary if its source evidence has drifted.""" + if not condition: + raise ValueError(message) + + +def identity(path): + """Stream exact bytes; never load a database into memory.""" + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + size += len(block) + digest.update(block) + return {"bytes": size, "sha256": digest.hexdigest()} + + +def checked(path, expected): + """Return a saved identity only after checking the actual file.""" + require(identity(path) == expected, f"Evidence changed: {path}") + return dict(expected) + + +def read_record(path): + """Read one already-completed evidence record without printing its payload.""" + with path.open() as stream: + return json.load(stream) + + +def collect(args): + """Build a bounded projection of recorded execution and current selected members.""" + summary_path = args.output_root / "canonical-production-summary.json" + audit_copy = args.output_root / "full-bactotraits-audit" / AUDIT + require(args.output_root.is_dir(), "Worker output root must already exist") + require( + not summary_path.exists() and not audit_copy.parent.exists(), "Preserve existing results" + ) + result = { + "profile": "m5-canonical-production-summary/v1", + "observed_at": datetime.now(timezone.utc).isoformat(), + "summary_helper": {"path": Path(__file__).name, **identity(Path(__file__))}, + "scope": "Three actual source producers; no merge or published-release success is asserted.", + "merged_graph_verified": False, + "published_release_verified": False, + "sources": {}, + } + original_inputs = {} + runtime = None + for source in SOURCES: + unit_name = f"canonical-{source}-r1" + chain_path = args.output_root / f"chain-{unit_name}.json" + chain_identity = identity(chain_path) + require(chain_identity["sha256"] == CHAIN_SHA256[source], "Reviewed chain receipt changed") + chain = read_record(chain_path) + plan = chain["plan"] + source_dir = args.kg_root / "data/transformed" / source + unit_path = args.kg_root / "data/validation-bactotraits-130" / unit_name / "receipt.json" + require(chain["returncode"] == 0 and plan["source"] == source, "Producer did not complete") + require(plan["canonical_source_directory"] == str(source_dir), "Unexpected source path") + require(chain["unit_receipt"]["path"] == str(unit_path), "Unexpected unit receipt path") + expected_unit = {key: chain["unit_receipt"][key] for key in ("bytes", "sha256")} + checked(unit_path, expected_unit) + unit = read_record(unit_path) + require(unit["returncode"] == 0, "Native CLI did not complete") + require(not unit["task_raw_changed_files"], "A pre-existing raw input changed") + require( + chain["upstream_unchanged"] and chain["ancillary_preexisting_unchanged"], "Input drift" + ) + require(plan["processes"] == 1, "Unexpected source concurrency") + require(all(plan["environment"][name] == "1" for name in CAPS), "Native cap drift") + recorded_runtime = { + "producer_commit": plan["git_revision"], + "python": plan["python"], + "packages": plan["packages"], + "poetry_lock": checked( + args.kg_root / "poetry.lock", + plan["implementation_and_policy_inputs"]["poetry.lock"], + ), + "kg_launcher": { + "path": LAUNCHER, + **checked( + args.kg_root / LAUNCHER, plan["implementation_and_policy_inputs"][LAUNCHER] + ), + }, + "canonical_wrapper": { + "path": "scripts/adhoc/" + Path(plan["wrapper"]["path"]).name, + **checked( + Path(plan["wrapper"]["path"]), + {key: plan["wrapper"][key] for key in ("bytes", "sha256")}, + ), + }, + "source_processes": 1, + "native_thread_caps": {name: plan["environment"][name] for name in CAPS}, + "authority_guards": { + name: plan["environment"][name] + for name in ( + "KG_SEMSQL_BUILD", + "KG_GO_VERSION_CHECK", + "KG_NCBITAXON_VERSION_CHECK", + "KG_CHEBI_VERSION_CHECK", + ) + }, + } + java_proofs = { + name: checked(Path(item["path"]), {key: item[key] for key in ("bytes", "sha256")}) + for name, item in plan["java_supplements"].items() + } + java_native = read_record(args.output_root / "java-runtime-implementation-r3.json") + java_implementation = {} + for role in ("actual_runtime_launcher", "actual_runtime_vm"): + item = java_native[role] + java_implementation[role] = { + "file": Path(item["path"]).name, + **checked(Path(item["path"]), {key: item[key] for key in ("bytes", "sha256")}), + } + recorded_runtime["java"] = { + "version_output": "\n".join( + line + for line in plan["java"]["stderr"].splitlines() + if not line.startswith("Picked up _JAVA_OPTIONS:") + ), + "implementation": java_implementation, + "supplement_files": java_proofs, + "original_go_record_java_metadata_issue": plan["java_provenance_issue"], + } + require( + plan["python"] == sys.version, "Summary interpreter differs from recorded execution" + ) + require( + plan["interpreter"]["executable"] == sys.executable, "Different interpreter selected" + ) + if runtime is None: + runtime = recorded_runtime + require(runtime == recorded_runtime, "Source runtime or implementation differs") + selected = { + "source_finalization.json": chain["canonical_source_artifacts"][ + "source_finalization.json" + ], + "source_fingerprint.json": chain["canonical_source_artifacts"][ + "source_fingerprint.json" + ], + **chain["finalization_record"]["members"], + **chain["finalization_record"]["audit_members"], + } + for name, expected in selected.items(): + require(Path(name).name == name, "Unexpected nested finalizer member") + checked(source_dir / name, expected) + result["sources"][source] = { + "producer_commit": plan["git_revision"], + "started_at": unit["started_at"], + "completed_at": unit["completed_at"], + "counts": chain["finalization_record"]["summaries"]["rows"], + "chain_receipt": {"path": chain_path.name, **chain_identity}, + "inner_receipt": {"path": str(unit_path.relative_to(args.kg_root)), **expected_unit}, + "source_directory": str(source_dir.relative_to(args.kg_root)), + "finalizer_graph_and_audit_members": selected, + "finalizer_code_fingerprint": chain["finalization_record"]["finalizer_code"], + "configuration_digest": hashlib.sha256( + json.dumps(plan["environment"], sort_keys=True, separators=(",", ":")).encode() + ).hexdigest(), + "configuration_digest_format": "sha256 of compact sorted-key environment JSON", + "original_inputs_unchanged_at_execution_end": True, + "existing_raw_inputs_unchanged": True, + "upstream_unchanged": chain["upstream_unchanged"], + "preexisting_ancillary_unchanged": chain["ancillary_preexisting_unchanged"], + } + original_inputs.update({Path(item["source"]).name: item for item in plan["originals"]}) + if source == "bactotraits": + fresh_lookup = plan["fresh_bacdive_lookup"] + result["runtime"] = runtime + result["runtime"]["interpreter_binary_observed_during_summary"] = { + "name": Path(sys.executable).resolve().name, + **identity(Path(sys.executable).resolve()), + } + inputs = {} + for role, name in ( + ("original_full_csv", "BactoTraits_databaseV2_Jun2022.csv"), + ("retained_ncbitaxon_db", "ncbitaxon.db"), + ("retained_ncbitaxon_owl_gzip", "ncbitaxon.owl.gz"), + ): + recorded = original_inputs[name] + expected = {key: recorded[key] for key in ("bytes", "sha256")} + inputs[role] = { + "path": "data/raw/" + name, + **checked(args.kg_root / "data/raw" / name, expected), + "original_retained_identity_matches_staged": True, + } + lookup_path = args.kg_root / "kg_microbe/transform_utils/bacdive/tmp/bacdive.tsv" + inputs["fresh_bacdive_lookup"] = { + "path": str(lookup_path.relative_to(args.kg_root)), + **checked(lookup_path, {key: fresh_lookup[key] for key in ("bytes", "sha256")}), + "source_receipt": Path(fresh_lookup["receipt"]).name, + } + go_cache = plan["retained_go_cache"] + go_members = {} + for name, item in go_cache["members"].items(): + path = args.kg_root / "data/raw" / name + require(item["path"] == str(path), "Unexpected GO cache location") + go_members[name] = { + "path": str(path.relative_to(args.kg_root)), + **checked(path, {key: item[key] for key in ("bytes", "sha256")}), + } + go_receipt = go_cache["receipt"] + go_receipt_path = Path(go_receipt["path"]) + inputs["current_go_cache"] = { + "members": go_members, + "fresh_build_receipt": { + "path": str(go_receipt_path.relative_to(args.kg_root)), + **checked(go_receipt_path, {key: go_receipt[key] for key in ("bytes", "sha256")}), + }, + "java_history_note": "Original toolchain record is preserved; separately pinned supplements identify the actual JVM.", + } + db = args.kg_root / "data/raw/ncbitaxon.db" + with closing(sqlite3.connect(db.as_uri() + "?mode=ro", uri=True)) as connection: + version = connection.execute( + "SELECT value FROM statements WHERE predicate = 'owl:versionInfo' " + "AND subject LIKE '%ncbitaxon%' AND value IS NOT NULL LIMIT 1" + ).fetchone() + require(version is not None, "Missing NCBITaxon version stamp") + db_version = re.search(r"\d{4}-\d{2}-\d{2}", str(version[0])) + with gzip.open(args.kg_root / "data/raw/ncbitaxon.owl.gz", "rt", errors="replace") as stream: + head = stream.read(2_000_000) + owl_version = re.search(r"versionIRI[^>]{0,160}?(\d{4}-\d{2}-\d{2})", head) + require( + db_version and owl_version and db_version[0] == owl_version[1], "Authority version drift" + ) + inputs["retained_ncbitaxon_db"].update( + { + "authority_version": db_version[0], + "matching_owl_gzip_version": owl_version[1], + "toolchain_record_present": db.with_name(db.name + ".toolchain.json").exists(), + "build_history": "Retained database has no historical toolchain record; no original builder is inferred.", + } + ) + result["inputs"] = inputs + source_audit = args.kg_root / "data/transformed/bactotraits" / AUDIT + with source_audit.open(newline="") as stream: + audits = list(csv.DictReader(stream, delimiter="\t")) + require(len(audits) == 1, "Unexpected quarantine audit count") + row = audits[0] + require(row["source_data_row"] == row["reviewed_source_data_row"] == "8521", "Wrong row") + require(row["bacdive_id"] == "17851", "Wrong BacDive identity") + result["quarantine_audit"] = { + key: row[key] + for key in ( + "rule_id", + "source_data_row", + "reviewed_source_data_row", + "bacdive_id", + "ncbitaxon_id", + "disposition", + "row_sha256", + "source_sha256", + "reviewed_source_sha256", + "policy_sha256", + ) + } + result["quarantine_audit"].update( + { + "row_count": 1, + "complete_raw_row_field_count": len(json.loads(row["raw_row_json"])), + "original_path": str(source_audit.relative_to(args.kg_root)), + "exact_copy_path": str(audit_copy.relative_to(args.output_root)), + **identity(source_audit), + } + ) + return result, summary_path, source_audit, audit_copy + + +def main(): + """Keep the plan read-only and both new output paths exclusive.""" + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--plan", action="store_true") + action.add_argument("--run", action="store_true") + parser.add_argument("--kg-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + result, summary_path, source_audit, audit_copy = collect(args) + if args.plan: + print( + json.dumps( + { + "action": "plan", + "writes": 0, + "planned_outputs": [ + summary_path.name, + str(audit_copy.relative_to(args.output_root)), + ], + **result, + }, + indent=2, + ) + ) + return + audit_copy.parent.mkdir(mode=0o700) + with audit_copy.open("xb") as stream: + stream.write(source_audit.read_bytes()) + require(identity(audit_copy) == identity(source_audit), "Audit copy changed") + with summary_path.open("x") as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({"summary": summary_path.name, **identity(summary_path)})) + + +if __name__ == "__main__": + main() diff --git a/tests/test_m5_release_chain.py b/tests/test_m5_release_chain.py new file mode 100644 index 0000000..dab81d0 --- /dev/null +++ b/tests/test_m5_release_chain.py @@ -0,0 +1,95 @@ +"""Keep canonical release units exclusive and reject scoped or tampered completion evidence.""" + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + +HELPER = Path(__file__).resolve().parents[1] / "scripts/adhoc/m5_release_chain_20261116.py" + + +@pytest.fixture +def chain(tmp_path): + """Use an isolated output root and restore process permissions after direct launcher calls.""" + spec = importlib.util.spec_from_file_location("m5_release_chain", HELPER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def require(condition, message): + if not condition: + raise ValueError(message) + + def identity(path): + raw = path.read_bytes() + return {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} + + root = tmp_path.resolve() + launcher = SimpleNamespace(ROOT=root, require=require, identity=identity, safe_path=lambda p: p) + previous = os.umask(0o077) + os.umask(previous) + try: + yield module, launcher, root + finally: + os.umask(previous) + + +def test_existing_canonical_source_is_refused_before_producer(chain): + """Even an empty source directory belongs to prior work and cannot be reused implicitly.""" + module, launcher, root = chain + existing = root / "data/transformed/ontologies" + existing.mkdir(parents=True) + sentinel = existing / "retained.tsv" + sentinel.write_text("previous source evidence\n") + launcher.prepare = lambda args: {"environment": {"ROBOT_JAVA_ARGS": "", "_JAVA_OPTIONS": ""}} + args = SimpleNamespace(kg_root=root, source="ontologies") + with pytest.raises(ValueError, match="Canonical source output already exists"): + module.prepare(args, launcher) + assert sentinel.read_text() == "previous source evidence\n" + + +@pytest.mark.parametrize("damage", ["scoped_only", "changed_member"]) +def test_cli_success_cannot_certify_incomplete_or_changed_source(chain, damage): + """A zero exit code with a scoped record or edited TSV never supplies whole-source evidence.""" + module, launcher, root = chain + source = root / "data/transformed/ontologies" + receipt_path = root / "chain.json" + plan = { + "canonical_output_base": str(source.parent), + "canonical_source_directory": str(source), + "upstream_before": {}, + "ancillary_before": {name: {} for name in module.ANCILLARY}, + "chain_receipt": str(receipt_path), + } + + def produced(_): + source.mkdir(parents=True) + nodes = source / "ro_nodes.tsv" + nodes.write_text("id\tcategory\n") + audit = source / "audit.tsv" + audit.write_text("record\n") + report = { + "members": {nodes.name: launcher.identity(nodes)}, + "audit_members": {audit.name: launcher.identity(audit)}, + } + filename = ( + "ro_source_finalization.json" if damage == "scoped_only" else "source_finalization.json" + ) + (source / filename).write_text(json.dumps(report)) + (source / "source_fingerprint.json").write_text('{"version": 3}\n') + if damage == "changed_member": + nodes.write_text("id\tcategory\nchanged:1\tbiolink:NamedThing\n") + return 0 + + launcher.run = produced + message = ( + "CLI success lacks actual completion evidence" + if damage == "scoped_only" + else "Finalizer member changed" + ) + with pytest.raises(ValueError, match=message): + module.run(plan, launcher) + assert not receipt_path.exists() diff --git a/tests/test_m5_release_gate_retry.py b/tests/test_m5_release_gate_retry.py new file mode 100644 index 0000000..ed3e78d --- /dev/null +++ b/tests/test_m5_release_gate_retry.py @@ -0,0 +1,44 @@ +"""Keep durable gate scratch outside ephemeral filesystem roots, including aliases.""" + +import importlib.util +from pathlib import Path + +import pytest + + +@pytest.fixture +def retry(): + """Load only the retry boundary helper, without creating an environment record.""" + path = Path(__file__).parents[1] / "scripts/adhoc/m5_release_gate_retry_20261116.py" + spec = importlib.util.spec_from_file_location("m5_gate_retry", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("name", ["/tmp", "/private/tmp", "/private/var/folders"]) +def test_exact_existing_ephemeral_roots_are_refused_without_writing_them(retry, name): + """The root itself must be rejected, not just strings containing its trailing slash.""" + path = Path(name) + if not path.exists(): + pytest.skip("Platform does not have this temporary root") + with pytest.raises(ValueError, match="durable"): + retry.durable_scratch(path) + + +def test_durable_named_alias_to_temporary_root_cannot_pass(retry, tmp_path): + """A durable-looking name does not change the resolved storage lifetime.""" + alias = tmp_path / "durable-gate-scratch" + alias.symlink_to(Path("/tmp"), target_is_directory=True) + with pytest.raises(ValueError, match="durable"): + retry.durable_scratch(alias) + + +def test_actual_durable_directory_remains_usable(retry): + """The boundary repair accepts task-owned durable scratch and does not require /tmp.""" + # Read only an existing directory; pytest itself may place tmp_path in /tmp. + scratch = Path(__file__).resolve().parent + ephemeral = (Path("/tmp").resolve(), Path("/private/tmp"), Path("/private/var/folders")) + if any(scratch.is_relative_to(root) for root in ephemeral): + pytest.skip("Checkout itself is in ephemeral storage") + assert retry.durable_scratch(scratch) == scratch diff --git a/tests/test_m5_release_merge.py b/tests/test_m5_release_merge.py new file mode 100644 index 0000000..b817bcf --- /dev/null +++ b/tests/test_m5_release_merge.py @@ -0,0 +1,284 @@ +"""Reject stale/scoped sources, unsafe config placement and laundered candidate archives.""" + +import copy +import csv +import hashlib +import importlib.util +import io +import json +import tarfile +from pathlib import Path + +import pytest + +HELPER = Path(__file__).resolve().parents[1] / "scripts/adhoc/m5_release_merge_20261116.py" + + +@pytest.fixture +def merge(): + """Load the standalone helper without importing KG's graph pipeline.""" + spec = importlib.util.spec_from_file_location("m5_release_merge", HELPER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_only_declared_path_placement_changes_and_existing_stats_remain(merge, tmp_path): + """Preserve every source/semantic option and never write tracked statistics during planning.""" + kg = tmp_path / "kg" + kg.mkdir() + stats = kg / "merged_graph_stats.yaml" + stats.write_bytes(b"retained historical statistics\n") + canonical = { + "configuration": {"output_directory": "data/merged", "checkpoint": False, + "preserve": ["primary_knowledge_source"]}, + "merged_graph": { + "source": {"bactotraits": {"name": "BactoTraits", "input": {"format": "tsv", "filename": [ + "data/transformed/bactotraits/nodes.tsv", "data/transformed/bactotraits/edges.tsv"]}}, + "madin_etal": {"input": {"format": "tsv", "filename": ["data/transformed/madin_etal/edges.tsv"]}}}, + "operations": [{"name": "kgx.graph_operations.summarize_graph.generate_graph_stats", "args": { + "filename": "merged_graph_stats.yaml", "edge_facet_properties": ["source"]}}], + "destination": {"merged-kg-tsv": {"format": "tsv", "compression": "tar.gz", "filename": "merged-kg"}}, + }, + } + retained = copy.deepcopy(canonical) + output = tmp_path / "exclusive-candidate" + configured, deltas = merge.relocated_configuration(canonical, kg, output) + assert canonical == retained + assert configured["merged_graph"]["destination"] == canonical["merged_graph"]["destination"] + assert configured["configuration"]["preserve"] == ["primary_knowledge_source"] + assert configured["merged_graph"]["operations"][0]["args"]["filename"] == str(output / stats.name) + assert configured["merged_graph"]["operations"][0]["args"]["edge_facet_properties"] == ["source"] + assert configured["merged_graph"]["source"]["bactotraits"]["input"]["filename"] == [ + str(kg / "data/transformed/bactotraits/nodes.tsv"), str(kg / "data/transformed/bactotraits/edges.tsv")] + assert len(deltas) == 4 + assert stats.read_bytes() == b"retained historical statistics\n" + assert not output.exists() + + +@pytest.mark.parametrize("diagnostic", [True, "false", 0, None]) +def test_diagnostic_or_ambiguous_finalization_flags_never_plan(merge, tmp_path, diagnostic): + """Only the exact false boolean or absent canonical setting can pass.""" + with pytest.raises(ValueError, match="opt-out"): + merge.relocated_configuration({"configuration": {"allow_unfinalized_sources": diagnostic}}, + tmp_path, tmp_path / "out") + + +def source_chain(merge, root): + """Create small whole-source contracts whose byte identities can be independently changed.""" + receipts, paths, snapshots = {}, {}, {} + units = root / "units" + units.mkdir() + for source in merge.SOURCES: + directory = root / "data/transformed" / source + directory.mkdir(parents=True) + names = ("ncbitaxon_nodes.tsv", "chebi_nodes.tsv") if source == "ontologies" else ("nodes.tsv", "edges.tsv") + for name in names: + (directory / name).write_text("id\tcategory\ntest:1\tbiolink:NamedThing\n") + (directory / "audit.tsv").write_text("check\tstatus\nsource\tpass\n") + finalization = {"source": source, "members": {name: merge.identity(directory / name) for name in names}, + "audit_members": {"audit.tsv": merge.identity(directory / "audit.tsv")}} + marker = {"version": 3, "source": source} + (directory / "source_finalization.json").write_text(json.dumps(finalization)) + (directory / "source_fingerprint.json").write_text(json.dumps(marker)) + unit_path = units / (source + ".json") + unit_path.write_text(json.dumps({"returncode": 0, "source": source, "git_revision": "producer-pin", + "task_raw_after": {}})) + record = {"profile": "m5-canonical-release-source-unit/v1", "returncode": 0, + "upstream_unchanged": True, "ancillary_preexisting_unchanged": True, + "plan": {"source": source, "canonical_source_directory": str(directory), + "git_revision": "producer-pin", "implementation_and_policy_inputs": {}, "originals": [], + "upstream_before": copy.deepcopy(snapshots)}, + "canonical_source_artifacts": merge.snapshot(directory), "finalization_record": finalization, + "producer_fingerprint": marker, "ancillary_after": {}, + "unit_receipt": {"path": str(unit_path), **merge.identity(unit_path)}} + path = root / (source + "-chain.json") + path.write_text(json.dumps(record)) + receipts[source], paths[source] = record, path + snapshots[source] = record["canonical_source_artifacts"] + return receipts, paths + + +@pytest.mark.parametrize("damage", ["scope", "source_bytes", "upstream", "failed_unit", "failed_chain"]) +def test_source_chain_must_bind_complete_actual_upstreams(merge, tmp_path, damage): + """A clean-looking latest receipt cannot launder an incomplete or changed earlier producer.""" + root = tmp_path / "kg" + root.mkdir() + records, paths = source_chain(merge, root) + merge.verify_chain(root, paths) + source = root / "data/transformed/ontologies" + if damage == "scope": + (source / "source_finalization.json").rename(source / "ro_source_finalization.json") + records["ontologies"]["canonical_source_artifacts"] = merge.snapshot(source) + paths["ontologies"].write_text(json.dumps(records["ontologies"])) + elif damage == "source_bytes": + (source / "ncbitaxon_nodes.tsv").write_text("tampered retained output\n") + elif damage == "upstream": + records["bacdive"]["plan"]["upstream_before"]["ontologies"] = {} + paths["bacdive"].write_text(json.dumps(records["bacdive"])) + elif damage == "failed_unit": + unit_path = Path(records["bactotraits"]["unit_receipt"]["path"]) + unit = json.loads(unit_path.read_text()) + unit["returncode"] = 1 + unit_path.write_text(json.dumps(unit)) + records["bactotraits"]["unit_receipt"].update(merge.identity(unit_path)) + paths["bactotraits"].write_text(json.dumps(records["bactotraits"])) + else: + records["ontologies"]["returncode"] = 1 + paths["ontologies"].write_text(json.dumps(records["ontologies"])) + with pytest.raises(ValueError): + merge.verify_chain(root, paths) + + +def graph_archive(path, *, target=False, provider="infores:bactotraits", diagnostic=False, tamper=False, duplicate=False): + """Write compact transport bytes with independently constructed manifest hashes.""" + obj = "METPO:1000703" if target else "METPO:9999999" + members = { + "merged-kg_nodes.tsv": b"id\tcategory\nNCBITaxon:1123071\tbiolink:OrganismTaxon\n", + "merged-kg_edges.tsv": ("subject\tpredicate\tobject\tprimary_knowledge_source\n" + f"NCBITaxon:1123071\tbiolink:has_phenotype\t{obj}\t{provider}\n").encode(), + "merged-kg_reference_resolution.tsv": b"check\tstatus\nendpoint_closure\tpass\n", + } + manifest = {"manifest_version": 1, "members": { + name: {"sha256": hashlib.sha256(value).hexdigest(), "bytes": len(value), "rows": value.count(b"\n") - 1} + for name, value in members.items()}, + "provenance": {"commit": "b631cacf", "python": "3.10.11", "merge_config": {"sha256": "a" * 64}, + "source_finalization": {"required": not diagnostic, "diagnostic_opt_out": diagnostic}}} + if tamper: + members["merged-kg_nodes.tsv"] += b"tampered:1\tbiolink:NamedThing\n" + with tarfile.open(path, "w:gz") as archive: + for name, payload in [*members.items(), ("manifest.json", json.dumps(manifest).encode())]: + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + if duplicate: + info = tarfile.TarInfo("merged-kg_edges.tsv") + info.size = len(members[info.name]) + archive.addfile(info, io.BytesIO(members[info.name])) + + +def test_actual_archive_counts_hashes_and_strict_provenance_are_verified(merge, tmp_path): + """A valid compressed candidate returns independently streamed member counts.""" + archive = tmp_path / "candidate.tar.gz" + graph_archive(archive) + result = merge.inspect_archive(archive, "a" * 64, "b631cacf", "3.10.11") + assert result["members"]["merged-kg_edges.tsv"]["rows"] == 1 + assert result["members"]["merged-kg_edges.tsv"]["target_spo_occurrences"] == 0 + assert result["archive"]["sha256"] == hashlib.sha256(archive.read_bytes()).hexdigest() + + +@pytest.mark.parametrize("options", [ + {"target": True}, {"target": True, "provider": ""}, {"diagnostic": True}, {"tamper": True}, {"duplicate": True}, +]) +def test_valid_exit_or_rehashed_manifest_cannot_hide_candidate_failure(merge, tmp_path, options): + """Reject surviving target assertions, provenance loss, diagnostics and archive tampering.""" + archive = tmp_path / "candidate.tar.gz" + graph_archive(archive, **options) + with pytest.raises(ValueError): + merge.inspect_archive(archive, "a" * 64, "b631cacf", "3.10.11") + + +def test_changed_plan_blocks_before_output_creation_or_cli(merge, tmp_path, monkeypatch): + """A stale plan cannot start any candidate process or reserve its output.""" + output = tmp_path / "must-stay-absent" + planned = {"profile": "m5-strict-bactotraits-merge-plan/v1", "binding": {"output": str(output)}, + "binding_sha256": "prior"} + plan_file = tmp_path / "old-plan.json" + plan_file.write_text(json.dumps(planned)) + current = {**planned, "binding_sha256": "changed-inputs"} + monkeypatch.setattr(merge.subprocess, "Popen", lambda *args, **kwargs: pytest.fail("CLI started with stale plan")) + with pytest.raises(ValueError, match="since --plan"): + merge.run(current, plan_file) + assert not output.exists() + + +def test_changed_protected_stats_are_a_required_failure(merge, tmp_path): + """Even an absent historical stats path is protected from new merge side effects.""" + stats = tmp_path / "merged_graph_stats.yaml" + binding = {"input_pins": {}, "source_trees": {}, "runtime": {}, "protected_original_outputs": {str(stats): None}} + stats.write_text("unexpected stats written in KG checkout\n") + with pytest.MonkeyPatch.context() as patch: + patch.setattr(merge, "runtime", lambda: {}) + with pytest.raises(ValueError, match="statistics were overwritten"): + merge.verify_after(binding) + + +def quarantine_tree(merge, root, monkeypatch): + """Use an explicitly synthetic CSV hash while preserving exact-row and complete-field contracts.""" + raw = root / "data/raw/BactoTraits_databaseV2_Jun2022.csv" + raw.parent.mkdir(parents=True) + row = {"strain": "fixture", "Bacdive_ID": "17851", "Full_name": "mixed identity fixture", + "culture collection codes": "DSM 19353, JCM 14586", "uninterpreted measurement": "0.5"} + with raw.open("w", newline="", encoding="ISO-8859-1") as stream: + writer = csv.writer(stream, delimiter=";") + writer.writerow(["fixture metadata1"]) + writer.writerow(["fixture metadata2"]) + writer.writerow(row) + for _ in range(8520): + writer.writerow(["neighbor", "7", "independent neighbor", "DSM 7", "1.0"]) + writer.writerow(row.values()) + monkeypatch.setattr(merge, "SOURCE_SHA", hashlib.sha256(raw.read_bytes()).hexdigest()) + monkeypatch.setattr(merge, "ROW_SHA", hashlib.sha256(json.dumps(row, ensure_ascii=False, indent=2, + sort_keys=True).encode()).hexdigest()) + policy = root / "mappings/canonical/bactotraits_identity_quarantine.json" + policy.parent.mkdir(parents=True) + decision = {"bacdive_id": "17851", "rule_id": "fixture-mixed-row", "full_name": row["Full_name"], + "culture_collection_codes": row["culture collection codes"], "rationale": "Fixture identity conflict", + "evidence_urls": ["https://example.test/fixture"]} + policy.write_text(json.dumps({"decisions": [decision]})) + source = root / "data/transformed/bactotraits" + source.mkdir(parents=True) + record = {**decision, "source_data_row": "8521", "reviewed_source_data_row": "8521", "ncbitaxon_id": merge.SPO[0], + "source_sha256": merge.SOURCE_SHA, "reviewed_source_sha256": merge.SOURCE_SHA, "row_sha256": merge.ROW_SHA, + "disposition": "quarantine_mixed_strain_row", "raw_row_json": json.dumps(row), + "policy_sha256": hashlib.sha256(policy.read_bytes()).hexdigest(), + "evidence_urls": "https://example.test/fixture"} + audit = source / merge.AUDIT + + def write_audit(records): + with audit.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(record), delimiter="\t") + writer.writeheader() + writer.writerows(records) + + write_audit([record]) + for folder, filename, key, bid, taxon in ( + ("bacdive", "bacdive.tsv", "bacdive_id", "bacdive:17851", merge.SPO[0]), + ("bactotraits", "bactotraits_mapping.tsv", "Bacdive_ID", "bacdive:17851", merge.SPO[0]), + ("bactotraits", "bactotraits.tsv", "Bacdive_ID", "bacdive:7", "NCBITaxon:7"), + ): + path = root / "kg_microbe/transform_utils" / folder / "tmp" / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{key}\tncbitaxon_id\n{bid}\t{taxon}\n") + (source / "edges.tsv").write_text("subject\tpredicate\tobject\tprimary_knowledge_source\n" + "NCBITaxon:7\tbiolink:has_phenotype\tMETPO:1000703\tinfores:bactotraits\n") + return record, write_audit + + +@pytest.mark.parametrize("damage", ["duplicate", "omitted_field", "source_hash", "identity_column", "retained_copy", "spo_leak"]) +def test_complete_audit_and_retained_joins_are_required(merge, tmp_path, monkeypatch, damage): + """A correct ordinal/hash label alone cannot hide row truncation or a leaking retained join.""" + record, write_audit = quarantine_tree(merge, tmp_path, monkeypatch) + result = merge.verify_quarantine(tmp_path) + assert result["audit"]["rows"] == 1 + assert result["source_edges"]["rows"] == 1 # Neighboring row remains present. + if damage == "duplicate": + write_audit([record, record]) + elif damage == "omitted_field": + raw_row = json.loads(record["raw_row_json"]) + del raw_row["uninterpreted measurement"] + write_audit([{**record, "raw_row_json": json.dumps(raw_row)}]) + elif damage == "source_hash": + write_audit([{**record, "source_sha256": "0" * 64}]) + elif damage == "identity_column": + write_audit([{**record, "full_name": "silent replacement identity"}]) + elif damage == "retained_copy": + path = tmp_path / "kg_microbe/transform_utils/bactotraits/tmp/bactotraits.tsv" + with path.open("a") as stream: + stream.write(f"bacdive:17851\t{merge.SPO[0]}\n") + else: + path = tmp_path / "data/transformed/bactotraits/edges.tsv" + with path.open("a") as stream: + stream.write("\t".join((*merge.SPO, "infores:bactotraits")) + "\n") + with pytest.raises(ValueError): + merge.verify_quarantine(tmp_path) diff --git a/tests/test_m5_release_progress.py b/tests/test_m5_release_progress.py new file mode 100644 index 0000000..bcd0144 --- /dev/null +++ b/tests/test_m5_release_progress.py @@ -0,0 +1,128 @@ +"""Retain command evidence when independent M5 launchers finish out of order.""" + +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + + +def test_stale_launcher_state_cannot_erase_another_command(tmp_path): + """Rebuild aggregate progress from atomic records, including overlapping commands.""" + helper = Path(__file__).parents[1] / "scripts/adhoc/m5_release_state_20261116.py" + spec = importlib.util.spec_from_file_location("release_progress", helper) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.ROOT = tmp_path + module.OUTPUT = tmp_path / ".temp/m5-worker-run-20261116" + logs = module.OUTPUT / "commands" + logs.mkdir(parents=True) + baseline = {"label": "baseline", "status": "running", "log": "baseline.log", + "started_at": "2026-11-16T20:00:00Z"} + preflight = {"label": "preflight", "status": "passed", "log": "preflight.log", + "started_at": "2026-11-16T20:01:00Z"} + stale_state = {"phase": "baseline", "pid": 1, "commands": [baseline]} + module.save_command(logs / "baseline.json", baseline) + module.save(stale_state) + module.save_command(logs / "preflight.json", preflight) + module.save({"phase": "preflight", "pid": 2, "blockers": ["current dependency"], + "results": {"preflight_passed": True}}) + baseline["status"] = "passed" + module.save_command(logs / "baseline.json", baseline) + module.save({}) + result = json.loads((module.OUTPUT / "progress.json").read_text()) + assert result["commands"] == [baseline, preflight] + assert result["phase"] == "preflight" + assert result["blockers"] == ["current dependency"] + assert result["results"] == {"preflight_passed": True} + assert "preflight: passed" in (module.ROOT / ".temp/M5_RELEASE_PROGRESS.md").read_text() + + +@pytest.fixture +def progress_module(tmp_path): + """Keep imported check validation separate from actual release progress.""" + helper = Path(__file__).parents[1] / "scripts/adhoc/m5_release_state_20261116.py" + spec = importlib.util.spec_from_file_location("release_progress_import", helper) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.ROOT = tmp_path + module.OUTPUT = tmp_path / ".temp/m5-worker-run-20261116" + return module + + +def check_receipt(tmp_path, status="complete", code=0): + """Write authentic-looking operational evidence with a byte-bound test log.""" + log = tmp_path / "command.log" + log.write_bytes(b"5 passed in 2.94s\n") + receipt = {"status": status, "command": ["pytest", "five-selected-cases"], "cwd": str(tmp_path), + "pid": 123, "started_at": "2026-11-16T21:14:40+00:00"} + if status != "running": + receipt.update(exit_code=code, completed_at="2026-11-16T21:14:44+00:00", seconds=4.0, + log_sha256=hashlib.sha256(log.read_bytes()).hexdigest()) + path = tmp_path / "receipt.json" + path.write_text(json.dumps(receipt)) + return path, receipt + + +@pytest.mark.parametrize(("status", "code", "expected"), [("complete", 0, "passed"), ("failed", 1, "failed"), + ("failed", -9, "failed"), ("running", None, "running")]) +def test_import_preserves_valid_observed_terminal_or_running_status(progress_module, tmp_path, status, code, expected): + """An active process stays running; a signal failure cannot become a passing result.""" + path, _ = check_receipt(tmp_path, status, code) + entry = progress_module.import_check_record(path, "actual-check") + assert entry["status"] == expected + assert entry["exit_code"] == code + assert entry["source_receipt_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.mark.parametrize(("status", "code"), [("complete", 1), ("failed", 0), ("complete", None), + ("complete", False), ("failed", True)]) +def test_contradictory_terminal_evidence_is_refused(progress_module, tmp_path, status, code): + """Boolean, missing and conflicting exit codes never justify terminal progress status.""" + path, _ = check_receipt(tmp_path, status, code) + with pytest.raises(ValueError, match="contradicts exit code"): + progress_module.import_check_record(path, "invalid-check") + + +@pytest.mark.parametrize("field", ["exit_code", "completed_at", "seconds", "log_sha256"]) +def test_running_receipt_cannot_carry_terminal_evidence(progress_module, tmp_path, field): + """A running label cannot conceal any already completed or contradictory field.""" + path, receipt = check_receipt(tmp_path, "running") + receipt[field] = None + path.write_text(json.dumps(receipt)) + with pytest.raises(ValueError, match="contains terminal fields"): + progress_module.import_check_record(path, "invalid-running") + + +@pytest.mark.parametrize("damage", ["changed", "missing_hash", "missing_log"]) +def test_terminal_import_requires_the_actual_receipt_bound_log(progress_module, tmp_path, damage): + """A claimed passing receipt cannot import changed or absent result evidence.""" + path, receipt = check_receipt(tmp_path) + log = tmp_path / "command.log" + if damage == "changed": + log.write_bytes(b"5 failed\n") + elif damage == "missing_hash": + receipt.pop("log_sha256") + path.write_text(json.dumps(receipt)) + else: + log.unlink() + with pytest.raises(ValueError, match="log"): + progress_module.import_check_record(path, "invalid-log") + + +def test_failed_import_cannot_replace_existing_command_progress(progress_module, tmp_path, monkeypatch): + """Validate before updating either the imported command row or aggregate state.""" + path, _ = check_receipt(tmp_path, "complete", 1) + commands = progress_module.OUTPUT / "commands" + commands.mkdir(parents=True) + existing = commands / "retained.json" + retained = b'{"label":"retained","status":"running"}\n' + existing.write_bytes(retained) + monkeypatch.setattr("sys.argv", ["m5_release_state", "--phase", "check", "--label", "retained", + "--import-check", str(path)]) + with pytest.raises(SystemExit) as error: + progress_module.main() + assert error.value.code == 2 + assert existing.read_bytes() == retained + assert not (tmp_path / ".temp/m5-release-progress.json").exists()