From 24f5e08a0d600721e59f0734ffa1c66fb2cb0bca Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Fri, 20 Nov 2026 17:02:49 +0800 Subject: [PATCH 1/2] Trace CANNBot round knowledge in managed optimization Record opt-in CANNBot searches and pinned read receipts beside local captures. Show metadata and explicit citations in Dashboard, preserve native local capture verification, and reserve final evaluation time when the external worker times out. Signed-off-by: jiangkuaixue123 Agent-assisted-by: Codex / implement_523 (implementer) --- docs/apps-module-migrations.json | 11 +- docs/examples/dashboard-profile-review.html | 57 ++++- docs/external-channels.md | 11 +- docs/operations.md | 1 + docs/optimization-benchmarks.md | 37 +++ src/inferrsi/apps/cli/__init__.py | 35 +-- src/inferrsi/apps/cli/arguments.py | 12 +- .../apps/integrations/cannbot/read_worker.py | 9 +- .../integrations/cannbot/round_knowledge.py | 116 +++++++++ src/inferrsi/apps/views/capture_delivery.py | 32 ++- src/inferrsi/apps/views/codex_search.py | 3 + src/inferrsi/apps/views/dashboard_data.py | 14 ++ .../apps/views/retrieval_projection.py | 68 +++++ .../apps/views/round_decision_view.html | 57 ++++- src/inferrsi/harness/optimization.py | 141 +++++++++-- src/inferrsi/harness/optimization_session.py | 22 +- tests/test_optimization_session.py | 234 +++++++++++++++++- tests/test_project_optimization.py | 86 ++++++- tests/test_round_citations.py | 25 +- tests/test_round_decisions_browser.py | 47 ++++ 20 files changed, 955 insertions(+), 63 deletions(-) create mode 100644 src/inferrsi/apps/integrations/cannbot/round_knowledge.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index b35a150dc..ecf092c9a 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -333,6 +333,7 @@ ], "consumers": [ "src/inferrsi/apps/cli/__init__.py", + "src/inferrsi/apps/integrations/cannbot/round_knowledge.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py" ], @@ -360,10 +361,14 @@ "disable" ], "consumers": [ + "src/inferrsi/apps/cli/arguments.py", "src/inferrsi/apps/integrations/cannbot/__init__.py", + "src/inferrsi/apps/integrations/cannbot/round_knowledge.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py", - "tests/test_cannbot_boundaries.py" + "tests/test_cannbot_boundaries.py", + "tests/test_optimization_session.py", + "tests/test_project_optimization.py" ] }, { @@ -378,9 +383,11 @@ ], "consumers": [ "src/inferrsi/apps/integrations/cannbot/project.py", + "src/inferrsi/apps/integrations/cannbot/round_knowledge.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py", - "tests/test_cannbot_boundaries.py" + "tests/test_cannbot_boundaries.py", + "tests/test_project_optimization.py" ] } ] diff --git a/docs/examples/dashboard-profile-review.html b/docs/examples/dashboard-profile-review.html index c092ec746..ce2c9cd14 100644 --- a/docs/examples/dashboard-profile-review.html +++ b/docs/examples/dashboard-profile-review.html @@ -875,6 +875,47 @@

} function item(box,title){const a=el('article',null,'rd-item');a.append(el('h4',title));box.append(a);return a;} function stage(grid,index,en,zh){const section=el('section',null,'rd-stage');section.dataset.rdStage=String(index);const h=el('h3');h.append(el('span',String(index).padStart(2,'0'),'rd-number'),document.createTextNode(' '+tr(en,zh)));section.append(h);grid.append(section);return section;} + function managedRetrieval(stage, record){ + const box=item(stage,tr('Round knowledge query','本轮知识查询')); + field(box,tr('Query','查询'),record.query); + const local=item(box,tr('Local knowledge','本地知识')); + field(local,tr('Returned items','返回条目'),record.items.length); + for(const entry of record.items){ + const card=item(local,entry.title||entry.ref); + field(card,tr('Reference','引用'),entry.ref); + field(card,tr('Delivery','交付'),entry.delivered?tr('Context prepared','已装配上下文'):tr('Context omitted','未装配上下文')); + } + const result=record.external; + if(!result)return; + const external=item(box,'CANNBot · '+result.platform); + field(external,tr('Channel state','通道状态'),result.state==='ok'&&result.count===0?tr('No hits','无命中'):result.state); + field(external,tr('Returned items','返回条目'),result.count); + if(result.count>result.items.length)p(external,tr('Some hits lack retainable identifiers.','部分命中缺少可留存的标识。'),'rd-note'); + if(result.revision)field(external,tr('Source revision','来源版本'),result.revision); + if(result.score_scale)field(external,tr('Score scale','打分体系'),result.score_scale); + for(const hit of result.items){ + const card=item(external,hit.locator),pages=result.reads.filter(r=>r.locator===hit.locator).map(r=>r.page); + field(card,tr('Search score','检索分数'),hit.score); + field(card,tr('Read receipts','读取凭据'),pages.length?tr('Pages ','页码 ')+[...new Set(pages)].join(', '):null); + detail(card,tr('Card identity','知识卡标识'),'managed:'+result.path+':'+hit.locator,b=>field(b,'SHA-256',hit.card_sha256)); + } + } + function knowledgeUses(box,candidate,hasQueries){ + const citations=candidate.knowledge_uses; + if(!citations||(!citations.length&&!hasQueries))return; + const name=tr('Agent-declared knowledge use','Agent 声明采用的知识'); + if(!citations.length)field(box,name,null); + for(const [index,citation] of citations.entries()){ + const source=citation.source==='local'?tr('Local','本地'):citation.source; + const origin=citation.retrieval_round?tr('Round ','第 ')+citation.retrieval_round+tr('',' 轮'):null; + const page=citation.page?tr('Page ','页码 ')+citation.page:null; + field(box,name,[source,citation.ref,origin,page].filter(Boolean).join(' · ')); + if(citation.revision)detail(box,tr('Source identity','来源标识'),'citation:'+identity(candidate)+':'+index,b=>{ + field(b,tr('Source revision','来源版本'),citation.revision); + if(citation.card_sha256)field(b,'SHA-256',citation.card_sha256); + }); + } + } function render(){ const root=document.getElementById('round-decisions');if(!root)return; const focus=root.contains(document.activeElement)?document.activeElement.dataset.rdFocus:null; @@ -884,7 +925,9 @@

p(root,tr('Retrieval results → Alternatives → Final selection → Execution results','检索结果 → 备选方案 → 最终选择 → 执行结果'),'rd-note'); const data=getData(),rows=data.candidates||[],story=data.story||{},knowledge=data.knowledge_trace||{}; const iterations=story.iterations||[],searches=knowledge.searches||[],captures=knowledge.retrievals||[]; - const numbers=[...new Set([...rows.map(r=>r.round),...iterations.map(i=>i.round),...searches.map(s=>s.round)].filter(validRound))].sort((a,b)=>a-b); + const managed=data.managed_retrievals||[]; + const pending=data.pending_candidate; + const numbers=[...new Set([...rows,...iterations,...searches,...managed,...(pending?[pending]:[])].map(r=>r.round).filter(validRound))].sort((a,b)=>a-b); const choices=numbers.map(String);if(rows.some(r=>!validRound(r.round)))choices.push('unassigned'); if(!choices.length)choices.push('unassigned'); if(!choices.includes(selected))selected=choices[0]; @@ -902,7 +945,9 @@

const contextIds=new Set(roundIterations.flatMap(i=>i.context_ids||[])); roundSearches.filter(s=>s.state==='recorded').forEach(s=>(s.context_ids||[]).forEach(id=>contextIds.add(id))); captures.filter(c=>number!==null&&c.links?.some(l=>l.round===number)).forEach(c=>contextIds.add(c.context_id)); - if(!contextIds.size)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); + const managedQueries=managed.filter(inRound); + for(const query of managedQueries)managedRetrieval(retrieved,query); + if(!contextIds.size&&!managedQueries.length)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); for(const id of contextIds){ const c=captures.find(c=>c.context_id===id),box=item(retrieved,id); if(!c||c.state!=='verified'){p(box,label(c?.state),'rd-warning');continue;} @@ -996,12 +1041,18 @@

p(selection,tr('Choice and selection rationale not recorded.','选中方案及选择理由未记录。'),'rd-note'); for(const row of roundRows){const box=item(selection,row.candidate_id);p(box,tr('Registered candidate; registration alone is not selection evidence.','已登记候选;登记本身不证明选择依据。'),'rd-note');field(box,tr('Submitted hypothesis','提交的假设'),row.optimization?.hypothesis,true);} } - if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); + if(pending&&inRound(pending)){ + const box=item(results,pending.candidate_id); + p(box,tr('Awaiting assessment','等待评测结果'),'rd-note'); + knowledgeUses(box,pending,managedQueries.length); + }else if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); for(const row of roundRows){ const key=identity(row),box=item(results,[row.candidate_id,row.run_id||unknown()].join(' · ')); p(box,label(row.verdict),'rd-badge');field(box,tr('Quality','质量'),label(row.quality)); const metric=v=>Number.isFinite(v)?String(v)+' '+(row.metric_unit||''):'—'; field(box,tr('Baseline → candidate','基线 → 候选'),metric(row.baseline_median)+' → '+metric(row.candidate_median)); + if(Number.isFinite(row.measured_score))field(box,tr('Measured score','实测成绩'),row.measured_score+' · '+row.metric_key); + knowledgeUses(box,row,managedQueries.length); field(box,tr('Recorded improvement','记录的改进幅度'),Number.isFinite(row.improvement_pct)?row.improvement_pct+'%':null); field(box,tr('Post-validation decision (record)','验证后决定(记录)'),label(row.decision)); const matches=rows.filter(r=>identity(r)===key); diff --git a/docs/external-channels.md b/docs/external-channels.md index 6db248f58..69ed69bb2 100644 --- a/docs/external-channels.md +++ b/docs/external-channels.md @@ -198,10 +198,12 @@ An external result is **not locally verified**. Cite it as `cannbot-knowledge @ ` and say so. It does not participate in a `PASS` verdict. -When a receipt is added (not in this release), it will hold the locator, the -card's `card_sha256` and the index revision — never card text, never -`frontmatter`, never a `local_path`. A digest proves which card was cited without -redistributing it. +[Managed round retrieval](optimization-benchmarks.md#include-cannbot-in-the-round) +records CANNBot search metadata and pinned read receipts. A receipt identifies +the search, locator, index revision, card SHA-256 and requested page. The +authorized read returns text directly to the caller. The campaign retains +metadata only. Candidate attribution requires an explicit citation when +submitting a round. Whether CANN OSL v2.0 permits even a local read-only copy is an open question for the organisation; see #476 Q1. This page documents the mechanism, not a legal @@ -213,6 +215,7 @@ conclusion. | --- | --- | | `cannbot enable / disable / describe / search / read` | implemented | | `find` integration with gating, budgets, per-channel states | implemented | +| Managed benchmark and AB/BA round query, read and Dashboard attribution | implemented | | MindIE as a second channel | not yet; `apps/mindie/` has the reader, not the adapter | | `cannbot serve` (stdio MCP proxying `discover` / `search` / `read_card`) | not yet | | `campaign import-cannbot` (receipts into evidence) | not yet | diff --git a/docs/operations.md b/docs/operations.md index 675686c37..85aec4829 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -38,6 +38,7 @@ Effect classes used below: | Inspect the package catalog | CLI `knowledge search` / `knowledge show`; catalog API | Read-only | Filtered inventory and existing catalog scripts. Uses the installed package's tier/repo/backend/scenario filters and IDs | | Prepare model-aware engine research | CLI `knowledge research`; Python `research_knowledge` | Read-only | Independent model/framework/task/modality/mechanism queries over the same package catalog; source-grounded known profiles, deduplicated matches and explicit target annotations; no engine/hardware qualification | | Enter an inference task or discover/read skills | CLI `skills enter` / `list` / `show` / `resource` / `reference` | Read-only | `enter` loads one primary owner's complete procedure before evidence search. `list`/ `show` inspect skill metadata and instructions. `resource`/`reference` resolve paths for your file reader. | +| Retrieve and cite external round knowledge | CLI `benchmarks session knowledge / knowledge-read`, AB/BA `campaign knowledge / knowledge-read` | Local evidence-writing | Requires an explicit `--cannbot-platform` at start and an approved project grant. Retains search and read metadata. Cite a delivered receipt with `round --knowledge-ref`. See [round retrieval](optimization-benchmarks.md#include-cannbot-in-the-round) | | Collect selected knowledge | CLI `collect`; Python `KnowledgeReader.collect` | Read-only | Batch selection and budgeted assembly by query or exact IDs, with revisions and omissions. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Read exact material/resources | CLI `read` / `resource`; Python `KnowledgeReader.read` / `KnowledgeReader.resource` | Read-only | Returns the requested revision or an explicit unavailable/stale error; never substitutes newer bytes | | Export legacy cited material | CLI `context` | Read-only | Cited package export for manual use with a coding agent; not a conversation store and not a full Collect equivalent | diff --git a/docs/optimization-benchmarks.md b/docs/optimization-benchmarks.md index 9ebc6fa20..5166b688a 100644 --- a/docs/optimization-benchmarks.md +++ b/docs/optimization-benchmarks.md @@ -77,6 +77,43 @@ selection method and the external Agent configuration are caller-controlled variables. A retained delivery establishes what was supplied. Model consumption and access to material outside this session require their own observation. +### Include CANNBot in the round + +Run from the project with an [approved CANNBot grant](external-channels.md). +Add `--cannbot-platform ` at `benchmarks session start` or AB/BA +`campaign start` to freeze the external source and an explicit Ascend +platform. Omit the option for a local-only session. The paired +`benchmarks compare` conditions keep their existing behavior. + +`knowledge` queries both sources for the next round. It returns the local +context and a separate `external` result, with its own ranking, channel state +and `reference.path`. A CANNBot timeout still returns the local result. The +external request and worker cleanup use the remaining search budget before the +final reserve. Use task keywords and keep private details out. +InferRSI applies its standard evidence redaction to the combined query before +searching and retaining it. Use a locator from this round's external result: + +```bash +inferrsi benchmarks session knowledge-read --session \ + --search-ref --locator --page 1 +``` + +The read verifies the revision and digest recorded by the search. Card text +returns to the authorized CLI caller. The session retains only the source, +locator, revision, digest, page and search reference. Read additional pages +with `--page`. Use the equivalent `campaign knowledge-read` command for AB/BA. + +At candidate submission, repeat `--knowledge-ref` for each delivery you used. +Pass the read's `reference.path` for an external card page, or +`#` for a local item with delivered resources. +A local capture path alone cites the whole delivery. You can cite an earlier +delivery from the same session. Search result paths are ineligible for citation. + +Open [Dashboard](dashboard.md) → **Round decisions** to see the queries and +channel states before submitting a candidate. Completed attempts show the +agent's explicit citations alongside the candidate ID, verdict and measured +score. A search hit or read receipt alone does not record a citation. + ## Submit a round and use its feedback ```bash diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index e6dc3d900..8baaa74fe 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -23,6 +23,25 @@ def _print(value: object) -> None: print(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False)) +def _optimization_session(args, *, task_source=None): + from ...harness.optimization_session import session_command + + settings = None + provider = None + if getattr(args, "cannbot_platform", None): + settings = {"channel": "cannbot-knowledge", "backend": "ascend", + "platform": args.cannbot_platform} + if args.session_action in ("knowledge", "knowledge-read"): + from ..integrations.cannbot.round_knowledge import CannbotRoundKnowledge + + provider = CannbotRoundKnowledge() + return session_command( + args, task_source=task_source, retain_context=capture_context, + knowledge_reader=KnowledgeReader() if args.session_action == "knowledge" else None, + external_knowledge=settings, external_provider=provider, + ) + + def _fixed_attempt_command(args) -> int: from ...harness.errors import PublicError from ..api import read_attempt_view @@ -356,7 +375,6 @@ def main(argv: list[str] | None = None) -> int: _print(compare_command(args)) return 0 if args.benchmark_action == "session": - from ...harness.optimization_session import session_command from ..delivery import after_final task_source = None @@ -364,12 +382,7 @@ def main(argv: list[str] | None = None) -> int: from ...benchmark.registry import task_resources task_source = Path(str(task_resources(args.task))) - result = session_command( - args, task_source=task_source, - retain_context=capture_context, - knowledge_reader=(KnowledgeReader() - if args.session_action == "knowledge" else None), - ) + result = _optimization_session(args, task_source=task_source) _print(after_final(args, result)) return 2 if result.get("state") == "blocked" else 0 if args.benchmark_action == "run": @@ -628,15 +641,9 @@ def main(argv: list[str] | None = None) -> int: _print({"context_directory": str(path), "reviewed_by_caller": False}) return 0 if getattr(args, "session_action", None) is not None: - from ...harness.optimization_session import session_command from ..delivery import after_final - result = session_command( - args, - retain_context=capture_context, - knowledge_reader=(KnowledgeReader() - if args.session_action == "knowledge" else None), - ) + result = _optimization_session(args) _print(after_final(args, result)) if "exit_code" in result: return result["exit_code"] diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index dd099a747..f25f8d4ce 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -99,9 +99,13 @@ def _optimization_actions(stages, *, project=False): start.add_argument("--condition", default="current-agent" if project else None, required=not project) start.add_argument("--knowledge", choices=("on", "off"), default="on") + from ..integrations.cannbot.project import PLATFORMS + + start.add_argument("--cannbot-platform", choices=PLATFORMS, + help="Opt into CANNBot retrieval on the declared Ascend platform") start.add_argument("--evaluation-prefix", default="", help="Quoted allocator argv prefix") start.add_argument("--python", default=sys.executable) - actions = ("status", "diagnose", "round", "final", "recover", "knowledge") + actions = ("status", "diagnose", "round", "final", "recover", "knowledge", "knowledge-read") if project: actions += ("baseline", "refresh-references", "resume", "profile") for name in actions: @@ -120,7 +124,7 @@ def _optimization_actions(stages, *, project=False): stage.add_argument("--hypothesis", required=True) stage.add_argument("--strategy", help="Agent-declared optimization strategy") stage.add_argument("--knowledge-ref", action="append", - help="Delivered context path cited by this round; repeatable") + help="Delivered path, path#item-ref, or CANNBot receipt; repeatable") if name == "final": stage.add_argument("--reason", help="Why the caller stopped an up-to-budget search") if name in ("round", "final"): @@ -146,6 +150,10 @@ def _optimization_actions(stages, *, project=False): type=Path, help="Investigation receipt proving interrupted resources were cleaned", ) + if name == "knowledge-read": + stage.add_argument("--search-ref", required=True, help="Recorded CANNBot search path") + stage.add_argument("--locator", required=True) + stage.add_argument("--page", type=int, default=1) if name == "knowledge": stage.add_argument("--query", required=True) stage.add_argument( diff --git a/src/inferrsi/apps/integrations/cannbot/read_worker.py b/src/inferrsi/apps/integrations/cannbot/read_worker.py index 0ff4055d4..615a133ea 100644 --- a/src/inferrsi/apps/integrations/cannbot/read_worker.py +++ b/src/inferrsi/apps/integrations/cannbot/read_worker.py @@ -78,6 +78,10 @@ class OutputLimitError(ValueError): """A pipe exceeded the bounded capture size.""" +_PROCESS_WAIT_SECONDS = 2.0 +_WORKER_CLEANUP_SECONDS = _PROCESS_WAIT_SECONDS * (2 if os.name == "nt" else 1) + + def _stop(process, *, own_group): if own_group and os.name == "posix": try: @@ -87,11 +91,12 @@ def _stop(process, *, own_group): elif own_group and os.name == "nt" and process.poll() is None: subprocess.run( ["taskkill", "/PID", str(process.pid), "/T", "/F"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2, check=False, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=_PROCESS_WAIT_SECONDS, + check=False, ) if process.poll() is None: process.kill() - process.wait(timeout=2) + process.wait(timeout=_PROCESS_WAIT_SECONDS) def capture(argv, *, cwd, timeout, environment=None, request=None, own_group=False): diff --git a/src/inferrsi/apps/integrations/cannbot/round_knowledge.py b/src/inferrsi/apps/integrations/cannbot/round_knowledge.py new file mode 100644 index 000000000..985130bc7 --- /dev/null +++ b/src/inferrsi/apps/integrations/cannbot/round_knowledge.py @@ -0,0 +1,116 @@ +"""Connect managed round knowledge to the approved read-only CANNBot worker.""" + +import math +import re + +from ....harness.records.evidence import relative_location +from ....knowledge.channels import CHANNEL_STATES, TIMEOUT, ChannelRegistry, search_channels +from . import SCORE_SCALE, CannbotChannel, project +from .read_worker import _WORKER_CLEANUP_SECONDS + +SCORE_SCALES = (SCORE_SCALE, "knowledge-search.v5") +_LOCATOR = re.compile(r"[\w][\w.+-]*(?:/[\w][\w.+-]*)*") + + +def _hex(value, size): + return (isinstance(value, str) and len(value) == size + and re.fullmatch("[0-9a-f]+", value) is not None) + + +def _card_identity(record): + if not isinstance(record, dict): + return None + locator, digest = record.get("locator"), record.get("card_sha256") + if (not isinstance(locator, str) or relative_location(locator) != locator + or not _LOCATOR.fullmatch(locator) or not _hex(digest, 64)): + return None + return {"locator": locator, "card_sha256": digest} + + +def search_metadata(record): + """Validate the scalar metadata published from a retained search.""" + if ( + not isinstance(record, dict) or record.get("channel") != "cannbot-knowledge" + or record.get("backend") != "ascend" or record.get("platform") not in project.PLATFORMS + or record.get("state") not in CHANNEL_STATES or not isinstance(record.get("items"), list) + ): + return None + revision, count = record.get("revision"), record.get("count") + elapsed, scale = record.get("elapsed_ms"), record.get("score_scale") + items = [] + if record["state"] == "ok": + for row in record["items"]: + card = _card_identity(row) + if card is None: + continue + score = row.get("score") + if type(score) not in (int, float) or not -math.inf < score < math.inf: + score = None + items.append({**card, "score": score}) + return { + **{key: record[key] for key in ("channel", "backend", "platform", "state")}, + "revision": revision if _hex(revision, 40) else None, + "score_scale": scale if scale in SCORE_SCALES else None, + "count": count if type(count) is int and len(items) <= count <= 200 else None, + "elapsed_ms": elapsed if type(elapsed) is int and elapsed >= 0 else None, + "items": items, + } + + +def read_metadata(record): + """Validate the identity and page of a retained read receipt.""" + card = _card_identity(record) + if (card is None or record.get("channel") != "cannbot-knowledge" + or not _hex(record.get("revision"), 40) + or type(record.get("page")) is not int or record["page"] < 1): + return None + return {**card, **{key: record[key] for key in ("channel", "revision", "page")}} + + +class CannbotRoundKnowledge: + def search(self, query, *, settings, limit, timeout): + # Process cleanup runs after a timeout and must fit in the search budget too. + timeout = min(8.0, timeout - _WORKER_CLEANUP_SECONDS) + if timeout <= 0: + return {"state": TIMEOUT, "items": [], "count": 0} + channel = CannbotChannel() + results = search_channels( + ChannelRegistry([channel]), query, backend=settings["backend"], + platform=settings["platform"], limit=limit, timeout=timeout, + ) + result, = results + # Select metadata explicitly. The channel view also contains prose, open + # commands and worker diagnostics, none of which belongs in a receipt. + return search_metadata({ + **settings, "state": result.status.state, "revision": result.status.revision, + "items": [{"locator": hit.locator, "card_sha256": hit.card_sha256, + "score": hit.score} for hit in result.items], + "count": len(result.items), "elapsed_ms": result.elapsed_ms, + "score_scale": result.score_scale, + }) + + def read(self, locator, *, revision, card_sha256, page, timeout): + timeout = min(15.0, timeout - _WORKER_CLEANUP_SECONDS) + if timeout <= 0: + raise ValueError("CANNBot read has insufficient time before the final reserve") + try: + result = project.read( + locator, expected_revision=revision, expected_sha256=card_sha256, page=page, + timeout=timeout, + ) + except project.ChannelFailure as exc: + state = exc.state if exc.state in CHANNEL_STATES else "rejected" + reason = f"CANNBot read failed ({state}). Check the grant or search again" + raise ValueError(reason) from None + if ( + result.get("state") != "ok" or result.get("locator") != locator + or result.get("card_sha256") != card_sha256 or result.get("revision") != revision + or type(result.get("page")) is not int or result["page"] != page + or type(result.get("pages")) is not int + ): + raise ValueError("CANNBot read did not match the recorded search") + if page > result["pages"]: + raise ValueError("CANNBot page is outside the card") + if not isinstance(result.get("text"), str) or not result["text"].strip(): + raise ValueError("CANNBot returned an empty card page") + return {key: result[key] for key in ("text", "page", "pages")} diff --git a/src/inferrsi/apps/views/capture_delivery.py b/src/inferrsi/apps/views/capture_delivery.py index 3fc27a20c..b0670e5c1 100644 --- a/src/inferrsi/apps/views/capture_delivery.py +++ b/src/inferrsi/apps/views/capture_delivery.py @@ -15,10 +15,13 @@ sanitized_text as _text, ) from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES +from inferrsi.harness.records.optimization_log import _unique from inferrsi.knowledge.context_projection import assemble_context_text from .codex_tool_returns import _output_lines, _response_text +_EXTERNAL_SUFFIX = ',\n "external": ' + def capture_return_texts(trace, lines, references, context_ids): """Read only tool-return text at the caller's line references in this round.""" @@ -78,8 +81,35 @@ def managed_envelopes(captures): }) for key, (doc, source, digest) in captures.items()} +def _joint_output_end(chunk, start): + try: + external, end = json.JSONDecoder(object_pairs_hook=_unique).raw_decode(chunk, start) + except (ValueError, RecursionError): + return None + if (isinstance(external, dict) and external.get("channel") == "cannbot-knowledge" + and chunk.startswith("\n}", end)): + return end + 2 + return None + + +def managed_local_output(output): + """Return the exact local envelope from a complete joint CLI response.""" + prefix, separator, _ = output.rpartition(_EXTERNAL_SUFFIX) + if separator and _joint_output_end(output, len(prefix) + len(separator)) == len(output): + return prefix + "\n}" + return output + + def _contains_envelope(chunk, body): - return f"\n{body}\n" in "\n" + chunk.replace("\r\n", "\n") + "\n" + chunk = "\n" + chunk.replace("\r\n", "\n") + "\n" + if f"\n{body}\n" in chunk: + return True + prefix = f"\n{body[:-2]}{_EXTERNAL_SUFFIX}" + start = chunk.find(prefix) + if start < 0: + return False + end = _joint_output_end(chunk, start + len(prefix)) + return end is not None and chunk[end:end + 1] == "\n" def managed_return_lines(trace, lines, envelopes, *, reserved): diff --git a/src/inferrsi/apps/views/codex_search.py b/src/inferrsi/apps/views/codex_search.py index f7e2946be..f87531d64 100644 --- a/src/inferrsi/apps/views/codex_search.py +++ b/src/inferrsi/apps/views/codex_search.py @@ -16,6 +16,7 @@ sanitized_text as _text, ) +from .capture_delivery import managed_local_output from .codex_knowledge import decode as decode_knowledge from .codex_knowledge import invocation as knowledge_invocation from .codex_tool_returns import _output_lines, _path, correlate_window @@ -617,6 +618,8 @@ def _matched_capture(spec, item, join, capture_outputs): return False output = "".join(join["aggregate_lines"]).replace("\r\n", "\n").rstrip("\n") root = capture_outputs.get((spec["context_id"], output)) + if root is None and spec["context_id"] is None: + root = capture_outputs.get((None, managed_local_output(output))) cwd = _path(item.get("cwd")) return (root is not None and cwd is not None and spec["directory"] is not None and (cwd / spec["directory"]).resolve() == root) diff --git a/src/inferrsi/apps/views/dashboard_data.py b/src/inferrsi/apps/views/dashboard_data.py index 113832e99..ffc0f21fe 100644 --- a/src/inferrsi/apps/views/dashboard_data.py +++ b/src/inferrsi/apps/views/dashboard_data.py @@ -694,8 +694,10 @@ def _managed_dashboard(root: Path, campaign: dict) -> dict: from inferrsi.harness.optimization import read_records from .optimization_story import build_story + from .retrieval_projection import managed_knowledge plan, state, summary = read_records(root) + retrievals, citations = managed_knowledge(root, state["contexts"]) rows, rounds, decisions = [], [], {} for index, record in enumerate(state["rounds"], 1): candidate_id = record["candidate_id"] @@ -730,12 +732,23 @@ def _managed_dashboard(root: Path, campaign: dict) -> dict: artifacts={**row["artifacts"], "execution": facts, "assessment": judged, "status": facts, "evaluation": judged}, usage=record["observations"]["usage"], + measured_score=_number(assessment.get("score")), + knowledge_uses=[citations[ref] for ref in record.get("knowledge_refs") or [] + if ref in citations], ) row.pop("_comparison_signature") rows.append(row) rounds.append({"round": index, "candidates": [candidate_id], "decision": {"candidate_id": candidate_id, "outcome": decision}}) decisions[(index, candidate_id)] = decision + pending = state["pending"] + pending_candidate = None + if pending and pending["kind"] == "round": + pending_candidate = { + "round": len(rows) + 1, "candidate_id": pending["candidate_id"], + "knowledge_uses": [citations[ref] for ref in pending.get("knowledge_refs") or [] + if ref in citations], + } displayed_summary = _managed_summary(summary) displayed_summary["final_reference"] = ( file_ref(root, root / state["final"]["path"] / "observation.json", "final_assessment") @@ -744,6 +757,7 @@ def _managed_dashboard(root: Path, campaign: dict) -> dict: "campaign_id": sanitized_text(campaign["campaign_id"]), "synthetic": False, "rounds": rounds, "rounds_state": "present", "corrupt_round_lines": [], "candidates": rows, "optimization_session": displayed_summary, + "managed_retrievals": retrievals, "pending_candidate": pending_candidate, "integration": {"state": "connected" if rows else "not_connected", "stage_declaration": "missing", "external_records": 0, "collection": {"state": "", "observed_at": ""}}, diff --git a/src/inferrsi/apps/views/retrieval_projection.py b/src/inferrsi/apps/views/retrieval_projection.py index a2252a716..edfef92ad 100644 --- a/src/inferrsi/apps/views/retrieval_projection.py +++ b/src/inferrsi/apps/views/retrieval_projection.py @@ -11,6 +11,8 @@ from inferrsi.harness.records.evidence import inside, relative_location, sanitized_text from inferrsi.harness.records.optimization_log import _IDENTIFIER, _unique +from ..integrations.cannbot.round_knowledge import read_metadata, search_metadata + _MAX_BYTES = 4 * 1024 * 1024 @@ -311,6 +313,72 @@ def read_searches(root: Path, campaign_id, rows, retrievals): return results +def managed_knowledge(root, contexts): + """Project managed deliveries and explicit citation targets without card bodies.""" + queries, external, citations = {}, {}, {} + for reference in contexts: + path, kind = reference["path"], reference["kind"] + number = reference.get("round") + if kind == "delivered_context": + record = json.loads((root / path).read_text("utf-8")) + delivered = {item["ref"] for item in record["contexts"] if item["resources"]} + items = [ + {**{key: item.get(key) for key in ("ref", "title", "revision")}, + "delivered": item["ref"] in delivered} + for item in record["items"] + ] + if number is not None: + request = record["request"] + queries[path] = { + "round": number, "path": path, + "query": request.get("relevance") or request.get("query"), + "items": items, + } + citation = {"source": "local", "retrieval_round": number} + citations[path] = {**citation, "ref": path} + for item in items: + if item["delivered"]: + citations[path + "#" + item["ref"]] = { + **citation, "ref": item["ref"], "revision": item["revision"], + } + elif kind in ("external_search", "external_read"): + if (type(number) is not int or number < 1 + or not re.fullmatch(rf"knowledge/{kind}-[0-9a-f]{{32}}\.json", path)): + continue + record, state, digest = _read(root, root / path) + if state != "present" or digest != reference["sha256"]: + continue + if kind == "external_search": + metadata = search_metadata(record) + parent = record.get("context_ref") + query = queries.get(parent) if isinstance(parent, str) else None + if metadata is not None and query is not None and number == query["round"]: + search = {**metadata, "path": path, "reads": []} + query["external"] = search + external[path] = (number, search) + else: + receipt = read_metadata(record) + parent = record.get("search_ref") + source = external.get(parent) if isinstance(parent, str) else None + if receipt is None or source is None or source[0] != number: + continue + search = source[1] + if search["revision"] != receipt["revision"] or not any( + hit["locator"] == receipt["locator"] + and hit["card_sha256"] == receipt["card_sha256"] for hit in search["items"] + ): + continue + search["reads"].append({key: receipt[key] for key in ("locator", "page")}) + citations[path] = { + "source": receipt["channel"], "retrieval_round": number, + "ref": receipt["locator"], "revision": receipt["revision"], + "card_sha256": receipt["card_sha256"], "page": receipt["page"], + } + return public_value(list(queries.values())), { + path: public_value(citation) for path, citation in citations.items() + } + + def project_knowledge(root, data): """One projection for both layouts, offline export and live refresh.""" story = data.get("story") or {} diff --git a/src/inferrsi/apps/views/round_decision_view.html b/src/inferrsi/apps/views/round_decision_view.html index c0ec511b3..c4bb814a4 100644 --- a/src/inferrsi/apps/views/round_decision_view.html +++ b/src/inferrsi/apps/views/round_decision_view.html @@ -36,6 +36,47 @@ } function item(box,title){const a=el('article',null,'rd-item');a.append(el('h4',title));box.append(a);return a;} function stage(grid,index,en,zh){const section=el('section',null,'rd-stage');section.dataset.rdStage=String(index);const h=el('h3');h.append(el('span',String(index).padStart(2,'0'),'rd-number'),document.createTextNode(' '+tr(en,zh)));section.append(h);grid.append(section);return section;} + function managedRetrieval(stage, record){ + const box=item(stage,tr('Round knowledge query','本轮知识查询')); + field(box,tr('Query','查询'),record.query); + const local=item(box,tr('Local knowledge','本地知识')); + field(local,tr('Returned items','返回条目'),record.items.length); + for(const entry of record.items){ + const card=item(local,entry.title||entry.ref); + field(card,tr('Reference','引用'),entry.ref); + field(card,tr('Delivery','交付'),entry.delivered?tr('Context prepared','已装配上下文'):tr('Context omitted','未装配上下文')); + } + const result=record.external; + if(!result)return; + const external=item(box,'CANNBot · '+result.platform); + field(external,tr('Channel state','通道状态'),result.state==='ok'&&result.count===0?tr('No hits','无命中'):result.state); + field(external,tr('Returned items','返回条目'),result.count); + if(result.count>result.items.length)p(external,tr('Some hits lack retainable identifiers.','部分命中缺少可留存的标识。'),'rd-note'); + if(result.revision)field(external,tr('Source revision','来源版本'),result.revision); + if(result.score_scale)field(external,tr('Score scale','打分体系'),result.score_scale); + for(const hit of result.items){ + const card=item(external,hit.locator),pages=result.reads.filter(r=>r.locator===hit.locator).map(r=>r.page); + field(card,tr('Search score','检索分数'),hit.score); + field(card,tr('Read receipts','读取凭据'),pages.length?tr('Pages ','页码 ')+[...new Set(pages)].join(', '):null); + detail(card,tr('Card identity','知识卡标识'),'managed:'+result.path+':'+hit.locator,b=>field(b,'SHA-256',hit.card_sha256)); + } + } + function knowledgeUses(box,candidate,hasQueries){ + const citations=candidate.knowledge_uses; + if(!citations||(!citations.length&&!hasQueries))return; + const name=tr('Agent-declared knowledge use','Agent 声明采用的知识'); + if(!citations.length)field(box,name,null); + for(const [index,citation] of citations.entries()){ + const source=citation.source==='local'?tr('Local','本地'):citation.source; + const origin=citation.retrieval_round?tr('Round ','第 ')+citation.retrieval_round+tr('',' 轮'):null; + const page=citation.page?tr('Page ','页码 ')+citation.page:null; + field(box,name,[source,citation.ref,origin,page].filter(Boolean).join(' · ')); + if(citation.revision)detail(box,tr('Source identity','来源标识'),'citation:'+identity(candidate)+':'+index,b=>{ + field(b,tr('Source revision','来源版本'),citation.revision); + if(citation.card_sha256)field(b,'SHA-256',citation.card_sha256); + }); + } + } function render(){ const root=document.getElementById('round-decisions');if(!root)return; const focus=root.contains(document.activeElement)?document.activeElement.dataset.rdFocus:null; @@ -45,7 +86,9 @@ p(root,tr('Retrieval results → Alternatives → Final selection → Execution results','检索结果 → 备选方案 → 最终选择 → 执行结果'),'rd-note'); const data=getData(),rows=data.candidates||[],story=data.story||{},knowledge=data.knowledge_trace||{}; const iterations=story.iterations||[],searches=knowledge.searches||[],captures=knowledge.retrievals||[]; - const numbers=[...new Set([...rows.map(r=>r.round),...iterations.map(i=>i.round),...searches.map(s=>s.round)].filter(validRound))].sort((a,b)=>a-b); + const managed=data.managed_retrievals||[]; + const pending=data.pending_candidate; + const numbers=[...new Set([...rows,...iterations,...searches,...managed,...(pending?[pending]:[])].map(r=>r.round).filter(validRound))].sort((a,b)=>a-b); const choices=numbers.map(String);if(rows.some(r=>!validRound(r.round)))choices.push('unassigned'); if(!choices.length)choices.push('unassigned'); if(!choices.includes(selected))selected=choices[0]; @@ -63,7 +106,9 @@ const contextIds=new Set(roundIterations.flatMap(i=>i.context_ids||[])); roundSearches.filter(s=>s.state==='recorded').forEach(s=>(s.context_ids||[]).forEach(id=>contextIds.add(id))); captures.filter(c=>number!==null&&c.links?.some(l=>l.round===number)).forEach(c=>contextIds.add(c.context_id)); - if(!contextIds.size)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); + const managedQueries=managed.filter(inRound); + for(const query of managedQueries)managedRetrieval(retrieved,query); + if(!contextIds.size&&!managedQueries.length)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); for(const id of contextIds){ const c=captures.find(c=>c.context_id===id),box=item(retrieved,id); if(!c||c.state!=='verified'){p(box,label(c?.state),'rd-warning');continue;} @@ -157,12 +202,18 @@ p(selection,tr('Choice and selection rationale not recorded.','选中方案及选择理由未记录。'),'rd-note'); for(const row of roundRows){const box=item(selection,row.candidate_id);p(box,tr('Registered candidate; registration alone is not selection evidence.','已登记候选;登记本身不证明选择依据。'),'rd-note');field(box,tr('Submitted hypothesis','提交的假设'),row.optimization?.hypothesis,true);} } - if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); + if(pending&&inRound(pending)){ + const box=item(results,pending.candidate_id); + p(box,tr('Awaiting assessment','等待评测结果'),'rd-note'); + knowledgeUses(box,pending,managedQueries.length); + }else if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); for(const row of roundRows){ const key=identity(row),box=item(results,[row.candidate_id,row.run_id||unknown()].join(' · ')); p(box,label(row.verdict),'rd-badge');field(box,tr('Quality','质量'),label(row.quality)); const metric=v=>Number.isFinite(v)?String(v)+' '+(row.metric_unit||''):'—'; field(box,tr('Baseline → candidate','基线 → 候选'),metric(row.baseline_median)+' → '+metric(row.candidate_median)); + if(Number.isFinite(row.measured_score))field(box,tr('Measured score','实测成绩'),row.measured_score+' · '+row.metric_key); + knowledgeUses(box,row,managedQueries.length); field(box,tr('Recorded improvement','记录的改进幅度'),Number.isFinite(row.improvement_pct)?row.improvement_pct+'%':null); field(box,tr('Post-validation decision (record)','验证后决定(记录)'),label(row.decision)); const matches=rows.filter(r=>identity(r)===key); diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index dd9bd19c8..1b00dde8c 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -10,6 +10,7 @@ from . import candidates from .artifact_primitives import digest, file_lock, sha256 from .atomic_json import atomic_write_json +from .records.evidence import sanitized_text from .task_protocol import TaskProtocolError from .usage import normalize_usage, usage_totals @@ -107,23 +108,34 @@ def _load(root, *, read_only=False): if candidates.identity(root / state["best"]["path"]) != state["best"]["sha256"]: raise ValueError("Selected candidate changed") diagnoses = _diagnosis_index(state) + contexts = {} for context in state["contexts"]: raw = (root / context["path"]).read_bytes() if hashlib.sha256(raw).hexdigest() != context["sha256"]: raise ValueError("Delivered knowledge context changed") - _check_context_diagnosis(plan, diagnoses, context, json.loads(raw)) + if context.get("kind") not in ("external_search", "external_read"): + _check_context_diagnosis(plan, diagnoses, context, json.loads(raw)) + contexts[context["path"]] = context for record in [*state["rounds"], *([state["final"]] if state["final"] else [])]: directory = root / record["path"] - observation = directory / "observation.json" - if sha256(observation) != record["observation_sha256"]: + raw = (directory / "observation.json").read_bytes() + if hashlib.sha256(raw).hexdigest() != record["observation_sha256"]: raise ValueError("Retained assessment record changed") + observation = json.loads(raw) + if any(observation.get(key) != record.get(key) for key in ("knowledge_refs", "contexts")): + raise ValueError("Round knowledge attribution differs from the retained observation") + if observation.get("knowledge_refs"): + submitted = {ref["path"]: ref for ref in observation["contexts"]} + for ref in observation["knowledge_refs"]: + path = ref.partition("#")[0] + if path not in submitted or submitted[path] != contexts.get(path): + raise ValueError("Cited knowledge context differs from the submitted snapshot") if record is not state["final"] and ( plan.get("diagnosis_required") or diagnoses or record.get("diagnosis_id") ): - retained = json.loads(observation.read_text(encoding="utf-8")) - if retained.get("diagnosis_id") != record.get("diagnosis_id"): + if observation.get("diagnosis_id") != record.get("diagnosis_id"): raise ValueError("Round diagnosis differs from the retained observation") - _check_activity_diagnosis(plan, diagnoses, retained) + _check_activity_diagnosis(plan, diagnoses, observation) if candidates.identity(directory / plan["candidate_name"]) != record["candidate_sha256"]: raise ValueError("Retained round candidate changed") if sha256(directory / "execution/execution.json") != record["execution_sha256"]: @@ -180,6 +192,8 @@ def status(root: Path) -> dict: "task_instructions": str(root / "frozen/TASK.md"), "working_candidate": str(root / plan["working_candidate"]), "best": state["best"], "pending": state["pending"], + **({"external_knowledge": plan["external_knowledge"]} if "external_knowledge" in plan + else {}), "latest_observation": state["rounds"][-1] if state["rounds"] else None, "diagnosis": _diagnosis(state), "summary": str(root / "summary.json") if state["final"] is not None else None, @@ -303,21 +317,115 @@ def record_diagnosis(root: Path, bottleneck: str, evidence: Path) -> dict: return _retain_diagnosis(root, plan, state, bottleneck, evidence) -def capture_context(root: Path, reader, *, retain_context, **request) -> dict: +def _knowledge_round(plan, state): + _active(state, reserve=plan["final_reserve_seconds"]) + if plan.get("knowledge_allowed") is False: + raise ValueError("Knowledge delivery is disabled by the frozen condition") + number = len(state["rounds"]) + 1 + return number if number <= plan["round_limit"] else None + + +def _knowledge_time(plan, state): + return state["deadline"] - time.time() - plan["final_reserve_seconds"] + + +def _store_knowledge(root, kind, number, record): + directory = root / "knowledge" + if directory.is_symlink(): + raise ValueError("Knowledge directory must not be a symbolic link") + directory.mkdir(exist_ok=True) + path = directory / f"{kind}-{uuid.uuid4().hex}.json" + atomic_write_json(path, record) + return {"path": path.relative_to(root).as_posix(), "sha256": sha256(path), + "kind": kind, "round": number} + + +def capture_context(root: Path, reader, *, retain_context, external_provider=None, + **request) -> dict: with _locked(root): plan, state = _load(root) - _active(state, reserve=plan["final_reserve_seconds"]) - if plan.get("knowledge_allowed") is False: - raise ValueError("Knowledge delivery is disabled by the frozen condition") + number = _knowledge_round(plan, state) + settings = plan.get("external_knowledge") + if settings: + if number is None: + raise ValueError("Round budget consumed, run independent final evaluation") + if external_provider is None: + raise ValueError("The frozen external knowledge provider is unavailable") + request["relevance"] = sanitized_text(request["relevance"], "", limit=None) binding = _context_diagnosis(plan, state) path, fingerprint = retain_context(root, context_id="context-" + uuid.uuid4().hex, reader=reader, diagnosis_binding=binding, **request) reference = {"path": path.relative_to(root).as_posix(), "sha256": fingerprint, "kind": "delivered_context"} - state["contexts"].append( - {**reference, "diagnosis_id": binding["diagnosis_id"]} if binding else reference) + # Keep the local tool response compatible with stored Codex trace verification. + retained = {**reference, "round": number} if number is not None else reference + if binding: + retained = {**retained, "diagnosis_id": binding["diagnosis_id"]} + state["contexts"].append(retained) + response = {"reference": reference, "context": json.loads(path.read_text("utf-8"))} + if settings: + remaining = _knowledge_time(plan, state) + result = ( + external_provider.search(request["relevance"], settings=settings, + limit=request["limit"], timeout=remaining) + if remaining > 0 else {"state": "timeout", "items": [], "count": 0} + ) + search = {**result, **settings, "context_ref": reference["path"]} + search_ref = _store_knowledge(root, "external_search", number, search) + state["contexts"].append(search_ref) + response["external"] = {**search, "reference": search_ref} + atomic_write_json(root / "state.json", state) + return response + + +def read_external_context(root: Path, provider, *, search_ref, locator, page): + with _locked(root): + plan, state = _load(root) + number = _knowledge_round(plan, state) + if not plan.get("external_knowledge") or provider is None: + raise ValueError("External knowledge is disabled or unavailable for this session") + if type(page) is not int or page < 1: + raise ValueError("CANNBot page must be a positive integer") + reference = next((entry for entry in state["contexts"] + if entry["path"] == search_ref and entry["kind"] == "external_search" + and entry["round"] == number), None) + if reference is None: + raise ValueError("Supply a CANNBot search reference from the current round") + search = json.loads((root / reference["path"]).read_text("utf-8")) + hit = next((item for item in search["items"] if item["locator"] == locator), None) + if search["state"] != "ok" or not search["revision"] or hit is None: + raise ValueError("Supply a versioned CANNBot hit from the recorded search") + remaining = _knowledge_time(plan, state) + if remaining <= 0: + raise ValueError("Optimization deadline exhausted, final evaluation time is reserved") + revision, card_sha256 = search["revision"], hit["card_sha256"] + body = provider.read(locator, revision=revision, card_sha256=card_sha256, + page=page, timeout=remaining) + receipt = {"search_ref": search_ref, "channel": search["channel"], "locator": locator, + "revision": revision, "card_sha256": card_sha256, "page": page} + delivered = _store_knowledge(root, "external_read", number, receipt) + state["contexts"].append(delivered) atomic_write_json(root / "state.json", state) - return {"reference": reference, "context": json.loads(path.read_text(encoding="utf-8"))} + return {"reference": delivered, "receipt": receipt, "context": body} + + +def _validate_knowledge_refs(root, contexts, refs): + if not isinstance(refs, list) or any(not isinstance(ref, str) for ref in refs): + raise ValueError("Knowledge references must be a list of delivered references") + if len(set(refs)) != len(refs): + raise ValueError("Cite unique delivered knowledge references") + delivered = {entry["path"]: entry["kind"] for entry in contexts + if entry["kind"] != "external_search"} + for ref in refs: + path, separator, item = ref.partition("#") + kind = delivered.get(path) + if kind is None or (separator and kind != "delivered_context"): + raise ValueError("Cite a delivered context or CANNBot read receipt from this session") + if separator: + capture = json.loads((root / path).read_text("utf-8")) + if not any(entry["ref"] == item and entry["resources"] + for entry in capture["contexts"]): + raise ValueError("Cite an item with delivered resources from this context") def _usage_record(source, *, fingerprint=True): @@ -491,12 +599,7 @@ def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str, if strategy is not None and (not isinstance(strategy, str) or not strategy.strip()): raise ValueError("Strategy must be a nonempty declaration") if knowledge_refs is not None: - available = {context["path"] for context in state["contexts"]} - if (not isinstance(knowledge_refs, list) - or any(not isinstance(ref, str) or ref not in available - for ref in knowledge_refs) - or len(set(knowledge_refs)) != len(knowledge_refs)): - raise ValueError("Cite unique delivered context paths from this campaign") + _validate_knowledge_refs(root, state["contexts"], knowledge_refs) source = root / state["best"]["path"] if final else Path(candidate) if (source.is_symlink() or (plan["candidate_kind"] == "file" and (not source.is_file() or source.suffix != ".py")) diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index e6d71bcb8..7549495f8 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -127,7 +127,7 @@ def _runtime_identity(): def session_command(args, *, task_source: Path | None = None, knowledge_reader=None, - retain_context=None) -> dict: + retain_context=None, external_knowledge=None, external_provider=None) -> dict: """Run one operation; discovery and knowledge access are supplied by the caller.""" from inferrsi.harness import optimization @@ -150,6 +150,11 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N if not math.isfinite(reserve) or not 0 < reserve < args.max_minutes * 60: raise ValueError("Reserve time for final within the total budget") evaluation_prefix = shlex.split(args.evaluation_prefix) + if external_knowledge and args.knowledge != "on": + raise ValueError("External knowledge requires --knowledge on") + if (external_knowledge and args.project_task + and args.evaluation_mode == "service_reuse"): + raise ValueError("External round knowledge requires a benchmark or AB/BA campaign") if args.project_task: names = ( "metric_key", @@ -241,6 +246,8 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N "evaluation_timeout_seconds": args.evaluation_timeout_seconds, "evaluation_prefix": evaluation_prefix, "python": args.python, "runtime_identity": _runtime_identity(), **candidate, **settings} + if external_knowledge: + plan["external_knowledge"] = external_knowledge result = optimization.initialize(root, plan) operation = "campaign" if args.project_task else "benchmarks session" result["instructions"] = ( @@ -252,6 +259,11 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N f"Follow the frozen round policy, then run {operation} final to retest independently. " "Report summary.json and its curve. Import actual usage exports when available." ) + if external_knowledge: + result["instructions"] += ( + f" Read external hits with {operation} knowledge-read. Cite the delivered " + "item or read receipt with --knowledge-ref when submitting a candidate." + ) return result import json @@ -274,6 +286,8 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N "status": "failed", "stage": "campaign", "message": message, }, } + if args.session_action == "knowledge-read": + raise ValueError("External round knowledge requires a benchmark or AB/BA campaign") return optimization.service_command(root, args, knowledge_reader=knowledge_reader) if ( args.session_action in ("baseline", "refresh-references", "resume", "profile") @@ -296,12 +310,18 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N root, knowledge_reader, retain_context=retain_context, + external_provider=external_provider, query="", relevance=args.query, relevance_method=args.relevance_method, context_bytes=args.context_bytes, limit=args.limit, ) + if args.session_action == "knowledge-read": + return optimization.read_external_context( + root, external_provider, search_ref=args.search_ref, locator=args.locator, + page=args.page, + ) if args.session_action == "recover": return optimization.recover(root, _session_assess) final = args.session_action == "final" diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index a8d0ecbbb..c647157c8 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -1,5 +1,6 @@ """Run the current-Agent protocol using real child processes and synthetic work.""" +import hashlib import json import os import subprocess @@ -449,6 +450,7 @@ def test_knowledge_delivery_is_retained_and_bound_to_round(session, capsys): "--limit", "1", "--context-bytes", "8192", ]) == 0 delivered = json.loads(capsys.readouterr().out) + assert set(delivered["reference"]) == {"path", "sha256", "kind"} path = session / delivered["reference"]["path"] capture, state = verify_capture(path.read_bytes()) assert state == "verified" @@ -470,9 +472,239 @@ def test_knowledge_delivery_is_retained_and_bound_to_round(session, capsys): result = submit(session, 2, capsys) observation = result["latest_observation"] assert observation["contexts"] == [ - {**delivered["reference"], "diagnosis_id": observation["diagnosis_id"]} + {**delivered["reference"], "round": 1, "diagnosis_id": observation["diagnosis_id"]} ] + from inferrsi.apps.views.dashboard import dashboard_snapshot + + data = dashboard_snapshot(session)["data"] + retrieval, = data["managed_retrievals"] + assert retrieval["round"] == 1 and "external" not in retrieval + assert data["candidates"][0]["knowledge_uses"] == [] + + +def test_cannbot_query_and_read_trace_to_candidate(tmp_path, task, capsys, monkeypatch): + from inferrsi.apps.integrations.cannbot import project + from inferrsi.apps.views.dashboard import dashboard_snapshot + + revision, digest = "a" * 40, "b" * 64 + locator = "ops/ascendc/apis/synthetic" + uncited = "ops/ascendc/apis/uncited" + card_text = "synthetic external body" + search_query = "cache per-step masks ran out of device memory" + phase = {"state": "ok", "items": [locator, uncited]} + monkeypatch.setattr(project, "describe", lambda *a, **kw: { + "state": "ok", "revision": revision, + }) + + def search(query, *, platform, limit, **kwargs): + assert (query, platform, limit) == (search_query, "a3", 2) + if phase["state"] != "ok": + raise project.ChannelFailure("private worker diagnostics", state=phase["state"]) + return { + "revision": revision, + "items": [ + {"locator": ref, "card_sha256": digest, "score": 7, "summary": card_text} + for ref in phase["items"] + ], + } + + def read(ref, *, expected_revision, expected_sha256, page, timeout): + assert ref in (locator, uncited) + assert (expected_revision, expected_sha256, page) == (revision, digest, 1) + return {"state": "ok", "locator": ref, "revision": revision, "card_sha256": digest, + "page": 1, "pages": 1, "text": card_text} + + monkeypatch.setattr(project, "search", search) + monkeypatch.setattr(project, "read", read) + root = tmp_path / "with-cannbot" + operation = ["benchmarks", "session"] + session_args = ["--session", str(root)] + start_args = [*operation, "start", task.name, *session_args, "--rounds", "3", + "--max-minutes", "5", "--evaluation-timeout-seconds", "15", + "--condition", "synthetic-combined", "--evaluation-args", "", + "--cannbot-platform", "a3"] + with pytest.raises(SystemExit): + main([*start_args, "--knowledge", "off"]) + assert "requires --knowledge on" in capsys.readouterr().err + assert not root.exists() + assert main(start_args) == 0 + started = json.loads(capsys.readouterr().out) + assert started["external_knowledge"]["platform"] == "a3" + + query_args = [*operation, "knowledge", *session_args, "--query", search_query, + "--context-bytes", "8192", "--limit", "2"] + with pytest.raises(SystemExit): + main(query_args) + assert "Record measured bottleneck evidence" in capsys.readouterr().err + diagnose(root, capsys) + assert main(query_args) == 0 + answer = json.loads(capsys.readouterr().out) + search_path = answer["external"]["reference"]["path"] + local_path = answer["reference"]["path"] + local_item = answer["context"]["contexts"][0]["ref"] + before = dashboard_snapshot(root)["data"] + query, = before["managed_retrievals"] + assert query["query"] == search_query and query["round"] == 1 + assert any(x["ref"] == local_item and x["delivered"] for x in query["items"]) + assert query["external"]["state"] == "ok" + assert query["external"]["count"] == 2 + assert query["external"]["reads"] == [] + assert not before["candidates"] + + read_args = [*operation, "knowledge-read", *session_args, "--search-ref", search_path, + "--locator"] + with pytest.raises(SystemExit): + main([*read_args, "ops/ascendc/not-from-search"]) + assert "from the recorded search" in capsys.readouterr().err + capsys.readouterr() + assert main([*read_args, uncited]) == 0 + capsys.readouterr() + assert main([*read_args, locator]) == 0 + delivery = json.loads(capsys.readouterr().out) + assert delivery["context"]["text"] == card_text + receipt_path = delivery["reference"]["path"] + + candidate = root / "work/candidate.py" + candidate.write_text("FACTOR = 2\n") + round_args = [*operation, "round", *session_args, "--candidate", str(candidate), + "--hypothesis", "Apply selected knowledge"] + with pytest.raises(SystemExit): + main([*round_args, "--knowledge-ref", search_path]) + assert "read receipt from this session" in capsys.readouterr().err + with pytest.raises(SystemExit): + main([*round_args, "--knowledge-ref", local_path + "#unknown"]) + assert "delivered resources" in capsys.readouterr().err + execute = optimization_session._session_execute + + def observe_pending(root, directory, plan, deadline): + current = dashboard_snapshot(root)["data"]["pending_candidate"] + assert current["candidate_id"] == "candidate-001" and current["round"] == 1 + assert [use["ref"] for use in current["knowledge_uses"]] == [locator, local_item] + return execute(root, directory, plan, deadline) + + monkeypatch.setattr(optimization_session, "_session_execute", observe_pending) + assert main([*round_args, "--knowledge-ref", receipt_path, + "--knowledge-ref", local_path + "#" + local_item]) == 0 + capsys.readouterr() + with pytest.raises(SystemExit): + main([*read_args, locator]) + assert "from the current round" in capsys.readouterr().err + capsys.readouterr() + + monkeypatch.setattr(optimization_session, "_session_execute", execute) + phase["items"] = [uncited] + assert main(query_args) == 0 + later_search = json.loads(capsys.readouterr().out)["external"]["reference"]["path"] + assert main([*operation, "knowledge-read", *session_args, "--search-ref", later_search, + "--locator", uncited]) == 0 + later_reference = json.loads(capsys.readouterr().out)["reference"] + later_path = later_reference["path"] + + phase["items"] = [] + assert main(query_args) == 0 + capsys.readouterr() + assert submit(root, -1, capsys)["latest_observation"]["assessment"]["score"] is None + phase["state"] = "unreachable" + assert main(query_args) == 0 + capsys.readouterr() + data = dashboard_snapshot(root)["data"] + first, later, empty, unavailable = data["managed_retrievals"] + assert [first["round"], later["round"], empty["round"], unavailable["round"]] == [1, 2, 2, 3] + assert {entry["locator"] for entry in first["external"]["reads"]} == {locator, uncited} + assert later["external"]["reads"] == [{"locator": uncited, "page": 1}] + assert empty["external"]["state"] == "ok" and empty["external"]["count"] == 0 + assert unavailable["external"]["state"] == "unreachable" + row, failed = data["candidates"] + assert (row["candidate_id"], row["round"], row["measured_score"], row["verdict"]) == ( + "candidate-001", 1, 2, "PASS", + ) + assert [(use["source"], use["ref"]) for use in row["knowledge_uses"]] == [ + ("cannbot-knowledge", locator), ("local", local_item), + ] + assert row["knowledge_uses"][0]["card_sha256"] == digest + assert failed["knowledge_uses"] == [] and failed["measured_score"] is None + assert card_text not in json.dumps(data) + assert "private worker diagnostics" not in json.dumps(data) + for path in root.rglob("*.json"): + retained = path.read_text(encoding="utf-8") + assert card_text not in retained + assert "private worker diagnostics" not in retained + + state_path = root / "state.json" + original_state = state_path.read_bytes() + state = json.loads(original_state) + first_round = state["rounds"][0] + original_refs = first_round["knowledge_refs"] + original_contexts = first_round["contexts"] + first_round["knowledge_refs"] = [later_path] + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="Round knowledge attribution"): + dashboard_snapshot(root) + first_round["knowledge_refs"] = original_refs + first_round["contexts"] = [*original_contexts, later_reference] + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="Round knowledge attribution"): + dashboard_snapshot(root) + first_round["contexts"] = original_contexts + + # Even an imported record with a matching digest must not widen the view. + for reference in state["contexts"]: + if reference["path"] not in (search_path, later_path): + continue + path = root / reference["path"] + record = json.loads(path.read_text()) + record["text"] = card_text + if "items" in record: + record["items"][0]["summary"] = card_text + raw = json.dumps(record).encode() + path.write_bytes(raw) + reference["sha256"] = hashlib.sha256(raw).hexdigest() + state_path.write_text(json.dumps(state)) + assert card_text not in json.dumps(dashboard_snapshot(root)) + + # Allowlisted keys can still carry card material or private worker details. + for reference in state["contexts"]: + if reference["path"] not in (search_path, later_path): + continue + path = root / reference["path"] + record = json.loads(path.read_text()) + nested = {"nested": [card_text, "private worker diagnostics"]} + if reference["path"] == search_path: + record.update(revision=nested, score_scale=nested, count=True, elapsed_ms="1") + record["items"][0]["locator"] = "/private/worker/details" + record["items"][1]["score"] = nested + record["items"].append({"locator": nested, "card_sha256": digest}) + else: + record["page"] = [nested] + raw = json.dumps(record).encode() + path.write_bytes(raw) + reference["sha256"] = hashlib.sha256(raw).hexdigest() + state_path.write_text(json.dumps(state)) + projection = dashboard_snapshot(root)["data"] + first_query, later_query, *_ = projection["managed_retrievals"] + external = first_query["external"] + assert set(external) == {"channel", "backend", "platform", "state", "revision", + "score_scale", "count", "elapsed_ms", "items", "path", "reads"} + assert all(external[key] is None for key in ("revision", "score_scale", "count", "elapsed_ms")) + assert external["items"] == [{"locator": uncited, "card_sha256": digest, "score": None}] + assert external["reads"] == later_query["external"]["reads"] == [] + assert [use["ref"] for use in projection["candidates"][0]["knowledge_uses"]] == [local_item] + for secret in (card_text, "private worker diagnostics", "/private/worker/details"): + assert secret not in json.dumps(projection) + + # Updating the session-wide receipt and its hash cannot rebind the citation. + path = root / receipt_path + receipt = json.loads(path.read_text()) + receipt["locator"] = uncited + raw = json.dumps(receipt).encode() + path.write_bytes(raw) + cited = next(entry for entry in state["contexts"] if entry["path"] == receipt_path) + cited["sha256"] = hashlib.sha256(raw).hexdigest() + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="Cited knowledge context differs"): + dashboard_snapshot(root) + def test_runtime_identity_covers_migrated_process_domain_and_not_removed_executor() -> None: from importlib.resources import files diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index 81ef920ec..92a9aa577 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -215,15 +215,32 @@ def test_fixed_rounds_and_frozen_protocol_are_not_bypassed(project, capsys): submit(project, capsys, 30) -def test_project_knowledge_uses_find_and_records_budget_omission(project, capsys): - start(project, capsys) +@pytest.mark.parametrize("cannbot", [False, True]) +def test_project_knowledge_uses_find_and_records_budget_omission( + project, capsys, monkeypatch, cannbot, +): + from inferrsi.apps.integrations.cannbot import project as channel + + def unavailable(*args, **kwargs): + if not cannbot: + pytest.fail("local-only retrieval called the external channel") + raise channel.ChannelFailure("synthetic missing grant", state="not_enabled") + + monkeypatch.setattr(channel, "describe", unavailable) + extra = ("--cannbot-platform", "a3") if cannbot else () + start(project, capsys, extra=extra) root = project[3] assert main([ "campaign", "knowledge", "--directory", str(root), "--query", "cache per-step masks ran out of device memory", "--limit", "1", "--context-bytes", "1", ]) == 0 - capture = json.loads(capsys.readouterr().out)["context"] + answer = json.loads(capsys.readouterr().out) + if cannbot: + assert answer["external"]["state"] == "not_enabled" + else: + assert "external" not in answer + capture = answer["context"] assert capture["selection"]["method"] == "find-lexical-v1" item, = capture["items"] assert item["ref"] == "checkout:case.qwen-ascend-rejected-directions" @@ -232,6 +249,69 @@ def test_project_knowledge_uses_find_and_records_budget_omission(project, capsys assert capture["contexts"] == [] +def test_cannbot_timeout_keeps_local_ranking_and_final_budget(project, capsys, monkeypatch): + from inferrsi.apps.integrations.cannbot import project as channel + from inferrsi.apps.integrations.cannbot.read_worker import _WORKER_CLEANUP_SECONDS + from inferrsi.harness import optimization + + source, protocol, instructions, root = project + local_only = source, protocol, instructions, root.with_name("local-only") + query = "cache per-step masks ran out of device memory" + budgets = [] + + def timeout_worker(*args, timeout, **kwargs): + budgets.append(timeout) + raise channel.ChannelFailure("synthetic worker timeout", state="timeout") + + def search(session): + assert main([ + "campaign", "knowledge", "--directory", str(session), "--query", query, + "--limit", "2", "--context-bytes", "8192", + ]) == 0 + return json.loads(capsys.readouterr().out) + + monkeypatch.setattr(channel, "describe", lambda *a, **kw: {"state": "ok"}) + monkeypatch.setattr(channel, "search", timeout_worker) + start(local_only, capsys) + local = search(local_only[-1]) + assert not budgets and "external" not in local + + start(project, capsys, extra=("--cannbot-platform", "a3")) + state_path = root / "state.json" + deadline = json.loads(state_path.read_bytes())["deadline"] + with monkeypatch.context() as short_search: + # Exercise a channel call close enough to final that the channel's + # ordinary command timeout would consume the reserved time. + short_search.setattr(optimization, "_knowledge_time", lambda plan, state: 6.0) + opted_in = search(root) + timeout, = budgets + assert 0 < timeout <= 6 - _WORKER_CLEANUP_SECONDS + assert json.loads(state_path.read_bytes())["deadline"] == deadline + assert opted_in["external"]["state"] == "timeout" + assert [item["ref"] for item in opted_in["context"]["items"]] == [ + item["ref"] for item in local["context"]["items"] + ] + + capture = opted_in["reference"]["path"] + ref = opted_in["context"]["contexts"][0]["ref"] + (source / "config.json").write_text('{"metric": 20}', encoding="utf-8") + assert main([ + "campaign", "round", "--directory", str(root), "--candidate", str(source), + "--hypothesis", "Improve the synthetic configuration", + "--knowledge-ref", f"{capture}#{ref}", + ]) == 0 + capsys.readouterr() + snapshot = dashboard_snapshot(root)["data"] + retrieval, = snapshot["managed_retrievals"] + candidate, = snapshot["candidates"] + assert retrieval["external"]["state"] == "timeout" + assert [use["ref"] for use in candidate["knowledge_uses"]] == [ref] + assert main([ + "campaign", "final", "--directory", str(root), "--reason", "Synthetic timeout check", + ]) == 0 + assert json.loads(capsys.readouterr().out)["state"] == "completed" + + def test_project_knowledge_off_and_manual_record_writes_are_rejected(project, capsys): start(project, capsys, extra=("--knowledge", "off")) root = project[3] diff --git a/tests/test_round_citations.py b/tests/test_round_citations.py index 67f063fa7..cc13c34db 100644 --- a/tests/test_round_citations.py +++ b/tests/test_round_citations.py @@ -21,20 +21,24 @@ @pytest.mark.parametrize("case", [ "read", "managed_context", "late_body", "clipped", "wrong_quote", - "long_line", "grouped_context", + "long_line", "grouped_context", "joint_context", "joint_context_clipped", ]) def test_managed_round_resolves_citations_from_original_tool_outputs( project, tmp_path, capsys, case, ): repo, _, _, root = project - start(project, capsys) + joint = case.startswith("joint_context") + start(project, capsys, extra=("--cannbot-platform", "a3") if joint else ()) context_ids = [] ref, quote, reason = REF, QUOTE, REASON - if case in ("managed_context", "grouped_context"): + if joint or case in ("managed_context", "grouped_context"): read_args = ["campaign", "knowledge", "--directory", str(root), "--query", QUERY, "--context-bytes", "8192", "--limit", "1"] read_argv, read_output = _cli(tmp_path, *read_args) - context_ids = [json.loads(read_output)["context"]["context_id"]] + answer = json.loads(read_output) + context_ids = [answer["context"]["context_id"]] + if joint: + assert answer["external"]["state"] == "not_enabled" elif case == "long_line": ref = "package:inferrsi.ascend.skills.ascend_aclgraph_capture_replay" quote = ( @@ -48,8 +52,12 @@ def test_managed_round_resolves_citations_from_original_tool_outputs( else: read_argv, read_output = _cli(tmp_path, "read", REF, "--material", "--omit-resources", "--installed-library") - read = _operation(tmp_path, read_argv, read_output, "read", - received="Warning: truncated output\n" if case == "clipped" else True) + received = True + if case == "clipped": + received = "Warning: truncated output\n" + elif case == "joint_context_clipped": + received = read_output.split('\n "external":', 1)[0] + '\n "external": {\n' + read = _operation(tmp_path, read_argv, read_output, "read", received=received) if case == "grouped_context": _, second = _cli(tmp_path, "campaign", "knowledge", "--directory", str(root), "--query", "zzunmatchedcaptureqv", "--context-bytes", "8192") @@ -68,9 +76,10 @@ def test_managed_round_resolves_citations_from_original_tool_outputs( rows = ([round_op[0], *read, *round_op[1:]] if case == "late_body" else [*read, *round_op]) _, archive = _import(tmp_path, rows, context_ids=context_ids) - coverage = "partial" if case in ("clipped", "grouped_context") else "complete" + coverage = ("partial" if case in ("clipped", "grouped_context", "joint_context_clipped") + else "complete") assert archive["coverage"] == coverage - matched = case in ("read", "managed_context", "long_line") + matched = case in ("read", "managed_context", "long_line", "joint_context") assert archive["observed"]["used"] == int(matched) assert archive["observed"]["searches"] == len(context_ids) [projected] = [record for record in dashboard_snapshot(root)["data"]["knowledge_trace"][ diff --git a/tests/test_round_decisions_browser.py b/tests/test_round_decisions_browser.py index c48a20dae..2daa856b7 100644 --- a/tests/test_round_decisions_browser.py +++ b/tests/test_round_decisions_browser.py @@ -332,3 +332,50 @@ def test_external_experiment_ids_disambiguate_missing_round_run(browser, tmp_pat expect(page.locator(f'[data-experiment-id="{experiment_id}"]')).to_be_focused() else: expect(page.locator("#run-review strong").first).to_have_text("B") + + +def test_managed_round_knowledge_and_measured_score(browser, tmp_path): + campaign(tmp_path) + snapshot = dashboard_snapshot(tmp_path) + data = snapshot["data"] + card, unused = "ops/example/tiling", "ops/example/unused" + external = { + "channel": "cannbot-knowledge", "platform": "a3", "state": "ok", + "revision": "a" * 40, "score_scale": "knowledge-search.v5", "count": 2, + "path": "knowledge/search.json", + "items": [{"locator": ref, "score": 5, "card_sha256": "b" * 64} + for ref in (card, unused)], + "reads": [{"locator": card, "page": 1}, {"locator": unused, "page": 1}], + } + data["managed_retrievals"] = [ + {"round": 1, "query": "tile sizes", "items": [ + {"ref": "package:tile", "title": "Tiling", "delivered": True}, + ], "external": external}, + {"round": 2, "query": "empty query", "items": [], + "external": {**external, "count": 0, "items": [], "reads": []}}, + ] + row = { + **data["candidates"][0], + "round": 1, "candidate_id": "candidate-001", "run_id": "attempt-one", + "metric_key": "speedup", "measured_score": 2, "verdict": "PASS", + "knowledge_uses": [{"source": "cannbot-knowledge", "ref": card, + "retrieval_round": 1, "page": 1}], + } + data["candidates"] = [row] + with dashboard(browser, tmp_path, False, snapshot=snapshot) as page: + root = page.locator("#round-decisions") + retrieval, result = (root.locator(f'[data-rd-stage="{i}"]') for i in (1, 4)) + expect(retrieval).to_contain_text("Query · tile sizes") + expect(retrieval).to_contain_text("Local knowledge") + expect(retrieval).to_contain_text("CANNBot · a3") + expect(retrieval).to_contain_text("Read receipts · Pages 1") + expect(retrieval).to_contain_text(unused) + expect(result).to_contain_text("candidate-001") + expect(result).to_contain_text("Measured score · 2 · speedup") + expect(result).to_contain_text( + "Agent-declared knowledge use · cannbot-knowledge · " + card, + ) + expect(result).not_to_contain_text(unused) + root.locator("select").select_option("2") + expect(retrieval).to_contain_text("Channel state · No hits") + expect(result).not_to_contain_text("candidate-001") From cc552e47bc5a707f715fcb36da91b7d42525e42f Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Fri, 20 Nov 2026 17:20:03 +0800 Subject: [PATCH 2/2] Verify bounded managed knowledge reads Use the contained regular-file reader and verify the retained SHA-256 before consuming managed knowledge. Cover escaping paths, symbolic links, the capture byte limit, and digest mismatch through the Dashboard. Signed-off-by: jiangkuaixue123 Agent-assisted-by: Codex / implement_523 (implementer) --- .../apps/views/retrieval_projection.py | 47 +++++++----- src/inferrsi/harness/optimization.py | 36 +++++++--- src/inferrsi/harness/records/evidence.py | 28 ++++++++ tests/test_optimization_session.py | 71 +++++++++++++++++++ 4 files changed, 152 insertions(+), 30 deletions(-) diff --git a/src/inferrsi/apps/views/retrieval_projection.py b/src/inferrsi/apps/views/retrieval_projection.py index edfef92ad..98851a97e 100644 --- a/src/inferrsi/apps/views/retrieval_projection.py +++ b/src/inferrsi/apps/views/retrieval_projection.py @@ -8,13 +8,17 @@ import re from pathlib import Path -from inferrsi.harness.records.evidence import inside, relative_location, sanitized_text +from inferrsi.harness.records.evidence import ( + bounded_record_bytes, + inside, + relative_location, + sanitized_text, +) +from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES from inferrsi.harness.records.optimization_log import _IDENTIFIER, _unique from ..integrations.cannbot.round_knowledge import read_metadata, search_metadata -_MAX_BYTES = 4 * 1024 * 1024 - def public_value(value): """Sanitize even verified records. Verification does not authorize publication.""" @@ -73,20 +77,19 @@ def retrieval_view(capture): def _read(root, path): - if not inside(root, path) or path.is_symlink(): + try: + relative = path.relative_to(root).as_posix() + except ValueError: return None, "unsafe_path", None + raw, state = bounded_record_bytes(root, relative, MAX_CONTEXT_CAPTURE_BYTES) + if raw is None: + return None, state, None try: - with path.open("rb") as stream: - raw = stream.read(_MAX_BYTES + 1) - if len(raw) > _MAX_BYTES: - return None, "too_large", None document = json.loads(raw, object_pairs_hook=_unique) if not isinstance(document, dict): return None, "invalid_format", None return document, "present", hashlib.sha256(raw).hexdigest() - except FileNotFoundError: - return None, "file_absent", None - except (OSError, ValueError, RecursionError): + except (ValueError, RecursionError): return None, "invalid_format", None @@ -319,8 +322,20 @@ def managed_knowledge(root, contexts): for reference in contexts: path, kind = reference["path"], reference["kind"] number = reference.get("round") + if not isinstance(path, str) or Path(path).is_absolute(): + continue + external_record = kind in ("external_search", "external_read") + if kind != "delivered_context" and not external_record: + continue + if external_record and ( + type(number) is not int or number < 1 + or not re.fullmatch(rf"knowledge/{kind}-[0-9a-f]{{32}}\.json", path) + ): + continue + record, state, digest = _read(root, root / path) + if state != "present" or digest != reference.get("sha256"): + continue if kind == "delivered_context": - record = json.loads((root / path).read_text("utf-8")) delivered = {item["ref"] for item in record["contexts"] if item["resources"]} items = [ {**{key: item.get(key) for key in ("ref", "title", "revision")}, @@ -341,13 +356,7 @@ def managed_knowledge(root, contexts): citations[path + "#" + item["ref"]] = { **citation, "ref": item["ref"], "revision": item["revision"], } - elif kind in ("external_search", "external_read"): - if (type(number) is not int or number < 1 - or not re.fullmatch(rf"knowledge/{kind}-[0-9a-f]{{32}}\.json", path)): - continue - record, state, digest = _read(root, root / path) - if state != "present" or digest != reference["sha256"]: - continue + else: if kind == "external_search": metadata = search_metadata(record) parent = record.get("context_ref") diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index 1b00dde8c..735def3eb 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -10,7 +10,8 @@ from . import candidates from .artifact_primitives import digest, file_lock, sha256 from .atomic_json import atomic_write_json -from .records.evidence import sanitized_text +from .records.evidence import bounded_record_bytes, sanitized_text +from .records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES from .task_protocol import TaskProtocolError from .usage import normalize_usage, usage_totals @@ -92,6 +93,21 @@ def _control(root): return plan, state +def _read_context(root, reference): + raw, state = bounded_record_bytes(root, reference.get("path"), MAX_CONTEXT_CAPTURE_BYTES) + if raw is None: + raise ValueError(f"Invalid delivered knowledge context ({state})") + if hashlib.sha256(raw).hexdigest() != reference.get("sha256"): + raise ValueError("Delivered knowledge context changed") + try: + record = json.loads(raw) + except (ValueError, RecursionError): + raise ValueError("Invalid delivered knowledge context (invalid_format)") from None + if not isinstance(record, dict): + raise ValueError("Invalid delivered knowledge context (invalid_format)") + return record + + def _load(root, *, read_only=False): plan, state = _control(root) if plan.get("evaluation_mode") == "service_reuse": @@ -110,11 +126,9 @@ def _load(root, *, read_only=False): diagnoses = _diagnosis_index(state) contexts = {} for context in state["contexts"]: - raw = (root / context["path"]).read_bytes() - if hashlib.sha256(raw).hexdigest() != context["sha256"]: - raise ValueError("Delivered knowledge context changed") + retained = _read_context(root, context) if context.get("kind") not in ("external_search", "external_read"): - _check_context_diagnosis(plan, diagnoses, context, json.loads(raw)) + _check_context_diagnosis(plan, diagnoses, context, retained) contexts[context["path"]] = context for record in [*state["rounds"], *([state["final"]] if state["final"] else [])]: directory = root / record["path"] @@ -362,7 +376,7 @@ def capture_context(root: Path, reader, *, retain_context, external_provider=Non if binding: retained = {**retained, "diagnosis_id": binding["diagnosis_id"]} state["contexts"].append(retained) - response = {"reference": reference, "context": json.loads(path.read_text("utf-8"))} + response = {"reference": reference, "context": _read_context(root, reference)} if settings: remaining = _knowledge_time(plan, state) result = ( @@ -391,7 +405,7 @@ def read_external_context(root: Path, provider, *, search_ref, locator, page): and entry["round"] == number), None) if reference is None: raise ValueError("Supply a CANNBot search reference from the current round") - search = json.loads((root / reference["path"]).read_text("utf-8")) + search = _read_context(root, reference) hit = next((item for item in search["items"] if item["locator"] == locator), None) if search["state"] != "ok" or not search["revision"] or hit is None: raise ValueError("Supply a versioned CANNBot hit from the recorded search") @@ -414,15 +428,15 @@ def _validate_knowledge_refs(root, contexts, refs): raise ValueError("Knowledge references must be a list of delivered references") if len(set(refs)) != len(refs): raise ValueError("Cite unique delivered knowledge references") - delivered = {entry["path"]: entry["kind"] for entry in contexts + delivered = {entry["path"]: entry for entry in contexts if entry["kind"] != "external_search"} for ref in refs: path, separator, item = ref.partition("#") - kind = delivered.get(path) - if kind is None or (separator and kind != "delivered_context"): + reference = delivered.get(path) + if reference is None or (separator and reference["kind"] != "delivered_context"): raise ValueError("Cite a delivered context or CANNBot read receipt from this session") if separator: - capture = json.loads((root / path).read_text("utf-8")) + capture = _read_context(root, reference) if not any(entry["ref"] == item and entry["resources"] for entry in capture["contexts"]): raise ValueError("Cite an item with delivered resources from this context") diff --git a/src/inferrsi/harness/records/evidence.py b/src/inferrsi/harness/records/evidence.py index eaafd61db..dd982b00a 100644 --- a/src/inferrsi/harness/records/evidence.py +++ b/src/inferrsi/harness/records/evidence.py @@ -11,6 +11,8 @@ import re from pathlib import Path, PurePosixPath, PureWindowsPath +from .optimization_log import read_regular + STAGES = ("diagnosis", "knowledge", "search", "execution", "validation", "review") @@ -155,6 +157,32 @@ def inside(root: Path, path: Path) -> bool: return False +def bounded_record_bytes(root: Path, relative: object, limit: int) -> tuple[bytes | None, str]: + """Read a bounded regular file under the campaign, rejecting symbolic paths.""" + if not isinstance(relative, str) or not relative or "\\" in relative or ":" in relative: + return None, "unsafe_path" + location = Path(relative) + if not location.parts or location.is_absolute() or ".." in location.parts: + return None, "unsafe_path" + try: + root = root.resolve() + path = root + for part in location.parts: + path = path / part + if path.is_symlink(): + return None, "unsafe_path" + if not inside(root, path): + return None, "unsafe_path" + raw = read_regular(path, limit) + except FileNotFoundError: + return None, "file_absent" + except (OSError, ValueError, RuntimeError): + return None, "invalid_format" + if len(raw) > limit: + return None, "too_large" + return raw, "present" + + def read_snapshot(root: Path, path: Path, kind: str) -> tuple[bytes | None, dict]: try: relative = path.resolve().relative_to(root.resolve()).as_posix() diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index c647157c8..c4d207c57 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -483,6 +483,77 @@ def test_knowledge_delivery_is_retained_and_bound_to_round(session, capsys): assert data["candidates"][0]["knowledge_uses"] == [] +@pytest.mark.parametrize("mutation", [ + "absolute", "outside", "symlink", "ancestor_symlink", "oversized", "hash_mismatch", +]) +def test_managed_knowledge_rejects_unverified_capture( + session, capsys, monkeypatch, mutation, +): + from inferrsi.apps.views.dashboard import dashboard_snapshot + from inferrsi.apps.views.retrieval_projection import managed_knowledge + from inferrsi.harness.records import evidence + from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES + + assert main([ + "benchmarks", "session", "knowledge", "--session", str(session), + "--query", "cache per-step masks ran out of device memory", + "--limit", "1", "--context-bytes", "8192", + ]) == 0 + capsys.readouterr() + state_path = session / "state.json" + state = json.loads(state_path.read_bytes()) + reference, = state["contexts"] + path = session / reference["path"] + raw = path.read_bytes() + expected = "unsafe_path" + if mutation in ("absolute", "outside", "symlink"): + outside = session.parent / "other-capture.json" + outside.write_bytes(raw) + if mutation == "absolute": + reference["path"] = outside.as_posix() + elif mutation == "outside": + reference["path"] = "../other-capture.json" + else: + path.unlink() + try: + path.symlink_to(outside) + except OSError: + pytest.skip("test requires permission to create symlinks") + elif mutation == "ancestor_symlink": + alias = session / "alias" + try: + alias.symlink_to(path.parent, target_is_directory=True) + except OSError: + pytest.skip("test requires permission to create symlinks") + reference["path"] = f"alias/{path.name}" + elif mutation == "oversized": + raw += b" " * MAX_CONTEXT_CAPTURE_BYTES + path.write_bytes(raw) + reference["sha256"] = hashlib.sha256(raw).hexdigest() + expected = "too_large" + else: + path.write_bytes(raw + b" ") + expected = "Delivered knowledge context changed" + state_path.write_text(json.dumps(state), encoding="utf-8") + + reads = [] + read_regular = evidence.read_regular + + def track_read(path, limit): + data = read_regular(path, limit) + reads.append(len(data)) + return data + + monkeypatch.setattr(evidence, "read_regular", track_read) + assert managed_knowledge(session, [reference]) == ([], {}) + with pytest.raises(ValueError, match=expected): + dashboard_snapshot(session) + if expected == "unsafe_path": + assert not reads + elif expected == "too_large": + assert reads == [MAX_CONTEXT_CAPTURE_BYTES + 1] * 2 + + def test_cannbot_query_and_read_trace_to_candidate(tmp_path, task, capsys, monkeypatch): from inferrsi.apps.integrations.cannbot import project from inferrsi.apps.views.dashboard import dashboard_snapshot