diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index b35a150dc..ecf092c9a 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -333,6 +333,7 @@ ], "consumers": [ "src/inferrsi/apps/cli/__init__.py", + "src/inferrsi/apps/integrations/cannbot/round_knowledge.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py" ], @@ -360,10 +361,14 @@ "disable" ], "consumers": [ + "src/inferrsi/apps/cli/arguments.py", "src/inferrsi/apps/integrations/cannbot/__init__.py", + "src/inferrsi/apps/integrations/cannbot/round_knowledge.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py", - "tests/test_cannbot_boundaries.py" + "tests/test_cannbot_boundaries.py", + "tests/test_optimization_session.py", + "tests/test_project_optimization.py" ] }, { @@ -378,9 +383,11 @@ ], "consumers": [ "src/inferrsi/apps/integrations/cannbot/project.py", + "src/inferrsi/apps/integrations/cannbot/round_knowledge.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py", - "tests/test_cannbot_boundaries.py" + "tests/test_cannbot_boundaries.py", + "tests/test_project_optimization.py" ] } ] diff --git a/docs/examples/dashboard-profile-review.html b/docs/examples/dashboard-profile-review.html index c092ec746..ce2c9cd14 100644 --- a/docs/examples/dashboard-profile-review.html +++ b/docs/examples/dashboard-profile-review.html @@ -875,6 +875,47 @@

} function item(box,title){const a=el('article',null,'rd-item');a.append(el('h4',title));box.append(a);return a;} function stage(grid,index,en,zh){const section=el('section',null,'rd-stage');section.dataset.rdStage=String(index);const h=el('h3');h.append(el('span',String(index).padStart(2,'0'),'rd-number'),document.createTextNode(' '+tr(en,zh)));section.append(h);grid.append(section);return section;} + function managedRetrieval(stage, record){ + const box=item(stage,tr('Round knowledge query','本轮知识查询')); + field(box,tr('Query','查询'),record.query); + const local=item(box,tr('Local knowledge','本地知识')); + field(local,tr('Returned items','返回条目'),record.items.length); + for(const entry of record.items){ + const card=item(local,entry.title||entry.ref); + field(card,tr('Reference','引用'),entry.ref); + field(card,tr('Delivery','交付'),entry.delivered?tr('Context prepared','已装配上下文'):tr('Context omitted','未装配上下文')); + } + const result=record.external; + if(!result)return; + const external=item(box,'CANNBot · '+result.platform); + field(external,tr('Channel state','通道状态'),result.state==='ok'&&result.count===0?tr('No hits','无命中'):result.state); + field(external,tr('Returned items','返回条目'),result.count); + if(result.count>result.items.length)p(external,tr('Some hits lack retainable identifiers.','部分命中缺少可留存的标识。'),'rd-note'); + if(result.revision)field(external,tr('Source revision','来源版本'),result.revision); + if(result.score_scale)field(external,tr('Score scale','打分体系'),result.score_scale); + for(const hit of result.items){ + const card=item(external,hit.locator),pages=result.reads.filter(r=>r.locator===hit.locator).map(r=>r.page); + field(card,tr('Search score','检索分数'),hit.score); + field(card,tr('Read receipts','读取凭据'),pages.length?tr('Pages ','页码 ')+[...new Set(pages)].join(', '):null); + detail(card,tr('Card identity','知识卡标识'),'managed:'+result.path+':'+hit.locator,b=>field(b,'SHA-256',hit.card_sha256)); + } + } + function knowledgeUses(box,candidate,hasQueries){ + const citations=candidate.knowledge_uses; + if(!citations||(!citations.length&&!hasQueries))return; + const name=tr('Agent-declared knowledge use','Agent 声明采用的知识'); + if(!citations.length)field(box,name,null); + for(const [index,citation] of citations.entries()){ + const source=citation.source==='local'?tr('Local','本地'):citation.source; + const origin=citation.retrieval_round?tr('Round ','第 ')+citation.retrieval_round+tr('',' 轮'):null; + const page=citation.page?tr('Page ','页码 ')+citation.page:null; + field(box,name,[source,citation.ref,origin,page].filter(Boolean).join(' · ')); + if(citation.revision)detail(box,tr('Source identity','来源标识'),'citation:'+identity(candidate)+':'+index,b=>{ + field(b,tr('Source revision','来源版本'),citation.revision); + if(citation.card_sha256)field(b,'SHA-256',citation.card_sha256); + }); + } + } function render(){ const root=document.getElementById('round-decisions');if(!root)return; const focus=root.contains(document.activeElement)?document.activeElement.dataset.rdFocus:null; @@ -884,7 +925,9 @@

p(root,tr('Retrieval results → Alternatives → Final selection → Execution results','检索结果 → 备选方案 → 最终选择 → 执行结果'),'rd-note'); const data=getData(),rows=data.candidates||[],story=data.story||{},knowledge=data.knowledge_trace||{}; const iterations=story.iterations||[],searches=knowledge.searches||[],captures=knowledge.retrievals||[]; - const numbers=[...new Set([...rows.map(r=>r.round),...iterations.map(i=>i.round),...searches.map(s=>s.round)].filter(validRound))].sort((a,b)=>a-b); + const managed=data.managed_retrievals||[]; + const pending=data.pending_candidate; + const numbers=[...new Set([...rows,...iterations,...searches,...managed,...(pending?[pending]:[])].map(r=>r.round).filter(validRound))].sort((a,b)=>a-b); const choices=numbers.map(String);if(rows.some(r=>!validRound(r.round)))choices.push('unassigned'); if(!choices.length)choices.push('unassigned'); if(!choices.includes(selected))selected=choices[0]; @@ -902,7 +945,9 @@

const contextIds=new Set(roundIterations.flatMap(i=>i.context_ids||[])); roundSearches.filter(s=>s.state==='recorded').forEach(s=>(s.context_ids||[]).forEach(id=>contextIds.add(id))); captures.filter(c=>number!==null&&c.links?.some(l=>l.round===number)).forEach(c=>contextIds.add(c.context_id)); - if(!contextIds.size)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); + const managedQueries=managed.filter(inRound); + for(const query of managedQueries)managedRetrieval(retrieved,query); + if(!contextIds.size&&!managedQueries.length)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); for(const id of contextIds){ const c=captures.find(c=>c.context_id===id),box=item(retrieved,id); if(!c||c.state!=='verified'){p(box,label(c?.state),'rd-warning');continue;} @@ -996,12 +1041,18 @@

p(selection,tr('Choice and selection rationale not recorded.','选中方案及选择理由未记录。'),'rd-note'); for(const row of roundRows){const box=item(selection,row.candidate_id);p(box,tr('Registered candidate; registration alone is not selection evidence.','已登记候选;登记本身不证明选择依据。'),'rd-note');field(box,tr('Submitted hypothesis','提交的假设'),row.optimization?.hypothesis,true);} } - if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); + if(pending&&inRound(pending)){ + const box=item(results,pending.candidate_id); + p(box,tr('Awaiting assessment','等待评测结果'),'rd-note'); + knowledgeUses(box,pending,managedQueries.length); + }else if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); for(const row of roundRows){ const key=identity(row),box=item(results,[row.candidate_id,row.run_id||unknown()].join(' · ')); p(box,label(row.verdict),'rd-badge');field(box,tr('Quality','质量'),label(row.quality)); const metric=v=>Number.isFinite(v)?String(v)+' '+(row.metric_unit||''):'—'; field(box,tr('Baseline → candidate','基线 → 候选'),metric(row.baseline_median)+' → '+metric(row.candidate_median)); + if(Number.isFinite(row.measured_score))field(box,tr('Measured score','实测成绩'),row.measured_score+' · '+row.metric_key); + knowledgeUses(box,row,managedQueries.length); field(box,tr('Recorded improvement','记录的改进幅度'),Number.isFinite(row.improvement_pct)?row.improvement_pct+'%':null); field(box,tr('Post-validation decision (record)','验证后决定(记录)'),label(row.decision)); const matches=rows.filter(r=>identity(r)===key); diff --git a/docs/external-channels.md b/docs/external-channels.md index 6db248f58..69ed69bb2 100644 --- a/docs/external-channels.md +++ b/docs/external-channels.md @@ -198,10 +198,12 @@ An external result is **not locally verified**. Cite it as `cannbot-knowledge @ ` and say so. It does not participate in a `PASS` verdict. -When a receipt is added (not in this release), it will hold the locator, the -card's `card_sha256` and the index revision — never card text, never -`frontmatter`, never a `local_path`. A digest proves which card was cited without -redistributing it. +[Managed round retrieval](optimization-benchmarks.md#include-cannbot-in-the-round) +records CANNBot search metadata and pinned read receipts. A receipt identifies +the search, locator, index revision, card SHA-256 and requested page. The +authorized read returns text directly to the caller. The campaign retains +metadata only. Candidate attribution requires an explicit citation when +submitting a round. Whether CANN OSL v2.0 permits even a local read-only copy is an open question for the organisation; see #476 Q1. This page documents the mechanism, not a legal @@ -213,6 +215,7 @@ conclusion. | --- | --- | | `cannbot enable / disable / describe / search / read` | implemented | | `find` integration with gating, budgets, per-channel states | implemented | +| Managed benchmark and AB/BA round query, read and Dashboard attribution | implemented | | MindIE as a second channel | not yet; `apps/mindie/` has the reader, not the adapter | | `cannbot serve` (stdio MCP proxying `discover` / `search` / `read_card`) | not yet | | `campaign import-cannbot` (receipts into evidence) | not yet | diff --git a/docs/operations.md b/docs/operations.md index 675686c37..85aec4829 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -38,6 +38,7 @@ Effect classes used below: | Inspect the package catalog | CLI `knowledge search` / `knowledge show`; catalog API | Read-only | Filtered inventory and existing catalog scripts. Uses the installed package's tier/repo/backend/scenario filters and IDs | | Prepare model-aware engine research | CLI `knowledge research`; Python `research_knowledge` | Read-only | Independent model/framework/task/modality/mechanism queries over the same package catalog; source-grounded known profiles, deduplicated matches and explicit target annotations; no engine/hardware qualification | | Enter an inference task or discover/read skills | CLI `skills enter` / `list` / `show` / `resource` / `reference` | Read-only | `enter` loads one primary owner's complete procedure before evidence search. `list`/ `show` inspect skill metadata and instructions. `resource`/`reference` resolve paths for your file reader. | +| Retrieve and cite external round knowledge | CLI `benchmarks session knowledge / knowledge-read`, AB/BA `campaign knowledge / knowledge-read` | Local evidence-writing | Requires an explicit `--cannbot-platform` at start and an approved project grant. Retains search and read metadata. Cite a delivered receipt with `round --knowledge-ref`. See [round retrieval](optimization-benchmarks.md#include-cannbot-in-the-round) | | Collect selected knowledge | CLI `collect`; Python `KnowledgeReader.collect` | Read-only | Batch selection and budgeted assembly by query or exact IDs, with revisions and omissions. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Read exact material/resources | CLI `read` / `resource`; Python `KnowledgeReader.read` / `KnowledgeReader.resource` | Read-only | Returns the requested revision or an explicit unavailable/stale error; never substitutes newer bytes | | Export legacy cited material | CLI `context` | Read-only | Cited package export for manual use with a coding agent; not a conversation store and not a full Collect equivalent | diff --git a/docs/optimization-benchmarks.md b/docs/optimization-benchmarks.md index 9ebc6fa20..5166b688a 100644 --- a/docs/optimization-benchmarks.md +++ b/docs/optimization-benchmarks.md @@ -77,6 +77,43 @@ selection method and the external Agent configuration are caller-controlled variables. A retained delivery establishes what was supplied. Model consumption and access to material outside this session require their own observation. +### Include CANNBot in the round + +Run from the project with an [approved CANNBot grant](external-channels.md). +Add `--cannbot-platform ` at `benchmarks session start` or AB/BA +`campaign start` to freeze the external source and an explicit Ascend +platform. Omit the option for a local-only session. The paired +`benchmarks compare` conditions keep their existing behavior. + +`knowledge` queries both sources for the next round. It returns the local +context and a separate `external` result, with its own ranking, channel state +and `reference.path`. A CANNBot timeout still returns the local result. The +external request and worker cleanup use the remaining search budget before the +final reserve. Use task keywords and keep private details out. +InferRSI applies its standard evidence redaction to the combined query before +searching and retaining it. Use a locator from this round's external result: + +```bash +inferrsi benchmarks session knowledge-read --session \ + --search-ref --locator --page 1 +``` + +The read verifies the revision and digest recorded by the search. Card text +returns to the authorized CLI caller. The session retains only the source, +locator, revision, digest, page and search reference. Read additional pages +with `--page`. Use the equivalent `campaign knowledge-read` command for AB/BA. + +At candidate submission, repeat `--knowledge-ref` for each delivery you used. +Pass the read's `reference.path` for an external card page, or +`#` for a local item with delivered resources. +A local capture path alone cites the whole delivery. You can cite an earlier +delivery from the same session. Search result paths are ineligible for citation. + +Open [Dashboard](dashboard.md) → **Round decisions** to see the queries and +channel states before submitting a candidate. Completed attempts show the +agent's explicit citations alongside the candidate ID, verdict and measured +score. A search hit or read receipt alone does not record a citation. + ## Submit a round and use its feedback ```bash diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index e6dc3d900..8baaa74fe 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -23,6 +23,25 @@ def _print(value: object) -> None: print(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False)) +def _optimization_session(args, *, task_source=None): + from ...harness.optimization_session import session_command + + settings = None + provider = None + if getattr(args, "cannbot_platform", None): + settings = {"channel": "cannbot-knowledge", "backend": "ascend", + "platform": args.cannbot_platform} + if args.session_action in ("knowledge", "knowledge-read"): + from ..integrations.cannbot.round_knowledge import CannbotRoundKnowledge + + provider = CannbotRoundKnowledge() + return session_command( + args, task_source=task_source, retain_context=capture_context, + knowledge_reader=KnowledgeReader() if args.session_action == "knowledge" else None, + external_knowledge=settings, external_provider=provider, + ) + + def _fixed_attempt_command(args) -> int: from ...harness.errors import PublicError from ..api import read_attempt_view @@ -356,7 +375,6 @@ def main(argv: list[str] | None = None) -> int: _print(compare_command(args)) return 0 if args.benchmark_action == "session": - from ...harness.optimization_session import session_command from ..delivery import after_final task_source = None @@ -364,12 +382,7 @@ def main(argv: list[str] | None = None) -> int: from ...benchmark.registry import task_resources task_source = Path(str(task_resources(args.task))) - result = session_command( - args, task_source=task_source, - retain_context=capture_context, - knowledge_reader=(KnowledgeReader() - if args.session_action == "knowledge" else None), - ) + result = _optimization_session(args, task_source=task_source) _print(after_final(args, result)) return 2 if result.get("state") == "blocked" else 0 if args.benchmark_action == "run": @@ -628,15 +641,9 @@ def main(argv: list[str] | None = None) -> int: _print({"context_directory": str(path), "reviewed_by_caller": False}) return 0 if getattr(args, "session_action", None) is not None: - from ...harness.optimization_session import session_command from ..delivery import after_final - result = session_command( - args, - retain_context=capture_context, - knowledge_reader=(KnowledgeReader() - if args.session_action == "knowledge" else None), - ) + result = _optimization_session(args) _print(after_final(args, result)) if "exit_code" in result: return result["exit_code"] diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index dd099a747..f25f8d4ce 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -99,9 +99,13 @@ def _optimization_actions(stages, *, project=False): start.add_argument("--condition", default="current-agent" if project else None, required=not project) start.add_argument("--knowledge", choices=("on", "off"), default="on") + from ..integrations.cannbot.project import PLATFORMS + + start.add_argument("--cannbot-platform", choices=PLATFORMS, + help="Opt into CANNBot retrieval on the declared Ascend platform") start.add_argument("--evaluation-prefix", default="", help="Quoted allocator argv prefix") start.add_argument("--python", default=sys.executable) - actions = ("status", "diagnose", "round", "final", "recover", "knowledge") + actions = ("status", "diagnose", "round", "final", "recover", "knowledge", "knowledge-read") if project: actions += ("baseline", "refresh-references", "resume", "profile") for name in actions: @@ -120,7 +124,7 @@ def _optimization_actions(stages, *, project=False): stage.add_argument("--hypothesis", required=True) stage.add_argument("--strategy", help="Agent-declared optimization strategy") stage.add_argument("--knowledge-ref", action="append", - help="Delivered context path cited by this round; repeatable") + help="Delivered path, path#item-ref, or CANNBot receipt; repeatable") if name == "final": stage.add_argument("--reason", help="Why the caller stopped an up-to-budget search") if name in ("round", "final"): @@ -146,6 +150,10 @@ def _optimization_actions(stages, *, project=False): type=Path, help="Investigation receipt proving interrupted resources were cleaned", ) + if name == "knowledge-read": + stage.add_argument("--search-ref", required=True, help="Recorded CANNBot search path") + stage.add_argument("--locator", required=True) + stage.add_argument("--page", type=int, default=1) if name == "knowledge": stage.add_argument("--query", required=True) stage.add_argument( diff --git a/src/inferrsi/apps/integrations/cannbot/read_worker.py b/src/inferrsi/apps/integrations/cannbot/read_worker.py index 0ff4055d4..615a133ea 100644 --- a/src/inferrsi/apps/integrations/cannbot/read_worker.py +++ b/src/inferrsi/apps/integrations/cannbot/read_worker.py @@ -78,6 +78,10 @@ class OutputLimitError(ValueError): """A pipe exceeded the bounded capture size.""" +_PROCESS_WAIT_SECONDS = 2.0 +_WORKER_CLEANUP_SECONDS = _PROCESS_WAIT_SECONDS * (2 if os.name == "nt" else 1) + + def _stop(process, *, own_group): if own_group and os.name == "posix": try: @@ -87,11 +91,12 @@ def _stop(process, *, own_group): elif own_group and os.name == "nt" and process.poll() is None: subprocess.run( ["taskkill", "/PID", str(process.pid), "/T", "/F"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2, check=False, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=_PROCESS_WAIT_SECONDS, + check=False, ) if process.poll() is None: process.kill() - process.wait(timeout=2) + process.wait(timeout=_PROCESS_WAIT_SECONDS) def capture(argv, *, cwd, timeout, environment=None, request=None, own_group=False): diff --git a/src/inferrsi/apps/integrations/cannbot/round_knowledge.py b/src/inferrsi/apps/integrations/cannbot/round_knowledge.py new file mode 100644 index 000000000..985130bc7 --- /dev/null +++ b/src/inferrsi/apps/integrations/cannbot/round_knowledge.py @@ -0,0 +1,116 @@ +"""Connect managed round knowledge to the approved read-only CANNBot worker.""" + +import math +import re + +from ....harness.records.evidence import relative_location +from ....knowledge.channels import CHANNEL_STATES, TIMEOUT, ChannelRegistry, search_channels +from . import SCORE_SCALE, CannbotChannel, project +from .read_worker import _WORKER_CLEANUP_SECONDS + +SCORE_SCALES = (SCORE_SCALE, "knowledge-search.v5") +_LOCATOR = re.compile(r"[\w][\w.+-]*(?:/[\w][\w.+-]*)*") + + +def _hex(value, size): + return (isinstance(value, str) and len(value) == size + and re.fullmatch("[0-9a-f]+", value) is not None) + + +def _card_identity(record): + if not isinstance(record, dict): + return None + locator, digest = record.get("locator"), record.get("card_sha256") + if (not isinstance(locator, str) or relative_location(locator) != locator + or not _LOCATOR.fullmatch(locator) or not _hex(digest, 64)): + return None + return {"locator": locator, "card_sha256": digest} + + +def search_metadata(record): + """Validate the scalar metadata published from a retained search.""" + if ( + not isinstance(record, dict) or record.get("channel") != "cannbot-knowledge" + or record.get("backend") != "ascend" or record.get("platform") not in project.PLATFORMS + or record.get("state") not in CHANNEL_STATES or not isinstance(record.get("items"), list) + ): + return None + revision, count = record.get("revision"), record.get("count") + elapsed, scale = record.get("elapsed_ms"), record.get("score_scale") + items = [] + if record["state"] == "ok": + for row in record["items"]: + card = _card_identity(row) + if card is None: + continue + score = row.get("score") + if type(score) not in (int, float) or not -math.inf < score < math.inf: + score = None + items.append({**card, "score": score}) + return { + **{key: record[key] for key in ("channel", "backend", "platform", "state")}, + "revision": revision if _hex(revision, 40) else None, + "score_scale": scale if scale in SCORE_SCALES else None, + "count": count if type(count) is int and len(items) <= count <= 200 else None, + "elapsed_ms": elapsed if type(elapsed) is int and elapsed >= 0 else None, + "items": items, + } + + +def read_metadata(record): + """Validate the identity and page of a retained read receipt.""" + card = _card_identity(record) + if (card is None or record.get("channel") != "cannbot-knowledge" + or not _hex(record.get("revision"), 40) + or type(record.get("page")) is not int or record["page"] < 1): + return None + return {**card, **{key: record[key] for key in ("channel", "revision", "page")}} + + +class CannbotRoundKnowledge: + def search(self, query, *, settings, limit, timeout): + # Process cleanup runs after a timeout and must fit in the search budget too. + timeout = min(8.0, timeout - _WORKER_CLEANUP_SECONDS) + if timeout <= 0: + return {"state": TIMEOUT, "items": [], "count": 0} + channel = CannbotChannel() + results = search_channels( + ChannelRegistry([channel]), query, backend=settings["backend"], + platform=settings["platform"], limit=limit, timeout=timeout, + ) + result, = results + # Select metadata explicitly. The channel view also contains prose, open + # commands and worker diagnostics, none of which belongs in a receipt. + return search_metadata({ + **settings, "state": result.status.state, "revision": result.status.revision, + "items": [{"locator": hit.locator, "card_sha256": hit.card_sha256, + "score": hit.score} for hit in result.items], + "count": len(result.items), "elapsed_ms": result.elapsed_ms, + "score_scale": result.score_scale, + }) + + def read(self, locator, *, revision, card_sha256, page, timeout): + timeout = min(15.0, timeout - _WORKER_CLEANUP_SECONDS) + if timeout <= 0: + raise ValueError("CANNBot read has insufficient time before the final reserve") + try: + result = project.read( + locator, expected_revision=revision, expected_sha256=card_sha256, page=page, + timeout=timeout, + ) + except project.ChannelFailure as exc: + state = exc.state if exc.state in CHANNEL_STATES else "rejected" + reason = f"CANNBot read failed ({state}). Check the grant or search again" + raise ValueError(reason) from None + if ( + result.get("state") != "ok" or result.get("locator") != locator + or result.get("card_sha256") != card_sha256 or result.get("revision") != revision + or type(result.get("page")) is not int or result["page"] != page + or type(result.get("pages")) is not int + ): + raise ValueError("CANNBot read did not match the recorded search") + if page > result["pages"]: + raise ValueError("CANNBot page is outside the card") + if not isinstance(result.get("text"), str) or not result["text"].strip(): + raise ValueError("CANNBot returned an empty card page") + return {key: result[key] for key in ("text", "page", "pages")} diff --git a/src/inferrsi/apps/views/capture_delivery.py b/src/inferrsi/apps/views/capture_delivery.py index 3fc27a20c..b0670e5c1 100644 --- a/src/inferrsi/apps/views/capture_delivery.py +++ b/src/inferrsi/apps/views/capture_delivery.py @@ -15,10 +15,13 @@ sanitized_text as _text, ) from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES +from inferrsi.harness.records.optimization_log import _unique from inferrsi.knowledge.context_projection import assemble_context_text from .codex_tool_returns import _output_lines, _response_text +_EXTERNAL_SUFFIX = ',\n "external": ' + def capture_return_texts(trace, lines, references, context_ids): """Read only tool-return text at the caller's line references in this round.""" @@ -78,8 +81,35 @@ def managed_envelopes(captures): }) for key, (doc, source, digest) in captures.items()} +def _joint_output_end(chunk, start): + try: + external, end = json.JSONDecoder(object_pairs_hook=_unique).raw_decode(chunk, start) + except (ValueError, RecursionError): + return None + if (isinstance(external, dict) and external.get("channel") == "cannbot-knowledge" + and chunk.startswith("\n}", end)): + return end + 2 + return None + + +def managed_local_output(output): + """Return the exact local envelope from a complete joint CLI response.""" + prefix, separator, _ = output.rpartition(_EXTERNAL_SUFFIX) + if separator and _joint_output_end(output, len(prefix) + len(separator)) == len(output): + return prefix + "\n}" + return output + + def _contains_envelope(chunk, body): - return f"\n{body}\n" in "\n" + chunk.replace("\r\n", "\n") + "\n" + chunk = "\n" + chunk.replace("\r\n", "\n") + "\n" + if f"\n{body}\n" in chunk: + return True + prefix = f"\n{body[:-2]}{_EXTERNAL_SUFFIX}" + start = chunk.find(prefix) + if start < 0: + return False + end = _joint_output_end(chunk, start + len(prefix)) + return end is not None and chunk[end:end + 1] == "\n" def managed_return_lines(trace, lines, envelopes, *, reserved): diff --git a/src/inferrsi/apps/views/codex_search.py b/src/inferrsi/apps/views/codex_search.py index f7e2946be..f87531d64 100644 --- a/src/inferrsi/apps/views/codex_search.py +++ b/src/inferrsi/apps/views/codex_search.py @@ -16,6 +16,7 @@ sanitized_text as _text, ) +from .capture_delivery import managed_local_output from .codex_knowledge import decode as decode_knowledge from .codex_knowledge import invocation as knowledge_invocation from .codex_tool_returns import _output_lines, _path, correlate_window @@ -617,6 +618,8 @@ def _matched_capture(spec, item, join, capture_outputs): return False output = "".join(join["aggregate_lines"]).replace("\r\n", "\n").rstrip("\n") root = capture_outputs.get((spec["context_id"], output)) + if root is None and spec["context_id"] is None: + root = capture_outputs.get((None, managed_local_output(output))) cwd = _path(item.get("cwd")) return (root is not None and cwd is not None and spec["directory"] is not None and (cwd / spec["directory"]).resolve() == root) diff --git a/src/inferrsi/apps/views/dashboard_data.py b/src/inferrsi/apps/views/dashboard_data.py index 113832e99..ffc0f21fe 100644 --- a/src/inferrsi/apps/views/dashboard_data.py +++ b/src/inferrsi/apps/views/dashboard_data.py @@ -694,8 +694,10 @@ def _managed_dashboard(root: Path, campaign: dict) -> dict: from inferrsi.harness.optimization import read_records from .optimization_story import build_story + from .retrieval_projection import managed_knowledge plan, state, summary = read_records(root) + retrievals, citations = managed_knowledge(root, state["contexts"]) rows, rounds, decisions = [], [], {} for index, record in enumerate(state["rounds"], 1): candidate_id = record["candidate_id"] @@ -730,12 +732,23 @@ def _managed_dashboard(root: Path, campaign: dict) -> dict: artifacts={**row["artifacts"], "execution": facts, "assessment": judged, "status": facts, "evaluation": judged}, usage=record["observations"]["usage"], + measured_score=_number(assessment.get("score")), + knowledge_uses=[citations[ref] for ref in record.get("knowledge_refs") or [] + if ref in citations], ) row.pop("_comparison_signature") rows.append(row) rounds.append({"round": index, "candidates": [candidate_id], "decision": {"candidate_id": candidate_id, "outcome": decision}}) decisions[(index, candidate_id)] = decision + pending = state["pending"] + pending_candidate = None + if pending and pending["kind"] == "round": + pending_candidate = { + "round": len(rows) + 1, "candidate_id": pending["candidate_id"], + "knowledge_uses": [citations[ref] for ref in pending.get("knowledge_refs") or [] + if ref in citations], + } displayed_summary = _managed_summary(summary) displayed_summary["final_reference"] = ( file_ref(root, root / state["final"]["path"] / "observation.json", "final_assessment") @@ -744,6 +757,7 @@ def _managed_dashboard(root: Path, campaign: dict) -> dict: "campaign_id": sanitized_text(campaign["campaign_id"]), "synthetic": False, "rounds": rounds, "rounds_state": "present", "corrupt_round_lines": [], "candidates": rows, "optimization_session": displayed_summary, + "managed_retrievals": retrievals, "pending_candidate": pending_candidate, "integration": {"state": "connected" if rows else "not_connected", "stage_declaration": "missing", "external_records": 0, "collection": {"state": "", "observed_at": ""}}, diff --git a/src/inferrsi/apps/views/retrieval_projection.py b/src/inferrsi/apps/views/retrieval_projection.py index a2252a716..98851a97e 100644 --- a/src/inferrsi/apps/views/retrieval_projection.py +++ b/src/inferrsi/apps/views/retrieval_projection.py @@ -8,10 +8,16 @@ import re from pathlib import Path -from inferrsi.harness.records.evidence import inside, relative_location, sanitized_text +from inferrsi.harness.records.evidence import ( + bounded_record_bytes, + inside, + relative_location, + sanitized_text, +) +from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES from inferrsi.harness.records.optimization_log import _IDENTIFIER, _unique -_MAX_BYTES = 4 * 1024 * 1024 +from ..integrations.cannbot.round_knowledge import read_metadata, search_metadata def public_value(value): @@ -71,20 +77,19 @@ def retrieval_view(capture): def _read(root, path): - if not inside(root, path) or path.is_symlink(): + try: + relative = path.relative_to(root).as_posix() + except ValueError: return None, "unsafe_path", None + raw, state = bounded_record_bytes(root, relative, MAX_CONTEXT_CAPTURE_BYTES) + if raw is None: + return None, state, None try: - with path.open("rb") as stream: - raw = stream.read(_MAX_BYTES + 1) - if len(raw) > _MAX_BYTES: - return None, "too_large", None document = json.loads(raw, object_pairs_hook=_unique) if not isinstance(document, dict): return None, "invalid_format", None return document, "present", hashlib.sha256(raw).hexdigest() - except FileNotFoundError: - return None, "file_absent", None - except (OSError, ValueError, RecursionError): + except (ValueError, RecursionError): return None, "invalid_format", None @@ -311,6 +316,78 @@ def read_searches(root: Path, campaign_id, rows, retrievals): return results +def managed_knowledge(root, contexts): + """Project managed deliveries and explicit citation targets without card bodies.""" + queries, external, citations = {}, {}, {} + for reference in contexts: + path, kind = reference["path"], reference["kind"] + number = reference.get("round") + if not isinstance(path, str) or Path(path).is_absolute(): + continue + external_record = kind in ("external_search", "external_read") + if kind != "delivered_context" and not external_record: + continue + if external_record and ( + type(number) is not int or number < 1 + or not re.fullmatch(rf"knowledge/{kind}-[0-9a-f]{{32}}\.json", path) + ): + continue + record, state, digest = _read(root, root / path) + if state != "present" or digest != reference.get("sha256"): + continue + if kind == "delivered_context": + delivered = {item["ref"] for item in record["contexts"] if item["resources"]} + items = [ + {**{key: item.get(key) for key in ("ref", "title", "revision")}, + "delivered": item["ref"] in delivered} + for item in record["items"] + ] + if number is not None: + request = record["request"] + queries[path] = { + "round": number, "path": path, + "query": request.get("relevance") or request.get("query"), + "items": items, + } + citation = {"source": "local", "retrieval_round": number} + citations[path] = {**citation, "ref": path} + for item in items: + if item["delivered"]: + citations[path + "#" + item["ref"]] = { + **citation, "ref": item["ref"], "revision": item["revision"], + } + else: + if kind == "external_search": + metadata = search_metadata(record) + parent = record.get("context_ref") + query = queries.get(parent) if isinstance(parent, str) else None + if metadata is not None and query is not None and number == query["round"]: + search = {**metadata, "path": path, "reads": []} + query["external"] = search + external[path] = (number, search) + else: + receipt = read_metadata(record) + parent = record.get("search_ref") + source = external.get(parent) if isinstance(parent, str) else None + if receipt is None or source is None or source[0] != number: + continue + search = source[1] + if search["revision"] != receipt["revision"] or not any( + hit["locator"] == receipt["locator"] + and hit["card_sha256"] == receipt["card_sha256"] for hit in search["items"] + ): + continue + search["reads"].append({key: receipt[key] for key in ("locator", "page")}) + citations[path] = { + "source": receipt["channel"], "retrieval_round": number, + "ref": receipt["locator"], "revision": receipt["revision"], + "card_sha256": receipt["card_sha256"], "page": receipt["page"], + } + return public_value(list(queries.values())), { + path: public_value(citation) for path, citation in citations.items() + } + + def project_knowledge(root, data): """One projection for both layouts, offline export and live refresh.""" story = data.get("story") or {} diff --git a/src/inferrsi/apps/views/round_decision_view.html b/src/inferrsi/apps/views/round_decision_view.html index c0ec511b3..c4bb814a4 100644 --- a/src/inferrsi/apps/views/round_decision_view.html +++ b/src/inferrsi/apps/views/round_decision_view.html @@ -36,6 +36,47 @@ } function item(box,title){const a=el('article',null,'rd-item');a.append(el('h4',title));box.append(a);return a;} function stage(grid,index,en,zh){const section=el('section',null,'rd-stage');section.dataset.rdStage=String(index);const h=el('h3');h.append(el('span',String(index).padStart(2,'0'),'rd-number'),document.createTextNode(' '+tr(en,zh)));section.append(h);grid.append(section);return section;} + function managedRetrieval(stage, record){ + const box=item(stage,tr('Round knowledge query','本轮知识查询')); + field(box,tr('Query','查询'),record.query); + const local=item(box,tr('Local knowledge','本地知识')); + field(local,tr('Returned items','返回条目'),record.items.length); + for(const entry of record.items){ + const card=item(local,entry.title||entry.ref); + field(card,tr('Reference','引用'),entry.ref); + field(card,tr('Delivery','交付'),entry.delivered?tr('Context prepared','已装配上下文'):tr('Context omitted','未装配上下文')); + } + const result=record.external; + if(!result)return; + const external=item(box,'CANNBot · '+result.platform); + field(external,tr('Channel state','通道状态'),result.state==='ok'&&result.count===0?tr('No hits','无命中'):result.state); + field(external,tr('Returned items','返回条目'),result.count); + if(result.count>result.items.length)p(external,tr('Some hits lack retainable identifiers.','部分命中缺少可留存的标识。'),'rd-note'); + if(result.revision)field(external,tr('Source revision','来源版本'),result.revision); + if(result.score_scale)field(external,tr('Score scale','打分体系'),result.score_scale); + for(const hit of result.items){ + const card=item(external,hit.locator),pages=result.reads.filter(r=>r.locator===hit.locator).map(r=>r.page); + field(card,tr('Search score','检索分数'),hit.score); + field(card,tr('Read receipts','读取凭据'),pages.length?tr('Pages ','页码 ')+[...new Set(pages)].join(', '):null); + detail(card,tr('Card identity','知识卡标识'),'managed:'+result.path+':'+hit.locator,b=>field(b,'SHA-256',hit.card_sha256)); + } + } + function knowledgeUses(box,candidate,hasQueries){ + const citations=candidate.knowledge_uses; + if(!citations||(!citations.length&&!hasQueries))return; + const name=tr('Agent-declared knowledge use','Agent 声明采用的知识'); + if(!citations.length)field(box,name,null); + for(const [index,citation] of citations.entries()){ + const source=citation.source==='local'?tr('Local','本地'):citation.source; + const origin=citation.retrieval_round?tr('Round ','第 ')+citation.retrieval_round+tr('',' 轮'):null; + const page=citation.page?tr('Page ','页码 ')+citation.page:null; + field(box,name,[source,citation.ref,origin,page].filter(Boolean).join(' · ')); + if(citation.revision)detail(box,tr('Source identity','来源标识'),'citation:'+identity(candidate)+':'+index,b=>{ + field(b,tr('Source revision','来源版本'),citation.revision); + if(citation.card_sha256)field(b,'SHA-256',citation.card_sha256); + }); + } + } function render(){ const root=document.getElementById('round-decisions');if(!root)return; const focus=root.contains(document.activeElement)?document.activeElement.dataset.rdFocus:null; @@ -45,7 +86,9 @@ p(root,tr('Retrieval results → Alternatives → Final selection → Execution results','检索结果 → 备选方案 → 最终选择 → 执行结果'),'rd-note'); const data=getData(),rows=data.candidates||[],story=data.story||{},knowledge=data.knowledge_trace||{}; const iterations=story.iterations||[],searches=knowledge.searches||[],captures=knowledge.retrievals||[]; - const numbers=[...new Set([...rows.map(r=>r.round),...iterations.map(i=>i.round),...searches.map(s=>s.round)].filter(validRound))].sort((a,b)=>a-b); + const managed=data.managed_retrievals||[]; + const pending=data.pending_candidate; + const numbers=[...new Set([...rows,...iterations,...searches,...managed,...(pending?[pending]:[])].map(r=>r.round).filter(validRound))].sort((a,b)=>a-b); const choices=numbers.map(String);if(rows.some(r=>!validRound(r.round)))choices.push('unassigned'); if(!choices.length)choices.push('unassigned'); if(!choices.includes(selected))selected=choices[0]; @@ -63,7 +106,9 @@ const contextIds=new Set(roundIterations.flatMap(i=>i.context_ids||[])); roundSearches.filter(s=>s.state==='recorded').forEach(s=>(s.context_ids||[]).forEach(id=>contextIds.add(id))); captures.filter(c=>number!==null&&c.links?.some(l=>l.round===number)).forEach(c=>contextIds.add(c.context_id)); - if(!contextIds.size)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); + const managedQueries=managed.filter(inRound); + for(const query of managedQueries)managedRetrieval(retrieved,query); + if(!contextIds.size&&!managedQueries.length)p(retrieved,tr('No retrieval snapshot linked to this round. This does not mean no search occurred.','本轮未关联检索快照,不代表没有发生检索。'),'rd-note'); for(const id of contextIds){ const c=captures.find(c=>c.context_id===id),box=item(retrieved,id); if(!c||c.state!=='verified'){p(box,label(c?.state),'rd-warning');continue;} @@ -157,12 +202,18 @@ p(selection,tr('Choice and selection rationale not recorded.','选中方案及选择理由未记录。'),'rd-note'); for(const row of roundRows){const box=item(selection,row.candidate_id);p(box,tr('Registered candidate; registration alone is not selection evidence.','已登记候选;登记本身不证明选择依据。'),'rd-note');field(box,tr('Submitted hypothesis','提交的假设'),row.optimization?.hypothesis,true);} } - if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); + if(pending&&inRound(pending)){ + const box=item(results,pending.candidate_id); + p(box,tr('Awaiting assessment','等待评测结果'),'rd-note'); + knowledgeUses(box,pending,managedQueries.length); + }else if(!roundRows.length)p(results,tr('No candidate execution linked to this round.','本轮未关联候选执行记录。'),'rd-note'); for(const row of roundRows){ const key=identity(row),box=item(results,[row.candidate_id,row.run_id||unknown()].join(' · ')); p(box,label(row.verdict),'rd-badge');field(box,tr('Quality','质量'),label(row.quality)); const metric=v=>Number.isFinite(v)?String(v)+' '+(row.metric_unit||''):'—'; field(box,tr('Baseline → candidate','基线 → 候选'),metric(row.baseline_median)+' → '+metric(row.candidate_median)); + if(Number.isFinite(row.measured_score))field(box,tr('Measured score','实测成绩'),row.measured_score+' · '+row.metric_key); + knowledgeUses(box,row,managedQueries.length); field(box,tr('Recorded improvement','记录的改进幅度'),Number.isFinite(row.improvement_pct)?row.improvement_pct+'%':null); field(box,tr('Post-validation decision (record)','验证后决定(记录)'),label(row.decision)); const matches=rows.filter(r=>identity(r)===key); diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index dd9bd19c8..735def3eb 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -10,6 +10,8 @@ from . import candidates from .artifact_primitives import digest, file_lock, sha256 from .atomic_json import atomic_write_json +from .records.evidence import bounded_record_bytes, sanitized_text +from .records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES from .task_protocol import TaskProtocolError from .usage import normalize_usage, usage_totals @@ -91,6 +93,21 @@ def _control(root): return plan, state +def _read_context(root, reference): + raw, state = bounded_record_bytes(root, reference.get("path"), MAX_CONTEXT_CAPTURE_BYTES) + if raw is None: + raise ValueError(f"Invalid delivered knowledge context ({state})") + if hashlib.sha256(raw).hexdigest() != reference.get("sha256"): + raise ValueError("Delivered knowledge context changed") + try: + record = json.loads(raw) + except (ValueError, RecursionError): + raise ValueError("Invalid delivered knowledge context (invalid_format)") from None + if not isinstance(record, dict): + raise ValueError("Invalid delivered knowledge context (invalid_format)") + return record + + def _load(root, *, read_only=False): plan, state = _control(root) if plan.get("evaluation_mode") == "service_reuse": @@ -107,23 +124,32 @@ def _load(root, *, read_only=False): if candidates.identity(root / state["best"]["path"]) != state["best"]["sha256"]: raise ValueError("Selected candidate changed") diagnoses = _diagnosis_index(state) + contexts = {} for context in state["contexts"]: - raw = (root / context["path"]).read_bytes() - if hashlib.sha256(raw).hexdigest() != context["sha256"]: - raise ValueError("Delivered knowledge context changed") - _check_context_diagnosis(plan, diagnoses, context, json.loads(raw)) + retained = _read_context(root, context) + if context.get("kind") not in ("external_search", "external_read"): + _check_context_diagnosis(plan, diagnoses, context, retained) + contexts[context["path"]] = context for record in [*state["rounds"], *([state["final"]] if state["final"] else [])]: directory = root / record["path"] - observation = directory / "observation.json" - if sha256(observation) != record["observation_sha256"]: + raw = (directory / "observation.json").read_bytes() + if hashlib.sha256(raw).hexdigest() != record["observation_sha256"]: raise ValueError("Retained assessment record changed") + observation = json.loads(raw) + if any(observation.get(key) != record.get(key) for key in ("knowledge_refs", "contexts")): + raise ValueError("Round knowledge attribution differs from the retained observation") + if observation.get("knowledge_refs"): + submitted = {ref["path"]: ref for ref in observation["contexts"]} + for ref in observation["knowledge_refs"]: + path = ref.partition("#")[0] + if path not in submitted or submitted[path] != contexts.get(path): + raise ValueError("Cited knowledge context differs from the submitted snapshot") if record is not state["final"] and ( plan.get("diagnosis_required") or diagnoses or record.get("diagnosis_id") ): - retained = json.loads(observation.read_text(encoding="utf-8")) - if retained.get("diagnosis_id") != record.get("diagnosis_id"): + if observation.get("diagnosis_id") != record.get("diagnosis_id"): raise ValueError("Round diagnosis differs from the retained observation") - _check_activity_diagnosis(plan, diagnoses, retained) + _check_activity_diagnosis(plan, diagnoses, observation) if candidates.identity(directory / plan["candidate_name"]) != record["candidate_sha256"]: raise ValueError("Retained round candidate changed") if sha256(directory / "execution/execution.json") != record["execution_sha256"]: @@ -180,6 +206,8 @@ def status(root: Path) -> dict: "task_instructions": str(root / "frozen/TASK.md"), "working_candidate": str(root / plan["working_candidate"]), "best": state["best"], "pending": state["pending"], + **({"external_knowledge": plan["external_knowledge"]} if "external_knowledge" in plan + else {}), "latest_observation": state["rounds"][-1] if state["rounds"] else None, "diagnosis": _diagnosis(state), "summary": str(root / "summary.json") if state["final"] is not None else None, @@ -303,21 +331,115 @@ def record_diagnosis(root: Path, bottleneck: str, evidence: Path) -> dict: return _retain_diagnosis(root, plan, state, bottleneck, evidence) -def capture_context(root: Path, reader, *, retain_context, **request) -> dict: +def _knowledge_round(plan, state): + _active(state, reserve=plan["final_reserve_seconds"]) + if plan.get("knowledge_allowed") is False: + raise ValueError("Knowledge delivery is disabled by the frozen condition") + number = len(state["rounds"]) + 1 + return number if number <= plan["round_limit"] else None + + +def _knowledge_time(plan, state): + return state["deadline"] - time.time() - plan["final_reserve_seconds"] + + +def _store_knowledge(root, kind, number, record): + directory = root / "knowledge" + if directory.is_symlink(): + raise ValueError("Knowledge directory must not be a symbolic link") + directory.mkdir(exist_ok=True) + path = directory / f"{kind}-{uuid.uuid4().hex}.json" + atomic_write_json(path, record) + return {"path": path.relative_to(root).as_posix(), "sha256": sha256(path), + "kind": kind, "round": number} + + +def capture_context(root: Path, reader, *, retain_context, external_provider=None, + **request) -> dict: with _locked(root): plan, state = _load(root) - _active(state, reserve=plan["final_reserve_seconds"]) - if plan.get("knowledge_allowed") is False: - raise ValueError("Knowledge delivery is disabled by the frozen condition") + number = _knowledge_round(plan, state) + settings = plan.get("external_knowledge") + if settings: + if number is None: + raise ValueError("Round budget consumed, run independent final evaluation") + if external_provider is None: + raise ValueError("The frozen external knowledge provider is unavailable") + request["relevance"] = sanitized_text(request["relevance"], "", limit=None) binding = _context_diagnosis(plan, state) path, fingerprint = retain_context(root, context_id="context-" + uuid.uuid4().hex, reader=reader, diagnosis_binding=binding, **request) reference = {"path": path.relative_to(root).as_posix(), "sha256": fingerprint, "kind": "delivered_context"} - state["contexts"].append( - {**reference, "diagnosis_id": binding["diagnosis_id"]} if binding else reference) + # Keep the local tool response compatible with stored Codex trace verification. + retained = {**reference, "round": number} if number is not None else reference + if binding: + retained = {**retained, "diagnosis_id": binding["diagnosis_id"]} + state["contexts"].append(retained) + response = {"reference": reference, "context": _read_context(root, reference)} + if settings: + remaining = _knowledge_time(plan, state) + result = ( + external_provider.search(request["relevance"], settings=settings, + limit=request["limit"], timeout=remaining) + if remaining > 0 else {"state": "timeout", "items": [], "count": 0} + ) + search = {**result, **settings, "context_ref": reference["path"]} + search_ref = _store_knowledge(root, "external_search", number, search) + state["contexts"].append(search_ref) + response["external"] = {**search, "reference": search_ref} + atomic_write_json(root / "state.json", state) + return response + + +def read_external_context(root: Path, provider, *, search_ref, locator, page): + with _locked(root): + plan, state = _load(root) + number = _knowledge_round(plan, state) + if not plan.get("external_knowledge") or provider is None: + raise ValueError("External knowledge is disabled or unavailable for this session") + if type(page) is not int or page < 1: + raise ValueError("CANNBot page must be a positive integer") + reference = next((entry for entry in state["contexts"] + if entry["path"] == search_ref and entry["kind"] == "external_search" + and entry["round"] == number), None) + if reference is None: + raise ValueError("Supply a CANNBot search reference from the current round") + search = _read_context(root, reference) + hit = next((item for item in search["items"] if item["locator"] == locator), None) + if search["state"] != "ok" or not search["revision"] or hit is None: + raise ValueError("Supply a versioned CANNBot hit from the recorded search") + remaining = _knowledge_time(plan, state) + if remaining <= 0: + raise ValueError("Optimization deadline exhausted, final evaluation time is reserved") + revision, card_sha256 = search["revision"], hit["card_sha256"] + body = provider.read(locator, revision=revision, card_sha256=card_sha256, + page=page, timeout=remaining) + receipt = {"search_ref": search_ref, "channel": search["channel"], "locator": locator, + "revision": revision, "card_sha256": card_sha256, "page": page} + delivered = _store_knowledge(root, "external_read", number, receipt) + state["contexts"].append(delivered) atomic_write_json(root / "state.json", state) - return {"reference": reference, "context": json.loads(path.read_text(encoding="utf-8"))} + return {"reference": delivered, "receipt": receipt, "context": body} + + +def _validate_knowledge_refs(root, contexts, refs): + if not isinstance(refs, list) or any(not isinstance(ref, str) for ref in refs): + raise ValueError("Knowledge references must be a list of delivered references") + if len(set(refs)) != len(refs): + raise ValueError("Cite unique delivered knowledge references") + delivered = {entry["path"]: entry for entry in contexts + if entry["kind"] != "external_search"} + for ref in refs: + path, separator, item = ref.partition("#") + reference = delivered.get(path) + if reference is None or (separator and reference["kind"] != "delivered_context"): + raise ValueError("Cite a delivered context or CANNBot read receipt from this session") + if separator: + capture = _read_context(root, reference) + if not any(entry["ref"] == item and entry["resources"] + for entry in capture["contexts"]): + raise ValueError("Cite an item with delivered resources from this context") def _usage_record(source, *, fingerprint=True): @@ -491,12 +613,7 @@ def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str, if strategy is not None and (not isinstance(strategy, str) or not strategy.strip()): raise ValueError("Strategy must be a nonempty declaration") if knowledge_refs is not None: - available = {context["path"] for context in state["contexts"]} - if (not isinstance(knowledge_refs, list) - or any(not isinstance(ref, str) or ref not in available - for ref in knowledge_refs) - or len(set(knowledge_refs)) != len(knowledge_refs)): - raise ValueError("Cite unique delivered context paths from this campaign") + _validate_knowledge_refs(root, state["contexts"], knowledge_refs) source = root / state["best"]["path"] if final else Path(candidate) if (source.is_symlink() or (plan["candidate_kind"] == "file" and (not source.is_file() or source.suffix != ".py")) diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index e6d71bcb8..7549495f8 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -127,7 +127,7 @@ def _runtime_identity(): def session_command(args, *, task_source: Path | None = None, knowledge_reader=None, - retain_context=None) -> dict: + retain_context=None, external_knowledge=None, external_provider=None) -> dict: """Run one operation; discovery and knowledge access are supplied by the caller.""" from inferrsi.harness import optimization @@ -150,6 +150,11 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N if not math.isfinite(reserve) or not 0 < reserve < args.max_minutes * 60: raise ValueError("Reserve time for final within the total budget") evaluation_prefix = shlex.split(args.evaluation_prefix) + if external_knowledge and args.knowledge != "on": + raise ValueError("External knowledge requires --knowledge on") + if (external_knowledge and args.project_task + and args.evaluation_mode == "service_reuse"): + raise ValueError("External round knowledge requires a benchmark or AB/BA campaign") if args.project_task: names = ( "metric_key", @@ -241,6 +246,8 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N "evaluation_timeout_seconds": args.evaluation_timeout_seconds, "evaluation_prefix": evaluation_prefix, "python": args.python, "runtime_identity": _runtime_identity(), **candidate, **settings} + if external_knowledge: + plan["external_knowledge"] = external_knowledge result = optimization.initialize(root, plan) operation = "campaign" if args.project_task else "benchmarks session" result["instructions"] = ( @@ -252,6 +259,11 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N f"Follow the frozen round policy, then run {operation} final to retest independently. " "Report summary.json and its curve. Import actual usage exports when available." ) + if external_knowledge: + result["instructions"] += ( + f" Read external hits with {operation} knowledge-read. Cite the delivered " + "item or read receipt with --knowledge-ref when submitting a candidate." + ) return result import json @@ -274,6 +286,8 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N "status": "failed", "stage": "campaign", "message": message, }, } + if args.session_action == "knowledge-read": + raise ValueError("External round knowledge requires a benchmark or AB/BA campaign") return optimization.service_command(root, args, knowledge_reader=knowledge_reader) if ( args.session_action in ("baseline", "refresh-references", "resume", "profile") @@ -296,12 +310,18 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N root, knowledge_reader, retain_context=retain_context, + external_provider=external_provider, query="", relevance=args.query, relevance_method=args.relevance_method, context_bytes=args.context_bytes, limit=args.limit, ) + if args.session_action == "knowledge-read": + return optimization.read_external_context( + root, external_provider, search_ref=args.search_ref, locator=args.locator, + page=args.page, + ) if args.session_action == "recover": return optimization.recover(root, _session_assess) final = args.session_action == "final" diff --git a/src/inferrsi/harness/records/evidence.py b/src/inferrsi/harness/records/evidence.py index eaafd61db..dd982b00a 100644 --- a/src/inferrsi/harness/records/evidence.py +++ b/src/inferrsi/harness/records/evidence.py @@ -11,6 +11,8 @@ import re from pathlib import Path, PurePosixPath, PureWindowsPath +from .optimization_log import read_regular + STAGES = ("diagnosis", "knowledge", "search", "execution", "validation", "review") @@ -155,6 +157,32 @@ def inside(root: Path, path: Path) -> bool: return False +def bounded_record_bytes(root: Path, relative: object, limit: int) -> tuple[bytes | None, str]: + """Read a bounded regular file under the campaign, rejecting symbolic paths.""" + if not isinstance(relative, str) or not relative or "\\" in relative or ":" in relative: + return None, "unsafe_path" + location = Path(relative) + if not location.parts or location.is_absolute() or ".." in location.parts: + return None, "unsafe_path" + try: + root = root.resolve() + path = root + for part in location.parts: + path = path / part + if path.is_symlink(): + return None, "unsafe_path" + if not inside(root, path): + return None, "unsafe_path" + raw = read_regular(path, limit) + except FileNotFoundError: + return None, "file_absent" + except (OSError, ValueError, RuntimeError): + return None, "invalid_format" + if len(raw) > limit: + return None, "too_large" + return raw, "present" + + def read_snapshot(root: Path, path: Path, kind: str) -> tuple[bytes | None, dict]: try: relative = path.resolve().relative_to(root.resolve()).as_posix() diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index a8d0ecbbb..c4d207c57 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -1,5 +1,6 @@ """Run the current-Agent protocol using real child processes and synthetic work.""" +import hashlib import json import os import subprocess @@ -449,6 +450,7 @@ def test_knowledge_delivery_is_retained_and_bound_to_round(session, capsys): "--limit", "1", "--context-bytes", "8192", ]) == 0 delivered = json.loads(capsys.readouterr().out) + assert set(delivered["reference"]) == {"path", "sha256", "kind"} path = session / delivered["reference"]["path"] capture, state = verify_capture(path.read_bytes()) assert state == "verified" @@ -470,9 +472,310 @@ def test_knowledge_delivery_is_retained_and_bound_to_round(session, capsys): result = submit(session, 2, capsys) observation = result["latest_observation"] assert observation["contexts"] == [ - {**delivered["reference"], "diagnosis_id": observation["diagnosis_id"]} + {**delivered["reference"], "round": 1, "diagnosis_id": observation["diagnosis_id"]} ] + from inferrsi.apps.views.dashboard import dashboard_snapshot + + data = dashboard_snapshot(session)["data"] + retrieval, = data["managed_retrievals"] + assert retrieval["round"] == 1 and "external" not in retrieval + assert data["candidates"][0]["knowledge_uses"] == [] + + +@pytest.mark.parametrize("mutation", [ + "absolute", "outside", "symlink", "ancestor_symlink", "oversized", "hash_mismatch", +]) +def test_managed_knowledge_rejects_unverified_capture( + session, capsys, monkeypatch, mutation, +): + from inferrsi.apps.views.dashboard import dashboard_snapshot + from inferrsi.apps.views.retrieval_projection import managed_knowledge + from inferrsi.harness.records import evidence + from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES + + assert main([ + "benchmarks", "session", "knowledge", "--session", str(session), + "--query", "cache per-step masks ran out of device memory", + "--limit", "1", "--context-bytes", "8192", + ]) == 0 + capsys.readouterr() + state_path = session / "state.json" + state = json.loads(state_path.read_bytes()) + reference, = state["contexts"] + path = session / reference["path"] + raw = path.read_bytes() + expected = "unsafe_path" + if mutation in ("absolute", "outside", "symlink"): + outside = session.parent / "other-capture.json" + outside.write_bytes(raw) + if mutation == "absolute": + reference["path"] = outside.as_posix() + elif mutation == "outside": + reference["path"] = "../other-capture.json" + else: + path.unlink() + try: + path.symlink_to(outside) + except OSError: + pytest.skip("test requires permission to create symlinks") + elif mutation == "ancestor_symlink": + alias = session / "alias" + try: + alias.symlink_to(path.parent, target_is_directory=True) + except OSError: + pytest.skip("test requires permission to create symlinks") + reference["path"] = f"alias/{path.name}" + elif mutation == "oversized": + raw += b" " * MAX_CONTEXT_CAPTURE_BYTES + path.write_bytes(raw) + reference["sha256"] = hashlib.sha256(raw).hexdigest() + expected = "too_large" + else: + path.write_bytes(raw + b" ") + expected = "Delivered knowledge context changed" + state_path.write_text(json.dumps(state), encoding="utf-8") + + reads = [] + read_regular = evidence.read_regular + + def track_read(path, limit): + data = read_regular(path, limit) + reads.append(len(data)) + return data + + monkeypatch.setattr(evidence, "read_regular", track_read) + assert managed_knowledge(session, [reference]) == ([], {}) + with pytest.raises(ValueError, match=expected): + dashboard_snapshot(session) + if expected == "unsafe_path": + assert not reads + elif expected == "too_large": + assert reads == [MAX_CONTEXT_CAPTURE_BYTES + 1] * 2 + + +def test_cannbot_query_and_read_trace_to_candidate(tmp_path, task, capsys, monkeypatch): + from inferrsi.apps.integrations.cannbot import project + from inferrsi.apps.views.dashboard import dashboard_snapshot + + revision, digest = "a" * 40, "b" * 64 + locator = "ops/ascendc/apis/synthetic" + uncited = "ops/ascendc/apis/uncited" + card_text = "synthetic external body" + search_query = "cache per-step masks ran out of device memory" + phase = {"state": "ok", "items": [locator, uncited]} + monkeypatch.setattr(project, "describe", lambda *a, **kw: { + "state": "ok", "revision": revision, + }) + + def search(query, *, platform, limit, **kwargs): + assert (query, platform, limit) == (search_query, "a3", 2) + if phase["state"] != "ok": + raise project.ChannelFailure("private worker diagnostics", state=phase["state"]) + return { + "revision": revision, + "items": [ + {"locator": ref, "card_sha256": digest, "score": 7, "summary": card_text} + for ref in phase["items"] + ], + } + + def read(ref, *, expected_revision, expected_sha256, page, timeout): + assert ref in (locator, uncited) + assert (expected_revision, expected_sha256, page) == (revision, digest, 1) + return {"state": "ok", "locator": ref, "revision": revision, "card_sha256": digest, + "page": 1, "pages": 1, "text": card_text} + + monkeypatch.setattr(project, "search", search) + monkeypatch.setattr(project, "read", read) + root = tmp_path / "with-cannbot" + operation = ["benchmarks", "session"] + session_args = ["--session", str(root)] + start_args = [*operation, "start", task.name, *session_args, "--rounds", "3", + "--max-minutes", "5", "--evaluation-timeout-seconds", "15", + "--condition", "synthetic-combined", "--evaluation-args", "", + "--cannbot-platform", "a3"] + with pytest.raises(SystemExit): + main([*start_args, "--knowledge", "off"]) + assert "requires --knowledge on" in capsys.readouterr().err + assert not root.exists() + assert main(start_args) == 0 + started = json.loads(capsys.readouterr().out) + assert started["external_knowledge"]["platform"] == "a3" + + query_args = [*operation, "knowledge", *session_args, "--query", search_query, + "--context-bytes", "8192", "--limit", "2"] + with pytest.raises(SystemExit): + main(query_args) + assert "Record measured bottleneck evidence" in capsys.readouterr().err + diagnose(root, capsys) + assert main(query_args) == 0 + answer = json.loads(capsys.readouterr().out) + search_path = answer["external"]["reference"]["path"] + local_path = answer["reference"]["path"] + local_item = answer["context"]["contexts"][0]["ref"] + before = dashboard_snapshot(root)["data"] + query, = before["managed_retrievals"] + assert query["query"] == search_query and query["round"] == 1 + assert any(x["ref"] == local_item and x["delivered"] for x in query["items"]) + assert query["external"]["state"] == "ok" + assert query["external"]["count"] == 2 + assert query["external"]["reads"] == [] + assert not before["candidates"] + + read_args = [*operation, "knowledge-read", *session_args, "--search-ref", search_path, + "--locator"] + with pytest.raises(SystemExit): + main([*read_args, "ops/ascendc/not-from-search"]) + assert "from the recorded search" in capsys.readouterr().err + capsys.readouterr() + assert main([*read_args, uncited]) == 0 + capsys.readouterr() + assert main([*read_args, locator]) == 0 + delivery = json.loads(capsys.readouterr().out) + assert delivery["context"]["text"] == card_text + receipt_path = delivery["reference"]["path"] + + candidate = root / "work/candidate.py" + candidate.write_text("FACTOR = 2\n") + round_args = [*operation, "round", *session_args, "--candidate", str(candidate), + "--hypothesis", "Apply selected knowledge"] + with pytest.raises(SystemExit): + main([*round_args, "--knowledge-ref", search_path]) + assert "read receipt from this session" in capsys.readouterr().err + with pytest.raises(SystemExit): + main([*round_args, "--knowledge-ref", local_path + "#unknown"]) + assert "delivered resources" in capsys.readouterr().err + execute = optimization_session._session_execute + + def observe_pending(root, directory, plan, deadline): + current = dashboard_snapshot(root)["data"]["pending_candidate"] + assert current["candidate_id"] == "candidate-001" and current["round"] == 1 + assert [use["ref"] for use in current["knowledge_uses"]] == [locator, local_item] + return execute(root, directory, plan, deadline) + + monkeypatch.setattr(optimization_session, "_session_execute", observe_pending) + assert main([*round_args, "--knowledge-ref", receipt_path, + "--knowledge-ref", local_path + "#" + local_item]) == 0 + capsys.readouterr() + with pytest.raises(SystemExit): + main([*read_args, locator]) + assert "from the current round" in capsys.readouterr().err + capsys.readouterr() + + monkeypatch.setattr(optimization_session, "_session_execute", execute) + phase["items"] = [uncited] + assert main(query_args) == 0 + later_search = json.loads(capsys.readouterr().out)["external"]["reference"]["path"] + assert main([*operation, "knowledge-read", *session_args, "--search-ref", later_search, + "--locator", uncited]) == 0 + later_reference = json.loads(capsys.readouterr().out)["reference"] + later_path = later_reference["path"] + + phase["items"] = [] + assert main(query_args) == 0 + capsys.readouterr() + assert submit(root, -1, capsys)["latest_observation"]["assessment"]["score"] is None + phase["state"] = "unreachable" + assert main(query_args) == 0 + capsys.readouterr() + data = dashboard_snapshot(root)["data"] + first, later, empty, unavailable = data["managed_retrievals"] + assert [first["round"], later["round"], empty["round"], unavailable["round"]] == [1, 2, 2, 3] + assert {entry["locator"] for entry in first["external"]["reads"]} == {locator, uncited} + assert later["external"]["reads"] == [{"locator": uncited, "page": 1}] + assert empty["external"]["state"] == "ok" and empty["external"]["count"] == 0 + assert unavailable["external"]["state"] == "unreachable" + row, failed = data["candidates"] + assert (row["candidate_id"], row["round"], row["measured_score"], row["verdict"]) == ( + "candidate-001", 1, 2, "PASS", + ) + assert [(use["source"], use["ref"]) for use in row["knowledge_uses"]] == [ + ("cannbot-knowledge", locator), ("local", local_item), + ] + assert row["knowledge_uses"][0]["card_sha256"] == digest + assert failed["knowledge_uses"] == [] and failed["measured_score"] is None + assert card_text not in json.dumps(data) + assert "private worker diagnostics" not in json.dumps(data) + for path in root.rglob("*.json"): + retained = path.read_text(encoding="utf-8") + assert card_text not in retained + assert "private worker diagnostics" not in retained + + state_path = root / "state.json" + original_state = state_path.read_bytes() + state = json.loads(original_state) + first_round = state["rounds"][0] + original_refs = first_round["knowledge_refs"] + original_contexts = first_round["contexts"] + first_round["knowledge_refs"] = [later_path] + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="Round knowledge attribution"): + dashboard_snapshot(root) + first_round["knowledge_refs"] = original_refs + first_round["contexts"] = [*original_contexts, later_reference] + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="Round knowledge attribution"): + dashboard_snapshot(root) + first_round["contexts"] = original_contexts + + # Even an imported record with a matching digest must not widen the view. + for reference in state["contexts"]: + if reference["path"] not in (search_path, later_path): + continue + path = root / reference["path"] + record = json.loads(path.read_text()) + record["text"] = card_text + if "items" in record: + record["items"][0]["summary"] = card_text + raw = json.dumps(record).encode() + path.write_bytes(raw) + reference["sha256"] = hashlib.sha256(raw).hexdigest() + state_path.write_text(json.dumps(state)) + assert card_text not in json.dumps(dashboard_snapshot(root)) + + # Allowlisted keys can still carry card material or private worker details. + for reference in state["contexts"]: + if reference["path"] not in (search_path, later_path): + continue + path = root / reference["path"] + record = json.loads(path.read_text()) + nested = {"nested": [card_text, "private worker diagnostics"]} + if reference["path"] == search_path: + record.update(revision=nested, score_scale=nested, count=True, elapsed_ms="1") + record["items"][0]["locator"] = "/private/worker/details" + record["items"][1]["score"] = nested + record["items"].append({"locator": nested, "card_sha256": digest}) + else: + record["page"] = [nested] + raw = json.dumps(record).encode() + path.write_bytes(raw) + reference["sha256"] = hashlib.sha256(raw).hexdigest() + state_path.write_text(json.dumps(state)) + projection = dashboard_snapshot(root)["data"] + first_query, later_query, *_ = projection["managed_retrievals"] + external = first_query["external"] + assert set(external) == {"channel", "backend", "platform", "state", "revision", + "score_scale", "count", "elapsed_ms", "items", "path", "reads"} + assert all(external[key] is None for key in ("revision", "score_scale", "count", "elapsed_ms")) + assert external["items"] == [{"locator": uncited, "card_sha256": digest, "score": None}] + assert external["reads"] == later_query["external"]["reads"] == [] + assert [use["ref"] for use in projection["candidates"][0]["knowledge_uses"]] == [local_item] + for secret in (card_text, "private worker diagnostics", "/private/worker/details"): + assert secret not in json.dumps(projection) + + # Updating the session-wide receipt and its hash cannot rebind the citation. + path = root / receipt_path + receipt = json.loads(path.read_text()) + receipt["locator"] = uncited + raw = json.dumps(receipt).encode() + path.write_bytes(raw) + cited = next(entry for entry in state["contexts"] if entry["path"] == receipt_path) + cited["sha256"] = hashlib.sha256(raw).hexdigest() + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="Cited knowledge context differs"): + dashboard_snapshot(root) + def test_runtime_identity_covers_migrated_process_domain_and_not_removed_executor() -> None: from importlib.resources import files diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index 81ef920ec..92a9aa577 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -215,15 +215,32 @@ def test_fixed_rounds_and_frozen_protocol_are_not_bypassed(project, capsys): submit(project, capsys, 30) -def test_project_knowledge_uses_find_and_records_budget_omission(project, capsys): - start(project, capsys) +@pytest.mark.parametrize("cannbot", [False, True]) +def test_project_knowledge_uses_find_and_records_budget_omission( + project, capsys, monkeypatch, cannbot, +): + from inferrsi.apps.integrations.cannbot import project as channel + + def unavailable(*args, **kwargs): + if not cannbot: + pytest.fail("local-only retrieval called the external channel") + raise channel.ChannelFailure("synthetic missing grant", state="not_enabled") + + monkeypatch.setattr(channel, "describe", unavailable) + extra = ("--cannbot-platform", "a3") if cannbot else () + start(project, capsys, extra=extra) root = project[3] assert main([ "campaign", "knowledge", "--directory", str(root), "--query", "cache per-step masks ran out of device memory", "--limit", "1", "--context-bytes", "1", ]) == 0 - capture = json.loads(capsys.readouterr().out)["context"] + answer = json.loads(capsys.readouterr().out) + if cannbot: + assert answer["external"]["state"] == "not_enabled" + else: + assert "external" not in answer + capture = answer["context"] assert capture["selection"]["method"] == "find-lexical-v1" item, = capture["items"] assert item["ref"] == "checkout:case.qwen-ascend-rejected-directions" @@ -232,6 +249,69 @@ def test_project_knowledge_uses_find_and_records_budget_omission(project, capsys assert capture["contexts"] == [] +def test_cannbot_timeout_keeps_local_ranking_and_final_budget(project, capsys, monkeypatch): + from inferrsi.apps.integrations.cannbot import project as channel + from inferrsi.apps.integrations.cannbot.read_worker import _WORKER_CLEANUP_SECONDS + from inferrsi.harness import optimization + + source, protocol, instructions, root = project + local_only = source, protocol, instructions, root.with_name("local-only") + query = "cache per-step masks ran out of device memory" + budgets = [] + + def timeout_worker(*args, timeout, **kwargs): + budgets.append(timeout) + raise channel.ChannelFailure("synthetic worker timeout", state="timeout") + + def search(session): + assert main([ + "campaign", "knowledge", "--directory", str(session), "--query", query, + "--limit", "2", "--context-bytes", "8192", + ]) == 0 + return json.loads(capsys.readouterr().out) + + monkeypatch.setattr(channel, "describe", lambda *a, **kw: {"state": "ok"}) + monkeypatch.setattr(channel, "search", timeout_worker) + start(local_only, capsys) + local = search(local_only[-1]) + assert not budgets and "external" not in local + + start(project, capsys, extra=("--cannbot-platform", "a3")) + state_path = root / "state.json" + deadline = json.loads(state_path.read_bytes())["deadline"] + with monkeypatch.context() as short_search: + # Exercise a channel call close enough to final that the channel's + # ordinary command timeout would consume the reserved time. + short_search.setattr(optimization, "_knowledge_time", lambda plan, state: 6.0) + opted_in = search(root) + timeout, = budgets + assert 0 < timeout <= 6 - _WORKER_CLEANUP_SECONDS + assert json.loads(state_path.read_bytes())["deadline"] == deadline + assert opted_in["external"]["state"] == "timeout" + assert [item["ref"] for item in opted_in["context"]["items"]] == [ + item["ref"] for item in local["context"]["items"] + ] + + capture = opted_in["reference"]["path"] + ref = opted_in["context"]["contexts"][0]["ref"] + (source / "config.json").write_text('{"metric": 20}', encoding="utf-8") + assert main([ + "campaign", "round", "--directory", str(root), "--candidate", str(source), + "--hypothesis", "Improve the synthetic configuration", + "--knowledge-ref", f"{capture}#{ref}", + ]) == 0 + capsys.readouterr() + snapshot = dashboard_snapshot(root)["data"] + retrieval, = snapshot["managed_retrievals"] + candidate, = snapshot["candidates"] + assert retrieval["external"]["state"] == "timeout" + assert [use["ref"] for use in candidate["knowledge_uses"]] == [ref] + assert main([ + "campaign", "final", "--directory", str(root), "--reason", "Synthetic timeout check", + ]) == 0 + assert json.loads(capsys.readouterr().out)["state"] == "completed" + + def test_project_knowledge_off_and_manual_record_writes_are_rejected(project, capsys): start(project, capsys, extra=("--knowledge", "off")) root = project[3] diff --git a/tests/test_round_citations.py b/tests/test_round_citations.py index 67f063fa7..cc13c34db 100644 --- a/tests/test_round_citations.py +++ b/tests/test_round_citations.py @@ -21,20 +21,24 @@ @pytest.mark.parametrize("case", [ "read", "managed_context", "late_body", "clipped", "wrong_quote", - "long_line", "grouped_context", + "long_line", "grouped_context", "joint_context", "joint_context_clipped", ]) def test_managed_round_resolves_citations_from_original_tool_outputs( project, tmp_path, capsys, case, ): repo, _, _, root = project - start(project, capsys) + joint = case.startswith("joint_context") + start(project, capsys, extra=("--cannbot-platform", "a3") if joint else ()) context_ids = [] ref, quote, reason = REF, QUOTE, REASON - if case in ("managed_context", "grouped_context"): + if joint or case in ("managed_context", "grouped_context"): read_args = ["campaign", "knowledge", "--directory", str(root), "--query", QUERY, "--context-bytes", "8192", "--limit", "1"] read_argv, read_output = _cli(tmp_path, *read_args) - context_ids = [json.loads(read_output)["context"]["context_id"]] + answer = json.loads(read_output) + context_ids = [answer["context"]["context_id"]] + if joint: + assert answer["external"]["state"] == "not_enabled" elif case == "long_line": ref = "package:inferrsi.ascend.skills.ascend_aclgraph_capture_replay" quote = ( @@ -48,8 +52,12 @@ def test_managed_round_resolves_citations_from_original_tool_outputs( else: read_argv, read_output = _cli(tmp_path, "read", REF, "--material", "--omit-resources", "--installed-library") - read = _operation(tmp_path, read_argv, read_output, "read", - received="Warning: truncated output\n" if case == "clipped" else True) + received = True + if case == "clipped": + received = "Warning: truncated output\n" + elif case == "joint_context_clipped": + received = read_output.split('\n "external":', 1)[0] + '\n "external": {\n' + read = _operation(tmp_path, read_argv, read_output, "read", received=received) if case == "grouped_context": _, second = _cli(tmp_path, "campaign", "knowledge", "--directory", str(root), "--query", "zzunmatchedcaptureqv", "--context-bytes", "8192") @@ -68,9 +76,10 @@ def test_managed_round_resolves_citations_from_original_tool_outputs( rows = ([round_op[0], *read, *round_op[1:]] if case == "late_body" else [*read, *round_op]) _, archive = _import(tmp_path, rows, context_ids=context_ids) - coverage = "partial" if case in ("clipped", "grouped_context") else "complete" + coverage = ("partial" if case in ("clipped", "grouped_context", "joint_context_clipped") + else "complete") assert archive["coverage"] == coverage - matched = case in ("read", "managed_context", "long_line") + matched = case in ("read", "managed_context", "long_line", "joint_context") assert archive["observed"]["used"] == int(matched) assert archive["observed"]["searches"] == len(context_ids) [projected] = [record for record in dashboard_snapshot(root)["data"]["knowledge_trace"][ diff --git a/tests/test_round_decisions_browser.py b/tests/test_round_decisions_browser.py index c48a20dae..2daa856b7 100644 --- a/tests/test_round_decisions_browser.py +++ b/tests/test_round_decisions_browser.py @@ -332,3 +332,50 @@ def test_external_experiment_ids_disambiguate_missing_round_run(browser, tmp_pat expect(page.locator(f'[data-experiment-id="{experiment_id}"]')).to_be_focused() else: expect(page.locator("#run-review strong").first).to_have_text("B") + + +def test_managed_round_knowledge_and_measured_score(browser, tmp_path): + campaign(tmp_path) + snapshot = dashboard_snapshot(tmp_path) + data = snapshot["data"] + card, unused = "ops/example/tiling", "ops/example/unused" + external = { + "channel": "cannbot-knowledge", "platform": "a3", "state": "ok", + "revision": "a" * 40, "score_scale": "knowledge-search.v5", "count": 2, + "path": "knowledge/search.json", + "items": [{"locator": ref, "score": 5, "card_sha256": "b" * 64} + for ref in (card, unused)], + "reads": [{"locator": card, "page": 1}, {"locator": unused, "page": 1}], + } + data["managed_retrievals"] = [ + {"round": 1, "query": "tile sizes", "items": [ + {"ref": "package:tile", "title": "Tiling", "delivered": True}, + ], "external": external}, + {"round": 2, "query": "empty query", "items": [], + "external": {**external, "count": 0, "items": [], "reads": []}}, + ] + row = { + **data["candidates"][0], + "round": 1, "candidate_id": "candidate-001", "run_id": "attempt-one", + "metric_key": "speedup", "measured_score": 2, "verdict": "PASS", + "knowledge_uses": [{"source": "cannbot-knowledge", "ref": card, + "retrieval_round": 1, "page": 1}], + } + data["candidates"] = [row] + with dashboard(browser, tmp_path, False, snapshot=snapshot) as page: + root = page.locator("#round-decisions") + retrieval, result = (root.locator(f'[data-rd-stage="{i}"]') for i in (1, 4)) + expect(retrieval).to_contain_text("Query · tile sizes") + expect(retrieval).to_contain_text("Local knowledge") + expect(retrieval).to_contain_text("CANNBot · a3") + expect(retrieval).to_contain_text("Read receipts · Pages 1") + expect(retrieval).to_contain_text(unused) + expect(result).to_contain_text("candidate-001") + expect(result).to_contain_text("Measured score · 2 · speedup") + expect(result).to_contain_text( + "Agent-declared knowledge use · cannbot-knowledge · " + card, + ) + expect(result).not_to_contain_text(unused) + root.locator("select").select_option("2") + expect(retrieval).to_contain_text("Channel state · No hits") + expect(result).not_to_contain_text("candidate-001")