diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index fcd8a41e..d06d65db 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -23,13 +23,17 @@ Discovery reads packaged resources and works without device libraries. ```bash inferrsi benchmarks list +inferrsi benchmarks list --iterative inferrsi benchmarks show moe-forward-npu inferrsi benchmarks show qwen21-denoise-npu inferrsi benchmarks show qwen3-tts-npu ``` The library's `index.json` registers the packages. Common Python reads this -registry without embedding case names. `show` returns the task manifest and +registry without embedding case names. The `--iterative` filter lists tasks +whose package root contains an `optimization.py` file. It checks only for the +file, so listing works without importing task code or probing a device. +`show` returns the task manifest and the package's files without interpreting workload-specific fields. `run ...` executes that package's `benchmark.py` with the remaining arguments unchanged. Each package owns its entry, measurement and assessment code. Review the implementation and diff --git a/docs/operations.md b/docs/operations.md index 9115ca36..79e8d565 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -32,7 +32,7 @@ Effect classes used below: | Optimize an ordinary project in the current Agent | CLI `campaign start/knowledge/round/status/final/recover` | Experiment-executing / local records | Harness freezes explicit caller-provided checks, manages multi-file candidates, executes and assesses retained results. This path does not depend on Benchmark or infer acceptance from the goal. See [shared workflow](optimization-workflow-unification.md) | | Reuse one project service across multi-scenario checks | CLI `campaign start --evaluation-mode service_reuse`, then `baseline/round/profile/status/recover/final` | Experiment-executing / local records | Caller supplies protocol and service scripts. An optional profiler can run inline or on a settled round. `resume`, `refresh-references` and explicit retry retain recovery decisions. See [service reuse](service-reuse-campaigns.md) | | Prepare and record a fresh native Agent | CLI `agents prepare/dispatch/bind/record/status` | Local records / explicit host handoff | The initiating Agent calls native host tools. Records prevent repeated dispatch; exported host observations do not establish task acceptance. See [native sessions](native-agent-sessions.md) | -| Discover packaged tasks | CLI `benchmarks list` / `benchmarks show`; Python `benchmark.registry.builtin_tasks` / `describe_task` | Read-only | Reads the task library's registry and opaque manifests from installed resources | +| Discover packaged tasks | CLI `benchmarks list [--iterative]` / `benchmarks show`; Python `benchmark.registry.builtin_tasks` / `describe_task` | Read-only | Reads the installed registry and task manifests. The optional filter selects task packages with an `optimization.py` file at their root | | Execute one packaged task | CLI `benchmarks run`; Python `apps.workflows.benchmark_operations.run_task` | Experiment-executing | Application launches the task entry, supplies the execution callback and preserves arguments and exit status. The task owns its device, measurement and assessment protocol | | Find relevant cases, skills and records | CLI `find` | Read-only | Default discovery across the registered library and package records. Follow the emitted command to read a hit and its revision | | Inspect the package catalog | CLI `knowledge search` / `knowledge show`; catalog API | Read-only | Filtered inventory and existing catalog scripts. Uses the installed package's tier/repo/backend/scenario filters and IDs | diff --git a/docs/optimization-benchmarks.md b/docs/optimization-benchmarks.md index 4e95b286..dbc8cc0c 100644 --- a/docs/optimization-benchmarks.md +++ b/docs/optimization-benchmarks.md @@ -27,9 +27,10 @@ only observes and collects its results. Benchmark supplies task resources and task-specific rules. Harness freezes those inputs, runs the checks, assesses retained outputs and manages the lifecycle; it does not invent project acceptance. -The iterative tasks are `qwen21-denoise-npu` and `moe-forward-npu`; TTS -currently exposes its single-candidate task entry. New session tasks provide -their own `optimization.py` alongside the task definition and evaluator. +Before starting a session, use `inferrsi benchmarks list --iterative` to list +installed tasks with an `optimization.py` file at the task package root. +The current list contains `moe-forward-npu` and `qwen21-denoise-npu`. +TTS currently exposes its single-candidate task entry. For paired knowledge on/off experiments over both tasks, use the Apps [`benchmarks compare` workflow](benchmark-comparison.md). It reuses this session diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 9cbdbb05..2e03d018 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -395,10 +395,13 @@ def main(argv: list[str] | None = None) -> int: from ..workflows.benchmark_operations import run_task return run_task(args.task, args.arguments) - from ...benchmark.registry import builtin_tasks, describe_task + from ...benchmark.registry import builtin_tasks, describe_task, has_iterative_protocol if args.benchmark_action == "list": - _print(builtin_tasks()) + tasks = builtin_tasks() + if args.iterative: + tasks = [task for task in tasks if has_iterative_protocol(task)] + _print(tasks) else: _print(describe_task(args.task)) return 0 diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index da3720cd..6d349b10 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -254,7 +254,11 @@ def parser(*, parser_class=argparse.ArgumentParser) -> argparse.ArgumentParser: action.add_argument("--artifact", type=Path, action="append", default=[]) benchmarks = commands.add_parser("benchmarks", help="Discover packaged engineering tasks") actions = benchmarks.add_subparsers(dest="benchmark_action", required=True) - actions.add_parser("list") + listing = actions.add_parser("list") + listing.add_argument( + "--iterative", action="store_true", + help="List only tasks with a packaged optimization.py file for iterative sessions", + ) compare = actions.add_parser( "compare", help="Compare complete RSI optimization sessions (default: knowledge on/off)", description=("Apps coordinates paired fresh-Agent optimizations and reports. Knowledge " diff --git a/src/inferrsi/apps/workflows/benchmark_compare.py b/src/inferrsi/apps/workflows/benchmark_compare.py index 8d66c98e..6f65056f 100644 --- a/src/inferrsi/apps/workflows/benchmark_compare.py +++ b/src/inferrsi/apps/workflows/benchmark_compare.py @@ -17,7 +17,7 @@ from tempfile import TemporaryDirectory from inferrsi.benchmark.comparison import DEFAULT_POLICY, DEFAULT_TASKS, summarize, validate_policy -from inferrsi.benchmark.registry import task_resources +from inferrsi.benchmark.registry import has_iterative_protocol, task_resources from inferrsi.benchmark.summary import POLICY, supported_policy from inferrsi.benchmark.summary import comparison as metrics_comparison from inferrsi.harness import agent_sessions, optimization @@ -123,8 +123,7 @@ def _validate_config(config): or len(set(tasks)) != len(tasks)): raise ValueError("tasks must be a nonempty list of unique registered task names") for task in tasks: - root = Path(str(task_resources(task))) - if not (root / "optimization.py").is_file(): + if not has_iterative_protocol(task): raise ValueError(f"Task has no iterative protocol: {task}") if not isinstance(config["task_arguments"].get(task), str): raise ValueError(f"Missing task arguments: {task}") diff --git a/src/inferrsi/benchmark/registry.py b/src/inferrsi/benchmark/registry.py index 2d9d6a8d..6d4454c9 100644 --- a/src/inferrsi/benchmark/registry.py +++ b/src/inferrsi/benchmark/registry.py @@ -35,6 +35,11 @@ def task_resources(name: str): return root +def has_iterative_protocol(name: str) -> bool: + """Check for the packaged protocol file without loading task code.""" + return task_resources(name).joinpath("optimization.py").is_file() + + def describe_task(name: str) -> dict: root = task_resources(name) return { diff --git a/tests/test_benchmark_operations.py b/tests/test_benchmark_operations.py index 0f23e917..fd523b45 100644 --- a/tests/test_benchmark_operations.py +++ b/tests/test_benchmark_operations.py @@ -19,6 +19,24 @@ def write_registry(library, value): (library / "index.json").write_text(json.dumps(value), encoding="utf-8") +def test_iterative_listing_uses_packaged_files_without_executing_them(library, capsys): + names = ["z-task", "directory-task", "a-task"] + write_registry(library, {"schema_version": 1, "packages": names}) + for name in names: + (library / name).mkdir() + (library / "directory-task" / "optimization.py").mkdir() + + assert main(["benchmarks", "list", "--iterative"]) == 0 + assert json.loads(capsys.readouterr().out) == [] + + for name in ("z-task", "a-task"): + (library / name / "optimization.py").write_text( + 'raise AssertionError("Discovery executed the task protocol")\n', encoding="utf-8", + ) + assert main(["benchmarks", "list", "--iterative"]) == 0 + assert json.loads(capsys.readouterr().out) == ["a-task", "z-task"] + + @pytest.mark.parametrize("exit_code", [0, 1, 2]) def test_new_task_needs_only_package_and_registry(library, capsys, exit_code): package = library / "new-workload" diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py index 6436f7e2..affa2590 100644 --- a/tests/test_engineering_benchmarks.py +++ b/tests/test_engineering_benchmarks.py @@ -25,6 +25,18 @@ def test_cli_lists_the_three_packaged_workloads(capsys): assert builtin_tasks() == sorted(EXPECTED) +def test_cli_lists_packaged_iterative_workloads(capsys): + assert main(["benchmarks", "list", "--iterative"]) == 0 + assert json.loads(capsys.readouterr().out) == ["moe-forward-npu", "qwen21-denoise-npu"] + + +def test_benchmark_list_help_documents_iterative(capsys): + with pytest.raises(SystemExit) as result: + main(["benchmarks", "list", "--help"]) + assert result.value.code == 0 + assert "--iterative" in capsys.readouterr().out + + @pytest.mark.parametrize("name,task_id", EXPECTED.items()) def test_cli_show_reads_the_relocated_manifest(name, task_id, capsys): assert main(["benchmarks", "show", name]) == 0 @@ -55,9 +67,11 @@ def test_unknown_packages_are_rejected(name): def test_discovery_does_not_import_device_libraries(): script = """ import sys +from inferrsi.apps.cli import main from inferrsi.benchmark.registry import builtin_tasks, describe_task for name in builtin_tasks(): describe_task(name) +assert main(["benchmarks", "list", "--iterative"]) == 0 assert not {'torch', 'torch_npu', 'qwen_tts', 'vllm', 'vllm_omni'} & sys.modules.keys() """ subprocess.run([sys.executable, "-c", script], check=True, timeout=15)