From c2d6be418738ae0c06f9e7b9997541d28a5c49ed Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Thu, 19 Nov 2026 14:05:42 +0100 Subject: [PATCH 01/11] feat(catalogue): add OSWorld desktop-env-v1.0.3 as a version of OSWorld1 The original pin (044bc33, no tag) stays the default version, unchanged, with its proofs. desktop-env-v1.0.3 (9d6f711) is 48 commits ahead: no agent script, lib_run_single or docker provider changes; evaluator fixes, key repeat turned off in the guest, and one live task retired (test_all 368, test_nogdrive 360). Its original setup is marked not comparable with the paper's or the default version's. At the tag the upstream's setup.py exits on purpose, so the driver's editable install of the project fails. install.virtual installs the lock's pins without the project, as uv sync does for a project with no build system; the driver already imports the upstream from its clone. Proofs: units on all four datasets; nop 0/8 on test_small. --- .../benchmarks/osworld1/benchmark.json | 5 +- .../benchmarks/osworld1/standard.json | 251 ++++++++++++++---- app/nullius/standard/definitions.py | 8 +- app/nullius/standard/install.py | 3 +- app/tests/test_standard_bench_options.py | 9 +- app/tests/test_standard_mcp_machine.py | 19 ++ docs/benchmarks/supported/osworld1.md | 6 +- docs/standard.md | 1 + 8 files changed, 250 insertions(+), 52 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json index 9377167d..c561fe43 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +++ b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json @@ -37,7 +37,8 @@ "The agent's python and upload calls do not mark OSWorld's desktop as used; each unit's desktop is reset once, before the agent's first call.", "Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach.", "The VMs run only on Docker's default socket, not on rootless Docker or a Docker context. Podman is untested.", - "A task run by one of OSWorld's own agents and still running after 2 hours is stopped and gets no score. For an agent, the task's setup may take 1 hour, the agent's turn 2 hours and the scoring 10 minutes; past any of them the task gets no score." + "A task run by one of OSWorld's own agents and still running after 2 hours is stopped and gets no score. For an agent, the task's setup may take 1 hour, the agent's turn 2 hours and the scoring 10 minutes; past any of them the task gets no score.", + "Version desktop-env-v1.0.3 fixes evaluators, turns off key repeat in the VM and retires one task (368 tasks, 360 without Google Drive): its scores compare with each other only, not with the paper's or the default version's." ], "datasets": [ { @@ -49,7 +50,7 @@ "registry": "", "hub": "", "verifier": "OSWorld's own evaluator, DesktopEnv.evaluate: the task's evaluator functions on the VM's final state.", - "notes": "369 tasks in 10 domains; 361 without Google Drive. Apache-2.0." + "notes": "369 tasks in 10 domains; 361 without Google Drive (368 and 360 in desktop-env-v1.0.3). Apache-2.0." } ], "citation": "@misc{OSWorld,\n title={OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments}, \n author={Tianbao Xie and Danyang Zhang and Jixuan Chen and Xiaochuan Li and Siheng Zhao and Ruisheng Cao and Toh Jing Hua and Zhoujun Cheng and Dongchan Shin and Fangyu Lei and Yitao Liu and Yiheng Xu and Shuyan Zhou and Silvio Savarese and Caiming Xiong and Victor Zhong and Tao Yu},\n year={2024},\n eprint={2404.07972},\n archivePrefix={arXiv},\n primaryClass={cs.AI}\n}", diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 0b588a1e..61450607 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -285,57 +285,216 @@ } } }, - "proven": { - "test_nogdrive": { - "units": { - "units": 361, - "passed": "361 ids, the same twice, unique", - "on": "2026-11-13" - }, - "nop": { - "passed": "0/8", - "value": 0.0, - "expect": { - "at_most": 0.0 + "default": "044bc33", + "versions": { + "044bc33": { + "proven": { + "test_nogdrive": { + "units": { + "units": 361, + "passed": "361 ids, the same twice, unique", + "on": "2026-11-13" + }, + "nop": { + "passed": "0/8", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 8, + "job": "2026-10-29__18-38-14__osworld1__test_nogdrive__prove-nop", + "source": "nullius", + "on": "2026-10-29", + "limit": 8 + } }, - "units": 8, - "job": "2026-10-29__18-38-14__osworld1__test_nogdrive__prove-nop", - "source": "nullius", - "on": "2026-10-29", - "limit": 8 - } - }, - "test_all": { - "units": { - "units": 369, - "passed": "369 ids, the same twice, unique", - "on": "2026-11-13" - } - }, - "test_infeasible": { - "units": { - "units": 25, - "passed": "25 ids, the same twice, unique", - "on": "2026-11-13" + "test_all": { + "units": { + "units": 369, + "passed": "369 ids, the same twice, unique", + "on": "2026-11-13" + } + }, + "test_infeasible": { + "units": { + "units": 25, + "passed": "25 ids, the same twice, unique", + "on": "2026-11-13" + } + }, + "test_small": { + "units": { + "units": 39, + "passed": "39 ids, the same twice, unique", + "on": "2026-11-13" + }, + "nop": { + "passed": "0/8", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 8, + "job": "2026-11-13__14-08-41__osworld1__test_small__prove-nop", + "source": "nullius", + "on": "2026-11-13", + "limit": 8 + } + } } }, - "test_small": { - "units": { - "units": 39, - "passed": "39 ids, the same twice, unique", - "on": "2026-11-13" + "desktop-env-v1.0.3": { + "upstream": { + "tag": "desktop-env-v1.0.3", + "tested": "9d6f711d234b84cbff1b5199211610baa03b000c", + "tested_date": "2026-11-19" }, - "nop": { - "passed": "0/8", - "value": 0.0, - "expect": { - "at_most": 0.0 + "datasets": [ + { + "id": "test_nogdrive", + "label": "Without Google Drive", + "summary": "The upstream's test_nogdrive.json: every task but the 8 that need a Google account's credentials. The live MBTA booking task is retired at this release.", + "path": "evaluation_examples/test_nogdrive.json", + "sha256": "9270c11fe809eb4c4e35a78a47cdb50717ea76557c1b9164c183e8e8b8d9ce91", + "units": 360 + }, + { + "id": "test_all", + "label": "All tasks", + "summary": "The upstream's test_all.json, its runners' default: the 8 Google Drive tasks need credentials under evaluation_examples/settings, which the upstream does not ship. The live MBTA booking task is retired at this release.", + "path": "evaluation_examples/test_all.json", + "sha256": "47c54203369bd438abf279f6b9836bffc5aa43de6cbaccdfed546993f3fe9149", + "units": 368 }, - "units": 8, - "job": "2026-11-13__14-08-41__osworld1__test_small__prove-nop", - "source": "nullius", - "on": "2026-11-13", - "limit": 8 + { + "id": "test_small", + "label": "Small", + "summary": "The upstream's test_small.json: a few tasks of each domain.", + "path": "evaluation_examples/test_small.json", + "sha256": "bb29275171b78728d71825b125af311d74e66bb2a6d693c45c497d380b1fc640", + "units": 39 + }, + { + "id": "test_infeasible", + "label": "Infeasible", + "summary": "The upstream's test_infeasible.json: tasks that cannot be done, passed by answering FAIL.", + "path": "evaluation_examples/test_infeasible.json", + "sha256": "1b447d0462b1bdac4b660e0dd6e6add6d01c17a9e684702c701abcffda409abb", + "units": 25 + } + ], + "driver": { + "command": [ + "{install}/.venv/bin/python", + "{here}/driver.py", + "{here}/agents.json", + "{here}/nop.py", + "{here}/runner.py" + ], + "install": { + "python": "3.12", + "project": ".", + "locked": true, + "virtual": true + }, + "participant": true, + "unit_end": true + }, + "setups": { + "original": { + "summary": "OSWorld's own agents as published: the agent option's script (scripts/python/run_*.py) on the task, with its own prompts, parser, history and step limit, asking one of the models it is published with, on the docker provider's VM, scored by DesktopEnv.evaluate. At OSWorld's desktop-env 1.0.3 release: its evaluator fixes change the score of some tasks and one task is retired, so its runs compare with each other only, not with the paper's or the earlier pin's.", + "models": [ + "MiniMax-M3", + "gpt-4o", + "claude-sonnet-4-5-20250929", + "claude-opus-4-5", + "claude-opus-4-6", + "claude-opus-4-7", + "gpt-5.4", + "computer-use-preview", + "qwen3.7-plus", + "gpt-5", + "EvoCUA-S2", + "EvoCUA-S1", + "autoglm-os", + "mano", + "muse-spark-1.1" + ], + "default": "MiniMax-M3", + "served_as": { + "MiniMax-M3": "minimax/minimax-m3", + "gpt-4o": "openai/gpt-4o", + "claude-sonnet-4-5-20250929": "anthropic/claude-sonnet-4.5", + "claude-opus-4-5": "anthropic/claude-opus-4.5", + "claude-opus-4-6": "anthropic/claude-opus-4.6", + "claude-opus-4-7": "anthropic/claude-opus-4.7", + "gpt-5.4": "openai/gpt-5.4", + "qwen3.7-plus": "qwen/qwen3.7-plus", + "gpt-5": "openai/gpt-5", + "UI-TARS-1.5-7B": "bytedance/ui-tars-1.5-7b" + }, + "providers": { + "openrouter": "", + "openai": "", + "custom": "" + }, + "turn_seconds": 7200, + "side_models": [ + "UI-TARS-1.5-7B" + ], + "reference_agent": "OSWorld's agents", + "comparable": false + }, + "model": { + "summary": "The same agent with any model in its own model's place: the upstream's code asks for the agent's first published model, with its prompt and action format, and the relay sends the model chosen, every side call included. Its scores compare with other runs of this setup.", + "runs_as": "MiniMax-M3" + } + }, + "proven": { + "test_nogdrive": { + "units": { + "units": 360, + "passed": "360 ids, the same twice, unique", + "on": "2026-11-19" + } + }, + "test_all": { + "units": { + "units": 368, + "passed": "368 ids, the same twice, unique", + "on": "2026-11-19" + } + }, + "test_small": { + "units": { + "units": 39, + "passed": "39 ids, the same twice, unique", + "on": "2026-11-19" + }, + "nop": { + "passed": "0/8", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 8, + "job": "2026-11-19__13-12-06__osworld1-desktop-env-v1.0.3__test_small__prove-nop", + "source": "nullius", + "options": { + "max_steps": "default", + "vm_provider": "docker" + }, + "on": "2026-11-19", + "limit": 8 + } + }, + "test_infeasible": { + "units": { + "units": 25, + "passed": "25 ids, the same twice, unique", + "on": "2026-11-19" + } + } } } } diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 76b84d7e..c12d934f 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -311,7 +311,9 @@ def _install(d: dict, where: str = "install") -> List[str]: project's resolution keeps them; each a name, or a name pinned), and ``prefetch``, models fetched from the Hugging Face hub at install time so a run needs no network for them; ``locked``, the project's own lock file (``uv.lock``) installed as its - pins, ``pip`` resolved beside them, rather than its loose requirements. + pins, ``pip`` resolved beside them, rather than its loose requirements; + ``virtual``, with ``locked``, those pins without the project itself, as + ``uv sync`` installs a project that declares no build system. A benchmark's driver has the same block, ``driver.install`` (:func:`driver_install`): ``where`` is the name its problems are said under.""" inst = d.get("install") or {} @@ -353,6 +355,10 @@ def _install_block(d: dict, inst: dict) -> List[str]: bad.append("install.locked: true or false") elif inst.get("locked") and "project" not in inst: bad.append("install.locked: the lock file of install.project, which is not set") + if "virtual" in inst and not isinstance(inst["virtual"], bool): + bad.append("install.virtual: true or false") + elif inst.get("virtual") and not inst.get("locked"): + bad.append("install.virtual: the lock's packages without the project, which needs install.locked") for i, x in enumerate(inst.get("indexes") or []): if not isinstance(x, dict) or not _HTTPS.match(str(x.get("url", ""))) or not x.get("packages") \ or any(not _PKG.match(str(p)) for p in x.get("packages") or []): diff --git a/app/nullius/standard/install.py b/app/nullius/standard/install.py index c4108856..3bb91c3b 100644 --- a/app/nullius/standard/install.py +++ b/app/nullius/standard/install.py @@ -258,7 +258,8 @@ def _steps(inst: dict, where: str, up: Optional[dict], src: Optional[str], log: _submodules(proj, lock, log) step(["uv", "--directory", proj, "pip", "install", "--quiet", "--python", py, "-r", lock] + list(inst.get("pip") or []) + wheels) - step(["uv", "pip", "install", "--quiet", "--python", py, "--no-deps", "-e", proj + extras]) + if not inst.get("virtual"): # virtual: the lock's packages only, as uv sync leaves a project with no build system + step(["uv", "pip", "install", "--quiet", "--python", py, "--no-deps", "-e", proj + extras]) else: step(["uv", "pip", "install", "--quiet", "--python", py, "-e", proj + extras]) if inst.get("prefetch"): diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index 94e4dfa4..6e4dc2da 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -261,7 +261,14 @@ def test_bundled_benchmarks_do_not_expose_integration_revisions(self): for path in root.glob("*/standard.json"): with self.subTest(benchmark=path.parent.name): data = json.loads(path.read_text()) - self.assertNotIn("versions", data) + shared = (data.get("upstream") or {}).get("repository") + for name, block in (data.get("versions") or {}).items(): + if name == data.get("default"): + continue + own = block.get("upstream") or {} + # a version beside the default is an upstream release (its tag) or a fork (another repository) + self.assertTrue(own.get("tag") or own.get("repository", shared) != shared, name) + self.assertEqual(len(own.get("tested") or ""), 40, name) class MachineDocker(unittest.TestCase): diff --git a/app/tests/test_standard_mcp_machine.py b/app/tests/test_standard_mcp_machine.py index 10ad6229..6fa46700 100644 --- a/app/tests/test_standard_mcp_machine.py +++ b/app/tests/test_standard_mcp_machine.py @@ -103,6 +103,12 @@ def test_locked_goes_with_a_project(self): self.assertIn("driver.install.locked: the lock file of install.project, which is not set", bad) bad = definitions.problems(dict(ok, driver=dict(ok["driver"], install={"project": ".", "locked": "yes"})), "benchmark") self.assertIn("driver.install.locked: true or false", bad) + self.assertEqual(definitions.problems(dict(ok, driver=dict(ok["driver"], install={"project": ".", "locked": True, + "virtual": True})), "benchmark"), []) + bad = definitions.problems(dict(ok, driver=dict(ok["driver"], install={"project": ".", "virtual": True})), "benchmark") + self.assertIn("driver.install.virtual: the lock's packages without the project, which needs install.locked", bad) + bad = definitions.problems(dict(ok, driver=dict(ok["driver"], install={"project": ".", "locked": True, "virtual": 1})), "benchmark") + self.assertIn("driver.install.virtual: true or false", bad) class Locked(unittest.TestCase): @@ -129,6 +135,19 @@ def run(cmd, cwd, log, env=None): self.assertEqual(ran[-1][-3:], ["--no-deps", "-e", "/up/clone[x]"]) self.assertFalse(any(c[:2] == ["git", "fetch"] for c in ran)) # the clone that was there, not a second one + def test_a_virtual_project_installs_its_lock_and_not_itself(self): + ran = [] + + def run(cmd, cwd, log, env=None): + ran.append(cmd) + if cmd[:2] == ["uv", "export"]: + open(cmd[cmd.index("-o") + 1], "w").close() + with mock.patch.object(install, "_run", side_effect=run): + install._packages({"python": "3.12", "project": ".", "locked": True, "virtual": True}, + os.path.join(self.tmp, "w"), {"repository": "r", "tested": "c"}, src="/up/clone") + self.assertTrue(any("-r" in c for c in ran)) # the lock's pins + self.assertFalse(any("-e" in c for c in ran)) # never the project's own build + def _install_with_lock(self, lines, made=()): proj = os.path.join(self.tmp, "clone") for d in ("sub/empty", "sub/held"): diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index cf9ac1b3..69a6ccf3 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -31,7 +31,10 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of | Version | Setups as published, whose model | Options | |---|---|---| -| `osworld1` | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `max_steps`, `pause`, `thinking`, `vm_provider` | +| `osworld1@044bc33` (default) | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `max_steps`, `pause`, `thinking`, `vm_provider` | +| `osworld1@desktop-env-v1.0.3` | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `max_steps`, `pause`, `thinking`, `vm_provider` | + +- `key@version`: a revision of the definition and driver; a recipe pins one, the builder offers the default. ## Options @@ -80,6 +83,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of - Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach. - The VMs run only on Docker's default socket, not on rootless Docker or a Docker context. Podman is untested. - A task run by one of OSWorld's own agents and still running after 2 hours is stopped and gets no score. For an agent, the task's setup may take 1 hour, the agent's turn 2 hours and the scoring 10 minutes; past any of them the task gets no score. +- Version desktop-env-v1.0.3 fixes evaluators, turns off key repeat in the VM and retires one task (368 tasks, 360 without Google Drive): its scores compare with each other only, not with the paper's or the default version's. ## Cite diff --git a/docs/standard.md b/docs/standard.md index 0c0cd9f1..c03039df 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -477,6 +477,7 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is | `indexes` | Packages from another index | | `wheels` | A wheel served by URL only (a model package such as spaCy's `en_core_web_sm`), `{url, sha256}`: refused unless the file matches, resolved with `pip` | | `project` | The upstream itself, from its clone; `locked: true` installs it at its own lock file's pins (`uv export --frozen`), `pip` resolved beside them | +| `virtual` | With `locked`: the lock's pins without the project itself, as `uv sync` leaves a project with no build system; the driver imports the upstream from its clone | | `extras`, `prefetch` | The project's extras; what to download ahead | #### Benchmark options From 309d97c6b272a9c66d8e5a30c7365e34931e3c93 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Thu, 19 Nov 2026 15:19:55 +0100 Subject: [PATCH 02/11] feat(catalogue): show a benchmark's versions on its page, one at a time, as an agent's page does --- app/templates/catalogue/benchmark.html | 8 ++++++++ app/ui_tests/test_catalogue_routes.py | 13 +++++++++++++ app/web/catalogue.py | 25 ++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/app/templates/catalogue/benchmark.html b/app/templates/catalogue/benchmark.html index 898b3ec2..471c4175 100644 --- a/app/templates/catalogue/benchmark.html +++ b/app/templates/catalogue/benchmark.html @@ -66,6 +66,13 @@

Original

{%- endif %}

Proofs

+ {% if versions %} + {#- Each version has its own pin and its own proofs, as an agent's page shows them. -#} + + {% endif %} {% if std.upstream.repository %}

{{ std.upstream.repository|replace('https://github.com/', '') }} @ {% if std.upstream.tag %}{{ std.upstream.tag }} · {% endif %}{{ std.upstream.tested[:7] }}

{% endif %}

Null and reference controls check the scorer; stability and leak checks validate the tasks.

@@ -89,6 +96,7 @@

Proofs

Run control checks
+ {% if versions %}{% endif %}

Check dataset stability, answer leaks and control scores. Results update the recorded proofs. {% if std.judged %}The judge needs a model key.{% else %}No model calls or key required.{% endif %}

diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index 874cebe7..4943eba4 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -186,6 +186,19 @@ def test_a_page_exists_only_for_a_listed_benchmark(self): class BenchmarkPage(PageCase): + def test_a_benchmark_with_versions_shows_one_at_a_time_with_its_own_pin_and_proofs(self): + html = self.get("/benchmarks/osworld1") + nav = re.search(r'
', cards["osworld1:openai-cua"]) # the Agents page's foot - self.assertIn('ACPBundled', cards["goose"]) + self.assertIn('ACPNullius', cards["goose"]) self.assertIn('
Belongs to OSWorld2
', cards["osworld2:m3"]) self.assertIn('data-f-from="elsewhere "', cards["osworld2:m3"]) self.assertIn(">Lacks computer", cards["goose"]) @@ -70,7 +70,7 @@ def test_osworld1_offers_its_own_agents_and_codex_and_says_why_not_the_rest(self self.assertIn(' Cannot', fit) self.assertLess(html.index("data-dropdown"), html.index('data-tools-btn="filter"')) self.assertIn('data-dropdown aria-haspopup="true" aria-expanded="false">Fit<', html) # an icon as Filter's - self.assertIn("> Bundled" + self.assertIn("> Nullius" "