From 5f9f051e5d36eb1bc5c267b9b2851cb917542309 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 18:23:43 +0100 Subject: [PATCH 01/20] refactor(standard): list the agents and capabilities with their fits in one place The Agents page built its entries, their ports and their fits in the web layer. recipe.entries now does it for a given list of benchmarks, so the builder's Agent step can draw the same entries. The Agents page output is unchanged. --- app/nullius/standard/recipe.py | 55 ++++++++++++++++++++++++++++ app/tests/test_benchmark_agents.py | 17 +++++++++ app/web/agents.py | 58 +++--------------------------- 3 files changed, 77 insertions(+), 53 deletions(-) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 2e2369de..afaaa9bd 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -213,6 +213,61 @@ def _other_page(o: dict) -> dict: if isinstance((o.get("proven") or {}).get("fidelity"), dict) else {})} +def _install_words(o: dict) -> list: + """What a definition installs, in a few words each.""" + inst, up = o.get("install") or {}, o.get("upstream") or {} + out = [f"npm {p}" for p in inst.get("npm") or []] + [f"pip {p}" for p in inst.get("pip") or []] + if inst.get("project"): + extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else "" + out.append(f"{inst['project']}{extras} from {up.get('repository', '').removeprefix('https://')} at {(up['tag'] + ' (' + str(up.get('tested', ''))[:12] + ')') if up.get('tag') else str(up.get('tested', ''))[:12]}") + if inst.get("python"): + out.append(f"on Python {inst['python']}") + out += [f"{', '.join(x['packages'])} from {x['url'].removeprefix('https://')}" for x in inst.get("indexes") or []] + out += [f"the model {m}" for m in inst.get("prefetch") or []] + return out + + +def entries(kind: str, benches: Optional[List[dict]] = None, every: bool = False, own: bool = False, jobs=()) -> dict: + """The agents (or capabilities) as the Agents page and the builder's Agent step list them, each with its ports + and its fits (``match.fit``) on ``benches`` (default: every benchmark of its family, default versions), and + the definitions with problems. ``own``: and the benchmarks' own agents (``definitions.benchmark_agents``); + ``jobs``: the job ids whose proof links to its job.""" + from nullius.standard import conform + others, bad = definitions.load(kind, every=every) + loaded = definitions.load("benchmark")[0] + benches = [b for b in loaded if b["family"] == kind] if benches is None else benches + if own and kind == "agent": + others = others + definitions.benchmark_agents(loaded) + out = [] + for o in others: + driver = "driver" in (o.get("start") or {}) # a benchmark's own: its driver runs it, it speaks no port + cand = set() if driver else set(conform.candidates(o)) + rows = [] + for p, spec in ports.PORTS.items() if not driver else (): + if ports.family(p) != kind or (kind == "capability" and p.split(".", 1)[1].split("/")[0] != o.get("kind")): + continue + rec = (o.get("proven") or {}).get(p) if p in (o.get("offers") or []) else None + state = "proven" if rec else ("not offered" if p in cand else ("no kit yet" if not spec["kit"] else "not for it")) + kit = definitions.kit(p) if spec["kit"] else None + rows.append({"port": p, "label": spec.get("label") or p, "summary": spec["summary"], "state": state, "record": rec, + "proof_kit": definitions.proof_kit(rec) if rec else None, + "kit": {"says": kit["summary"], "checks": kit["datasets"][0]["units"], "key": kit["key"], "version": definitions.kit_version(kit)} if kit else None, + "job": rec["job"] if rec and rec.get("job") in jobs else None, + "older_kit": definitions.older_kit(p, rec), + "skills": skills_of(p, rec)}) + fits = [{"key": b["key"], "ref": definitions.ref_of(b) if "_version" in b else b["key"], "label": b["label"], "needs": b["needs"], "why": match.fit(b, o)} for b in benches] + fid = (o.get("proven") or {}).get("fidelity") + out.append({"d": o, "protocol": protocol(o), "ports": rows, "fits": fits, + "install": _install_words(o), + # its options, each with its default and what else it takes; its fidelity proof, when it has one + "options": [{"name": n, "summary": x.get("summary", ""), "default": x.get("default"), + "values": list(x.get("values") or []), "pattern": x.get("pattern", ""), "example": x.get("example", "")} + for n, x in (o.get("options") or {}).items()], + "fidelity": fid if isinstance(fid, dict) else None}) + out.sort(key=lambda e: (not e["d"].get("example", False), definitions.title(e["d"]).casefold())) + return {"entries": out, "problems": bad} + + def choices() -> dict: """What the page offers - choices and the reasons a pair does not fit, never a command.""" diff --git a/app/tests/test_benchmark_agents.py b/app/tests/test_benchmark_agents.py index 76a585ad..68811bc4 100644 --- a/app/tests/test_benchmark_agents.py +++ b/app/tests/test_benchmark_agents.py @@ -115,6 +115,23 @@ def test_its_own_benchmark_alone_with_no_port_reason(self): self.assertEqual(t["gsm8k"]["osworld2:muse-spark"], ["Muse Spark belongs to OSWorld2"]) self.assertNotIn("osworld1:openai-cua", t.get("locomo", {})) # a capability benchmark takes none + def test_one_entry_list_whose_fits_are_the_builders(self): + # recipe.entries: the Agents page's list and the builder's Agent step grid; its fits are the builder payload's + every = definitions.load("benchmark", every=True)[0] + c = recipe.choices() + fits, offered = c["fits"], {o["key"] for o in c["agents"] + c["capabilities"]} # the builder never offers an example + for kind in ("agent", "capability"): + for b in (x for x in every if x["family"] == kind): + ref = definitions.ref_of(b) if "_version" in b else b["key"] + with self.subTest(ref): + got = {e["d"]["_ref"]: e["fits"][0] for e in recipe.entries(kind, benches=[b], every=True, own=True)["entries"] + if not e["d"].get("example")} + self.assertEqual({k: f["why"] for k, f in got.items()}, {k: v for k, v in fits[ref].items() if k in offered}) + self.assertEqual({f["ref"] for f in got.values()}, {ref}) + page = {e["d"]["_ref"]: e for e in recipe.entries("agent", own=True)["entries"]} + self.assertEqual([f["why"] for f in page["osworld1:openai-cua"]["fits"] if f["key"] == "osworld1"], [[]]) + self.assertEqual(page["osworld1:openai-cua"]["protocol"], "its own script") + RECIPE = {"harness": "standard", "name": "os", "benchmark": "osworld1", "datasets": ["test_small"], "provider": "openrouter", diff --git a/app/web/agents.py b/app/web/agents.py index 1adcee24..30230c9c 100644 --- a/app/web/agents.py +++ b/app/web/agents.py @@ -28,7 +28,7 @@ from flask import abort, flash, redirect, render_template, request, url_for from nullius import execution, paths, providers from nullius.catalogue import storage -from nullius.standard import conform, definitions, howto, match, ports +from nullius.standard import conform, definitions, howto, ports from nullius.standard import recipe as standard_recipe from .core import app @@ -38,62 +38,14 @@ from . import settings as settings_web -def _install_words(o: dict) -> list: - """What a definition installs, in a few words each.""" - inst, up = o.get("install") or {}, o.get("upstream") or {} - out = [f"npm {p}" for p in inst.get("npm") or []] + [f"pip {p}" for p in inst.get("pip") or []] - if inst.get("project"): - extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else "" - out.append(f"{inst['project']}{extras} from {up.get('repository', '').removeprefix('https://')} at {(up['tag'] + ' (' + str(up.get('tested', ''))[:12] + ')') if up.get('tag') else str(up.get('tested', ''))[:12]}") - if inst.get("python"): - out.append(f"on Python {inst['python']}") - out += [f"{', '.join(x['packages'])} from {x['url'].removeprefix('https://')}" for x in inst.get("indexes") or []] - out += [f"the model {m}" for m in inst.get("prefetch") or []] - return out - - -def _entries(kind: str, every: bool = False, own: bool = False) -> dict: - """Its entries; ``own``: and the benchmarks' own agents (``definitions.own_agents``), which have no page.""" - others, bad = definitions.load(kind, every=every) - loaded = definitions.load("benchmark")[0] - benches = [b for b in loaded if b["family"] == kind] - if own and kind == "agent": - others = others + definitions.benchmark_agents(loaded) - jobs = {j.job_id for j in shared._jobs_for_request()} - out = [] - for o in others: - driver = "driver" in (o.get("start") or {}) # a benchmark's own: its driver runs it, it speaks no port - cand = set() if driver else set(conform.candidates(o)) - rows = [] - for p, spec in ports.PORTS.items() if not driver else (): - if ports.family(p) != kind or (kind == "capability" and p.split(".", 1)[1].split("/")[0] != o.get("kind")): - continue - rec = (o.get("proven") or {}).get(p) if p in (o.get("offers") or []) else None - state = "proven" if rec else ("not offered" if p in cand else ("no kit yet" if not spec["kit"] else "not for it")) - kit = definitions.kit(p) if spec["kit"] else None - rows.append({"port": p, "label": spec.get("label") or p, "summary": spec["summary"], "state": state, "record": rec, - "proof_kit": definitions.proof_kit(rec) if rec else None, - "kit": {"says": kit["summary"], "checks": kit["datasets"][0]["units"], "key": kit["key"], "version": definitions.kit_version(kit)} if kit else None, - "job": rec["job"] if rec and rec.get("job") in jobs else None, - "older_kit": definitions.older_kit(p, rec), - "skills": standard_recipe.skills_of(p, rec)}) - fits = [{"key": b["key"], "label": b["label"], "needs": b["needs"], "why": match.fit(b, o)} for b in benches] - fid = (o.get("proven") or {}).get("fidelity") - out.append({"d": o, "protocol": standard_recipe.protocol(o), "ports": rows, "fits": fits, - "install": _install_words(o), - # its options, each with its default and what else it takes; its fidelity proof, when it has one - "options": [{"name": n, "summary": x.get("summary", ""), "default": x.get("default"), - "values": list(x.get("values") or []), "pattern": x.get("pattern", ""), "example": x.get("example", "")} - for n, x in (o.get("options") or {}).items()], - "fidelity": fid if isinstance(fid, dict) else None}) - out.sort(key=lambda e: (not e["d"].get("example", False), definitions.title(e["d"]).casefold())) - return {"entries": out, "problems": bad} +def _job_ids() -> set: + return {j.job_id for j in shared._jobs_for_request()} def _page(kind: str): example = os.path.join(definitions.CATALOGUE, "agents", "reference") if kind == "agent" else \ os.path.join(definitions.CATALOGUE, "capabilities", "keyword-memory") - found = _entries(kind, own=True) + found = standard_recipe.entries(kind, own=True, jobs=_job_ids()) return render_template("agents/standard.html", kind=kind, how=howto.steps(kind), prompts=howto.prompts(), catalogue_dir=paths.catalogue_dir(), example_dir=example, problems=found["problems"], items=[_card(kind, e) for e in found["entries"]], choices=_choices(kind, found["entries"]), only=ONLY.get(kind, {})) @@ -149,7 +101,7 @@ def _card(kind: str, e: dict) -> dict: def _entry(kind: str, key: str): - every = _entries(kind, every=True)["entries"] + every = standard_recipe.entries(kind, every=True, jobs=_job_ids())["entries"] mine = [x for x in every if x["d"]["key"] == key] v = request.args.get("v") e = next((x for x in mine if x["d"]["_version"] == v), None) if v else \ From 2c66a039ccdde4d7ab0dd9205d7bbed75e0be021 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 18:23:50 +0100 Subject: [PATCH 02/20] feat(web): let a card of the card toolkit be a toggle A card's pick ({value, on, disabled}) makes its title a button with aria-pressed instead of a link; tables.js flips it on a click, Space or Enter and sends nullius-pick, and the page keeps the choice. A disabled card's reason goes in its alert slot, which now takes a tooltip, and a row of filter pills can name its first pill (all). --- app/static/catalogue/catalogue.css | 6 ++++++ app/static/tables.js | 15 +++++++++++++-- app/templates/_cards.html | 10 +++++++--- app/templates/_tables.html | 4 ++-- app/ui_tests/test_cards.py | 29 +++++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 7 deletions(-) diff --git a/app/static/catalogue/catalogue.css b/app/static/catalogue/catalogue.css index 7b61581d..81aa9f8c 100644 --- a/app/static/catalogue/catalogue.css +++ b/app/static/catalogue/catalogue.css @@ -24,6 +24,12 @@ .card-link::after { content: ""; position: absolute; inset: -1px; border-radius: 12px; } .card-link:focus-visible { outline: none; } .card-link:focus-visible::after { box-shadow: 0 0 0 3px var(--accent-soft), 0 0 0 1px var(--accent); } +/* a pick card (_cards.html `pick`): its title a toggle; chosen, it is marked; disabled, its reason says why */ +button.card-link { font: inherit; color: inherit; background: none; border: 0; padding: 0; margin: 0; text-align: left; cursor: pointer; } +button.card-link:disabled { cursor: not-allowed; } +.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) { border-color: var(--accent); background: var(--accent-soft); } +.bench-card:has(> .bench-card-title [data-pick]:disabled) { opacity: .55; } +.bench-card:has(> .bench-card-title [data-pick]:disabled):hover { border-color: var(--border-soft); } .bench-card :is(input, button, .btn, [title]) { position: relative; z-index: 1; } .bench-card [title]:not(input, button, .btn) { cursor: pointer; } /* raised for its tooltip, it still opens the card */ .bench-card-foot .bench-harness { margin-left: auto; } diff --git a/app/static/tables.js b/app/static/tables.js index d966c587..a23327d3 100644 --- a/app/static/tables.js +++ b/app/static/tables.js @@ -229,19 +229,30 @@ window.NulliusTable = { how many show once the changes pause, counted in the live region's data-noun. A click anywhere on a card but on a control of its own follows the card's link, in a new tab or window with the keys or the - button a link takes; a click that ends a text selection does not. Sort by, - when it has one, sorts the cards within their grids as a table's rows. */ + button a link takes; a click that ends a text selection does not. A pick + card's title is a toggle (data-pick, aria-pressed): a click, Space or + Enter flips it and sends nullius-pick {value, pressed} from it; the page + keeps the choice. Sort by, when it has one, sorts the cards within their + grids as a table's rows. */ cards(root) { const items = [...root.querySelectorAll('[data-card]')], empty = document.querySelector(`[data-empty="${CSS.escape(root.id)}"]`); const live = document.querySelector(`[data-live="${CSS.escape(root.id)}"]`), say = this.voice(live); const follow = e => { const link = e.target.closest('[data-card]')?.querySelector('.card-link'); if (!link || e.target.closest('a, button, input, select, textarea, label, summary, [role=button]') || !getSelection().isCollapsed) return; + if (link.dataset.pick !== undefined) { if (e.button === 0) link.click(); return; } if (e.button === 1 || e.ctrlKey || e.metaKey || e.shiftKey) window.open(link.href, '_blank', 'noopener'); else if (e.button === 0) link.click(); }; root.addEventListener('click', follow); root.addEventListener('auxclick', follow); + root.addEventListener('click', e => { + const b = e.target.closest('[data-pick]'); + if (!b || b.disabled) return; + const pressed = b.getAttribute('aria-pressed') !== 'true'; + b.setAttribute('aria-pressed', String(pressed)); + b.dispatchEvent(new CustomEvent('nullius-pick', {bubbles: true, detail: {value: b.dataset.pick, pressed}})); + }); const draw = () => { items.forEach(c => { c.hidden = !passes(c); }); root.querySelectorAll('[data-group]').forEach(g => { diff --git a/app/templates/_cards.html b/app/templates/_cards.html index aef7dc2c..f16e6378 100644 --- a/app/templates/_cards.html +++ b/app/templates/_cards.html @@ -8,7 +8,9 @@ An item is a dict of display values by field key: a string, a dict {text, class, title, label, href, value, attrs} or a list of them, empty for none; a filter and Sort by read a value's `value`, else its text; an action's `attrs` are its button's own attributes, its `class` added to the button's, and a `delete` takes the same shape (a quiet - trash button). Plus `href` (its title's link), + trash button). Plus `href` (its title's link), or `pick` {value, on, disabled}: the card is a toggle, its title a + button[data-pick][aria-pressed] that tables.js flips, sending `nullius-pick` {value, pressed} (a disabled one's + reason goes in its `alert`); a title's `title` is the tooltip of its link or button; `id`, `search` (what the search reads) and `keys`. A title's `after` is markup in the flow of its name, after its last word (a control of its own), and its `class` goes on the title row. A field's `class` goes on each of its values; `card_label` names a stat when its column header does not, and a @@ -83,7 +85,9 @@ {%- set title = by.title[0] %}{% set v = item.get(title.key) %}{% set v = v if v is mapping else {'text': v} %}{% set name = v.text %} {%- set cls = [title.class, v.class]|select|join(' ') %}
{% if icons %}{{ mark(icon, item.get(icon.key), name) }}{% endif -%} - {%- set link %}{{ name }}{% endset %} + {%- set pick = item.get('pick') %} + {%- set link %}{% if pick %} + {%- else %}{{ name }}{% endif %}{% endset %} {%- if v.after %}{{ link }} {{ v.after }}{% else %}{{ link }}{% endif %} {%- set corner %}{% call(f, v) each(item, by.check) %}{% endcall %} {%- call(f, v) each(item, by.delete) %}{% endcall %}{% endset %} @@ -98,7 +102,7 @@
{% for f in stats %}{% set v = item.get(f.key) %}{% set v = v if v is mapping else {'text': v} -%}
{{ v.label or f.card_label or f.label }}
{{ '—' if v.text is none or v.text == '' else v.text }}
{% endfor %}
{% endif %} {%- call(f, v) each(item, by.alert) %} -
{{ v.text }}
{% endcall %} +
{{ v.text }}
{% endcall %} {%- set actions %}{% call(f, v) each(item, by.action) %}{% if v.href %}{{ v.text }}{% else %}{% endif %}{% endcall %}{% endset %}{% if actions %}
{{ actions }}
{% endif %} {%- set foot %}{% call(f, v) each(item, by.status) %}{{ pill(f, v) }}{% endcall %} diff --git a/app/templates/_tables.html b/app/templates/_tables.html index 5bb42046..bb8d2209 100644 --- a/app/templates/_tables.html +++ b/app/templates/_tables.html @@ -74,7 +74,7 @@ page's other one, its other query kept, at the end; `default` the view a page shows without ?view=, the other one's link carrying ?view=. The search goes left, the buttons right. A column's `filter` says where its choices go, one form or a list of them: 'button' (the default), in the Filter menu; - 'dropdown', a menu of its own; 'pills', a row of toggles under the controls. A column's `filter_icon`, a sprite id + 'dropdown', a menu of its own; 'pills', a row of toggles under the controls, the first named by `all` (All). A column's `filter_icon`, a sprite id without its `i-`, leads its dropdown's button and its pills. `id`: the table's. #} {% macro controls(id, cols, search='', columns=false, choices={}, only={}, views=none, actions=none, sort=true, layout=none, default='table') %} {%- set by = {'button': [], 'dropdown': [], 'pills': []} %}{% for c in cols if c.key in choices %}{% for f in ([c.filter] if c.filter is string else c.filter or ['button']) %}{% set _ = by[f].append(c) %}{% endfor %}{% endfor %} @@ -120,7 +120,7 @@
{% endif %} {% for c in by.pills %}
- {{ icon(c) }} + {{ icon(c) }} {%- for value, text in choices[c.key] %}{% endfor %}
{% endfor %} diff --git a/app/ui_tests/test_cards.py b/app/ui_tests/test_cards.py index 69890aac..2775d130 100644 --- a/app/ui_tests/test_cards.py +++ b/app/ui_tests/test_cards.py @@ -125,6 +125,18 @@ def test_two_collections_with_every_form_keep_their_ids_apart(self): self.assertEqual(html.count('data-dropdown'), 2) + def test_a_pick_card_is_a_toggle_and_a_disabled_one_says_why(self): + cols = [dict(key='title', label='Name', card='title'), dict(key='why', label='Why not', card='alert')] + html = self.render(cols, [{'title': {'text': 'Alpha', 'title': 'What it is'}, 'pick': {'value': 'a'}}, + {'title': 'Beta', 'pick': {'value': 'b', 'on': True}}, + {'title': 'Gamma', 'pick': {'value': 'g', 'disabled': True}, 'why': {'text': 'Lacks tools', 'title': 'needs agent.tools/v1'}}]) + self.assertIn('', html) + self.assertIn('', html) + self.assertIn('' + '
Lacks tools
', html) + self.assertNotIn(' el('article', {'data-card': ''}, el('div', {}, el('button', {class: 'card-link', 'data-pick': v, 'aria-pressed': 'false'}, v)), el('span', {title: 't'}, 'pill')); +const root = el('div', {id: 'c', 'data-cards': ''}, card('a'), card('g')); +root.querySelectorAll('[data-pick]').forEach(b => { b.dispatchEvent = ev => b.dispatch(ev.type, {detail: ev.detail}); }); +root.querySelector('[data-pick="g"]').disabled = true; +document.appendChild(el('main', {}, el('div', {id: 'c-tools'}), root, el('p', {'data-empty': 'c', hidden: true}), el('span', {'data-live': 'c', 'data-noun': 'cards'}))); +global.getSelection = () => ({isCollapsed: true}); +T.cards(root); +root.addEventListener('nullius-pick', e => picks.push([e.detail.value, e.detail.pressed])); +const a = root.querySelector('[data-pick="a"]'), g = root.querySelector('[data-pick="g"]'); +a.click(); root.querySelectorAll('[title="t"]')[0].dispatch('click', {button: 0}); a.click(); g.click(); +process.stdout.write(JSON.stringify([picks, a.getAttribute('aria-pressed'), g.getAttribute('aria-pressed')])); +""") + self.assertEqual(got, [[["a", True], ["a", False], ["a", True]], "true", "false"]) # its pill too: the card is the toggle + if __name__ == "__main__": unittest.main() From 54c3d0ab46754187d83ce14887da38f7b7482184 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 18:23:51 +0100 Subject: [PATCH 03/20] refactor(builder): draw the Agent step's grid with the card toolkit The Agent step fetched nothing and drew its own tiles, search and chips from the builder payload. It now loads /new-run/agents for the recipe's benchmark: the Agents page's entries (recipe.entries) as pick cards, with Protocol and Fit pills and a Filter menu for From and Proves, a skeleton while loading and one line on failure. The grid is kept while the benchmark stays the same, so its filters survive each render, and its cards follow the recipe. Other benchmarks' agents are part of From. --- app/static/builder/command-builder-core.js | 2 +- app/static/builder/command-builder.css | 8 ++ app/static/builder/command-builder.js | 66 ++++++++-------- app/templates/builder/agents.html | 10 +++ app/templates/builder/new_run.html | 6 +- app/ui_tests/test_agents.py | 5 +- app/ui_tests/test_builder_agents.py | 91 ++++++++++++++++++++++ app/ui_tests/test_command_builder_state.py | 4 +- app/ui_tests/test_every_route.py | 6 +- app/web/builder.py | 71 +++++++++++++++++ docs/builder.md | 2 +- 11 files changed, 229 insertions(+), 42 deletions(-) create mode 100644 app/templates/builder/agents.html create mode 100644 app/ui_tests/test_builder_agents.py diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index a76008d3..25fe1b83 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -48,7 +48,7 @@ const own = ownAgentsOf(b).map(o => ({...o, why: why[o.key] || [], setup: true, own: true})); const setups = own.length ? [] : (b.setups || []).map(x => ({...x, protocol: 'its own code', version: '', offers: [], proven: {}, options: {}, setup: true, why: []})); const rest = b.family === 'agent' ? standard().agents.filter(o => o.benchmark !== b.key) : standard().capabilities; - return own.concat(rest.map(o => ({...o, why: why[o.key] || [], elsewhere: !!o.benchmark})), setups, controls); + return own.concat(rest.map(o => ({...o, why: why[o.key] || []})), setups, controls); } // the chosen ones of a benchmark's own agents, and what each choice runs as: its own agent, its Original setup const ownChosen = r => ownAgentsOf(standardBench(r)).filter(a => chosenOf(r).includes(a.key)); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 769c48da..5c6f33e2 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -291,3 +291,11 @@ #nullius-builder .std-isolation small:empty { display: none; } #nullius-builder .std-boat-sizes .nb-tile { min-height: 60px; } #nullius-builder .std-boat-sizes .nb-tile-note { font-size: 13px; line-height: 1.4; } +/* the Agent step's grid: the card toolkit's (catalogue.css), fitted to the form */ +#nullius-builder .nb-step { border-radius: 0; margin: 0; } /* catalogue.css has a .nb-step of its own */ +#nullius-builder #nb-agent-grid { margin-bottom: 20px; } +#nullius-builder #nb-agent-grid .bench-grid { grid-template-columns: repeat(auto-fill, minmax(min(100%, 230px), 1fr)); gap: 12px; } +#nullius-builder #nb-agent-grid .bench-card { padding: 16px; gap: 10px; } +#nullius-builder #nb-agent-grid .bench-card-title { font-size: 15px; } +#nullius-builder #nb-agent-grid .bench-card-foot { flex-wrap: wrap; } +#nullius-builder #nb-agent-grid .std-tag { white-space: nowrap; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 7d4fd188..bd4b987e 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -93,14 +93,6 @@ const schema = JSON.parse(document.getElementById('nb-schema').textContent); const C = window.NulliusRecipe(schema), $ = s => root.querySelector(s); const esc = value => String(value ?? '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); - const agentMarks = JSON.parse(root.dataset.agentMarks || '{}'); - const agentMark = (key, label) => { - const source = agentMarks[String(key).split('@')[0]]; - if (!source) return esc(label[0]); - return source.includes('#') - ? `` - : ``; - }; const steps = C.STEPS, S = C.STEP, REVIEW = S.review; const query = Object.fromEntries(new URLSearchParams(location.search)); let recipe = C.prefill(query); @@ -646,7 +638,7 @@ return !t.length ? '' : t.every(n => n === t[0]) && t.length > 1 ? `${t[0]} tasks each` : `${t.reduce((a, n) => a + n, 0)} tasks`; }; const pickChip = (attr, v, label, on) => ``; // one tile: {attr, key, on, title, mark, cat, origin, line, meta, foot, find, fam, data (more attributes), tip (its title)} - const tile = t => ` diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 269baafa..683d1e1f 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -588,8 +588,9 @@ def test_the_builder_gets_the_skills_and_a_benchmark_needing_one_says_who_lacks_ [("decoys", False, "1/3"), ("chain", True, "3/3"), ("retry", False, "0/3")]) self.assertEqual(schema["standard"]["fits"]["fakebench"]["fake"], ["Fake agent has not shown the decoys skill of agent.tools/v1 (chooses the right tool among similar ones)"]) - js = (Path(web.app.static_folder) / "builder" / "command-builder.js").read_text() - self.assertIn('.filter(s => !s.shown).map(s => `decoys ✗', card[:card.index("")]) def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_here(self): from nullius.standard import boat diff --git a/app/ui_tests/test_builder_agents.py b/app/ui_tests/test_builder_agents.py new file mode 100644 index 00000000..3dc347ba --- /dev/null +++ b/app/ui_tests/test_builder_agents.py @@ -0,0 +1,91 @@ +"""The builder's Agent step grid (/new-run/agents): the card toolkit's pick cards for one benchmark, from the +entries the Agents page lists, greyed as the builder's payload says.""" +import json +import os +import re +import shutil +import tempfile +import unittest +from html import unescape +from unittest.mock import patch + +from .test_routes import web + + +class AgentGrid(unittest.TestCase): + def setUp(self): + tmp = tempfile.mkdtemp(prefix="nullius-grid-") + self.addCleanup(shutil.rmtree, tmp, True) + env = patch.dict(os.environ, {"NULLIUS_HOME": tmp, "HOSTS": "", "RUNS_DIR": os.path.join(tmp, "runs")}) + env.start(); self.addCleanup(env.stop) + os.environ.pop("NULLIUS_CATALOGUE", None) + self.client = web.app.test_client() + + def grid(self, ref): + r = self.client.get("/new-run/agents?benchmark=" + ref) + self.assertEqual(r.status_code, 200, ref) + html = r.get_data(as_text=True) + cards = {m.group(1): m.group(0) for m in re.finditer(r'
', html, re.S)} + return html, cards + + def schema(self): + html = self.client.get("/new-run").get_data(as_text=True) + return json.loads(re.search(r'', html, re.S).group(1))["standard"] + + def test_its_greying_is_the_builders_for_every_benchmark(self): + st = self.schema() + offered = {o["key"] for o in st["agents"] + st["capabilities"]} + for b in st["benchmarks"]: + with self.subTest(b["key"]): + _, cards = self.grid(b["key"]) + fits = {k: v for k, v in st["fits"][b["key"]].items() if k in offered} + self.assertEqual({k for k in cards if k != "original"}, set(fits)) + self.assertEqual({k for k, c in cards.items() if " disabled" in c[:c.index("")]}, {k for k, v in fits.items() if v}) + + def test_osworld1_offers_its_own_agents_and_codex_and_says_why_not_the_rest(self): + html, cards = self.grid("osworld1") + on = [k for k, c in cards.items() if " disabled" not in c[:c.index("")]] + self.assertEqual(len([k for k in on if k.startswith("osworld1:")]), 12) + self.assertEqual([k for k in on if not k.startswith("osworld1:")], ["codex@2.2.2"]) + self.assertTrue(list(cards)[0].startswith("osworld1:")) # its own first + self.assertNotIn("original", cards) # they are its Original + self.assertIn('", cards["original"]) + for k, c in cards.items(): + if k.startswith("osworld"): + self.assertIn(" disabled", c) + self.assertRegex(c, r">Belongs to OSWorld[12]") + + def test_a_capability_benchmark_lists_capabilities(self): + _, cards = self.grid("locomo") + self.assertIn("original", cards) + self.assertIn("mem0", cards) + self.assertFalse([k for k in cards if k.startswith("osworld") or k.startswith("codex")]) + + def test_an_unknown_benchmark_is_not_found_and_the_page_loads_the_toolkit(self): + self.assertEqual(self.client.get("/new-run/agents?benchmark=nope").status_code, 404) + page = self.client.get("/new-run").get_data(as_text=True) + self.assertIn("/static/tables.js", page) + self.assertIn("/static/catalogue/catalogue.css", page) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_command_builder_state.py b/app/ui_tests/test_command_builder_state.py index d0fc9aa9..3c696392 100644 --- a/app/ui_tests/test_command_builder_state.py +++ b/app/ui_tests/test_command_builder_state.py @@ -20,10 +20,10 @@ def test_a_benchmarks_own_agents_are_its_agents_and_greyed_elsewhere(self): out = self.js('const os=C.standardOthers(C.createStandard("osworld1")),g=C.standardOthers(C.createStandard("gsm8k"));' 'const cua=g.find(o=>o.key==="osworld1:openai-cua");' 'process.stdout.write(JSON.stringify([os.filter(o=>o.own).map(o=>o.key).slice(0,2),os.findIndex(o=>!o.own),os.filter(o=>o.own).length,' - 'os.some(o=>o.setup&&!o.own),cua.why,cua.elsewhere,os.find(o=>o.key==="osworld2:m3").why]));') + 'os.some(o=>o.setup&&!o.own),cua.why,cua.benchmark,os.find(o=>o.key==="osworld2:m3").why]));') self.assertEqual(out[0], ["osworld1:m3", "osworld1:prompt"]) self.assertEqual((out[1], out[2], out[3]), (12, 12, False)) # first, and no Original tile beside them - self.assertEqual((out[4], out[5]), (["OpenAI CUA belongs to OSWorld1"], True)) + self.assertEqual((out[4], out[5]), (["OpenAI CUA belongs to OSWorld1"], "osworld1")) # another benchmark's: the grid's From says so self.assertEqual(out[6], ["MiniMax M3 belongs to OSWorld2"]) tags = self.js('const os=C.standardOthers(C.createStandard("osworld1"));' 'process.stdout.write(JSON.stringify([...new Set(os.filter(o=>o.benchmark).map(o=>o.protocol))]));') diff --git a/app/ui_tests/test_every_route.py b/app/ui_tests/test_every_route.py index 200f581c..905adab8 100644 --- a/app/ui_tests/test_every_route.py +++ b/app/ui_tests/test_every_route.py @@ -60,9 +60,11 @@ # The Harbor source routes answer 400 without the source they are asked # about; asked with one, offline: refs with git unreachable (NO_GIT), the -# agents of a release pin from the cache only (a GET never builds). +# agents of a release pin from the cache only (a GET never builds); the +# builder's agent grid answers 404 without the benchmark it draws. HARBOR_QUERY = {"/new-run/harbor/refs": "repo=example/harbor", - "/new-run/harbor/agents": "kind=release&version=0.24.0"} + "/new-run/harbor/agents": "kind=release&version=0.24.0", + "/new-run/agents": "benchmark=gsm8k"} NO_GIT = "/nonexistent/nullius-route-test/git" diff --git a/app/web/builder.py b/app/web/builder.py index d3bb2d7e..79c9ad03 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -9,6 +9,7 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor from flask import abort, jsonify, render_template, request +from nullius.standard import ports from nullius import constraints, settings, execution, gitrefs, hosts, prerequisites, providers from nullius.harbor import source as harbor_source from nullius.catalogue import catalog @@ -18,6 +19,7 @@ from nullius.standard import recipe as standard_recipe from .core import app +from .filters import agent_mark_url # Harbor agents that call their vendor with a key of their own, beside the model's (read from Harbor's agents): each # entry's names are alternatives, any one of them will do. A recipe naming one whose key Settings lacks says so on Review. @@ -59,6 +61,75 @@ def new_run(): return render_template("builder/new_run.html", builder_data=data) +@app.route("/new-run/agents") +def new_run_agents(): + """The Agent step's grid for one benchmark of the standard (?benchmark=key or key@version), as a fragment the + builder puts in its page: its agents or capabilities as pick cards (templates/builder/agents.html), from the + entries the Agents page lists (``recipe.entries``), each greyed with the reasons it does not fit.""" + benches = definitions.load("benchmark", every=True)[0] + b = definitions.find(benches, request.args.get("benchmark", "")) + if b is None: + abort(404) + b = dict(b, label=definitions.title(b)) + kind = b["family"] + found = [e for e in standard_recipe.entries(kind, benches=[b], every=True, own=True)["entries"] if not e["d"].get("example")] + found.sort(key=lambda e: 0 if e["d"].get("benchmark") == b["key"] else 2 if e["d"].get("benchmark") else 1) # its own, the catalogue's, others' own + # a benchmark with agents of its own runs them as its Original setup; another offers that setup as one more card + setups = [] if any(e["d"].get("benchmark") == b["key"] for e in found) else [k for k in (b.get("setups") or {}) if k == "original"] + items = [_setup_card(b, k) for k in setups] + [_pick_card(b, e) for e in found] + names = dict(definitions.FROM, benchmark=f"{b['label']}'s own", elsewhere="Other benchmarks' agents") + choices = {"protocol": [(p, p[:1].upper() + p[1:]) for p in ("reference", "ACP", "a command", "its own script", "JSON lines") + if any(i["protocol"] and i["protocol"]["value"] == p for i in items)], + "fit": [(v, t) for v, t in (("fit", "Can run it"), ("unfit", "Cannot")) if any(i["fit"]["value"] == v for i in items)], + "from": [(k, v) for k, v in names.items() if any(i["from"]["value"] == k for i in items)], + "proves": [(p, s.get("label") or p) for p, s in ports.PORTS.items() if any(x["value"] == p for i in items for x in i["proves"])]} + # a filter only where its cards differ + return render_template("builder/agents.html", items=items, kind=kind, choices={k: v for k, v in choices.items() if len(v) > 1}) + + +def _setup_card(b: dict, key: str) -> dict: + """The benchmark as published, run by its own code: a pick card beside the agents or capabilities.""" + x = b["setups"][key] + title = x.get("reference_agent") or ("No agent · Original" if b["family"] == "agent" else "Original " + _kind_of(b)) + return {"pick": {"value": key}, "id": "pick-" + key, "icon": "#i-origin", "title": {"text": title, "title": x.get("summary", "")}, + "search": " ".join([title, b["label"], "reference"]), "published": "", "protocol": {"text": "reference", "value": "reference"}, + "fit": {"text": "Can run it", "value": "fit"}, "from": {"text": f"{b['label']}'s own", "value": "benchmark"}, + "proves": [], "needs": [], "ticks": [], "alert": None} + + +def _kind_of(b: dict) -> str: + kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) + return kinds[0] if kinds else "capability" + + +def _pick_card(b: dict, e: dict) -> dict: + """An agent's or a capability's pick card for the benchmark ``b``: its name and mark, its protocol, where it comes + from, the ports it has proven that ``b`` needs and a tick for each doubt; greyed, why (whole in its tooltip).""" + d, why = e["d"], e["fits"][0]["why"] + rows = {r["port"]: r for r in e["ports"] if r["state"] == "proven"} + needed = [rows[p] for p in b.get("needs") or [] if p in rows] + title = definitions.title(d) + version = str((d.get("upstream") or {}).get("version") or "") + title += f" {version}" if version and version not in title else "" + fid = (e["fidelity"] or {}).get("verdict") + ticks = ([{"text": "ACP ≠ CLI", "class": "no", "title": f"ACP differs from its own CLI: {fid}"}] if fid and fid != "same" else []) \ + + [{"text": f"{s['id']} ✗", "class": "no", "title": s["says"]} for r in needed for s in r["skills"] if not s["shown"]] \ + + ([{"text": "older kit", "class": "no", "title": "proven on an older kit: worth running again"}] if any(r["older_kit"] for r in needed) else []) + lacks = [p.split(".", 1)[-1].rsplit("/v", 1)[0].replace("-", " ").replace("_", " ") for p in b.get("needs") or [] if p not in rows] + note = ("Belongs to " + d["benchmark_label"]) if d.get("benchmark") else "Lacks " + (", ".join(lacks) or "a port it needs") + origin = d.get("from") or "bundled" + there = {"text": f"{d['benchmark_label']}'s own", "value": "benchmark" if d["benchmark"] == b["key"] else "elsewhere"} \ + if d.get("benchmark") else {"text": definitions.FROM[origin], "value": origin} + return {"pick": {"value": d["_ref"], "disabled": bool(why)}, "id": "pick-" + d["_ref"].replace(":", "-").replace("@", "-"), + "icon": agent_mark_url(d["key"]) if b["family"] == "agent" else "", "title": {"text": title, "title": d.get("summary", "")}, + "search": " ".join([title, d["key"], version, e["protocol"], d.get("summary", "")]), + "protocol": {"text": e["protocol"], "value": e["protocol"]}, "published": "", + "fit": {"text": "Cannot" if why else "Can run it", "value": "unfit" if why else "fit"}, "from": there, + "proves": [{"text": r["label"], "value": r["port"]} for r in rows.values()], + "needs": [{"text": r["label"], "title": r["summary"]} for r in needed], "ticks": ticks, + "alert": {"text": note, "title": "; ".join(why)} if why else None} + + @app.route("/recipes") def recipes_page(): """The saved recipes as a table, a column per step. The page checks each one as the builder checks a loaded diff --git a/docs/builder.md b/docs/builder.md index 34b648ca..37054127 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -25,7 +25,7 @@ Then [read its results](results.md). | Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`) | | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A benchmark option its datasets need (`step: dataset`) is asked here, for the datasets it applies to. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | -| Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. A benchmark with [agents of its own](standard.md#a-benchmarks-own-agents) (OSWorld's) lists them first, in place of its Original tile; several may be chosen, a job each, and a setting for one of them is in its tab. Other benchmarks' own agents are greyed with the benchmark they belong to; *Other benchmarks' agents* hides them. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | +| Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. A benchmark with [agents of its own](standard.md#a-benchmarks-own-agents) (OSWorld's) lists them first, in place of its Original tile; several may be chosen, a job each, and a setting for one of them is in its tab. The cards are the Agents page's, each greyed with why it cannot run; Protocol and Fit pills and the Filter menu (From, Proves) narrow them, and *From* hides other benchmarks' own agents. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, attempts per unit, retries per attempt, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | | Where | **Hosts**: this machine or Settings hosts. **Sandboxes**: Boat VMs, within the account’s sizes and capacity. Choose isolation where available; VM benchmarks keep their own VM provider | From 376d7d76f55db505b6aaeaaa5c76de229daf1d54 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 18:24:00 +0100 Subject: [PATCH 04/20] feat(builder): say a benchmark's Original runs as published, not as a protocol The Original card carried "reference" as its protocol, so the Protocol pills offered it. It now reads "as published" and is left out of them. --- app/ui_tests/test_builder_agents.py | 4 ++++ app/web/builder.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/ui_tests/test_builder_agents.py b/app/ui_tests/test_builder_agents.py index 3dc347ba..6a56638a 100644 --- a/app/ui_tests/test_builder_agents.py +++ b/app/ui_tests/test_builder_agents.py @@ -69,6 +69,10 @@ def test_a_benchmark_without_own_agents_keeps_its_original_and_greys_osworlds(se _, cards = self.grid("gsm8k") self.assertEqual(list(cards)[0], "original") self.assertIn(">No agent · Original", cards["original"]) + self.assertIn('as published', cards["original"]) # how it runs, not a protocol + self.assertIn('data-f-protocol=""', cards["original"]) + html, _ = self.grid("gsm8k") + self.assertNotIn('data-value="reference"', html) for k, c in cards.items(): if k.startswith("osworld"): self.assertIn(" disabled", c) diff --git a/app/web/builder.py b/app/web/builder.py index 79c9ad03..7aa9e45f 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -78,7 +78,7 @@ def new_run_agents(): setups = [] if any(e["d"].get("benchmark") == b["key"] for e in found) else [k for k in (b.get("setups") or {}) if k == "original"] items = [_setup_card(b, k) for k in setups] + [_pick_card(b, e) for e in found] names = dict(definitions.FROM, benchmark=f"{b['label']}'s own", elsewhere="Other benchmarks' agents") - choices = {"protocol": [(p, p[:1].upper() + p[1:]) for p in ("reference", "ACP", "a command", "its own script", "JSON lines") + choices = {"protocol": [(p, p[:1].upper() + p[1:]) for p in ("ACP", "a command", "its own script", "JSON lines") if any(i["protocol"] and i["protocol"]["value"] == p for i in items)], "fit": [(v, t) for v, t in (("fit", "Can run it"), ("unfit", "Cannot")) if any(i["fit"]["value"] == v for i in items)], "from": [(k, v) for k, v in names.items() if any(i["from"]["value"] == k for i in items)], @@ -92,7 +92,7 @@ def _setup_card(b: dict, key: str) -> dict: x = b["setups"][key] title = x.get("reference_agent") or ("No agent · Original" if b["family"] == "agent" else "Original " + _kind_of(b)) return {"pick": {"value": key}, "id": "pick-" + key, "icon": "#i-origin", "title": {"text": title, "title": x.get("summary", "")}, - "search": " ".join([title, b["label"], "reference"]), "published": "", "protocol": {"text": "reference", "value": "reference"}, + "search": " ".join([title, b["label"], "as published"]), "published": "as published", "protocol": "", "fit": {"text": "Can run it", "value": "fit"}, "from": {"text": f"{b['label']}'s own", "value": "benchmark"}, "proves": [], "needs": [], "ticks": [], "alert": None} From 048ac1352ca2823f4870ac4ff1bc79e34aa996cf Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 18:24:54 +0100 Subject: [PATCH 05/20] feat(web): show a benchmark's own agents when Runs on chooses that benchmark Runs on: OSWorld 1 showed Codex alone, since From hides a benchmark's own agents by default. Their cards now pass From whenever Runs on narrows to their benchmark (a card's unless in the card toolkit), so the 12 show beside Codex without changing From. --- app/static/tables.js | 8 +++++--- app/templates/_cards.html | 7 ++++--- app/ui_tests/test_agents.py | 3 +++ app/ui_tests/test_cards.py | 16 ++++++++++++++++ app/web/agents.py | 5 +++-- docs/standard.md | 2 +- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/app/static/tables.js b/app/static/tables.js index a23327d3..12e6adc7 100644 --- a/app/static/tables.js +++ b/app/static/tables.js @@ -60,7 +60,8 @@ window.NulliusTable = { (data-filter-menu) and rows of pills (data-pills). `allowed`: column -> the Set of values chosen, or null when all are; every list and row of a column shows that one state. A row passes when, for each column, one of - its values (data-f-, one per line) is chosen; a choice made at load + its values (data-f-, one per line) is chosen, or its data-unless- + names a column that narrows and passes it; a choice made at load filters from the start. `changed` runs after each change, a list's change event included. */ filters(scope, changed) { @@ -97,8 +98,9 @@ window.NulliusTable = { }); }); draw(); - const passes = row => Object.entries(allowed).every(([col, set]) => - !set || (row.getAttribute('data-f-' + col) ?? '').split('\n').some(v => set.has(v))); + // data-unless-="": a row the other column narrows to passes this one whatever it holds + const ok = (row, col) => !allowed[col] || (row.getAttribute('data-f-' + col) ?? '').split('\n').some(v => allowed[col].has(v)); + const passes = row => Object.keys(allowed).every(col => ok(row, col) || (by => !!by && !!allowed[by] && ok(row, by))(row.getAttribute('data-unless-' + col))); const clear = () => { Object.keys(allowed).forEach(c => { allowed[c] = null; }); draw(); }; return {allowed, passes, clear}; }, diff --git a/app/templates/_cards.html b/app/templates/_cards.html index f16e6378..dfa95795 100644 --- a/app/templates/_cards.html +++ b/app/templates/_cards.html @@ -11,7 +11,8 @@ trash button). Plus `href` (its title's link), or `pick` {value, on, disabled}: the card is a toggle, its title a button[data-pick][aria-pressed] that tables.js flips, sending `nullius-pick` {value, pressed} (a disabled one's reason goes in its `alert`); a title's `title` is the tooltip of its link or button; - `id`, `search` (what the search reads) and `keys`. A title's `after` is markup in the flow of its name, after its + `id`, `search` (what the search reads), `keys`, and `unless` {field: other}: the item passes the field's filter + whenever the other field's filter narrows and passes it. A title's `after` is markup in the flow of its name, after its last word (a control of its own), and its `class` goes on the title row. A field's `class` goes on each of its values; `card_label` names a stat when its column header does not, and a value's `label` names it for that item; `style` 'code' sets the text as code; `icon` puts #i- before a @@ -57,7 +58,7 @@ {%- set icons = icon and items|selectattr(icon.key)|list|length > 0 %} {%- set shown = [] %}{% for c in cols if c.table is not sameas false and (c.key in choices or c.type == 'num' or items|selectattr(c.key)|list) %}{% set _ = shown.append(dict(c, sticky=c.card == 'title')) %}{% endfor %} {%- set rs = [] %}{% for item in items %}{% set a %}{% if item.get('id') %} id="{{ item.id }}"{% endif %} data-search="{{ item.get('search', '') }}" - {%- for f in cols if f.key in choices %} data-f-{{ f.key }}="{% call(f, v) each(item, [f]) %}{{ v.get('value', v.text) }} {% endcall %}"{% endfor %}{% endset %} + {%- for f in cols if f.key in choices %} data-f-{{ f.key }}="{% call(f, v) each(item, [f]) %}{{ v.get('value', v.text) }} {% endcall %}"{% endfor %}{% for k, by in (item.get('unless') or {}).items() %} data-unless-{{ k }}="{{ by }}"{% endfor %}{% endset %} {%- set _ = rs.append({'item': item, 'attrs': a}) %}{% endfor %} {% call(r, c) table(id, shown, rs, attrs, search=search, columns=true, choices=choices, only=only, empty=empty, noun=noun, sort=sort, layout='table', default=default) %} {%- set v = r.item.get(c.key) %}{% set by %}{% call(f, x) each(r.item, [c]) %}{{ x.get('value', x.text) }} {% endcall %}{% endset %} @@ -80,7 +81,7 @@
{%- for item in items %}
{%- set title = by.title[0] %}{% set v = item.get(title.key) %}{% set v = v if v is mapping else {'text': v} %}{% set name = v.text %} {%- set cls = [title.class, v.class]|select|join(' ') %} diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 683d1e1f..03d7e4a0 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -209,6 +209,9 @@ def test_runs_on_filters_by_the_benchmarks_each_agent_runs(self): self.assertEqual(on, sorted(["codex"] + ["osworld1-" + k for k in ("m3", "prompt", "claude", "gpt54", "openai-cua", "qwen", "os-symphony", "evocua", "autoglm", "autoglm-v", "mano", "muse-spark")])) self.assertNotIn("osworld1", dict(cards)["osworld2-m3"].split(" ")) # another benchmark's own runs it alone + own = html[html.index('id="osworld1-m3"'):] + self.assertIn('data-unless-from="runs_on"', own[:own.index(">")]) # Runs on its benchmark shows it, whatever From says + self.assertNotIn("data-unless-from", html[html.index('id="codex"'):][:200]) # the count the card shows and the filter read one record card = html[html.index('id="codex"'):] n = card[:card.index(">")].split('data-f-runs_on="', 1)[1].split('"', 1)[0].count(" ") diff --git a/app/ui_tests/test_cards.py b/app/ui_tests/test_cards.py index 2775d130..ce3bdf39 100644 --- a/app/ui_tests/test_cards.py +++ b/app/ui_tests/test_cards.py @@ -235,6 +235,22 @@ def test_a_pick_flips_and_says_so_and_a_disabled_one_stays(self): """) self.assertEqual(got, [[["a", True], ["a", False], ["a", True]], "true", "false"]) # its pill too: the card is the toggle + def test_a_card_passes_a_filter_its_unless_column_narrows_to(self): + got = node(r""" +const card = (id, from, runs, unless) => el('article', Object.assign({'data-card': '', id, 'data-f-from': from + '\n', 'data-f-runs': runs + '\n'}, unless ? {'data-unless-from': 'runs'} : {}), id); +const menu = (col, values, on) => el('fieldset', {'data-filter-menu': col}, ...values.map(v => el('input', Object.assign({type: 'checkbox', 'data-value': v}, on.includes(v) ? {checked: true} : {})))); +const tools = el('div', {id: 'c-tools'}, menu('from', ['bundled', 'benchmark'], ['bundled']), menu('runs', ['os', 'gsm'], ['os', 'gsm'])); +tools.querySelectorAll('input').forEach(i => { i.checked = i.hasAttribute('checked'); }); +const root = el('div', {id: 'c', 'data-cards': ''}, card('codex', 'bundled', 'os'), card('m3', 'benchmark', 'os', true), card('other', 'benchmark', 'gsm', true)); +document.appendChild(el('main', {}, tools, root, el('p', {'data-empty': 'c', hidden: true}), el('span', {'data-live': 'c', 'data-noun': 'cards'}))); +T.cards(root); +const shown = () => root.querySelectorAll('[data-card]').filter(c => !c.hidden).map(c => c.id).join(' '); +const out = [shown()]; +tools.querySelector('[data-value="gsm"]').click(); out.push(shown()); +process.stdout.write(JSON.stringify(out)); +""") + self.assertEqual(got, ["codex", "codex m3"]) # Runs on everything narrows nothing: From's default holds + if __name__ == "__main__": unittest.main() diff --git a/app/web/agents.py b/app/web/agents.py index 30230c9c..e8cc6951 100644 --- a/app/web/agents.py +++ b/app/web/agents.py @@ -64,7 +64,8 @@ def _choices(kind: str, entries: list) -> dict: "from": [(k, v) for k, v in definitions.FROM.items() if k in have], "runs_on": runs_on} -# The From filter's default: the catalogue's own agents; a benchmark's own are one click away. +# The From filter's default: the catalogue's own agents; a benchmark's own are one click away, and shown whenever +# Runs on chooses their benchmark (their card's `unless`). ONLY = {"agent": {"from": ["bundled", "yours"]}} @@ -85,7 +86,7 @@ def _card(kind: str, e: dict) -> dict: "protocol": e["protocol"], "icon": agent_mark_url(d["key"]), "title": d["label"], "summary": d.get("summary"), "from": {"text": definitions.FROM["benchmark"], "value": "benchmark"}, "benchmark": d["benchmark_label"], "uses": "Computer use" if computer else None, "models": ", ".join(d.get("models") or []), - "runs_on": _runs_on(e), "hide": ["ports", "runs", "installs"]} + "runs_on": _runs_on(e), "unless": {"from": "runs_on"}, "hide": ["ports", "runs", "installs"]} version, n = (d.get("upstream") or {}).get("version"), len(d.get("_versions") or ()) return {"href": url_for("agent_page" if kind == "agent" else "feature_page", key=d["key"]), "id": d["key"], "search": " ".join([d["label"], d["key"], d.get("kind", ""), e["protocol"], d.get("summary", "")]), diff --git a/docs/standard.md b/docs/standard.md index 2be346cd..e552f315 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -658,7 +658,7 @@ it is published with. It speaks no port, so it has no `offers` or `proven`. | Derived, never written | Where it comes from (`from`: `benchmark`; a catalogue agent's is `bundled` or `yours`) and `only_for`, its benchmark. A written `only_for` names that benchmark alone | | `options.agent` | `standard.json` writes its `label`, `summary` and `setups`; the loader rebuilds `values` (each entry's `start.driver.agent`, in order), `names`, `about` and `default` (the first), so `when`, `by`, `--bench-option agent=...` and `NULLIUS_OPTION_AGENT` read as before | | Checked | Each entry's `models` are among `setups.original.models`; each upstream name once | -| Offered | On the Agents page behind its From filter (its Runs on filter lists each benchmark's agents, as the builder enables them), and in the builder beside the catalogue's agents; another benchmark greys it with the reason | +| Offered | On the Agents page behind its From filter, and shown whenever Runs on chooses its benchmark (Runs on lists each benchmark's agents, as the builder enables them), and in the builder beside the catalogue's agents; another benchmark greys it with the reason | | Run | As the Original setup, its upstream name as the agent option: `nullius standard run osworld1 original --bench-option agent=openai_cua`. Its jobs name it by its key (`openai-cua`); an older job, recorded as `original`, is read the same way from its agent option | ### Every setting, inventoried From cde611e8aa3fc426b0c8aa952b75fba34ff100fc Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 21:31:06 +0100 Subject: [PATCH 06/20] fix(web): give each piece of an agent card one slot, on the builder and the Agents page Every agent and capability card now reads the same whatever it comes from and whether it can be chosen: the name and its version on the title row, the reason it cannot be chosen under it, the ports it has proven (Computer use) and what it is as badges, and the foot on the card's last line with its protocol on the left and where it comes from on the right (Bundled, Yours, or the benchmark it belongs to). The protocol is in sentence case where it is made (recipe.protocol), so a card's pill reads as its filter does. The foot no longer wraps in the builder, which put the protocol on a line of its own on some cards. --- app/nullius/standard/recipe.py | 6 +++--- app/static/builder/command-builder.css | 1 - app/static/catalogue/catalogue.css | 4 +++- app/templates/agents/standard.html | 13 +++++++------ app/tests/test_benchmark_agents.py | 4 ++-- app/ui_tests/test_agents.py | 10 ++++++---- app/ui_tests/test_builder_agents.py | 6 ++++-- app/ui_tests/test_command_builder_state.py | 2 +- app/web/agents.py | 6 +++--- app/web/builder.py | 8 ++++---- 10 files changed, 33 insertions(+), 27 deletions(-) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index afaaa9bd..edb2b133 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -171,13 +171,13 @@ def _bench_page(b: dict) -> dict: def protocol(o: dict) -> str: - """How Nullius talks to it, in a word or two for a page.""" + """How Nullius talks to it, in a word or two for a page, in sentence case as its filter names it.""" if o["_kind"] == "control": - return "no agent" + return "No agent" if o["_kind"] == "capability": return "JSON lines" st = o.get("start") or {} - return "ACP" if "acp" in st else "its own script" if "driver" in st else "a command" + return "ACP" if "acp" in st else "Its own script" if "driver" in st else "A command" def skills_of(port: str, record: Optional[dict]) -> List[dict]: diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 5c6f33e2..590a7dfe 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -297,5 +297,4 @@ #nullius-builder #nb-agent-grid .bench-grid { grid-template-columns: repeat(auto-fill, minmax(min(100%, 230px), 1fr)); gap: 12px; } #nullius-builder #nb-agent-grid .bench-card { padding: 16px; gap: 10px; } #nullius-builder #nb-agent-grid .bench-card-title { font-size: 15px; } -#nullius-builder #nb-agent-grid .bench-card-foot { flex-wrap: wrap; } #nullius-builder #nb-agent-grid .std-tag { white-space: nowrap; } diff --git a/app/static/catalogue/catalogue.css b/app/static/catalogue/catalogue.css index 81aa9f8c..04ca3b5b 100644 --- a/app/static/catalogue/catalogue.css +++ b/app/static/catalogue/catalogue.css @@ -32,7 +32,9 @@ button.card-link:disabled { cursor: not-allowed; } .bench-card:has(> .bench-card-title [data-pick]:disabled):hover { border-color: var(--border-soft); } .bench-card :is(input, button, .btn, [title]) { position: relative; z-index: 1; } .bench-card [title]:not(input, button, .btn) { cursor: pointer; } /* raised for its tooltip, it still opens the card */ -.bench-card-foot .bench-harness { margin-left: auto; } +.bench-card-foot .bench-harness { margin-left: auto; text-align: right; } +/* the foot on a card's last line, wherever its badges or stats would put it */ +.bench-card-foot:not(.bench-card-runs ~ *, .bench-stats ~ *) { margin-top: auto; } .bench-card-actions { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 8px; } .bench-shelf.st-community .bench-card { background: transparent; border-style: dashed; border-color: var(--border); } .bench-shelf.st-community .bench-card:hover { border-color: var(--accent); } diff --git a/app/templates/agents/standard.html b/app/templates/agents/standard.html index 04550ecb..64aca41f 100644 --- a/app/templates/agents/standard.html +++ b/app/templates/agents/standard.html @@ -34,15 +34,16 @@ {% if problems %}
Not offered: a definition with problems
    {% for p in problems %}
  • {{ p }}
  • {% endfor %}
{% endif %} {% if not items %}

No {{ kind }} yet.

{% endif %} - {#- A card's fields (web/agents.py `_card`); an agent's protocol, the ports it proves and where it comes from as filters. #} + {#- A card's fields (web/agents.py `_card`), in the slots of the builder's Agent step (builder/agents.html): the name and + its version, what it is and proves as badges, its protocol and where it comes from at the foot; an agent's protocol, + the ports it proves and where it comes from as filters. #} {% set cols = [dict(key='icon', label='Logo', card='icon', table=false, class='std-mark small'), dict(key='title', label='Name', card='title'), - dict(key='kind', label='Kind', card='tag', class='cat-tag'), dict(key='benchmark', label='Benchmark', card='tag', class='cat-tag'), - dict(key='protocol', label='Protocol', card='tag', class='std-tag'), dict(key='uses', label='Uses', card='tag', class='std-tag', off=true), - dict(key='from', label='From'), dict(key='runs_on', label='Runs on', class='cat-tag', off=true), dict(key='models', label='Models', card='note', width='expand'), - dict(key='proves', label='Proves', class='std-tag', off=true), dict(key='example', label='Example', card='tag', class='std-tag', off=true), + dict(key='kind', label='Kind', card='badge', class='cat-tag'), dict(key='protocol', label='Protocol', card='tag', class='std-tag'), + dict(key='uses', label='Uses', card='badge', class='std-tag', off=true), dict(key='from', label='From', card='meta'), dict(key='runs_on', label='Runs on', class='cat-tag', off=true), dict(key='models', label='Models', card='note', width='expand'), + dict(key='proves', label='Proves', class='std-tag', off=true), dict(key='example', label='Example', card='badge', class='std-tag', off=true), dict(key='summary', label='Summary', card='text', class='std-clamp', width='expand', off=true), dict(key='ports', label='Ports proven', card='stat', type='num'), dict(key='runs', label='Benchmarks it runs', card='stat', type='num'), dict(key='installs', label='Installs', card='stat', class='word'), - dict(key='version', label='Version', card='meta')] %} + dict(key='version', label='Version')] %} {% if items %}{{ cards(plural|lower, cols, items, search='Name, protocol, summary', choices=choices, only=only, sort=['title', 'ports', 'runs'], empty='No ' ~ kind ~ ' matches.', noun=plural|lower, layout='table' if request.args.get('view') == 'table' else 'cards', default='cards') }}{% endif %}
{% endblock %} diff --git a/app/tests/test_benchmark_agents.py b/app/tests/test_benchmark_agents.py index 68811bc4..ee8afcf6 100644 --- a/app/tests/test_benchmark_agents.py +++ b/app/tests/test_benchmark_agents.py @@ -130,7 +130,7 @@ def test_one_entry_list_whose_fits_are_the_builders(self): self.assertEqual({f["ref"] for f in got.values()}, {ref}) page = {e["d"]["_ref"]: e for e in recipe.entries("agent", own=True)["entries"]} self.assertEqual([f["why"] for f in page["osworld1:openai-cua"]["fits"] if f["key"] == "osworld1"], [[]]) - self.assertEqual(page["osworld1:openai-cua"]["protocol"], "its own script") + self.assertEqual(page["osworld1:openai-cua"]["protocol"], "Its own script") @@ -147,7 +147,7 @@ def test_the_builder_offers_them_as_agents(self): got = {a["key"]: a for a in recipe.choices()["agents"]} cua = got["osworld1:openai-cua"] self.assertEqual((cua["benchmark"], cua["agent"], cua["models"], cua["protocol"]), - ("osworld1", "openai_cua", ["computer-use-preview"], "its own script")) + ("osworld1", "openai_cua", ["computer-use-preview"], "Its own script")) def test_an_own_agent_compiles_to_the_command_of_the_agent_option(self): old = self.compiled(others=["original"], bench_options={"agent": "openai_cua", "max_steps": "20"}) diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 03d7e4a0..0be840b6 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -83,7 +83,7 @@ def test_bundled_agents_use_shared_marks(self): self.assertIn('/static/brand/marks.svg#openai', html) self.assertIn('/static/brand/marks.svg#harbor', html) # Beside the logos, an agent without one shows its initials; where none has a logo, no card has an icon. - self.assertIn('
', html) + self.assertIn('Example agent 1', html) fixture_html = self.get("/agents").get_data(as_text=True) self.assertIn('', fixture_html) self.assertNotIn('std-mark', fixture_html) @@ -190,9 +190,10 @@ def test_a_benchmarks_own_agents_are_listed_behind_the_from_filter(self): card = card[:card.index("
")] self.assertIn('data-f-from="benchmark "', card) self.assertIn('href="/benchmarks/osworld1"', card) - self.assertIn('OSWorld1', card) self.assertIn('
computer-use-preview
', card) - self.assertIn("Computer use", card) + self.assertIn('
Computer use
', card) + self.assertIn('
Its own script' + 'OSWorld1
', card) # the builder's foot: its protocol, where it comes from self.assertNotIn("Ports proven", card) self.assertIn('id="osworld2-muse-spark"', table) self.assertEqual(self.get("/agents/openai-cua").status_code, 404) # its benchmark's page is its page @@ -383,7 +384,8 @@ def test_an_agent_with_versions_is_one_card_and_a_page_per_version(self): self.versioned() html = self.get("/agents").get_data(as_text=True) self.assertEqual(html.count('id="fake"'), 1) - self.assertIn("v1.0 · 2 versions", html) + self.assertIn('Fake agent 1.0', html) # its default version beside its name + self.assertIn("v1.0 · 2 versions", self.get("/agents?view=table").get_data(as_text=True)) html = self.get("/agents/fake").get_data(as_text=True) # the default self.assertIn('