`
+ `${t.mark ? `${t.mark} ` : ''}${esc(t.title)} ${t.on ? '✓' : ''} `
+ (t.cat || t.origin || t.fam ? `${t.cat ? `${esc(t.cat)} ` : ''}${t.origin === 'community' ? '' : ''}${t.famLabel ? `${esc(t.famLabel)} ` : ''} ` : '')
+ (t.line ? `${esc(t.line)} ` : '') + (t.note ? `${esc(t.note)} ` : '')
@@ -661,8 +653,6 @@
+ ` `
+ (cats.length > 1 ? `${pickChip('pick-cat', '', 'All', !pick.cat)}${cats.map(c => pickChip('pick-cat', c, c, pick.cat === c)).join('')}
` : '')
+ (fams ? `${pickChip('pick-fam', '', 'Any', !pick.fam)}${fams.map(([v, l]) => pickChip('pick-fam', v, l, pick.fam === v)).join('')}
` : '')
- // other benchmarks' own agents, greyed: shown until the viewer hides them
- + (tiles.some(t => t.elsewhere) ? `${pickChip('pick-elsewhere', '', "Other benchmarks' agents", !pick.hide)}
` : '')
+ `${tiles.map(tile).join('')}
No ${noun} matches.
`;
};
function applyPick() {
@@ -671,14 +661,61 @@
let shown = 0;
box.querySelectorAll('.nb-tile').forEach(t => {
const on = t.getAttribute('aria-pressed') === 'true';
- const fits = (!pick.cat || t.dataset.cat === pick.cat) && (!pick.fam || t.dataset.fam === pick.fam) && !(pick.hide && t.dataset.elsewhere) && words.every(w => t.dataset.find.includes(w));
+ const fits = (!pick.cat || t.dataset.cat === pick.cat) && (!pick.fam || t.dataset.fam === pick.fam) && words.every(w => t.dataset.find.includes(w));
t.hidden = !fits && !on; if (!t.hidden) shown++;
});
box.querySelectorAll('[data-pick-cat]').forEach(b => b.setAttribute('aria-pressed', b.dataset.pickCat === pick.cat));
box.querySelectorAll('[data-pick-fam]').forEach(b => b.setAttribute('aria-pressed', b.dataset.pickFam === pick.fam));
- box.querySelectorAll('[data-pick-elsewhere]').forEach(b => b.setAttribute('aria-pressed', !pick.hide));
box.querySelector('.nb-pick-none').hidden = shown > 0;
}
+ // The standard's Agent step grid: drawn by the server for the benchmark (web/builder.py, /new-run/agents) from the
+ // Agents page's entries, run by the card toolkit (tables.js); kept while the benchmark is the same, so a search or a
+ // filter survives each render, and its cards pressed as the recipe chooses (nullius-pick changes the recipe).
+ // Its view, cards or a table, is the viewer's: the toggle switches it in place, never leaving the page.
+ let agentView = 'cards';
+ try { agentView = localStorage.getItem('nb-agent-view') === 'table' ? 'table' : 'cards'; } catch {}
+ const grid = {ref: '', node: null};
+ function placeAgentGrid() {
+ const slot = $('#nb-agent-grid'), rec = r();
+ if (!slot) return;
+ if (grid.ref !== rec.benchmark + ' ' + agentView) { grid.ref = rec.benchmark + ' ' + agentView; grid.node = null; loadAgentGrid(grid.ref); }
+ if (!grid.node) { slot.setAttribute('aria-busy', 'true'); slot.innerHTML = ''; return; }
+ slot.replaceWith(grid.node);
+ const chosen = C.chosenOf(rec);
+ grid.node.querySelectorAll('[data-pick]').forEach(b => { // a card's toggle, or a row's box
+ const on = !b.disabled && chosen.includes(b.dataset.pick);
+ if (b.type === 'checkbox') b.checked = on; else b.setAttribute('aria-pressed', String(on));
+ });
+ // a choice made elsewhere (a recipe loaded) that the filters hide: the toolkit draws again, and it shows
+ if (grid.node.dataset.wired && [...grid.node.querySelectorAll('[data-pick]')].some(b => b.matches('[aria-pressed="true"], :checked') && b.closest('[data-card], tr').hidden))
+ grid.node.querySelector('[data-search]')?.dispatchEvent(new Event('input'));
+ if (!grid.node.dataset.wired) {
+ grid.node.dataset.wired = '1';
+ const cards = grid.node.querySelector('[data-cards]'), table = grid.node.querySelector('table[data-table]');
+ if (cards) NulliusTable.cards(cards);
+ if (table) { NulliusTable.sticky(table); NulliusTable.wire(table); }
+ grid.node.querySelectorAll('.seg-toggle a').forEach(a => a.addEventListener('click', e => {
+ e.preventDefault();
+ agentView = new URL(a.href).searchParams.get('view') === 'table' ? 'table' : 'cards';
+ try { localStorage.setItem('nb-agent-view', agentView); } catch {}
+ placeAgentGrid();
+ }));
+ }
+ }
+ async function loadAgentGrid(ref) {
+ const [benchmark, view] = ref.split(' ');
+ const res = await nulliusFetch('/new-run/agents?' + new URLSearchParams({benchmark, ...(view === 'table' ? {view} : {})})).catch(() => null);
+ const html = res && res.ok ? await res.text().catch(() => null) : null;
+ if (grid.ref !== ref) return; // another benchmark was chosen while it loaded
+ if (html === null) { // said once; the next render asks again
+ grid.ref = '';
+ const slot = $('#nb-agent-grid');
+ if (slot) { slot.removeAttribute('aria-busy'); slot.innerHTML = 'Could not load the list. Change step and come back to try again.
'; }
+ return;
+ }
+ grid.node = Object.assign(document.createElement('div'), {id: 'nb-agent-grid', innerHTML: html});
+ placeAgentGrid();
+ }
function modelChoiceCards(rec) {
const hasPaper = C.paperOffered(rec), on = rec.model_choice || 'any';
return `${tile({attr:'model-choice',key:'paper',title:'Like the paper',on:on==='paper',disabled:!hasPaper,why:'Paper models unavailable',note:hasPaper?'':'Paper models unavailable'})}`
@@ -700,18 +737,6 @@
const agent = b.family === 'agent', all = C.standardOthers(rec), others = all.filter(o => !o.control && (!o.setup || o.own));
const setups = all.filter(o => o.setup && !o.own), controls = all.filter(o => o.control);
const ds = b.datasets.find(x => C.datasetsOf(rec).includes(x.id)) || b.datasets[0];
- // an agent or a capability as a tile: its name, a skill it failed or a fidelity that differs as a tick; one that has not
- // proved a port the benchmark needs greyed, the ports it lacks named (the reasons whole in its title)
- const tileOf = o => {
- const ok = !o.why.length, lacks = b.needs.filter(p => !o.proven[p]);
- const skills = b.needs.flatMap(p => o.proven[p] ? o.proven[p].skills || [] : []).filter(s => !s.shown).map(s => `${esc(s.id)} ✗ `).join('');
- const fid = o.fidelity && o.fidelity.verdict !== 'same' ? `ACP ${o.fidelity.verdict === 'same' ? '= CLI ✓' : '≠ CLI'} ` : '';
- const older = b.needs.some(p => o.proven[p] && o.proven[p].older) ? 'older kit ' : '';
- return {attr: 'std-other', key: o.key, on: ok && C.chosenOf(rec).includes(o.key), disabled: !ok, why: o.why.join('; '), title: o.label + (o.version && !o.label.includes(o.version) ? ' ' + o.version : ''),
- mark: agent ? agentMark(o.key, o.label) : esc(o.label[0]), cat: o.protocol, fam: ok ? 'fit' : 'unfit', tip: o.summary, elsewhere: o.elsewhere,
- note: ok ? '' : o.benchmark ? 'Belongs to ' + o.benchmark_label : 'Lacks ' + (lacks.map(p => p.replace(/^[^.]+\./, '').replace(/\/v\d+$/, '').replace(/[-_]/g, ' ')).join(', ') || 'a port it needs'),
- foot: fid + skills + older, find: [o.label, o.key, o.version, o.protocol, o.summary].join(' ')};
- };
// each chosen agent's options, at its vendor's default unless changed: {ref: {name: word}}
const chosen = others.filter(o => C.chosenOf(rec).includes(o.key) && !o.why.length), mine = chosen.filter(o => !o.own);
const optionsOf = o => { const mine = (rec.options || {})[o.key] || {}, opts = Object.entries(o.options || {});
@@ -723,9 +748,6 @@
}).join('') + '
'; };
// the chosen ones' settings, one tab each: the benchmark's own way, the benchmark's settings every agent takes, each one's own
const yoursWord = agent ? 'agent' : (b.kind || 'capability');
- // the benchmark's own way of running it (its reference, as published) is one more choice beside yours
- const setupTile = o => ({attr: 'std-other', key: o.key, on: C.chosenOf(rec).includes(o.key), mark: o.key === 'original' ? '◆' : '◇',
- title: o.reference_agent || (agent ? 'No agent · Original' : 'Original ' + (b.kind || 'capability')), cat: 'reference', fam: 'fit', tip: o.summary, find: [b.label, o.label, 'reference'].join(' ')});
const ds2 = ds;
const controlTile = o => { const c = ((ds2 || {}).controls || []).find(x => x.key === o.key) || {...o, from: o.summary};
return tile({attr: 'std-other', key: o.key, on: C.chosenOf(rec).includes(o.key), mark: o.key === 'nop' ? '∅' : '≡', title: o.label,
@@ -733,7 +755,7 @@
const setupsNamed = ls => ls.filter(l => !ls.some(m => m !== l && l.startsWith(m))).join(' and ');
const onSetups = setups.filter(o => C.chosenOf(rec).includes(o.key)).concat(chosen.some(o => o.own) ? [{label: 'Original'}] : []);
return `
Who runs it? `
- + picker([...setups.map(setupTile), ...others.map(tileOf)], [['fit', 'Can run it'], ['unfit', 'Cannot']], agent ? 'agent' : 'capability')
+ + '
'
+ (others.length || setups.length ? '' : note('None yet', 'Bring one in with the /import-standard skill, or by hand: docs/standard.md.'))
+ settingsTabs([
...(chosen.length || onSetups.length ? [['shared', 'Every ' + yoursWord, benchOptions(rec, b, 'shared')]] : []),
@@ -939,7 +961,7 @@
const rows=own?[...((runnerOut&&runnerOut.review)||[]),...(C.isStandard(rec)?[]:[[S.where,'Where',C.whereLabel(rec)]])]:C.review(rec);
html='
Review the experiment. '+rows.map(([i,t,v])=>`${t} ${esc(v)}
${button('Edit',`data-step="${i}"`)}
`).join('')+'
'+(schema.local?'':'')+runPanel();
}
- $('#nb-form').innerHTML=html+'
';applyPick();
+ $('#nb-form').innerHTML=html+'
';applyPick();placeAgentGrid();
if(step===REVIEW&&schema.local){drawPrereq();checkPrereq();}
$('#nb-back').hidden=step===S.benchmark;
$('#nb-next').textContent=step===REVIEW?'Export recipe JSON':stepName(steps[nextShown(step,1)],nextShown(step,1))+' →';
@@ -1097,7 +1119,6 @@
if(d.copyCode!==undefined){const t=b.closest('.std-code').querySelector('code').textContent;try{await navigator.clipboard.writeText(t);b.textContent='Copied';}catch{b.textContent='Select it';}setTimeout(()=>{b.textContent='Copy';},1500);return;}
if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;}
if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;}
- if(d.pickElsewhere!==undefined){pickOf().hide=!pickOf().hide;applyPick();return;}
if(d.agent){const on=r().agents.includes(d.agent);r().agents=on?r().agents.filter(id=>id!==d.agent):[...r().agents,d.agent];renderForm();refreshPlan();return;}
else if(d.href){location.href=d.href;return;}
else if(b.id==='nb-add-custom-agent'){r().customAgents.push('');r().agents.push('custom-'+(r().customAgents.length-1));renderForm();refreshPlan();root.querySelector('[data-custom-agent="'+(r().customAgents.length-1)+'"]')?.focus();return;}
@@ -1110,8 +1131,8 @@
else if(d.runner){toRunner(d.runner,C.benchKey(r()));}
else if(d.bench){toRunner(C.runnerFor(r(),d.bench),d.bench);}
else if(d.stdBench){C.setStandardBenchmark(r(),d.stdBench);}
+ else if(d.stdOther){const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(d.stdOther)?c.filter(k=>k!==d.stdOther):[...c,d.stdOther];delete rec.other;if(rec.options)delete rec.options[d.stdOther];} // a check
else if(d.stdDataset){const rec=r(),c=C.datasetsOf(rec);rec.datasets=c.includes(d.stdDataset)?(c.length>1?c.filter(k=>k!==d.stdDataset):c):[...c,d.stdDataset];delete rec.dataset;}
- else if(d.stdOther){const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(d.stdOther)?c.filter(k=>k!==d.stdOther):[...c,d.stdOther];delete rec.other;if(rec.options)delete rec.options[d.stdOther];}
// the standard's machines: each a toggle, one at least; several split the run into shards
else if(d.stdWhere){const rec=r(),list=C.whereList(rec),on=list.includes(d.stdWhere),next=on?list.filter(w=>w!==d.stdWhere):[...list,d.stdWhere];if(!next.length)return;rec.where=next.length>1?next:next[0];}
// hosts or Boat sandboxes: a sandbox run starts from this machine alone
@@ -1149,6 +1170,8 @@
else return;
render(d.step!==undefined||['nb-next','nb-back'].includes(b.id));
});
+ // an agent, a capability or the setup picked or let go on the Agent step's grid; the focus stays on its card
+ root.addEventListener('nullius-pick',event=>{const k=event.detail.value,rec=r(),c=C.chosenOf(rec);rec.others=c.includes(k)?c.filter(x=>x!==k):[...c,k];delete rec.other;if(rec.options)delete rec.options[k];render();event.target.focus();});
root.addEventListener('input',event=>{
const el=event.target,d=el.dataset,rec=r();
if(el.id==='nb-recipe-title'){updateSavePreview();return;}
diff --git a/app/static/catalogue/catalogue.css b/app/static/catalogue/catalogue.css
index 7b61581d..c758a460 100644
--- a/app/static/catalogue/catalogue.css
+++ b/app/static/catalogue/catalogue.css
@@ -24,9 +24,26 @@
.card-link::after { content: ""; position: absolute; inset: -1px; border-radius: 12px; }
.card-link:focus-visible { outline: none; }
.card-link:focus-visible::after { box-shadow: 0 0 0 3px var(--accent-soft), 0 0 0 1px var(--accent); }
+/* a pick card (_cards.html `pick`): its title a toggle; chosen, it is marked; disabled, its reason says why */
+button.card-link { font: inherit; color: inherit; background: none; border: 0; padding: 0; margin: 0; text-align: left; cursor: pointer; }
+button.card-link:disabled { cursor: not-allowed; }
+/* a pick row: chosen, tinted to its pinned name; disabled, faint (not translucent: its pinned name covers what scrolls under) */
+.table-wrap .grid tbody tr:has([data-pick]:checked) > td { background: var(--accent-soft); }
+.table-wrap .grid tbody tr:has([data-pick]:disabled) > td { color: var(--text-faint); }
+tr:has([data-pick]:disabled) .row-name > :first-child { opacity: .55; }
+tr:has([data-pick]:not(:disabled)) { cursor: pointer; }
+.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) { border-color: var(--accent); background: var(--accent-soft); }
+.pick-check { flex: none; width: 20px; height: 20px; border-radius: 50%; display: grid; place-items: center; background: var(--accent); color: var(--btn-fg); visibility: hidden; }
+.pick-check .ic { width: 12px; height: 12px; stroke-width: 3; }
+.bench-card:has([data-pick][aria-pressed="true"]) .pick-check { visibility: visible; }
+.bench-card:has(> .bench-card-title [data-pick]:disabled) { opacity: .55; }
+.bench-card:has(> .bench-card-title [data-pick]:disabled):hover { border-color: var(--border-soft); }
.bench-card :is(input, button, .btn, [title]) { position: relative; z-index: 1; }
.bench-card [title]:not(input, button, .btn) { cursor: pointer; } /* raised for its tooltip, it still opens the card */
-.bench-card-foot .bench-harness { margin-left: auto; }
+.bench-card .card-link { position: static; z-index: auto; } /* with a tooltip too: its ::after stretches over the card, not round its name */
+.bench-card-foot .bench-harness { margin-left: auto; text-align: right; }
+/* the foot on a card's last line, wherever its badges or stats would put it */
+.bench-card-foot:not(.bench-card-runs ~ *, .bench-stats ~ *) { margin-top: auto; }
.bench-card-actions { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 8px; }
.bench-shelf.st-community .bench-card { background: transparent; border-style: dashed; border-color: var(--border); }
.bench-shelf.st-community .bench-card:hover { border-color: var(--accent); }
diff --git a/app/static/tables.js b/app/static/tables.js
index d966c587..586fea11 100644
--- a/app/static/tables.js
+++ b/app/static/tables.js
@@ -60,7 +60,8 @@ window.NulliusTable = {
(data-filter-menu) and rows of pills (data-pills). `allowed`: column ->
the Set of values chosen, or null when all are; every list and row of a
column shows that one state. A row passes when, for each column, one of
- its values (data-f- , one per line) is chosen; a choice made at load
+ its values (data-f- , one per line) is chosen, or its data-unless-
+ names a column that narrows and passes it; a choice made at load
filters from the start. `changed` runs after each change, a list's
change event included. */
filters(scope, changed) {
@@ -97,8 +98,9 @@ window.NulliusTable = {
});
});
draw();
- const passes = row => Object.entries(allowed).every(([col, set]) =>
- !set || (row.getAttribute('data-f-' + col) ?? '').split('\n').some(v => set.has(v)));
+ // data-unless- ="": a row the other column narrows to passes this one whatever it holds
+ const ok = (row, col) => !allowed[col] || (row.getAttribute('data-f-' + col) ?? '').split('\n').some(v => allowed[col].has(v));
+ const passes = row => Object.keys(allowed).every(col => ok(row, col) || (by => !!by && !!allowed[by] && ok(row, by))(row.getAttribute('data-unless-' + col)));
const clear = () => { Object.keys(allowed).forEach(c => { allowed[c] = null; }); draw(); };
return {allowed, passes, clear};
},
@@ -136,6 +138,9 @@ window.NulliusTable = {
const open = menu.hidden;
menu.hidden = !open;
btn.setAttribute('aria-expanded', String(open));
+ // one that opens leftwards past the page's edge (a narrow screen) opens rightwards instead
+ menu.style.left = menu.style.right = '';
+ if (open && menu.getBoundingClientRect().left < 0) Object.assign(menu.style, {left: '0', right: 'auto'});
if (open) menu.querySelector('input:not(:disabled), button, a[href]')?.focus();
});
document.addEventListener('click', e => { if (!menu.hidden && away(e.target)) close(); });
@@ -151,7 +156,7 @@ window.NulliusTable = {
Escape clears it. A column narrows when it does not show every value;
the Filter button or a dropdown holding such a column is marked
.narrowed and counts: the Filter button the columns it narrows, a
- dropdown the values chosen (after None, 0). Clear filters clears every
+ dropdown the values chosen (after None, 0), or the one chosen by name. Clear filters clears every
column. `changed` runs after any of them. Returns passes(row). */
narrow(tools, changed) {
const [fbtn, fmenu] = [tools.querySelector('[data-tools-btn="filter"]'), tools.querySelector('[data-tools="filter"]')];
@@ -162,12 +167,13 @@ window.NulliusTable = {
}
const found = row => !q || this.matches(row.dataset.search, q.value);
const menus = [...(fmenu ? [[fbtn, fmenu, () => 1]] : []),
- ...[...tools.querySelectorAll('[data-dropdown]')].map(b => [b, b.parentNode.querySelector('.menu'), set => set.size])];
+ ...[...tools.querySelectorAll('[data-dropdown]')].map(b => [b, b.parentNode.querySelector('.menu'),
+ set => set.size === 1 ? b.parentNode.querySelector(`[data-value="${CSS.escape([...set][0])}"]`)?.parentNode.textContent.trim() ?? 1 : set.size])];
const mark = () => menus.forEach(([btn, menu, n]) => {
const sets = [...menu.querySelectorAll('[data-filter-menu]')].map(v => f.allowed[v.dataset.filterMenu]).filter(Boolean);
const count = btn.querySelector('.tools-count');
btn.classList.toggle('narrowed', sets.length > 0);
- count.textContent = sets.length ? sets.reduce((sum, set) => sum + n(set), 0) : '';
+ count.textContent = sets.length === 1 ? n(sets[0]) : sets.length ? sets.reduce((sum, set) => sum + n(set), 0) : '';
count.hidden = !sets.length;
});
const narrowed = () => { mark(); changed(); };
@@ -208,7 +214,8 @@ window.NulliusTable = {
/* A table of the table macro (data-table): its controls row (id
-tools), its Columns menu and checkbox column when it has
them, and its pages; choices kept per viewer under .cols and
- .per. */
+ .per. A row holding a pick toggle (the cards macro's rows) toggles it
+ and, chosen, shows whatever the search and filters say. */
wire(table) {
const tools = document.getElementById(table.id + '-tools');
const cmenu = tools.querySelector('[data-tools="columns"]');
@@ -219,7 +226,9 @@ window.NulliusTable = {
const ticks = table.querySelector('[data-check-all]') ? this.checks(table) : () => {};
let first = () => {};
const t = this.toolbar(table, tools, () => first());
- first = this.pager(table, table.id + '.per', t.passes, ticks);
+ const pick = Boolean(table.querySelector('[data-pick]'));
+ if (pick) this.picks(table, 'tr');
+ first = this.pager(table, table.id + '.per', pick ? r => t.passes(r) || this.chosen(r) : t.passes, ticks);
},
/* A collection of the cards macro (data-cards): its controls row (id
@@ -229,21 +238,25 @@ window.NulliusTable = {
how many show once the changes pause, counted in the live region's
data-noun. A click anywhere on a card but on a control of its own
follows the card's link, in a new tab or window with the keys or the
- button a link takes; a click that ends a text selection does not. Sort by,
- when it has one, sorts the cards within their grids as a table's rows. */
+ button a link takes; a click that ends a text selection does not. A pick
+ card's title is a toggle (`picks`); the page keeps the choice, and a
+ chosen card shows whatever the search and filters say. Sort by, when it
+ has one, sorts the cards within their grids as a table's rows. */
cards(root) {
const items = [...root.querySelectorAll('[data-card]')], empty = document.querySelector(`[data-empty="${CSS.escape(root.id)}"]`);
const live = document.querySelector(`[data-live="${CSS.escape(root.id)}"]`), say = this.voice(live);
const follow = e => {
const link = e.target.closest('[data-card]')?.querySelector('.card-link');
if (!link || e.target.closest('a, button, input, select, textarea, label, summary, [role=button]') || !getSelection().isCollapsed) return;
+ if (link.dataset.pick !== undefined) { if (e.button === 0) link.click(); return; }
if (e.button === 1 || e.ctrlKey || e.metaKey || e.shiftKey) window.open(link.href, '_blank', 'noopener');
else if (e.button === 0) link.click();
};
root.addEventListener('click', follow);
root.addEventListener('auxclick', follow);
+ this.picks(root);
const draw = () => {
- items.forEach(c => { c.hidden = !passes(c); });
+ items.forEach(c => { c.hidden = !passes(c) && !this.chosen(c); });
root.querySelectorAll('[data-group]').forEach(g => {
const n = [...g.querySelectorAll('[data-card]')].filter(c => !c.hidden).length, count = g.querySelector('[data-group-n]');
g.hidden = !n;
@@ -271,6 +284,27 @@ window.NulliusTable = {
draw();
},
+ /* The pick toggles under `root`, buttons (data-pick, aria-pressed) or
+ checkboxes (data-pick): a click, Space or Enter flips one and sends nullius-pick {value, pressed} from it; with
+ `row`, a click elsewhere in the row (a selector) that holds one, but on a
+ control of its own or at the end of a text selection, clicks it. */
+ picks(root, row) {
+ root.addEventListener('click', e => {
+ if (row && !e.target.closest('a, button, input, select, textarea, label, summary') && getSelection().isCollapsed) {
+ e.target.closest(row)?.querySelector('[data-pick]')?.click();
+ return;
+ }
+ const b = e.target.closest('[data-pick]');
+ if (!b || b.disabled) return;
+ const pressed = b.type === 'checkbox' ? b.checked : b.getAttribute('aria-pressed') !== 'true'; // a box has ticked itself
+ if (b.type !== 'checkbox') b.setAttribute('aria-pressed', String(pressed));
+ b.dispatchEvent(new CustomEvent('nullius-pick', {bubbles: true, detail: {value: b.dataset.pick, pressed}}));
+ });
+ },
+
+ /* Whether a card or a row holds a pressed pick: chosen, it shows whatever the search and filters say. */
+ chosen(el) { return Boolean(el.querySelector('[data-pick][aria-pressed="true"], [data-pick]:checked')); },
+
/* A checkbox column (a `check` column, check_cell in each row): the header
ticks or unticks every visible row, and shows all, none or some of them
ticked. Returns sync(), to run when rows show or hide. */
diff --git a/app/templates/_cards.html b/app/templates/_cards.html
index aef7dc2c..c695d4e9 100644
--- a/app/templates/_cards.html
+++ b/app/templates/_cards.html
@@ -8,8 +8,11 @@
An item is a dict of display values by field key: a string, a dict {text, class, title, label, href, value, attrs}
or a list of them, empty for none; a filter and Sort by read a value's `value`, else its text; an action's
`attrs` are its button's own attributes, its `class` added to the button's, and a `delete` takes the same shape (a quiet
- trash button). Plus `href` (its title's link),
- `id`, `search` (what the search reads) and `keys`. A title's `after` is markup in the flow of its name, after its
+ trash button). Plus `href` (its title's link), or `pick` {value, on, disabled}: the card is a toggle, its title a
+ button[data-pick][aria-pressed] that tables.js flips, sending `nullius-pick` {value, pressed} (a disabled one's
+ reason goes in its `alert`), and an enabled one's corner holds the check that marks it chosen; a title's `title` is the tooltip of its link or button;
+ `id`, `search` (what the search reads), `keys`, and `unless` {field: other}: the item passes the field's filter
+ whenever the other field's filter narrows and passes it. A title's `after` is markup in the flow of its name, after its
last word (a control of its own), and its `class` goes on the title row.
A field's `class` goes on each of its values; `card_label` names a stat when its column header does not, and a
value's `label` names it for that item; `style` 'code' sets the text as code; `icon` puts #i- before a
@@ -49,19 +52,23 @@
{# The items of a cards collection as a table (the table toolkit's, with its Columns menu): a column per field the
table takes that some item fills (a filtered or 'num' one always), in the order of `cols`; the title pinned, its
- icon before it and a link to `href`, the others as on the card. A field `off` starts hidden. #}
+ icon before it and a link to `href`, the others as on the card; picks put a checkbox in a leftmost column of no
+ name (its header no tick-all: a pick is one at a time), the name then plain. A field `off` starts hidden. #}
{% macro rows(id, cols, items, search, choices, only, empty, noun, sort, default) %}
{%- set icon = (cols|selectattr('card', 'equalto', 'icon')|list + [none])[0] %}
{%- set icons = icon and items|selectattr(icon.key)|list|length > 0 %}
-{%- set shown = [] %}{% for c in cols if c.table is not sameas false and (c.key in choices or c.type == 'num' or items|selectattr(c.key)|list) %}{% set _ = shown.append(dict(c, sticky=c.card == 'title')) %}{% endfor %}
+{%- set shown = [dict(key='_pick', label='Chosen', blank=true, pick=true, sticky=true)] if items|selectattr('pick')|list else [] %}{% for c in cols if c.table is not sameas false and (c.key in choices or c.type == 'num' or items|selectattr(c.key)|list) %}{% set _ = shown.append(dict(c, sticky=c.card == 'title')) %}{% endfor %}
{%- set rs = [] %}{% for item in items %}{% set a %}{% if item.get('id') %} id="{{ item.id }}"{% endif %} data-search="{{ item.get('search', '') }}"
- {%- for f in cols if f.key in choices %} data-f-{{ f.key }}="{% call(f, v) each(item, [f]) %}{{ v.get('value', v.text) }}
{% endcall %}"{% endfor %}{% endset %}
+ {%- for f in cols if f.key in choices %} data-f-{{ f.key }}="{% call(f, v) each(item, [f]) %}{{ v.get('value', v.text) }}
{% endcall %}"{% endfor %}{% for k, by in (item.get('unless') or {}).items() %} data-unless-{{ k }}="{{ by }}"{% endfor %}{% endset %}
{%- set _ = rs.append({'item': item, 'attrs': a}) %}{% endfor %}
{% call(r, c) table(id, shown, rs, attrs, search=search, columns=true, choices=choices, only=only, empty=empty, noun=noun, sort=sort, layout='table', default=default) %}
{%- set v = r.item.get(c.key) %}{% set by %}{% call(f, x) each(r.item, [c]) %}{{ x.get('value', x.text) }} {% endcall %}{% endset %}
- {%- call cell(c, sort=by|trim) %}{% if c.card == 'title' %}{% set name = v.text if v is mapping else v %}{% if icons %}{{ mark(icon, r.item.get(icon.key), name) }}{% endif %}{{ name }}
+ {%- set pick = r.item.get('pick') %}{% set t = r.item.get((cols|selectattr('card', 'equalto', 'title')|first).key) %}{% set t = t if t is mapping else {'text': t} %}
+ {%- if c.key == '_pick' %}{% call cell(c, class='pick-col') %} {% endcall %}
+ {%- else %}{% call cell(c, sort=by|trim) %}{% if c.card == 'title' %}{% set name = t.text %}{% if icons %}{{ mark(icon, r.item.get(icon.key), name) }}{% endif %}
+ {%- if pick %}{{ name }}{% else %}{{ name }} {% endif %}
{%- elif c.style == 'code' %}{{ v }}
- {%- else %}{% call(f, x) each(r.item, [c]) %}{{ pill(f, x) }} {% endcall %}{% endif %}{% endcall %}
+ {%- else %}{% call(f, x) each(r.item, [c]) %}{{ pill(f, x) }} {% endcall %}{% endif %}{% endcall %}{% endif %}
{%- endcall %}
{% endmacro %}
@@ -78,14 +85,16 @@
{%- for item in items %}
{%- set title = by.title[0] %}{% set v = item.get(title.key) %}{% set v = v if v is mapping else {'text': v} %}{% set name = v.text %}
{%- set cls = [title.class, v.class]|select|join(' ') %}
{% if icons %}{{ mark(icon, item.get(icon.key), name) }}{% endif -%}
- {%- set link %}
{{ name }} {% endset %}
+ {%- set pick = item.get('pick') %}
+ {%- set link %}{% if pick %}
{{ name }}
+ {%- else %}
{{ name }} {% endif %}{% endset %}
{%- if v.after %}
{{ link }} {{ v.after }} {% else %}{{ link }}{% endif %}
- {%- set corner %}{% call(f, v) each(item, by.check) %}
{% endcall %}
+ {%- set corner %}{% if pick and not pick.disabled %}
{% endif %}{% call(f, v) each(item, by.check) %}
{% endcall %}
{%- call(f, v) each(item, by.delete) %}
{% endcall %}{% endset %}
{%- if corner %}
{{ corner }} {% endif %}
{%- call(f, v) each(item, by.text) %}
@@ -98,7 +107,7 @@
{% for f in stats %}{% set v = item.get(f.key) %}{% set v = v if v is mapping else {'text': v} -%}
{{ v.label or f.card_label or f.label }} {{ '—' if v.text is none or v.text == '' else v.text }} {% endfor %} {% endif %}
{%- call(f, v) each(item, by.alert) %}
- {{ v.text }}
{% endcall %}
+ {{ v.text }}
{% endcall %}
{%- set actions %}{% call(f, v) each(item, by.action) %}{% if v.href %}{{ v.text }} {% else %}{{ v.text }} {% endif %}{% endcall %}{% endset %}{% if actions %}
{{ actions }}
{% endif %}
{%- set foot %}{% call(f, v) each(item, by.status) %}{{ pill(f, v) }}{% endcall %}
diff --git a/app/templates/_tables.html b/app/templates/_tables.html
index 5bb42046..db81311f 100644
--- a/app/templates/_tables.html
+++ b/app/templates/_tables.html
@@ -5,7 +5,7 @@
all read: `key` (the data-col), `label`; `type` 'text' (default) or 'num', how it sorts; `align` 'left', 'center'
or 'right' (a 'num' column's default), of its header and cells; `sticky`: pinned while the table scrolls, the last
one drawing the edge; `blank`: no visible name (a status, the actions), neither sorted nor in the Columns menu;
- `check`: blank, a checkbox that ticks every visible row (check_cell in each row); `sr`: the label for a screen
+ `check`: blank, a checkbox that ticks every visible row (check_cell in each row); `pick`: blank, as narrow, with no such box; `sr`: the label for a screen
reader only; `title`: its tooltip; `menu` false: not in the Columns menu; `fixed`: in it, always on; `off`: off
until the viewer chooses; `width` in a table macro's table: 'fit' (default), as wide as its widest cell on one
line, or 'expand', the rest of the width, wrapping; `table` false: on cards only (_cards.html, where `card` names
@@ -23,7 +23,7 @@
{# The header cells of `cols`. #}
{% macro heads(cols) %}{% set edge = edge_of(cols) %}
{%- for c in cols %}{% set named = not (c.blank or c.check) %}
-
+
{%- if c.check %} {% elif c.sr or c.blank %}{{ c.label }} {% else %}{{ c.label }}{% endif %}
{%- endfor %}
{%- endmacro %}
@@ -74,7 +74,7 @@
page's other one, its other query kept, at the end; `default` the view a page shows without ?view=, the other
one's link carrying ?view=. The search goes left, the buttons right. A column's
`filter` says where its choices go, one form or a list of them: 'button' (the default), in the Filter menu;
- 'dropdown', a menu of its own; 'pills', a row of toggles under the controls. A column's `filter_icon`, a sprite id
+ 'dropdown', a menu of its own; 'pills', a row of toggles under the controls, the first named by `all` (All). A column's `filter_icon`, a sprite id
without its `i-`, leads its dropdown's button and its pills. `id`: the table's. #}
{% macro controls(id, cols, search='', columns=false, choices={}, only={}, views=none, actions=none, sort=true, layout=none, default='table') %}
{%- set by = {'button': [], 'dropdown': [], 'pills': []} %}{% for c in cols if c.key in choices %}{% for f in ([c.filter] if c.filter is string else c.filter or ['button']) %}{% set _ = by[f].append(c) %}{% endfor %}{% endfor %}
@@ -120,7 +120,7 @@
{% endif %}
{% for c in by.pills %}
- {{ icon(c) }}All
+ {{ icon(c) }}{{ c.all or 'All' }}
{%- for value, text in choices[c.key] %}{{ text }} {% endfor %}
{% endfor %}
diff --git a/app/templates/agents/standard.html b/app/templates/agents/standard.html
index 04550ecb..64aca41f 100644
--- a/app/templates/agents/standard.html
+++ b/app/templates/agents/standard.html
@@ -34,15 +34,16 @@
{% if problems %}Not offered: a definition with problems {% for p in problems %}{{ p }} {% endfor %} {% endif %}
{% if not items %}No {{ kind }} yet.
{% endif %}
- {#- A card's fields (web/agents.py `_card`); an agent's protocol, the ports it proves and where it comes from as filters. #}
+ {#- A card's fields (web/agents.py `_card`), in the slots of the builder's Agent step (builder/agents.html): the name and
+ its version, what it is and proves as badges, its protocol and where it comes from at the foot; an agent's protocol,
+ the ports it proves and where it comes from as filters. #}
{% set cols = [dict(key='icon', label='Logo', card='icon', table=false, class='std-mark small'), dict(key='title', label='Name', card='title'),
- dict(key='kind', label='Kind', card='tag', class='cat-tag'), dict(key='benchmark', label='Benchmark', card='tag', class='cat-tag'),
- dict(key='protocol', label='Protocol', card='tag', class='std-tag'), dict(key='uses', label='Uses', card='tag', class='std-tag', off=true),
- dict(key='from', label='From'), dict(key='runs_on', label='Runs on', class='cat-tag', off=true), dict(key='models', label='Models', card='note', width='expand'),
- dict(key='proves', label='Proves', class='std-tag', off=true), dict(key='example', label='Example', card='tag', class='std-tag', off=true),
+ dict(key='kind', label='Kind', card='badge', class='cat-tag'), dict(key='protocol', label='Protocol', card='tag', class='std-tag'),
+ dict(key='uses', label='Uses', card='badge', class='std-tag', off=true), dict(key='from', label='From', card='meta'), dict(key='runs_on', label='Runs on', class='cat-tag', off=true), dict(key='models', label='Models', card='note', width='expand'),
+ dict(key='proves', label='Proves', class='std-tag', off=true), dict(key='example', label='Example', card='badge', class='std-tag', off=true),
dict(key='summary', label='Summary', card='text', class='std-clamp', width='expand', off=true), dict(key='ports', label='Ports proven', card='stat', type='num'),
dict(key='runs', label='Benchmarks it runs', card='stat', type='num'), dict(key='installs', label='Installs', card='stat', class='word'),
- dict(key='version', label='Version', card='meta')] %}
+ dict(key='version', label='Version')] %}
{% if items %}{{ cards(plural|lower, cols, items, search='Name, protocol, summary', choices=choices, only=only, sort=['title', 'ports', 'runs'], empty='No ' ~ kind ~ ' matches.', noun=plural|lower, layout='table' if request.args.get('view') == 'table' else 'cards', default='cards') }}{% endif %}
{% endblock %}
diff --git a/app/templates/base.html b/app/templates/base.html
index 59a29d81..eeadbdaa 100644
--- a/app/templates/base.html
+++ b/app/templates/base.html
@@ -71,6 +71,7 @@
+
diff --git a/app/templates/builder/agents.html b/app/templates/builder/agents.html
new file mode 100644
index 00000000..524d5033
--- /dev/null
+++ b/app/templates/builder/agents.html
@@ -0,0 +1,13 @@
+{#- The builder's Agent step grid (web/builder.py, new_run_agents): the benchmark's agents or capabilities as pick
+ cards of the card toolkit (_cards.html), or its rows (`layout` 'table'), loaded by command-builder.js, which keeps
+ the choice and switches the view in place. -#}
+{% from '_cards.html' import cards %}
+{% set noun = 'agents' if kind == 'agent' else 'capabilities' %}
+{#- the card's slots as the Agents page's (agents/standard.html); in a table, why not after the name, fit and proves
+ off until asked; Fit a menu of its own, on Can run it at first -#}
+{% set cols = [dict(key='icon', label='Logo', card='icon', table=false, class='std-mark small'), dict(key='title', label='Name', card='title', width='expand'),
+ dict(key='alert', label='Why not', card='alert'), dict(key='protocol', label='Protocol', card='tag', class='std-tag'),
+ dict(key='published', label='Setup', card='tag', class='std-tag'), dict(key='needs', label='Proven', card='badge', class='std-tag'),
+ dict(key='ticks', label='Checks', card='badge', class='nb-tick'), dict(key='fit', label='Fit', filter='dropdown', filter_icon='puzzle', off=true),
+ dict(key='from', label='From', card='meta'), dict(key='proves', label='Proves', class='std-tag', off=true)] %}
+{{ cards('nb-agents', cols, items, search='Search ' ~ noun, choices=choices, empty='No ' ~ kind ~ ' matches.', noun=noun, layout=layout, default='cards', only={'fit': 'fit'}) }}
diff --git a/app/templates/builder/new_run.html b/app/templates/builder/new_run.html
index 5e4408a1..752202b4 100644
--- a/app/templates/builder/new_run.html
+++ b/app/templates/builder/new_run.html
@@ -2,9 +2,11 @@
{% from '_page.html' import page_head %}
{% block title %}Run builder — Nullius{% endblock %}
{% block crumb %}/ run builder{% endblock %}
-{% block head %} {% endblock %}
+{% block head %} {# the Agent step's cards #}
+
+{% endblock %}
{% block body %}
-
+
{% set actions %}
Saved recipes
Import recipe
diff --git a/app/tests/test_benchmark_agents.py b/app/tests/test_benchmark_agents.py
index 76a585ad..ee8afcf6 100644
--- a/app/tests/test_benchmark_agents.py
+++ b/app/tests/test_benchmark_agents.py
@@ -115,6 +115,23 @@ def test_its_own_benchmark_alone_with_no_port_reason(self):
self.assertEqual(t["gsm8k"]["osworld2:muse-spark"], ["Muse Spark belongs to OSWorld2"])
self.assertNotIn("osworld1:openai-cua", t.get("locomo", {})) # a capability benchmark takes none
+ def test_one_entry_list_whose_fits_are_the_builders(self):
+ # recipe.entries: the Agents page's list and the builder's Agent step grid; its fits are the builder payload's
+ every = definitions.load("benchmark", every=True)[0]
+ c = recipe.choices()
+ fits, offered = c["fits"], {o["key"] for o in c["agents"] + c["capabilities"]} # the builder never offers an example
+ for kind in ("agent", "capability"):
+ for b in (x for x in every if x["family"] == kind):
+ ref = definitions.ref_of(b) if "_version" in b else b["key"]
+ with self.subTest(ref):
+ got = {e["d"]["_ref"]: e["fits"][0] for e in recipe.entries(kind, benches=[b], every=True, own=True)["entries"]
+ if not e["d"].get("example")}
+ self.assertEqual({k: f["why"] for k, f in got.items()}, {k: v for k, v in fits[ref].items() if k in offered})
+ self.assertEqual({f["ref"] for f in got.values()}, {ref})
+ page = {e["d"]["_ref"]: e for e in recipe.entries("agent", own=True)["entries"]}
+ self.assertEqual([f["why"] for f in page["osworld1:openai-cua"]["fits"] if f["key"] == "osworld1"], [[]])
+ self.assertEqual(page["osworld1:openai-cua"]["protocol"], "Its own script")
+
RECIPE = {"harness": "standard", "name": "os", "benchmark": "osworld1", "datasets": ["test_small"], "provider": "openrouter",
@@ -130,7 +147,7 @@ def test_the_builder_offers_them_as_agents(self):
got = {a["key"]: a for a in recipe.choices()["agents"]}
cua = got["osworld1:openai-cua"]
self.assertEqual((cua["benchmark"], cua["agent"], cua["models"], cua["protocol"]),
- ("osworld1", "openai_cua", ["computer-use-preview"], "its own script"))
+ ("osworld1", "openai_cua", ["computer-use-preview"], "Its own script"))
def test_an_own_agent_compiles_to_the_command_of_the_agent_option(self):
old = self.compiled(others=["original"], bench_options={"agent": "openai_cua", "max_steps": "20"})
diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py
index 269baafa..0be840b6 100644
--- a/app/ui_tests/test_agents.py
+++ b/app/ui_tests/test_agents.py
@@ -83,7 +83,7 @@ def test_bundled_agents_use_shared_marks(self):
self.assertIn('/static/brand/marks.svg#openai', html)
self.assertIn('/static/brand/marks.svg#harbor', html)
# Beside the logos, an agent without one shows its initials; where none has a logo, no card has an icon.
- self.assertIn('
EX ', html)
+ self.assertIn('EX Example agent 1 ', html)
fixture_html = self.get("/agents").get_data(as_text=True)
self.assertIn('', fixture_html)
self.assertNotIn('std-mark', fixture_html)
@@ -190,9 +190,10 @@ def test_a_benchmarks_own_agents_are_listed_behind_the_from_filter(self):
card = card[:card.index("")]
self.assertIn('data-f-from="benchmark
"', card)
self.assertIn('href="/benchmarks/osworld1"', card)
- self.assertIn('OSWorld1 ', card)
self.assertIn('computer-use-preview
', card)
- self.assertIn("Computer use", card)
+ self.assertIn('Computer use
', card)
+ self.assertIn('', card) # the builder's foot: its protocol, where it comes from
self.assertNotIn("Ports proven", card)
self.assertIn('id="osworld2-muse-spark"', table)
self.assertEqual(self.get("/agents/openai-cua").status_code, 404) # its benchmark's page is its page
@@ -209,6 +210,9 @@ def test_runs_on_filters_by_the_benchmarks_each_agent_runs(self):
self.assertEqual(on, sorted(["codex"] + ["osworld1-" + k for k in ("m3", "prompt", "claude", "gpt54", "openai-cua", "qwen", "os-symphony",
"evocua", "autoglm", "autoglm-v", "mano", "muse-spark")]))
self.assertNotIn("osworld1", dict(cards)["osworld2-m3"].split("
")) # another benchmark's own runs it alone
+ own = html[html.index('id="osworld1-m3"'):]
+ self.assertIn('data-unless-from="runs_on"', own[:own.index(">")]) # Runs on its benchmark shows it, whatever From says
+ self.assertNotIn("data-unless-from", html[html.index('id="codex"'):][:200])
# the count the card shows and the filter read one record
card = html[html.index('id="codex"'):]
n = card[:card.index(">")].split('data-f-runs_on="', 1)[1].split('"', 1)[0].count("
")
@@ -380,7 +384,8 @@ def test_an_agent_with_versions_is_one_card_and_a_page_per_version(self):
self.versioned()
html = self.get("/agents").get_data(as_text=True)
self.assertEqual(html.count('id="fake"'), 1)
- self.assertIn("v1.0 · 2 versions", html)
+ self.assertIn('Fake agent 1.0', html) # its default version beside its name
+ self.assertIn("v1.0 · 2 versions", self.get("/agents?view=table").get_data(as_text=True))
html = self.get("/agents/fake").get_data(as_text=True) # the default
self.assertIn('
1.0 2/7 proven · default ', html)
@@ -588,8 +593,9 @@ def test_the_builder_gets_the_skills_and_a_benchmark_needing_one_says_who_lacks_
[("decoys", False, "1/3"), ("chain", True, "3/3"), ("retry", False, "0/3")])
self.assertEqual(schema["standard"]["fits"]["fakebench"]["fake"],
["Fake agent has not shown the decoys skill of agent.tools/v1 (chooses the right tool among similar ones)"])
- js = (Path(web.app.static_folder) / "builder" / "command-builder.js").read_text()
- self.assertIn('.filter(s => !s.shown).map(s => `decoys ✗ ', card[:card.index("")])
def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_here(self):
from nullius.standard import boat
diff --git a/app/ui_tests/test_builder_agents.py b/app/ui_tests/test_builder_agents.py
new file mode 100644
index 00000000..2d441a2e
--- /dev/null
+++ b/app/ui_tests/test_builder_agents.py
@@ -0,0 +1,124 @@
+"""The builder's Agent step grid (/new-run/agents): the card toolkit's pick cards for one benchmark, from the
+entries the Agents page lists, greyed as the builder's payload says."""
+import json
+import os
+import re
+import shutil
+import tempfile
+import unittest
+from html import unescape
+from unittest.mock import patch
+
+from .test_routes import web
+
+
+class AgentGrid(unittest.TestCase):
+ def setUp(self):
+ tmp = tempfile.mkdtemp(prefix="nullius-grid-")
+ self.addCleanup(shutil.rmtree, tmp, True)
+ env = patch.dict(os.environ, {"NULLIUS_HOME": tmp, "HOSTS": "", "RUNS_DIR": os.path.join(tmp, "runs")})
+ env.start(); self.addCleanup(env.stop)
+ os.environ.pop("NULLIUS_CATALOGUE", None)
+ self.client = web.app.test_client()
+
+ def grid(self, ref):
+ r = self.client.get("/new-run/agents?benchmark=" + ref)
+ self.assertEqual(r.status_code, 200, ref)
+ html = r.get_data(as_text=True)
+ cards = {m.group(1): m.group(0) for m in re.finditer(r'', html, re.S)}
+ return html, cards
+
+ def schema(self):
+ html = self.client.get("/new-run").get_data(as_text=True)
+ return json.loads(re.search(r'', html, re.S).group(1))["standard"]
+
+ def test_its_greying_is_the_builders_for_every_benchmark(self):
+ st = self.schema()
+ offered = {o["key"] for o in st["agents"] + st["capabilities"]}
+ for b in st["benchmarks"]:
+ with self.subTest(b["key"]):
+ _, cards = self.grid(b["key"])
+ fits = {k: v for k, v in st["fits"][b["key"]].items() if k in offered}
+ self.assertEqual({k for k in cards if k != "original"}, set(fits))
+ self.assertEqual({k for k, c in cards.items() if " disabled" in c[:c.index("")]}, {k for k, v in fits.items() if v})
+
+ def test_osworld1_offers_its_own_agents_and_codex_and_says_why_not_the_rest(self):
+ html, cards = self.grid("osworld1")
+ on = [k for k, c in cards.items() if " disabled" not in c[:c.index("")]]
+ self.assertEqual(len([k for k in on if k.startswith("osworld1:")]), 12)
+ self.assertEqual([k for k in on if not k.startswith("osworld1:")], ["codex@2.2.2"])
+ self.assertTrue(list(cards)[0].startswith("osworld1:")) # its own first
+ self.assertNotIn("original", cards) # they are its Original
+ self.assertIn('Its own script '
+ 'OSWorld1 ', cards["osworld1:openai-cua"]) # the Agents page's foot
+ self.assertIn('
ACP Bundled ', cards["goose"])
+ self.assertIn('Belongs to OSWorld2
', cards["osworld2:m3"])
+ self.assertIn('data-f-from="elsewhere
"', cards["osworld2:m3"])
+ self.assertIn(">Lacks computer", cards["goose"])
+ check = ' '
+ self.assertIn('>OpenAI CUA' + check, cards["osworld1:openai-cua"]) # the mark a chosen card shows (CSS, by aria-pressed)
+ self.assertNotIn("pick-check", cards["goose"]) # none on one that cannot be chosen
+ self.assertIn('Computer use ", cards["codex@2.2.2"])
+ self.assertNotIn("data-pills", html) # every filter in the one Filter menu
+ menu = unescape(html[html.index('data-tools="filter"'):])
+ values = lambda col: re.findall(r'data-value="[^"]*" checked> ([^<]*)<', menu[menu.index(f'data-filter-menu="{col}"'):].split("")[0])
+ self.assertEqual(values("protocol"), ["ACP", "A command", "Its own script"]) # every value on: any protocol
+ self.assertNotIn('data-filter-menu="fit"', menu[:menu.index("")]) # Fit: a menu of its own, beside Filter
+ fit = unescape(html[html.index('")[0]
+ self.assertIn(' Can run it', fit) # what can run, at first
+ self.assertIn(' Cannot', fit)
+ self.assertLess(html.index("data-dropdown"), html.index('data-tools-btn="filter"'))
+ self.assertIn('data-dropdown aria-haspopup="true" aria-expanded="false"> Fit<', html) # an icon as Filter's
+ self.assertIn("> Bundled OSWorld1's own "
+ " Other benchmarks' agents", menu)
+ self.assertIn('data-filter-menu="proves"', menu)
+ self.assertNotIn("data-tools-btn=\"sort\"", html)
+
+ def test_a_benchmark_without_own_agents_keeps_its_original_and_greys_osworlds(self):
+ _, cards = self.grid("gsm8k")
+ self.assertEqual(list(cards)[0], "original")
+ self.assertIn(">No agent · Original", cards["original"])
+ self.assertIn('As published ', cards["original"]) # how it runs, not a protocol
+ self.assertIn('data-f-protocol=""', cards["original"])
+ html, _ = self.grid("gsm8k")
+ self.assertNotIn('data-value="reference"', html)
+ for k, c in cards.items():
+ if k.startswith("osworld"):
+ self.assertIn(" disabled", c)
+ self.assertRegex(c, r">Belongs to OSWorld[12]")
+
+ def test_its_table_view_picks_as_its_cards_do(self):
+ cards, _ = self.grid("osworld1")
+ self.assertIn('href="/new-run/agents?benchmark=osworld1&view=table" aria-label="Table"', cards) # the toggle, cards by default
+ r = self.client.get("/new-run/agents?benchmark=osworld1&view=table")
+ self.assertEqual(r.status_code, 200)
+ html = r.get_data(as_text=True)
+ self.assertIn('