Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ and its resources. The source layout stays under `app/`.
`cli` and `__main__` (the unified `nullius` and `python -m nullius` commands),
`paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request and before a capped run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `constraints` (what Nullius may use on each machine: one table, the container and systemd scope probes, the warnings), `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine
says is done, and where each step is done), `gitrefs` (a repository's branches and tags,
by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository, and the Hugging Face token of Data providers), `sandbox` (Docker's state, and the Docker Hub login Data providers uses), `charts` (categorical charts, bars, dots and a scatter, and their style registry: Compare's Chart view) and `redact` at the top.
by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository, and the Hugging Face token of Data providers), `sandbox` (the Docker Hub login Data providers uses), `charts` (categorical charts, bars, dots and a scatter, and their style registry: Compare's Chart view) and `redact` at the top.
- `app/templates/`, `app/static/` — the interface, grouped by the same capability
names (`runs/`, `traces/`, `builder/`, …; `base.html`, `style.css` and the fonts at the top), including the command
builder's JavaScript, which owns the generated recipes. `app/templates/_tables.html` and
Expand Down
40 changes: 2 additions & 38 deletions app/nullius/sandbox.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
"""The Settings page's Sandbox: the container backend's engine on this machine,
docker or podman (``container.DOCKER``), and its Docker Hub login under Data
providers.
"""The container engine's Docker Hub login, under Data providers: docker or
podman (``container.DOCKER``).

Two things, each something a user acts on:

- **Engine**: whether it answers, and its name and version (``container.unusable``).
- **Registry login**: Docker Hub limits anonymous pulls, which stops a
benchmark with an image per unit part-way. The username and an access token
go to ``docker login --password-stdin`` (``podman login ... docker.io``)
Expand All @@ -19,7 +15,6 @@
import json
import os
import re
import shutil
import subprocess
from typing import List, Optional

Expand All @@ -31,37 +26,6 @@
_TOKEN_MAX = 4096


def docker_state() -> dict:
"""{ok, version} when the engine answers - its name and version, ``podman
6.1.4`` - else {ok: False, error}; with ``also``, the other engine when both
are installed (``_other``), so the page says why this one is in use."""
why = container.unusable()
state = {"ok": False, "error": why} if why else None
if state is None:
r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
state = {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"}
also = _other()
return dict(state, also=also) if also else state


def _other() -> Optional[dict]:
"""The engine not in use, when it is installed too and is not the same program
(podman's ``docker``): {name, ok, said}, ``said`` its version or why it does not answer."""
other = "podman" if container.DOCKER == "docker" else "docker"
path = shutil.which(other)
if not path or (other == "docker" and container._runs_podman(path)):
return None
try:
r = subprocess.run([other, "version", "--format", "{{.Server.Version}}"], stdin=subprocess.DEVNULL,
capture_output=True, text=True, timeout=10)
except (OSError, subprocess.SubprocessError) as e:
return {"name": other, "ok": False, "said": str(e)[:200]}
if r.returncode == 0 and r.stdout.strip():
return {"name": other, "ok": True, "said": f"{other} {r.stdout.strip()}"}
lines = (r.stderr or r.stdout).strip().splitlines()
return {"name": other, "ok": False, "said": lines[-1][:200] if lines else f"exit {r.returncode}"}


def _config_dir() -> str:
return os.environ.get("DOCKER_CONFIG") or os.path.join(os.path.expanduser("~"), ".docker")

Expand Down
11 changes: 6 additions & 5 deletions app/nullius/standard/recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -392,9 +392,10 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *,
the run sends it as."""
d = defs or _all()
errors, notes = [], []
# each problem once: several picks of one kind (a benchmark's own agents) can each find it
add = lambda step, message: None if {"step": STEP[step], "message": message} in errors \
else errors.append({"step": STEP[step], "message": message})
# each problem once; shown: the step already says it elsewhere, so only Review lists it
def add(step, message, shown=False):
if not any(e["step"] == STEP[step] and e["message"] == message for e in errors):
errors.append({"step": STEP[step], "message": message, **({"shown": True} if shown else {})})
out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": "",
"served_models": {}}
name = str(recipe.get("name") or "")
Expand Down Expand Up @@ -652,7 +653,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *,
elif backend == "boat" and targets != ["local"]:
add("where", "Hosts and Boat sandboxes do not mix: choose Hosts, or Sandboxes alone.")
elif backend == "boat" and boat_unusable is not None and (why := boat_unusable(True)):
add("where", f"Boat cannot run: {why}.")
add("where", f"Boat cannot run: {why}.", shown=not boat.key())
elif backend == "boat" and boat_size_problem is not None and str(recipe.get("boat_size") or boat.SIZE) in boat.SIZES:
why = boat_size_problem(str(recipe.get("boat_size") or boat.SIZE))
if why:
Expand All @@ -676,7 +677,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *,
elsewhere = [definitions.setup_label(b, "original") if p.get("setup") else p["o"]["label"] for p in picks
if not whole and (p["control"] or p.get("setup") or (computer and side == "agent"))]
if backend == "boat" and elsewhere:
add("where", f"{', '.join(elsewhere)} cannot run in a Boat VM yet: choose Hosts.")
add("where", f"{', '.join(elsewhere)} cannot run in a Boat VM: choose Hosts.")
# a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share
spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"])
paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None])
Expand Down
11 changes: 7 additions & 4 deletions app/static/builder/command-builder.css

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 6 additions & 8 deletions app/static/builder/command-builder.js
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@
// An unsupported draft opens nothing; its rejection is shown as a toast.
else { let d = null; try { d = C.unpackDraft(localStorage.getItem(draftKey), query); } catch {} if (d && d.recipe) { recipe = d.recipe; step = d.step; tab = d.tab; savedId = d.saved; } else if (d && d.gone) opened = d.gone; }
const saveDraft = () => { try { localStorage.setItem(draftKey, C.packDraft({recipe, step, tab, saved: savedId})); } catch {} };
// Settings closed: reload from the draft to read what changed
document.addEventListener('close', e => { if (e.target.id === 'settings-modal') { saveDraft(); location.replace(location.pathname); } }, true);
let compiled, pendingDirs, dirsSequence = 0;
const r = () => recipe;
const store = recipeStore({request: (...a) => nulliusFetch(...a), csrf: schema.csrf, storage: (() => { try { return localStorage; } catch { return {length: 0}; } })()});
Expand Down Expand Up @@ -480,9 +482,6 @@
const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy
// a VM benchmark's provider of one value goes on the fixed line; several are its cards (benchOptions)
const provider = vms && Object.values(b.options || {}).find(o => o.step === 'where' && o.values && o.values.length === 1 && Object.values(o.needs || {}).some(n => n && n.vm));
// what a sandbox cannot run (recipe.py): a setup or a check runs on a machine; the computer port's socket is on this one
// a benchmark booting VMs of its own runs each unit whole in a VM, with a relay container in each VM (boat_units)
const outside = !sandboxes ? [] : selected.filter(o => !vms && (o.setup || o.control || ((b.needs || []).some(p => p.startsWith('agent.computer/')) && b.family === 'agent'))).map(o => o.label);
let sizes = '';
if (sandboxes && !here.boat) {
const offered = boatSizes(rec, b), fit = offered.find(z => !z.why);
Expand All @@ -493,8 +492,8 @@
disabled: !!z.why, why: z.why, note: z.note || z.spec, title: z.label})).join('')}</div>`;
}
return '<div class="nb-sectionhead"><h2>Where does it run?</h2></div>'
+ `<div class="std-machines" role="group" aria-label="Machines">${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boat || '')}</div>`
+ (sandboxes ? `<div class="std-boat-card"><strong>Boat</strong><span>One VM per ${esc(noun)}</span>${here.boatSlots ? `<span class="std-boat-slots">${here.boatSlots} at once on this account</span>` : ''}${here.boat ? `<span class="nb-error">${esc(here.boat)}</span>` : ''}</div>` + sizes
+ `<div class="std-machines" role="group" aria-label="Machines">${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boatKey ? here.boat || '' : '')}</div>`
+ (sandboxes ? `<div class="std-boat-card${here.boat ? ' off' : ''}"><strong><img class="std-boat-logo on-light-bg" src="/static/brand/boat.svg" alt="" aria-hidden="true"><img class="std-boat-logo on-dark-bg" src="/static/brand/boat-dark.svg" alt="" aria-hidden="true">Boat</strong>${here.boat ? (here.boatKey ? `<span class="nb-error">${esc(here.boat)}</span>` : button('Add API Key', 'data-href="/settings#sandbox-boat-key"', 'nb-inlinebutton std-boat-key')) : ''}</div>` + sizes
: `<div class="nb-pick-grid nb-where std-hosts">${machine('local', 'This machine', 'Runs here')}${H.map(h => machine(h.entry, h.label, 'Over SSH')).join('')}</div>`
+ (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' <span id="nb-hosttest" class="nb-note"></span>' : ''))
+ '<div class="nb-sectionhead std-wherehead"><h3>Isolation</h3></div>'
Expand All @@ -506,7 +505,6 @@
: 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + '</div>')
+ (b ? benchOptions(rec, b, 'where') : '')
+ (agentRun && backend === 'local' ? consentBox(rec) : '')
+ (outside.length ? `<p class="nb-note nb-error">${esc(outside.join(', '))} cannot run in a Boat VM yet: choose Hosts.</p>` : '')
+ (dockerOnly.length && backend !== 'container' && agentRun ? `<p class="nb-note nb-error">${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - ${sandboxes ? 'in a Boat VM' : 'on local processes'} their units are recorded as not run.</p>` : '')
+ (sandboxes ? '' : hostSetup(rec));
}
Expand Down Expand Up @@ -974,7 +972,7 @@
const out=runnerOut||{errors:[],notes:[]};
compiled={errors:out.errors||[],notes:out.notes||[]};
}else{compiled=C.compile(rec);}
$('#nb-localerrors').textContent=compiled.errors.filter(e=>e.step===step).map(e=>e.message).join(' ');
$('#nb-localerrors').textContent=compiled.errors.filter(e=>e.step===step&&!e.shown).map(e=>e.message).join(' ');
const errors=$('#nb-errors');
if(errors){
errors.hidden=!compiled.errors.length;
Expand Down Expand Up @@ -1120,7 +1118,7 @@
if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;}
if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;}
if(d.agent){const on=r().agents.includes(d.agent);r().agents=on?r().agents.filter(id=>id!==d.agent):[...r().agents,d.agent];renderForm();refreshPlan();return;}
else if(d.href){location.href=d.href;return;}
else if(d.href){if(!window.nulliusOpenSettings?.(d.href,b))location.href=d.href;return;}
else if(b.id==='nb-add-custom-agent'){r().customAgents.push('');r().agents.push('custom-'+(r().customAgents.length-1));renderForm();refreshPlan();root.querySelector('[data-custom-agent="'+(r().customAgents.length-1)+'"]')?.focus();return;}
else if(d.removeCustomAgent!==undefined){const i=Number(d.removeCustomAgent),rec=r(),kept=rec.agents.filter(id=>id.startsWith('custom-')&&id!=='custom-'+i).map(id=>Number(id.slice(7)));rec.customAgents.splice(i,1);rec.agents=rec.agents.filter(id=>!id.startsWith('custom-')).concat(kept.map(k=>'custom-'+(k>i?k-1:k)));renderForm();refreshPlan();return;}
// The Harbor panel (Runner step): a release or a repository, a ref of it, installed and asked for its agents.
Expand Down
18 changes: 11 additions & 7 deletions app/static/settings/modal.js
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,10 @@
proxy.remove();
});
});
document.addEventListener('click', event => {
const link = event.target.closest('a[href]');
if (!link || event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey || link.target || link.download) return;
const url = new URL(link.href, location.href);
if (url.origin !== location.origin || url.pathname !== '/settings' || location.pathname === '/settings') return;
event.preventDefault();
opener = link;
function open(href, from) {
const url = new URL(href, location.href);
if (url.origin !== location.origin || url.pathname !== '/settings' || location.pathname === '/settings') return false;
opener = from;
baseline = '';
closing = false;
closeButton.disabled = false;
Expand All @@ -87,5 +84,12 @@
frame.src = url.href;
modal.showModal();
document.getElementById('settings-modal-title').focus();
return true;
}
window.nulliusOpenSettings = open;
document.addEventListener('click', event => {
const link = event.target.closest('a[href]');
if (!link || event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey || link.target || link.download) return;
if (open(link.href, link)) event.preventDefault();
});
})();
16 changes: 7 additions & 9 deletions app/static/settings/sandbox.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// The Sandbox section: docker's state and whether a Boat key is saved, asked once when the section is first shown.
// The Sandbox section: whether a Boat key is saved, asked once when the section is first shown.
(() => {
const section = document.getElementById('sandbox');
if (!section) return;
Expand Down Expand Up @@ -26,13 +26,7 @@
const load = async () => {
asked = true;
let j;
try { j = await (await nulliusFetch(section.dataset.sandbox)).json(); } catch (err) { j = {docker: {ok: false, error: String(err)}, boat: {set: false, env: false}}; }
const docker = $('[data-docker-state]');
docker.textContent = j.docker.ok ? '✓ ' + j.docker.version : '✕ ' + j.docker.error;
docker.className = 'st-row-value ' + (j.docker.ok ? 'ok' : 'bad');
const also = j.docker.also, other = $('[data-docker-also]');
other.hidden = !also;
if (also) other.lastElementChild.textContent = also.ok ? also.said + ' is installed too' : also.name + ' is installed and does not answer: ' + also.said;
try { j = await (await nulliusFetch(section.dataset.sandbox)).json(); } catch (err) { j = {boat: {set: false, env: false}}; }
boat = j.boat || {set: false, env: false};
render();
};
Expand All @@ -56,7 +50,10 @@
section.querySelectorAll('button').forEach(b => { b.disabled = false; });
}
};
if (edit) edit.addEventListener('click', () => { editing = true; render(); form.querySelector('input').focus(); });
const open = () => { editing = true; render(); form.querySelector('input').focus(); };
if (edit) edit.addEventListener('click', open);
const fromHash = () => { if (form && location.hash === '#sandbox-boat-key') open(); };
window.addEventListener('hashchange', fromHash);
if (cancel) cancel.addEventListener('click', () => { form.reset(); editing = false; render(); edit.focus(); });
if (form) form.addEventListener('submit', e => { e.preventDefault(); change('save'); });
if (remove) remove.addEventListener('click', () => asking(removeAsk.hidden));
Expand All @@ -68,4 +65,5 @@
const ask = () => { if (!asked && !section.hidden) load(); };
new MutationObserver(ask).observe(section, {attributes: true, attributeFilter: ['hidden']});
ask();
fromHash();
})();
Loading