Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions app/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@

import argparse # noqa: E402

from werkzeug.serving import is_running_from_reloader # noqa: E402

if __package__: # the wheel's private web namespace
from .web.core import app
from .web import shared
Expand Down Expand Up @@ -64,6 +66,13 @@ def parser() -> argparse.ArgumentParser:

def main(argv=None) -> None:
args = parser().parse_args(argv)
# Runs started from the builder before this start: the ones still going
# are followed again (they run detached on their target), the rest listed.
# With --debug the reloader's watcher runs this too; the server it starts follows them.
try:
back = execution.restore() if not args.debug or is_running_from_reloader() else []
except execution.Followed as exc:
sys.exit(f"nullius: {exc}")

values = args.paths or [
p for p in os.environ.get("JOB_ROOTS", "").split(os.pathsep) if p
Expand All @@ -86,9 +95,6 @@ def main(argv=None) -> None:
print(f"Workspace {ws.name!r} ({ws.dir}) reopened")
else:
print("No job path given — the web page will ask for one, or offer a workspace.")
# Runs started from the builder before this start: the ones still going
# are followed again (they run detached on their target), the rest listed.
back = execution.restore()
for ex in back:
print(f"Run {ex.plan.name} picked up again on {ex.plan.host.label if ex.plan.host else 'this machine'} ({ex.state})")
shared.adopt_roots([ex.jobs], into=ex.workspace) # its job root back in the workspace it started in
Expand Down
38 changes: 35 additions & 3 deletions app/nullius/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,9 @@
asks: continue or stop. Every state change lands in ``runs/<name>/run.json``;
``restore`` reads those records when Nullius starts and reattaches to what
is still running, waits again where it was paused, and settles what ended
while nobody was looking. A record is checked against the part's files and
processes on the target before it is trusted (``_settle``): a part whose pid
while nobody was looking; one process at a time, by the runs root's lock.
A record is checked against the part's files and processes on the target
before it is trusted (``_settle``): a part whose pid
was never saved, but which was claimed or started, is followed by the pid it
wrote itself or by its processes - never started a second time; a record
from before a part's end was one save, naming the next part with the ended
Expand Down Expand Up @@ -109,6 +110,7 @@
from __future__ import annotations

import base64
import fcntl
import hashlib
import json
import os
Expand Down Expand Up @@ -143,6 +145,7 @@
LAUNCH_GRACE = 5.0 # seconds a restart waits for a part a launcher claimed to start (_settle)
UV_INSTALL = "curl -LsSf https://astral.sh/uv/install.sh | sh"
RECORD = "run.json"
LOCK = ".lock" # in the runs root, held by the one process that follows its runs
STANDARD = "standard" # the Nullius standard's runner key (standard/recipe.py HARNESS)
REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # this repository: app/ is in it
_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
Expand Down Expand Up @@ -1716,12 +1719,41 @@ def ended_for(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Opti
return ex.job_state(job_id) if ex is not None else None


class Followed(RuntimeError):
"""Another process follows the runs under this root."""


_FOLLOWED: Dict[str, object] = {} # runs root -> its lock, open for the rest of this process


def _follow(root: str) -> None:
"""Hold *root*'s lock for the rest of this process, or raise ``Followed``:
two processes following one run both resume it and both write its record.
The lock goes with the process, however it ends."""
key = os.path.realpath(root)
if key in _FOLLOWED:
return
try:
os.makedirs(key, exist_ok=True)
fh = open(os.path.join(key, LOCK), "a")
except OSError:
return # no runs root here: nothing to resume
try:
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
fh.close()
raise Followed(f"another Nullius follows the runs in {key}: stop it before starting this one") from None
_FOLLOWED[key] = fh


def restore(root: Optional[str] = None, **kw) -> List[Execution]:
"""At start-up: every recorded run under the root. The ones still alive
are reattached (their thread follows the part that was running, or
waits at the pause it was at); the finished ones are listed as they
ended. Returns the ones reattached."""
ended. Returns the ones reattached. ``Followed`` when another process
holds the root (``_follow``)."""
root = root or runs_root(kw.get("env", os.environ))
_follow(root)
back = []
try:
names = sorted(os.listdir(root))
Expand Down
25 changes: 25 additions & 0 deletions app/tests/test_execution_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import shutil
import subprocess
import sys
import tempfile
import threading
import time
Expand All @@ -16,6 +17,8 @@

from tests.execution_support import Base, HostBase, gone

APP = str(Path(__file__).resolve().parents[1])


class Restart(Base):
"""A run outlives Nullius: the record on disk, the part still running
Expand Down Expand Up @@ -53,6 +56,28 @@ def test_a_running_part_is_followed_again_after_a_restart(self):
text = "\n".join(again.lines)
self.assertIn("Nullius is back; picking the run up at slow run", text); self.assertIn("slow done", text); self.assertIn("next", (d / "run-next.log").read_text())

def test_a_second_process_on_one_runs_root_resumes_nothing(self):
"""Two Nullius started on one profile: one follows the run, the other
is refused, and the lock goes with the process that held it."""
launches = self.tmp / "launches"
parts = [{"id": "slow", "label": "slow", "script": "sleep 2; echo slow done"},
{"id": "next", "label": "next", "script": f"echo launched >> {launches}"}]
ex = execution.Execution(execution.Plan(name="r", script="", parts=parts), root=str(self.tmp)).start()
self.until(ex, "running"); self.recorded(self.tmp / "r", "pid")
self.abandon(ex) # the first Nullius is gone; the part runs on
child = ("import sys; from nullius import execution; execution.SYNC_EVERY = 0.05\n"
"try: back = execution.restore(sys.argv[1])\n"
"except execution.Followed: back = None\n"
"print(back and [e.plan.name for e in back], flush=True); [e.thread.join(20) for e in back or ()]")
kids = [subprocess.Popen([sys.executable, "-c", child, str(self.tmp)], env=dict(os.environ, PYTHONPATH=APP),
stdout=subprocess.PIPE, text=True) for _ in range(2)]
said = sorted(k.communicate(timeout=60)[0].strip() for k in kids)
log = (self.tmp / "r" / "nullius.log").read_text()
self.assertEqual(said, ["None", "['r']"], log) # one follows it, the other is refused
self.assertEqual(log.count("Nullius is back"), 1, log)
self.assertEqual(launches.read_text().split(), ["launched"])
self.assertEqual(execution.restore(str(self.tmp)), []) # not refused once that process is gone; the run has ended

def test_a_trial_line_is_said_once_across_a_restart(self):
"""The trials already said are part of the record: a restart says the
ones that end after it, not again the ones the log holds already."""
Expand Down
9 changes: 9 additions & 0 deletions app/ui_tests/test_run_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,15 @@ def test_a_workspace_whose_roots_are_gone_opens_as_the_open_action_does(self):
self.root.rmdir()
self.assertEqual(self.restart().roots, [str(self.root)])

def test_a_second_server_on_the_runs_folder_stops_with_one_line(self):
refused = execution.Followed("another Nullius follows the runs in /r: stop it before starting this one")
with patch.object(execution, "restore", side_effect=refused), self.assertRaises(SystemExit) as end:
self.restart()
self.assertEqual(end.exception.code, f"nullius: {refused}")
with patch.object(execution, "restore") as restore:
self.restart(["--debug"]) # the reloader's watcher: the server it starts follows the runs
restore.assert_not_called()

def test_job_paths_given_win(self):
workspace.create(self.home, ["/elsewhere"], label="witty pelican")
self.assertEqual(self.restart([str(self.root)]).roots, [str(self.root)])
Expand Down
2 changes: 1 addition & 1 deletion docs/builder.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ and Delete work only from a browser on the application host.
| Docker or Podman | The engine must answer `ps` (`docker ps`, or `podman ps` for the standard's container backend), or the run stops before it starts |
| Stop | Ends the current part (exit 130), records `stopped`; the next part never starts |
| Delete launch | On the launch in Runs, *By launch*, once every run of it has ended; asks first. The job page links to it. Deletes each run's `<RUNS_DIR>/<name>/` (a shard per machine included), every job of the launch, their judgements in every workspace, and each run's copy on its host when ticked. A run still going is refused: stop it first. A launch with jobs from elsewhere has no Delete |
| Restart of Nullius | `execution.restore` reattaches from `runs/<name>/run.json`; the job goes back into the workspace it started in, under the name a rename gave it, while that workspace exists, without opening it. The workspace open before the restart opens again |
| Restart of Nullius | `execution.restore` reattaches from `runs/<name>/run.json`; the job goes back into the workspace it started in, under the name a rename gave it, while that workspace exists, without opening it. The workspace open before the restart opens again. One process follows a runs folder: a second `nullius serve` on it stops with a message |
| Cannot run from the page | Generated tasks and custom agents are not supported by Run |
| Refused | Reads *refused* on the runs page, with why; the job was never written |
| Files | `<RUNS_DIR>/<name>/`: script, log, `run.json`, `jobs/`; default `runs/` under the user data directory (`nullius paths`) |
Expand Down
Loading