diff --git a/app/app.py b/app/app.py index dd8bf3e2..2200af07 100644 --- a/app/app.py +++ b/app/app.py @@ -25,6 +25,8 @@ import argparse # noqa: E402 +from werkzeug.serving import is_running_from_reloader # noqa: E402 + if __package__: # the wheel's private web namespace from .web.core import app from .web import shared @@ -64,6 +66,13 @@ def parser() -> argparse.ArgumentParser: def main(argv=None) -> None: args = parser().parse_args(argv) + # Runs started from the builder before this start: the ones still going + # are followed again (they run detached on their target), the rest listed. + # With --debug the reloader's watcher runs this too; the server it starts follows them. + try: + back = execution.restore() if not args.debug or is_running_from_reloader() else [] + except execution.Followed as exc: + sys.exit(f"nullius: {exc}") values = args.paths or [ p for p in os.environ.get("JOB_ROOTS", "").split(os.pathsep) if p @@ -86,9 +95,6 @@ def main(argv=None) -> None: print(f"Workspace {ws.name!r} ({ws.dir}) reopened") else: print("No job path given — the web page will ask for one, or offer a workspace.") - # Runs started from the builder before this start: the ones still going - # are followed again (they run detached on their target), the rest listed. - back = execution.restore() for ex in back: print(f"Run {ex.plan.name} picked up again on {ex.plan.host.label if ex.plan.host else 'this machine'} ({ex.state})") shared.adopt_roots([ex.jobs], into=ex.workspace) # its job root back in the workspace it started in diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 4998f745..02527fc2 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -53,8 +53,9 @@ asks: continue or stop. Every state change lands in ``runs//run.json``; ``restore`` reads those records when Nullius starts and reattaches to what is still running, waits again where it was paused, and settles what ended -while nobody was looking. A record is checked against the part's files and -processes on the target before it is trusted (``_settle``): a part whose pid +while nobody was looking; one process at a time, by the runs root's lock. +A record is checked against the part's files and processes on the target +before it is trusted (``_settle``): a part whose pid was never saved, but which was claimed or started, is followed by the pid it wrote itself or by its processes - never started a second time; a record from before a part's end was one save, naming the next part with the ended @@ -109,6 +110,7 @@ from __future__ import annotations import base64 +import fcntl import hashlib import json import os @@ -143,6 +145,7 @@ LAUNCH_GRACE = 5.0 # seconds a restart waits for a part a launcher claimed to start (_settle) UV_INSTALL = "curl -LsSf https://astral.sh/uv/install.sh | sh" RECORD = "run.json" +LOCK = ".lock" # in the runs root, held by the one process that follows its runs STANDARD = "standard" # the Nullius standard's runner key (standard/recipe.py HARNESS) REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # this repository: app/ is in it _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$") @@ -1716,12 +1719,41 @@ def ended_for(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Opti return ex.job_state(job_id) if ex is not None else None +class Followed(RuntimeError): + """Another process follows the runs under this root.""" + + +_FOLLOWED: Dict[str, object] = {} # runs root -> its lock, open for the rest of this process + + +def _follow(root: str) -> None: + """Hold *root*'s lock for the rest of this process, or raise ``Followed``: + two processes following one run both resume it and both write its record. + The lock goes with the process, however it ends.""" + key = os.path.realpath(root) + if key in _FOLLOWED: + return + try: + os.makedirs(key, exist_ok=True) + fh = open(os.path.join(key, LOCK), "a") + except OSError: + return # no runs root here: nothing to resume + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + fh.close() + raise Followed(f"another Nullius follows the runs in {key}: stop it before starting this one") from None + _FOLLOWED[key] = fh + + def restore(root: Optional[str] = None, **kw) -> List[Execution]: """At start-up: every recorded run under the root. The ones still alive are reattached (their thread follows the part that was running, or waits at the pause it was at); the finished ones are listed as they - ended. Returns the ones reattached.""" + ended. Returns the ones reattached. ``Followed`` when another process + holds the root (``_follow``).""" root = root or runs_root(kw.get("env", os.environ)) + _follow(root) back = [] try: names = sorted(os.listdir(root)) diff --git a/app/tests/test_execution_recovery.py b/app/tests/test_execution_recovery.py index b5ec5db6..c3b35ba7 100644 --- a/app/tests/test_execution_recovery.py +++ b/app/tests/test_execution_recovery.py @@ -5,6 +5,7 @@ import os import shutil import subprocess +import sys import tempfile import threading import time @@ -16,6 +17,8 @@ from tests.execution_support import Base, HostBase, gone +APP = str(Path(__file__).resolve().parents[1]) + class Restart(Base): """A run outlives Nullius: the record on disk, the part still running @@ -53,6 +56,28 @@ def test_a_running_part_is_followed_again_after_a_restart(self): text = "\n".join(again.lines) self.assertIn("Nullius is back; picking the run up at slow run", text); self.assertIn("slow done", text); self.assertIn("next", (d / "run-next.log").read_text()) + def test_a_second_process_on_one_runs_root_resumes_nothing(self): + """Two Nullius started on one profile: one follows the run, the other + is refused, and the lock goes with the process that held it.""" + launches = self.tmp / "launches" + parts = [{"id": "slow", "label": "slow", "script": "sleep 2; echo slow done"}, + {"id": "next", "label": "next", "script": f"echo launched >> {launches}"}] + ex = execution.Execution(execution.Plan(name="r", script="", parts=parts), root=str(self.tmp)).start() + self.until(ex, "running"); self.recorded(self.tmp / "r", "pid") + self.abandon(ex) # the first Nullius is gone; the part runs on + child = ("import sys; from nullius import execution; execution.SYNC_EVERY = 0.05\n" + "try: back = execution.restore(sys.argv[1])\n" + "except execution.Followed: back = None\n" + "print(back and [e.plan.name for e in back], flush=True); [e.thread.join(20) for e in back or ()]") + kids = [subprocess.Popen([sys.executable, "-c", child, str(self.tmp)], env=dict(os.environ, PYTHONPATH=APP), + stdout=subprocess.PIPE, text=True) for _ in range(2)] + said = sorted(k.communicate(timeout=60)[0].strip() for k in kids) + log = (self.tmp / "r" / "nullius.log").read_text() + self.assertEqual(said, ["None", "['r']"], log) # one follows it, the other is refused + self.assertEqual(log.count("Nullius is back"), 1, log) + self.assertEqual(launches.read_text().split(), ["launched"]) + self.assertEqual(execution.restore(str(self.tmp)), []) # not refused once that process is gone; the run has ended + def test_a_trial_line_is_said_once_across_a_restart(self): """The trials already said are part of the record: a restart says the ones that end after it, not again the ones the log holds already.""" diff --git a/app/ui_tests/test_run_routes.py b/app/ui_tests/test_run_routes.py index 8db8447c..d719d81d 100644 --- a/app/ui_tests/test_run_routes.py +++ b/app/ui_tests/test_run_routes.py @@ -387,6 +387,15 @@ def test_a_workspace_whose_roots_are_gone_opens_as_the_open_action_does(self): self.root.rmdir() self.assertEqual(self.restart().roots, [str(self.root)]) + def test_a_second_server_on_the_runs_folder_stops_with_one_line(self): + refused = execution.Followed("another Nullius follows the runs in /r: stop it before starting this one") + with patch.object(execution, "restore", side_effect=refused), self.assertRaises(SystemExit) as end: + self.restart() + self.assertEqual(end.exception.code, f"nullius: {refused}") + with patch.object(execution, "restore") as restore: + self.restart(["--debug"]) # the reloader's watcher: the server it starts follows the runs + restore.assert_not_called() + def test_job_paths_given_win(self): workspace.create(self.home, ["/elsewhere"], label="witty pelican") self.assertEqual(self.restart([str(self.root)]).roots, [str(self.root)]) diff --git a/docs/builder.md b/docs/builder.md index d049cef3..56bd7da9 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -116,7 +116,7 @@ and Delete work only from a browser on the application host. | Docker or Podman | The engine must answer `ps` (`docker ps`, or `podman ps` for the standard's container backend), or the run stops before it starts | | Stop | Ends the current part (exit 130), records `stopped`; the next part never starts | | Delete launch | On the launch in Runs, *By launch*, once every run of it has ended; asks first. The job page links to it. Deletes each run's `//` (a shard per machine included), every job of the launch, their judgements in every workspace, and each run's copy on its host when ticked. A run still going is refused: stop it first. A launch with jobs from elsewhere has no Delete | -| Restart of Nullius | `execution.restore` reattaches from `runs//run.json`; the job goes back into the workspace it started in, under the name a rename gave it, while that workspace exists, without opening it. The workspace open before the restart opens again | +| Restart of Nullius | `execution.restore` reattaches from `runs//run.json`; the job goes back into the workspace it started in, under the name a rename gave it, while that workspace exists, without opening it. The workspace open before the restart opens again. One process follows a runs folder: a second `nullius serve` on it stops with a message | | Cannot run from the page | Generated tasks and custom agents are not supported by Run | | Refused | Reads *refused* on the runs page, with why; the job was never written | | Files | `//`: script, log, `run.json`, `jobs/`; default `runs/` under the user data directory (`nullius paths`) |