Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions agentdescent/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -813,9 +813,13 @@ def demo_payload(
# #1145: do not point at evolve agentdescent.{algo}.spec.json — that
# file is only written when `algorithms run` succeeds, and that
# command already prints `evolve` with the absolute path it wrote.
# #1201: `--out spec.json` gives the next two steps a real path; the
# cap is a placeholder because the footer has no estimate yet.
own_steps = [
f"agentdescent algorithms run {algo} {target} <your-cases.jsonl> "
f"--model <model> [--base-url <url>] --rounds 1 --workers 1",
f"--model <model> [--base-url <url>] --rounds 1 --workers 1 --out spec.json",
"agentdescent plan spec.json --probe",
"agentdescent evolve spec.json --max-calls <N>",
"agentdescent doctor",
]
elif used_kind == "text":
Expand Down Expand Up @@ -4871,7 +4875,18 @@ def algorithms_run_payload(
obj["warnings"] = list(payload.get("warnings") or []) + [
t for k, t in report if k == "warning"]
obj["notes"] = [t for k, t in report if k == "note"]
obj["next"] = f"agentdescent evolve {out}" if ok else retry_line
if ok:
# #1201: the line a user follows names the preflight and the per-call
# cap `docs/start.md` documents, at the plan's own upper bound -- the
# bare `evolve <spec>` was the one path that named neither.
calls = (payload.get("estimate") or {}).get("agent_calls_upper_bound")
cap = (f" --max-calls {int(calls)}"
if isinstance(calls, (int, float)) and calls > 0
else " --max-calls <N>")
obj["next"] = (f"agentdescent plan {out} --probe, then "
f"agentdescent evolve {out}{cap}")
else:
obj["next"] = retry_line
obj["_stage"] = "done"
obj["_report"] = report
obj["_plan"] = payload
Expand Down
7 changes: 7 additions & 0 deletions agentdescent/integrations/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,13 @@ after `doctor`. Do not invent a keyless path for those cards.
call `start` -- a refused probe is the run failing in round one, found one
step earlier. A refusal prints `ok: false` and `ready: false` on both
surfaces, so the same check reads the same answer whichever one you drive.
Give the first real run a **per-call cap** too: `max_calls` (and `max_usd`
with `usd_per_call` when a price is known) on the MCP `plan` / `start` call,
or `--max-calls N` (`--max-usd` beside `--usd-per-call`) on the CLI. The call
that would cross the cap is never made, so one round cannot outspend it;
start at the plan's `agent_calls_upper_bound`, smaller for a trial.
`--budget` is the older dollar bound: it is checked between rounds and can
overshoot by a whole round.
To keep improving across sessions, offer `agentdescent campaign <spec>
--sessions N`; it works on a copy, reports the workspace/final target and
never touches the live target. Run its `--dry-run` first. A campaign has no
Expand Down
12 changes: 12 additions & 0 deletions changelog.d/1201-first-run-caps.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
Added
The first-run surfaces now name the preflight and the per-call cap that
`docs/start.md` already documents: the shared host skill (and its Claude Code
and DSH renditions) tells a host to pass `max_calls` / `max_usd` on MCP
`plan` / `start` or `--max-calls` / `--max-usd` on the CLI, the
plugin-quickstart habits and the `docs/plugins.md` budget section point at the
same cap beside the older `--budget` overshoot warning, and the `next:` line
from `algorithms run` now reads `plan <spec> --probe, then evolve <spec>
--max-calls N` with N from the plan's own `agent_calls_upper_bound`. The
`demo --algorithm` footer gains the same two steps (`<N>` there: the footer
runs before a spec exists) and `--out spec.json`, so the steps have a path
(#1201).
6 changes: 6 additions & 0 deletions docs/plugin-quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,12 @@ Two habits worth keeping from the first run:
prints the same pair, so one check works on both surfaces.
CLI agents are reported as `probe: … skipped (cli agent)` because one
call would cost a whole session.
* `--max-calls N` on that same first run (`--max-usd` beside `--usd-per-call`
when a price is known; MCP `plan` / `start` take `max_calls` / `max_usd`).
This one holds **per call**: the call that would cross the number is never
made, so the first round cannot outspend it. Start at the plan's
`agent_calls_upper_bound`, or smaller for a trial. `--budget` below is the
weaker bound, not a replacement.
* `--budget 5 --usd-per-call 0.02` on the first real run. A dollar budget
needs a per-call price (the engine counts calls; a tool-using agent's price
is not known in advance). The run stops **between rounds** once
Expand Down
8 changes: 8 additions & 0 deletions docs/plugins.md
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,14 @@ One round (plus the seed held-out scan) can overshoot by up to its call bound;
`plan` / `evolve` / MCP `start` warn when the budget is already below that
bound so the first round alone can blow past it.

The tighter bound is the **per-call cap** from
[Get started](start.md#real-model): `--max-calls N` on `evolve` (`--max-usd`
beside `--usd-per-call` when a price is known), or `max_calls` / `max_usd` on
MCP `plan` / `start`. It refuses the call that would cross the number instead
of stopping between rounds, so it cannot overshoot; a first trial can use the
plan's `agent_calls_upper_bound` or less. `--budget` remains the dollar bound
for a run whose price per call is known up front.

## Evolving the plugins themselves

A host **plugin** — the DSH Cordis package that adds tools and skills to `dsh`,
Expand Down
7 changes: 7 additions & 0 deletions integrations/claude-code/skills/agentdescent/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,13 @@ after `doctor`. Do not invent a keyless path for those cards.
call `start` -- a refused probe is the run failing in round one, found one
step earlier. A refusal prints `ok: false` and `ready: false` on both
surfaces, so the same check reads the same answer whichever one you drive.
Give the first real run a **per-call cap** too: `max_calls` (and `max_usd`
with `usd_per_call` when a price is known) on the MCP `plan` / `start` call,
or `--max-calls N` (`--max-usd` beside `--usd-per-call`) on the CLI. The call
that would cross the cap is never made, so one round cannot outspend it;
start at the plan's `agent_calls_upper_bound`, smaller for a trial.
`--budget` is the older dollar bound: it is checked between rounds and can
overshoot by a whole round.
To keep improving across sessions, offer `agentdescent campaign <spec>
--sessions N`; it works on a copy, reports the workspace/final target and
never touches the live target. Run its `--dry-run` first. A campaign has no
Expand Down
2 changes: 1 addition & 1 deletion integrations/dsh-agentdescent/lib/index.js

Large diffs are not rendered by default.

18 changes: 14 additions & 4 deletions tests/test_cli_algorithms_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,11 @@ def test_algorithms_run_writes_spec_and_plans(tmp_path, capsys, monkeypatch):
]) == 0
captured = capsys.readouterr()
assert f"wrote {out}" in captured.out
assert "next: agentdescent evolve" in captured.out
# #1201: the next step is the preflight plus the per-call cap, not a bare
# `evolve` that names neither.
assert "next: agentdescent plan" in captured.out
assert "--probe" in captured.out
assert "--max-calls" in captured.out
assert str(out) in captured.out
spec = json.loads(out.read_text())
assert spec["algorithm"] == "gepa"
Expand All @@ -49,7 +53,9 @@ def test_algorithms_run_default_out_path(tmp_path, capsys, monkeypatch):
out = tmp_path / "agentdescent.gepa.spec.json"
assert out.exists()
assert json.loads(out.read_text())["algorithm"] == "gepa"
assert f"next: agentdescent evolve {out.name}" in capsys.readouterr().out
printed = capsys.readouterr().out
assert f"next: agentdescent plan {out.name} --probe" in printed
assert f"then agentdescent evolve {out.name} --max-calls " in printed


def test_algorithms_run_json_one_object(tmp_path, capsys, monkeypatch):
Expand All @@ -65,7 +71,9 @@ def test_algorithms_run_json_one_object(tmp_path, capsys, monkeypatch):
assert payload["ok"] is True
assert payload["spec_path"] == str(out)
assert payload["algorithm"]["name"] == "gepa"
assert payload["next"] == f"agentdescent evolve {out}"
nxt = payload["next"]
assert nxt.startswith(f"agentdescent plan {out} --probe, then ")
assert f"agentdescent evolve {out} --max-calls " in nxt
assert payload["tasks"] >= 4


Expand Down Expand Up @@ -414,7 +422,9 @@ def test_algorithms_run_resolves_a_case_variant_to_the_registered_name(
out = tmp_path / "agentdescent.gepa.spec.json"
assert out.exists()
assert json.loads(out.read_text())["algorithm"] == "gepa"
assert f"next: agentdescent evolve {out.name}" in capsys.readouterr().out
printed = capsys.readouterr().out
assert f"next: agentdescent plan {out.name} --probe" in printed
assert f"then agentdescent evolve {out.name} --max-calls " in printed


def test_algorithms_run_plan_failure_keeps_backup_note(tmp_path, capsys, monkeypatch):
Expand Down
7 changes: 6 additions & 1 deletion tests/test_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,12 @@ def test_demo_algorithm_next_steps_keep_the_algorithm(tmp_path, monkeypatch, nam
assert "agentdescent doctor" in out
# #1145: that spec is not on disk yet; algorithms run prints evolve later.
assert f"agentdescent evolve agentdescent.{name}.spec.json" not in out
assert "--out spec.json" not in out
# #1201: the demo does not claim the spec exists -- `--out spec.json` is
# where the user's own `algorithms run` above puts it, and the two steps
# that consume it name the preflight and the per-call cap.
assert "--out spec.json" in out
assert "agentdescent plan spec.json --probe" in out
assert "--max-calls" in out

buf = io.StringIO()
with redirect_stdout(buf):
Expand Down
123 changes: 123 additions & 0 deletions tests/test_first_run_caps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
"""The first-run surfaces name the probe and the per-call cap (#1201).

#1166 / #1184 added the per-call cap (`agentdescent evolve --max-calls` /
`--max-usd`), and `docs/start.md` documents it. The surfaces a new user or host
agent actually follows did not name it: the shared host skill, the
plugin-quickstart habits, the budget paragraph in `docs/plugins.md`, and the
`next:` line `algorithms run` / `demo --algorithm` print. A first real run is
started from those lines, so the preflight and the cap have to be readable
there, together with the `--budget` overshoot warning they do not replace.

The rendered plugin copies are not compared here: `tests/test_integrations.py`
re-renders and diffs them already, and these rows only pin the words a reader
sees.
"""

import io
import json
import os
import re
from contextlib import redirect_stdout

import pytest

from agentdescent import cli
from agentdescent.integrations import skill_text

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ALGORITHMS = ("gepa", "archive_selection")


def _read(*parts):
with open(os.path.join(ROOT, *parts), encoding="utf-8") as fh:
return fh.read()


def _run(argv):
buf = io.StringIO()
with redirect_stdout(buf):
code = cli.main(argv)
return code, buf.getvalue()


def _prompt_and_cases(tmp_path):
prompt = tmp_path / "prompt.txt"
prompt.write_text("Answer the question.\n", encoding="utf-8")
cases = tmp_path / "cases.jsonl"
cases.write_text("".join(
json.dumps({"prompt": f"q{i}", "gold": f"a{i}"}) + "\n" for i in range(4)),
encoding="utf-8")
return prompt, cases


def test_shared_host_skill_names_the_per_call_cap():
text = skill_text()
assert "--max-calls" in text
assert "--max-usd" in text
assert "--probe" in text # the preflight stays the first step


def test_plugin_quickstart_habits_name_the_per_call_cap():
text = _read("docs", "plugin-quickstart.md")
assert "--max-calls" in text
assert "--max-usd" in text
assert "overshoot" in text # the weaker cap keeps its warning


def test_plugins_budget_paragraph_names_the_per_call_cap():
text = _read("docs", "plugins.md")
budget = text.split("A dollar budget", 1)[1].split("\n## ", 1)[0]
assert "--max-calls" in budget
assert "overshoot" in budget


@pytest.mark.parametrize("name", ALGORITHMS)
def test_algorithms_run_next_names_the_probe_then_the_cap(tmp_path, monkeypatch, name):
monkeypatch.chdir(tmp_path)
prompt, cases = _prompt_and_cases(tmp_path)
out = tmp_path / f"{name}.spec.json"
code, text = _run([
"algorithms", "run", name, str(prompt), str(cases),
"--model", "m", "--base-url", "https://example.test/v1",
"--rounds", "1", "--workers", "1", "--out", str(out),
])
assert code == 0, text
nxt = [line for line in text.splitlines() if line.startswith("next:")]
assert nxt, text
line = nxt[-1]
assert "plan" in line and "--probe" in line
# The cap is the plan's own upper bound, not a placeholder.
bound = re.search(r"up to (\d+) agent calls", text)
assert bound, text
assert f"--max-calls {bound.group(1)}" in line

code, text = _run([
"--json", "algorithms", "run", name, str(prompt), str(cases),
"--model", "m", "--base-url", "https://example.test/v1",
"--rounds", "1", "--workers", "1", "--out", str(out),
])
assert code == 0, text
payload = json.loads(text)
assert "--probe" in payload["next"]
assert f"--max-calls {bound.group(1)}" in payload["next"]


@pytest.mark.parametrize("name", ALGORITHMS)
def test_demo_algorithm_footer_names_the_probe_then_the_cap(tmp_path, monkeypatch, name):
monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home"))
code, text = _run(["--store", str(tmp_path / "runs"), "demo",
"--dir", str(tmp_path / "work"), "--algorithm", name])
assert code == 0, text
assert f"agentdescent algorithms run {name} " in text
assert "--out spec.json" in text
assert "agentdescent plan spec.json --probe" in text
assert "agentdescent evolve spec.json --max-calls <N>" in text
assert "agentdescent doctor" in text

code, text = _run(["--store", str(tmp_path / "runs2"), "demo",
"--dir", str(tmp_path / "work2"), "--algorithm", name,
"--json"])
assert code == 0, text
steps = json.loads(text)["next"]
assert "agentdescent plan spec.json --probe" in steps
assert "agentdescent evolve spec.json --max-calls <N>" in steps
5 changes: 4 additions & 1 deletion tests/test_plan_text_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,10 @@ def test_algorithms_run_plain_ends_with_the_human_form(tmp_path, capsys, monkeyp
json.loads(out)
assert f"wrote {out_path}" in out
assert "plan: 8 tasks (2 held out), 1 round x 1 worker, up to 11 agent calls" in out
assert f"next: agentdescent evolve {out_path}" in out
# #1201: the preflight and the per-call cap on the line the user follows,
# at the plan's own bound (11 calls above).
assert f"next: agentdescent plan {out_path} --probe, then " in out
assert f"agentdescent evolve {out_path} --max-calls 11" in out
assert "warning: the held-out set has only 2 task(s)" in err
assert not (tmp_path / ".agentdescent" / "runs").exists()

Expand Down
Loading