diff --git a/api/app/autopilot.py b/api/app/autopilot.py
index 65f1847..9868dd7 100644
--- a/api/app/autopilot.py
+++ b/api/app/autopilot.py
@@ -387,6 +387,12 @@ async def decide(filled: dict) -> bool:
_keep_screenshot(run, "confirm", final.get("confirm_screenshot") or "")
if final.get("submitted"):
outcome, reasons = "submitted", []
+ elif final.get("code_asked"):
+ # The site emailed the candidate a code to prove a person is applying (Catapult, on Greenhouse: "enter the
+ # 8-character code to confirm you're a human"). That check is theirs to pass: nothing went, so it waits.
+ outcome, reasons = "needs_you", [f"The job site emailed you a code to confirm a person is applying, so this "
+ f"wasn't sent. It says: “{final['code_asked']}” Apply on the job's page "
+ f"with this run's CV and cover letter."[:600]]
elif final.get("submit_attempted"):
detail = final.get("submit_detail") or "no confirmation shown"
outcome, reasons = "unconfirmed", [f"The submit button was clicked, but the page didn't confirm it ({detail}). "
@@ -409,6 +415,8 @@ async def decide(filled: dict) -> bool:
if app_id:
note = {"submitted": "Sent by autopilot.", "unconfirmed": "Autopilot clicked submit but saw no confirmation.",
"needs_you": "Autopilot filled this in and held it for you."}[outcome]
+ if final.get("code_asked"):
+ note = "Autopilot filled this in, but the job site asked for a code it emailed you: apply on its page."
tracker.update_status(app_id, status, note, user_id=user_id)
tracker.set_application_questions(app_id, questions, user_id=user_id)
reused = (f"Used the CV and cover letter from the {'practice ' if earlier['practice'] else ''}run of "
diff --git a/api/app/browser_agent.py b/api/app/browser_agent.py
index 660aacf..e4551b1 100644
--- a/api/app/browser_agent.py
+++ b/api/app/browser_agent.py
@@ -1319,6 +1319,39 @@ async def _empty_required_fields(browser_session) -> list[str]:
}"""
+# After a submit the page didn't confirm: the site's request for a code it emailed the candidate (Greenhouse: "A
+# verification code was sent to … enter the 8-character code to confirm you're a human"). The line that says it was
+# sent, else one that asks for such a code; "" when there's neither.
+_CODE_ASKED_JS = r"""() => {
+ const docs = [document];
+ for (const frame of document.querySelectorAll('iframe')) {
+ try { if (frame.contentDocument) docs.push(frame.contentDocument); } catch (e) {}
+ }
+ const sent = /\bcode\b[^.]*\b(sent|emailed)\b|\b(sent|emailed)\b[^.]*\bcode\b/i;
+ const asks = /\b(verification|security|confirmation|one[- ]time) code\b|confirm you'?re (a )?human/i;
+ let found = '';
+ for (const doc of docs) {
+ const text = (doc.body && doc.body.innerText) || '';
+ for (const line of text.split(/\n+/).map(l => l.trim()).filter(Boolean)) {
+ if (sent.test(line)) return JSON.stringify(line.slice(0, 300));
+ if (!found && asks.test(line)) found = line.slice(0, 300);
+ }
+ }
+ return JSON.stringify(found);
+}"""
+
+
+async def _code_asked(browser_session) -> str:
+ """The page's request for a code the site emailed the candidate (see _CODE_ASKED_JS); "" if none or unreadable."""
+ try:
+ page, _ = await _focused_page(browser_session)
+ asked = json.loads(await page.evaluate(_CODE_ASKED_JS))
+ return asked if isinstance(asked, str) else ""
+ except Exception as exc:
+ logger.debug("code check failed: %s", exc)
+ return ""
+
+
async def _submit_button_shown(browser_session) -> bool:
"""Whether the focused page shows the button that sends the application; False if it can't be read."""
try:
@@ -1486,7 +1519,8 @@ async def _finish_unattended(decide: Decider, filled: dict, browser_session, llm
"""
Send the filled form if `decide` says to, else leave it. The last event is done=True and carries the
fill's results plus `form_screenshot` (the filled form, before anything is sent); when the submit agent
- ran, also `submit_attempted`, `submitted` (only if the page confirmed it), `submit_detail` and
+ ran, also `submit_attempted`, `submitted` (only if the page confirmed it), `submit_detail`, `code_asked` (the
+ page's request for a code the site emailed the candidate, when it made one instead of confirming) and
`confirm_screenshot` (the page after the click).
"""
final = {"action": "Filled in; not sent", "screenshot": None, "blocked": False, "reason": None, "done": True,
@@ -1501,9 +1535,11 @@ async def _finish_unattended(decide: Decider, filled: dict, browser_session, llm
yield {"action": "Submitting the application…", "screenshot": None, "blocked": False, "reason": None,
"done": False}
ok, detail = await _submit_with_agent(llm, browser_session, sensitive_data, secret)
- final.update(submit_attempted=True, submitted=ok, submit_detail=detail,
+ code = "" if ok else _redact(await _code_asked(browser_session), secret)
+ final.update(submit_attempted=True, submitted=ok, submit_detail=detail, code_asked=code,
confirm_screenshot=await _screenshot_b64(browser_session),
- action="Application submitted" if ok else "Couldn't confirm the submission")
+ action="Application submitted" if ok else
+ "The site asked for a code it emailed you" if code else "Couldn't confirm the submission")
yield final
@@ -1651,8 +1687,13 @@ async def on_step(browser_state, agent_output, step_number):
notes = _with_empty_fields(summary, empty, left_blank, self_answered)[:_MAX_NOTES_CHARS]
if decide is not None:
- filled = {"ready": ready, "error": agent_error, "notes": notes, "questions": questions, "empty": empty,
- "self_answered": self_answered}
+ # The agent's summary is its own account, written as it stopped, and can disagree with the form as it ends
+ # up: Catapult's said the consent box was left unticked, yet the form showed it ticked from the saved
+ # answer. Unattended, nobody checks it against the page, so the notes keep what the final form shows; the
+ # summary stays only when the agent couldn't finish, to say why.
+ checked = _with_empty_fields("" if ready else summary, empty, left_blank, self_answered)
+ filled = {"ready": ready, "error": agent_error, "notes": checked[:_MAX_NOTES_CHARS],
+ "questions": questions, "empty": empty, "self_answered": self_answered}
async for step in _finish_unattended(decide, filled, browser_session, llm, sensitive_data, job_password):
yield step
return
diff --git a/api/tests/browser/site/code.html b/api/tests/browser/site/code.html
new file mode 100644
index 0000000..4196599
--- /dev/null
+++ b/api/tests/browser/site/code.html
@@ -0,0 +1,13 @@
+
+Test Application Form, after submit
+
+
+
+
diff --git a/api/tests/browser/test_browser_agent.py b/api/tests/browser/test_browser_agent.py
index 028ce0e..92c577c 100644
--- a/api/tests/browser/test_browser_agent.py
+++ b/api/tests/browser/test_browser_agent.py
@@ -480,8 +480,8 @@ async def flow():
llm, shown, events = run(send=True)
final = events[-1]
assert not any(e.get("interactive") for e in events) # nobody is asked anything
- assert shown == [{"ready": True, "error": "", "notes": "Filled every field", "questions": [], "empty": [],
- "self_answered": []}]
+ # A filled form's notes are what Windrush checked on it, not the agent's own summary ("Filled every field").
+ assert shown == [{"ready": True, "error": "", "notes": "", "questions": [], "empty": [], "self_answered": []}]
assert llm.tasks == ["fill", "submit"]
assert final["done"] and final["submitted"] and final["submit_attempted"]
assert _is_jpeg(final["form_screenshot"]) and _is_jpeg(final["confirm_screenshot"])
@@ -494,6 +494,25 @@ async def flow():
assert_no_chromium_left()
+@pytest.mark.parametrize("page, asked", [
+ ("code.html", "A verification code was sent to ada@example.com. To submit your application, enter the "
+ "8-character code to confirm you're a human."),
+ ("form.html", ""), # unconfirmed, but no code asked for
+])
+def test_a_submit_the_site_answers_with_an_emailed_code_is_reported(ba, site, monkeypatch, page, asked):
+ llm = ScriptedLLM(submit_success=False)
+ monkeypatch.setattr(ba, "_make_browser_llm", lambda: llm)
+
+ async def flow():
+ return [event async for event in ba.apply_with_browser(
+ f"{site}/{page}", {"core_info": {"first_name": "Ada"}}, "", asyncio.Queue(),
+ decide=lambda filled: asyncio.sleep(0, result=True))]
+ final = asyncio.run(flow())[-1]
+ assert final["submit_attempted"] and not final["submitted"] and final["code_asked"] == asked
+ assert final["action"] == ("The site asked for a code it emailed you" if asked else "Couldn't confirm the submission")
+ assert_no_chromium_left()
+
+
class GreenhouseLLM(ScriptedLLM):
"""Fills greenhouse.html the way the task asks: applicant details with fill_fields, questions through
answer_questions with their indexes, then tries to fill a question that was left for the candidate."""
diff --git a/api/tests/test_autopilot.py b/api/tests/test_autopilot.py
index 042fef1..df8c604 100644
--- a/api/tests/test_autopilot.py
+++ b/api/tests/test_autopilot.py
@@ -185,7 +185,7 @@ async def apply_with_browser(job_url, persona, cover_letter, queue, frames, *, j
w.decided[company] = send
if send:
final.update(submit_attempted=True, submitted=form["confirm"], submit_detail="Thanks for applying",
- confirm_screenshot=JPEG)
+ confirm_screenshot=JPEG, code_asked=form.get("code", ""))
yield final
monkeypatch.setattr(agent, "execute_tool", execute_tool)
@@ -271,6 +271,18 @@ def test_a_click_the_page_doesnt_confirm_is_not_counted_as_applied(ada, world):
assert autopilot.sent_last_day(ada) == 3 # still counts toward the limit
+def test_a_site_that_emails_a_code_holds_the_application_for_the_user(ada, world):
+ asked = ("A verification code was sent to ada@example.com. To submit your application, enter the 8-character "
+ "code to confirm you're a human.")
+ world.forms["alpha"].update(confirm=False, code=asked)
+ _run(ada, live=True)
+ run = _runs(ada)["alpha"]
+ assert run["outcome"] == "needs_you" and "wasn't sent" in run["reasons"][0] and asked in run["reasons"][0]
+ app = {a["company"]: a for a in tracker.list_applications(ada)}["alpha"]
+ assert app["status"] == "Pending Review" and "asked for a code it emailed you" in app["notes"]
+ assert autopilot.sent_last_day(ada) == 2 # beta and gamma: alpha didn't go
+
+
def test_poor_fits_and_stretches_are_skipped_without_filling(ada, world):
world.fit["alpha"] = (60, "strong")
world.fit["beta"] = (95, "reach")