fix: a site that emails a code holds the application, and the run note matches the form

api/app/autopilot.py

@@ -387,6 +387,12 @@ async def decide(filled: dict) -> bool:
     _keep_screenshot(run, "confirm", final.get("confirm_screenshot") or "")
     if final.get("submitted"):
         outcome, reasons = "submitted", []
+    elif final.get("code_asked"):
+        # The site emailed the candidate a code to prove a person is applying (Catapult, on Greenhouse: "enter the
+        # 8-character code to confirm you're a human"). That check is theirs to pass: nothing went, so it waits.
+        outcome, reasons = "needs_you", [f"The job site emailed you a code to confirm a person is applying, so this "
+                                         f"wasn't sent. It says: “{final['code_asked']}” Apply on the job's page "
+                                         f"with this run's CV and cover letter."[:600]]
     elif final.get("submit_attempted"):
         detail = final.get("submit_detail") or "no confirmation shown"
         outcome, reasons = "unconfirmed", [f"The submit button was clicked, but the page didn't confirm it ({detail}). "
@@ -409,6 +415,8 @@ async def decide(filled: dict) -> bool:
         if app_id:
             note = {"submitted": "Sent by autopilot.", "unconfirmed": "Autopilot clicked submit but saw no confirmation.",
                     "needs_you": "Autopilot filled this in and held it for you."}[outcome]
+            if final.get("code_asked"):
+                note = "Autopilot filled this in, but the job site asked for a code it emailed you: apply on its page."
             tracker.update_status(app_id, status, note, user_id=user_id)
             tracker.set_application_questions(app_id, questions, user_id=user_id)
     reused = (f"Used the CV and cover letter from the {'practice ' if earlier['practice'] else ''}run of "

api/app/browser_agent.py

@@ -1319,6 +1319,39 @@ async def _empty_required_fields(browser_session) -> list[str]:
 }"""
 
 
+# After a submit the page didn't confirm: the site's request for a code it emailed the candidate (Greenhouse: "A
+# verification code was sent to … enter the 8-character code to confirm you're a human"). The line that says it was
+# sent, else one that asks for such a code; "" when there's neither.
+_CODE_ASKED_JS = r"""() => {
+  const docs = [document];
+  for (const frame of document.querySelectorAll('iframe')) {
+    try { if (frame.contentDocument) docs.push(frame.contentDocument); } catch (e) {}
+  }
+  const sent = /\bcode\b[^.]*\b(sent|emailed)\b|\b(sent|emailed)\b[^.]*\bcode\b/i;
+  const asks = /\b(verification|security|confirmation|one[- ]time) code\b|confirm you'?re (a )?human/i;
+  let found = '';
+  for (const doc of docs) {
+    const text = (doc.body && doc.body.innerText) || '';
+    for (const line of text.split(/\n+/).map(l => l.trim()).filter(Boolean)) {
+      if (sent.test(line)) return JSON.stringify(line.slice(0, 300));
+      if (!found && asks.test(line)) found = line.slice(0, 300);
+    }
+  }
+  return JSON.stringify(found);
+}"""
+
+
+async def _code_asked(browser_session) -> str:
+    """The page's request for a code the site emailed the candidate (see _CODE_ASKED_JS); "" if none or unreadable."""
+    try:
+        page, _ = await _focused_page(browser_session)
+        asked = json.loads(await page.evaluate(_CODE_ASKED_JS))
+        return asked if isinstance(asked, str) else ""
+    except Exception as exc:
+        logger.debug("code check failed: %s", exc)
+        return ""
+
+
 async def _submit_button_shown(browser_session) -> bool:
     """Whether the focused page shows the button that sends the application; False if it can't be read."""
     try:
@@ -1486,7 +1519,8 @@ async def _finish_unattended(decide: Decider, filled: dict, browser_session, llm
     """
     Send the filled form if `decide` says to, else leave it. The last event is done=True and carries the
     fill's results plus `form_screenshot` (the filled form, before anything is sent); when the submit agent
-    ran, also `submit_attempted`, `submitted` (only if the page confirmed it), `submit_detail` and
+    ran, also `submit_attempted`, `submitted` (only if the page confirmed it), `submit_detail`, `code_asked` (the
+    page's request for a code the site emailed the candidate, when it made one instead of confirming) and
     `confirm_screenshot` (the page after the click).
     """
     final = {"action": "Filled in; not sent", "screenshot": None, "blocked": False, "reason": None, "done": True,
@@ -1501,9 +1535,11 @@ async def _finish_unattended(decide: Decider, filled: dict, browser_session, llm
         yield {"action": "Submitting the application…", "screenshot": None, "blocked": False, "reason": None,
                "done": False}
         ok, detail = await _submit_with_agent(llm, browser_session, sensitive_data, secret)
-        final.update(submit_attempted=True, submitted=ok, submit_detail=detail,
+        code = "" if ok else _redact(await _code_asked(browser_session), secret)
+        final.update(submit_attempted=True, submitted=ok, submit_detail=detail, code_asked=code,
                      confirm_screenshot=await _screenshot_b64(browser_session),
-                     action="Application submitted" if ok else "Couldn't confirm the submission")
+                     action="Application submitted" if ok else
+                     "The site asked for a code it emailed you" if code else "Couldn't confirm the submission")
     yield final
 
 
@@ -1651,8 +1687,13 @@ async def on_step(browser_state, agent_output, step_number):
         notes = _with_empty_fields(summary, empty, left_blank, self_answered)[:_MAX_NOTES_CHARS]
 
         if decide is not None:
-            filled = {"ready": ready, "error": agent_error, "notes": notes, "questions": questions, "empty": empty,
-                      "self_answered": self_answered}
+            # The agent's summary is its own account, written as it stopped, and can disagree with the form as it ends
+            # up: Catapult's said the consent box was left unticked, yet the form showed it ticked from the saved
+            # answer. Unattended, nobody checks it against the page, so the notes keep what the final form shows; the
+            # summary stays only when the agent couldn't finish, to say why.
+            checked = _with_empty_fields("" if ready else summary, empty, left_blank, self_answered)
+            filled = {"ready": ready, "error": agent_error, "notes": checked[:_MAX_NOTES_CHARS],
+                      "questions": questions, "empty": empty, "self_answered": self_answered}
             async for step in _finish_unattended(decide, filled, browser_session, llm, sensitive_data, job_password):
                 yield step
             return

api/tests/browser/site/code.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html><head><meta charset="utf-8"><title>Test Application Form, after submit</title></head>
+<body style="margin:0;font-family:sans-serif">
+<!-- Greenhouse after the submit click on Catapult's form: it emails a code instead of confirming. -->
+<form id="f" onsubmit="event.preventDefault();">
+  <label style="position:absolute;left:20px;top:105px">Full name</label>
+  <input id="name" name="name" style="position:absolute;left:100px;top:100px;width:300px;height:30px">
+  <p id="code-sent" style="position:absolute;left:20px;top:160px;width:600px">A verification code was sent to ada@example.com. To submit your application, enter the 8-character code to confirm you're a human.</p>
+  <label style="position:absolute;left:20px;top:230px">Security code</label>
+  <input id="code" name="code" style="position:absolute;left:140px;top:225px;width:200px;height:30px">
+  <button id="submit" type="submit" style="position:absolute;left:100px;top:300px;width:200px;height:40px">Submit application</button>
+</form>
+</body></html>

api/tests/browser/test_browser_agent.py

@@ -480,8 +480,8 @@ async def flow():
     llm, shown, events = run(send=True)
     final = events[-1]
     assert not any(e.get("interactive") for e in events)                 # nobody is asked anything
-    assert shown == [{"ready": True, "error": "", "notes": "Filled every field", "questions": [], "empty": [],
-                      "self_answered": []}]
+    # A filled form's notes are what Windrush checked on it, not the agent's own summary ("Filled every field").
+    assert shown == [{"ready": True, "error": "", "notes": "", "questions": [], "empty": [], "self_answered": []}]
     assert llm.tasks == ["fill", "submit"]
     assert final["done"] and final["submitted"] and final["submit_attempted"]
     assert _is_jpeg(final["form_screenshot"]) and _is_jpeg(final["confirm_screenshot"])
@@ -494,6 +494,25 @@ async def flow():
     assert_no_chromium_left()
 
 
+@pytest.mark.parametrize("page, asked", [
+    ("code.html", "A verification code was sent to ada@example.com. To submit your application, enter the "
+                  "8-character code to confirm you're a human."),
+    ("form.html", ""),                                                    # unconfirmed, but no code asked for
+])
+def test_a_submit_the_site_answers_with_an_emailed_code_is_reported(ba, site, monkeypatch, page, asked):
+    llm = ScriptedLLM(submit_success=False)
+    monkeypatch.setattr(ba, "_make_browser_llm", lambda: llm)
+
+    async def flow():
+        return [event async for event in ba.apply_with_browser(
+            f"{site}/{page}", {"core_info": {"first_name": "Ada"}}, "", asyncio.Queue(),
+            decide=lambda filled: asyncio.sleep(0, result=True))]
+    final = asyncio.run(flow())[-1]
+    assert final["submit_attempted"] and not final["submitted"] and final["code_asked"] == asked
+    assert final["action"] == ("The site asked for a code it emailed you" if asked else "Couldn't confirm the submission")
+    assert_no_chromium_left()
+
+
 class GreenhouseLLM(ScriptedLLM):
     """Fills greenhouse.html the way the task asks: applicant details with fill_fields, questions through
     answer_questions with their indexes, then tries to fill a question that was left for the candidate."""

api/tests/test_autopilot.py

@@ -185,7 +185,7 @@ async def apply_with_browser(job_url, persona, cover_letter, queue, frames, *, j
         w.decided[company] = send
         if send:
             final.update(submit_attempted=True, submitted=form["confirm"], submit_detail="Thanks for applying",
-                         confirm_screenshot=JPEG)
+                         confirm_screenshot=JPEG, code_asked=form.get("code", ""))
         yield final
 
     monkeypatch.setattr(agent, "execute_tool", execute_tool)
@@ -271,6 +271,18 @@ def test_a_click_the_page_doesnt_confirm_is_not_counted_as_applied(ada, world):
     assert autopilot.sent_last_day(ada) == 3                              # still counts toward the limit
 
 
+def test_a_site_that_emails_a_code_holds_the_application_for_the_user(ada, world):
+    asked = ("A verification code was sent to ada@example.com. To submit your application, enter the 8-character "
+             "code to confirm you're a human.")
+    world.forms["alpha"].update(confirm=False, code=asked)
+    _run(ada, live=True)
+    run = _runs(ada)["alpha"]
+    assert run["outcome"] == "needs_you" and "wasn't sent" in run["reasons"][0] and asked in run["reasons"][0]
+    app = {a["company"]: a for a in tracker.list_applications(ada)}["alpha"]
+    assert app["status"] == "Pending Review" and "asked for a code it emailed you" in app["notes"]
+    assert autopilot.sent_last_day(ada) == 2                              # beta and gamma: alpha didn't go
+
+
 def test_poor_fits_and_stretches_are_skipped_without_filling(ada, world):
     world.fit["alpha"] = (60, "strong")
     world.fit["beta"] = (95, "reach")