feat: type the code a job site emails into Windrush, and it sends the form

api/app/autopilot.py

@@ -226,6 +226,17 @@ class _Run:
 
 _runs: dict[str, _Run] = {}           # user id → the run in progress
 _last: dict[str, dict] = {}           # user id → how the last run ended
+_codes: dict[str, asyncio.Future] = {}   # run id → the code the user types in, while its form waits for one
+_CODE_WAIT_S = 10 * 60                # how long a form held open for an emailed code waits for the user's
+
+
+def give_code(run_id: str, code: str) -> bool:
+    """Hand the code the user typed into Windrush to the form waiting for it; False when none is waiting."""
+    waiting = _codes.get(run_id)
+    if waiting is None or waiting.done():
+        return False
+    waiting.set_result(code)
+    return True
 
 
 class _Stop(Exception):
@@ -368,6 +379,7 @@ async def _apply(state: _Run, user_id: str, job: dict, verdict: dict, cfg: dict,
     run = tracker.get_autopilot_run(user_id, run_id)
     state.job = f"{job.get('title', '')} at {job.get('company', '')}"
     held: list[str] = []
+    given: list[str] = []   # codes the user typed in for this form
     final: dict = {}
     questions: list[dict] = []
     try:
@@ -395,12 +407,32 @@ async def decide(filled: dict) -> bool:
                 held.append(f"Your limit of {cfg['daily_cap']} a day was reached before this one could go.")
             return live and not held
 
+        async def ask_code(asked: str, attempt: int) -> str | None:
+            # The site emailed the user a code to confirm a person is applying: the form waits, open, for them to
+            # type it into Windrush (give_code). It never comes from anywhere else (see browser_agent.CodeAsker).
+            again = " The last code didn't work: check it's the newest one." if attempt > 1 else ""
+            tracker.update_autopilot_run(run_id, user_id=user_id, outcome="awaiting_code", reasons=[
+                f"The job site emailed you a code: “{asked}”{again} Type it in here within {_CODE_WAIT_S // 60} "
+                f"minutes and Windrush will enter it and send the application."[:600]])
+            waiting = asyncio.get_running_loop().create_future()
+            _codes[run_id] = waiting
+            try:
+                code = await asyncio.wait_for(waiting, _CODE_WAIT_S)
+            except asyncio.TimeoutError:
+                return None
+            finally:
+                _codes.pop(run_id, None)
+            given.append(code)
+            tracker.update_autopilot_run(run_id, user_id=user_id, outcome="running", reasons=[])
+            return code
+
         state.step = "Filling in the form"
         core = persona.get("core_info") or {}
         async with aclosing(browser_agent.apply_with_browser(
             job["url"], persona, text, asyncio.Queue(), None,
             job_email=core.get("job_email", ""), job_password=tracker.get_job_password(user_id),
             cv_path=_pdf(cv_doc_id), cover_letter_path=_pdf(letter_doc_id), answerer=answerer, decide=decide,
+            ask_code=ask_code if live else None,
         )) as steps:
             async for step in steps:
                 if step.get("done"):
@@ -425,10 +457,12 @@ async def decide(filled: dict) -> bool:
         outcome, reasons = "submitted", []
     elif final.get("code_asked"):
         # The site emailed the candidate a code to prove a person is applying (Catapult, on Greenhouse: "enter the
-        # 8-character code to confirm you're a human"). That check is theirs to pass: nothing went, so it waits.
-        outcome, reasons = "needs_you", [f"The job site emailed you a code to confirm a person is applying, so this "
-                                         f"wasn't sent. It says: “{final['code_asked']}” Apply on the job's page "
-                                         f"with this run's CV and cover letter."[:600]]
+        # 8-character code to confirm you're a human"), and no code they gave got it through: nothing went.
+        why = ("the code you typed in didn't take" if given else
+               f"no code was typed in within {_CODE_WAIT_S // 60} minutes")
+        outcome, reasons = "needs_you", [f"The job site emailed you a code to confirm a person is applying, and "
+                                         f"{why}, so this wasn't sent. It says: “{final['code_asked']}” Apply on the "
+                                         f"job's page with this run's handover pack."[:600]]
     elif final.get("submit_attempted"):
         detail = final.get("submit_detail") or "no confirmation shown"
         outcome, reasons = "unconfirmed", [f"The submit button was clicked, but the page didn't confirm it ({detail}). "
@@ -457,6 +491,8 @@ async def decide(filled: dict) -> bool:
             tracker.set_application_questions(app_id, questions, user_id=user_id)
     reused = (f"Used the CV and cover letter from the {'practice ' if earlier['practice'] else ''}run of "
               f"{earlier['updated_at'][:10]}. " if earlier else "")
+    if given and outcome == "submitted":
+        reused += "Sent after you typed in the code the job site emailed you. "
     tracker.update_autopilot_run(run_id, user_id=user_id, outcome=outcome, reasons=reasons, questions=questions,
                                  notes=(reused + str(final.get("notes") or ""))[:2000], application_id=app_id)
     logger.info("autopilot: %s -> %s (%d reasons)", state.job, outcome, len(reasons))

api/app/browser_agent.py

@@ -1352,6 +1352,77 @@ async def _code_asked(browser_session) -> str:
         return ""
 
 
+# Where the emailed code goes: one box per character (Greenhouse: eight boxes of one character), else one field whose
+# label, name or placeholder says code. Marks them data-windrush-code=0..n-1 and returns how many it found.
+_CODE_BOXES_JS = r"""() => {
+  const docs = [document];
+  for (const frame of document.querySelectorAll('iframe')) {
+    try { if (frame.contentDocument) docs.push(frame.contentDocument); } catch (e) {}
+  }
+  const skip = ['hidden', 'checkbox', 'radio', 'file', 'submit', 'button', 'email', 'password'];
+  const inputs = [];
+  for (const doc of docs) {
+    for (const el of doc.querySelectorAll('input')) {
+      const style = doc.defaultView.getComputedStyle(el);
+      const box = el.getBoundingClientRect();
+      if (!skip.includes(el.type) && !el.disabled && style.display !== 'none' && style.visibility !== 'hidden'
+          && (box.width > 0 || box.height > 0)) inputs.push(el);
+    }
+  }
+  let boxes = inputs.filter((el) => el.maxLength === 1);
+  if (boxes.length < 4) {
+    const says = (el) => /code/i.test([el.name, el.id, el.placeholder, el.getAttribute('aria-label') || '',
+      el.getAttribute('autocomplete') || '', ...[...(el.labels || [])].map((l) => l.textContent)].join(' '));
+    boxes = inputs.filter(says).slice(0, 1);
+  }
+  boxes.forEach((el, i) => el.setAttribute('data-windrush-code', String(i)));
+  return JSON.stringify(boxes.length);
+}"""
+
+# Focus marked box __INDEX__ and select what it holds, so the next insertText replaces it.
+_FOCUS_CODE_BOX_JS = r"""() => {
+  const docs = [document];
+  for (const frame of document.querySelectorAll('iframe')) {
+    try { if (frame.contentDocument) docs.push(frame.contentDocument); } catch (e) {}
+  }
+  for (const doc of docs) {
+    const el = doc.querySelector('[data-windrush-code="__INDEX__"]');
+    if (el) { el.focus(); if (el.select) el.select(); return JSON.stringify(true); }
+  }
+  return JSON.stringify(false);
+}"""
+
+# What the marked boxes hold, in order.
+_CODE_TYPED_JS = r"""() => {
+  const docs = [document];
+  for (const frame of document.querySelectorAll('iframe')) {
+    try { if (frame.contentDocument) docs.push(frame.contentDocument); } catch (e) {}
+  }
+  const boxes = docs.flatMap((doc) => [...doc.querySelectorAll('[data-windrush-code]')]);
+  boxes.sort((a, b) => Number(a.dataset.windrushCode) - Number(b.dataset.windrushCode));
+  return JSON.stringify(boxes.map((el) => el.value).join(''));
+}"""
+
+
+async def _enter_code(browser_session, code: str) -> bool:
+    """Type the code the user gave into the page's code box(es), one character a box when there's a box each
+    (insertText, so React-controlled boxes take it like typing). True when the boxes then hold the code."""
+    try:
+        page, cdp_session = await _focused_page(browser_session)
+        count = json.loads(await page.evaluate(_CODE_BOXES_JS))
+        if not count:
+            return False
+        parts = list(code) if count > 1 else [code]
+        for i, part in enumerate(parts[:count]):
+            if json.loads(await page.evaluate(_FOCUS_CODE_BOX_JS.replace("__INDEX__", str(i)))) is not True:
+                return False
+            await _insert_text(browser_session, cdp_session, part)
+        return str(json.loads(await page.evaluate(_CODE_TYPED_JS))).upper() == code.upper()
+    except Exception as exc:
+        logger.warning("couldn't enter the code: %s", exc)
+        return False
+
+
 async def _submit_button_shown(browser_session) -> bool:
     """Whether the focused page shows the button that sends the application; False if it can't be read."""
     try:
@@ -1513,15 +1584,23 @@ def _with_empty_fields(notes: str, empty: list[str], left_blank: list[str] = (),
 # {ready, error, notes, questions, empty} — and says whether to send it. Nobody reviews it in between.
 Decider = Callable[[dict], Awaitable[bool]]
 
+# When the send stops at a page wanting a code the site emailed the candidate, `ask_code` gets that page's line and
+# the attempt (1, 2, …) and returns the code the user typed into Windrush, or None when none came. The code always
+# comes from the user: the check is there to confirm a person is applying, so Windrush never fetches it itself,
+# not even from a connected Gmail inbox.
+CodeAsker = Callable[[str, int], Awaitable[str | None]]
+_MAX_CODE_TRIES = 3
+
 
 async def _finish_unattended(decide: Decider, filled: dict, browser_session, llm, sensitive_data: dict | None,
-                             secret: str) -> AsyncGenerator[dict, None]:
+                             secret: str, ask_code: CodeAsker | None = None) -> AsyncGenerator[dict, None]:
     """
     Send the filled form if `decide` says to, else leave it. The last event is done=True and carries the
     fill's results plus `form_screenshot` (the filled form, before anything is sent); when the submit agent
     ran, also `submit_attempted`, `submitted` (only if the page confirmed it), `submit_detail`, `code_asked` (the
-    page's request for a code the site emailed the candidate, when it made one instead of confirming) and
-    `confirm_screenshot` (the page after the click).
+    page's request for a code the site emailed the candidate, when it still wants one) and `confirm_screenshot`
+    (the page after the click). With `ask_code`, a page that wants such a code waits for the user's: up to
+    _MAX_CODE_TRIES codes are entered and the form sent again.
     """
     final = {"action": "Filled in; not sent", "screenshot": None, "blocked": False, "reason": None, "done": True,
              "submitted": False, "submit_attempted": False, **filled,
@@ -1536,6 +1615,21 @@ async def _finish_unattended(decide: Decider, filled: dict, browser_session, llm
                "done": False}
         ok, detail = await _submit_with_agent(llm, browser_session, sensitive_data, secret)
         code = "" if ok else _redact(await _code_asked(browser_session), secret)
+        tries = 0
+        while code and ask_code is not None and tries < _MAX_CODE_TRIES:
+            tries += 1
+            yield {"action": "Waiting for the code the site emailed you", "screenshot": None, "blocked": False,
+                   "reason": None, "done": False}
+            given = await ask_code(code, tries)
+            if not given:
+                break
+            yield {"action": "Entering your code and sending the application…", "screenshot": None, "blocked": False,
+                   "reason": None, "done": False}
+            if not await _enter_code(browser_session, given):
+                detail = "Couldn't find where the code goes on the page."
+                break
+            ok, detail = await _submit_with_agent(llm, browser_session, sensitive_data, secret)
+            code = "" if ok else _redact(await _code_asked(browser_session), secret)
         final.update(submit_attempted=True, submitted=ok, submit_detail=detail, code_asked=code,
                      confirm_screenshot=await _screenshot_b64(browser_session),
                      action="Application submitted" if ok else
@@ -1555,14 +1649,16 @@ async def apply_with_browser(
     cover_letter_path: str = "",
     answerer: Answerer | None = None,
     decide: Decider | None = None,
+    ask_code: CodeAsker | None = None,
 ) -> AsyncGenerator[dict, None]:
     """
     Async generator: the browser-use agent fills the application, then the user reviews it and
     decides whether to submit (see module docstring). The CV and cover-letter PDFs are uploaded
     under the candidate's name ("Ada_Lovelace_CV.pdf"), not their stored ids. `answerer` answers the
     form's questions (answers.answer_questions); without one, every question is left for the user.
     With `decide` the run is unattended: there is no review, and `decide` says whether the filled form is
-    sent (see _finish_unattended for what the last event then carries).
+    sent (see _finish_unattended for what the last event then carries); `ask_code` gets the user's code when
+    the site emails one before it takes the application (see CodeAsker).
     Yields dicts:
       { action, screenshot (base64|None), blocked (bool), reason (str|None), done (bool),
         interactive (bool, optional), submitted (bool, only on the final done event),
@@ -1694,7 +1790,8 @@ async def on_step(browser_state, agent_output, step_number):
             checked = _with_empty_fields("" if ready else summary, empty, left_blank, self_answered)
             filled = {"ready": ready, "error": agent_error, "notes": checked[:_MAX_NOTES_CHARS],
                       "questions": questions, "empty": empty, "self_answered": self_answered}
-            async for step in _finish_unattended(decide, filled, browser_session, llm, sensitive_data, job_password):
+            async for step in _finish_unattended(decide, filled, browser_session, llm, sensitive_data, job_password,
+                                                 ask_code):
                 yield step
             return
 

api/app/main.py

@@ -1052,6 +1052,19 @@ async def autopilot_dismiss(run_id: str, current_user: Annotated[auth.User, Depe
     return {"dismissed": run_id}
 
 
+@app.post("/autopilot/runs/{run_id}/code", status_code=202)
+async def autopilot_code(run_id: str, body: dict, current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
+    """The code a job site emailed you, typed in for the form autopilot holds open for it: it's entered on the form
+    and the application sent. The only way a code reaches a form: Windrush never fetches one itself."""
+    run = _own_run(current_user.id, run_id)
+    code = re.sub(r"[\s-]+", "", str(body.get("code") or ""))
+    if not re.fullmatch(r"[A-Za-z0-9]{4,12}", code):
+        raise HTTPException(status_code=400, detail="That doesn't look like the code: letters and digits only.")
+    if run["outcome"] != "awaiting_code" or not autopilot.give_code(run_id, code):
+        raise HTTPException(status_code=409, detail="That form isn't waiting for a code any more.")
+    return {"accepted": True}
+
+
 @app.get("/autopilot/runs/{run_id}/pack")
 async def autopilot_pack(run_id: str, current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
     """A run's handover pack, for applying yourself: a ZIP of its CV and cover letter under the names autopilot

api/app/tracker.py

@@ -129,10 +129,12 @@
 # unconfirmed - the submit button was clicked but the page didn't confirm it: the user checks
 # ready     - practice mode: everything checked out, and it would have been sent
 # needs_you - filled in but held back (see reasons)
+# awaiting_code - the form is open on a page wanting a code the site emailed the user; it waits for them to type it in
 # skipped   - not tried: the fit was too low, say
 # failed    - something broke before the form was filled
 # dismissed - the user cleared it from the queue
-AUTOPILOT_OUTCOMES = ("running", "submitted", "unconfirmed", "ready", "needs_you", "skipped", "failed", "dismissed")
+AUTOPILOT_OUTCOMES = ("running", "submitted", "unconfirmed", "ready", "needs_you", "awaiting_code", "skipped", "failed",
+                      "dismissed")
 
 
 def init_db(db_path: str = "") -> None:
@@ -786,11 +788,16 @@ def autopilot_sent_since(user_id: str, since: str) -> int:
 
 
 def finish_stale_autopilot_runs() -> None:
-    """Runs left 'running' by a restart are over: nothing is working on them now."""
+    """Runs left 'running' or 'awaiting_code' by a restart are over: nothing is working on them now, and a form
+    waiting for a code closed with the browser."""
     con = sqlite3.connect(_DB_PATH)
     with con:
         con.execute("UPDATE autopilot_runs SET outcome='failed', reasons=?, updated_at=? WHERE outcome='running'",
                     (json.dumps(["Windrush restarted while this was in progress."]), _now()))
+        con.execute("UPDATE autopilot_runs SET outcome='needs_you', reasons=?, updated_at=? "
+                    "WHERE outcome='awaiting_code'",
+                    (json.dumps(["Windrush restarted while the form waited for the code the job site emailed you, "
+                                 "so it wasn't sent. Apply on the job's page with this run's handover pack."]), _now()))
     con.close()
 
 

api/tests/browser/browser_support.py

@@ -76,8 +76,10 @@ class ScriptedLLM:
     name = property(lambda self: "scripted")
     model_name = property(lambda self: "scripted")
 
-    def __init__(self, fill_success: bool = True, submit_success: bool = True):
-        self.fill_success, self.submit_success = fill_success, submit_success
+    def __init__(self, fill_success: bool = True, submit_success: bool | list[bool] = True):
+        # submit_success as a list: one result per submit agent, in order (the last repeats).
+        self.fill_success = fill_success
+        self.submit_results = list(submit_success) if isinstance(submit_success, list) else [submit_success]
         self.tasks: list[str] = []
         self.seen: list[str] = []
 
@@ -88,7 +90,8 @@ def _text(messages) -> str:
     def next_action(self, text: str) -> list[dict]:
         is_submit = "candidate has reviewed" in text
         self.tasks.append("submit" if is_submit else "fill")
-        ok = self.submit_success if is_submit else self.fill_success
+        ok = (self.submit_results.pop(0) if len(self.submit_results) > 1 else self.submit_results[0]) if is_submit \
+            else self.fill_success
         return [{"done": {"text": "submit agent ran" if is_submit else "Filled every field", "success": ok}}]
 
     async def ainvoke(self, messages, output_format=None, **kwargs):

api/tests/browser/site/code_boxes.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<html><head><meta charset="utf-8"><title>Test Application Form, wanting the emailed code</title></head>
+<body style="margin:0;font-family:sans-serif">
+<!-- Greenhouse after the first submit: eight one-character boxes for the emailed code, each moving on to the next. -->
+<form id="f" onsubmit="event.preventDefault();">
+  <p style="position:absolute;left:20px;top:40px;width:600px">A verification code was sent to ada@example.com. To submit your application, enter the 8-character code to confirm you're a human.</p>
+  <label style="position:absolute;left:20px;top:110px">Security code</label>
+  <div id="boxes" style="position:absolute;left:20px;top:140px"></div>
+  <button id="submit" type="submit" style="position:absolute;left:20px;top:220px;width:200px;height:40px">Submit application</button>
+</form>
+<script>
+  const holder = document.getElementById("boxes");
+  for (let i = 0; i < 8; i++) {
+    const box = document.createElement("input");
+    box.maxLength = 1;
+    box.setAttribute("aria-label", `Security code character ${i + 1}`);
+    box.style.cssText = "width:30px;height:36px;margin-right:6px;text-align:center";
+    box.addEventListener("input", () => { if (box.value && box.nextElementSibling) box.nextElementSibling.focus(); });
+    holder.appendChild(box);
+  }
+</script>
+</body></html>