diff --git a/pkg/create/templates.go b/pkg/create/templates.go index 0627ad3c..b14c08d2 100644 --- a/pkg/create/templates.go +++ b/pkg/create/templates.go @@ -87,8 +87,8 @@ var Templates = map[string]TemplateInfo{ Languages: []string{LanguageTypeScript, LanguagePython}, }, TemplateYutoriComputerUse: { - Name: "Yutori n1.5 Computer Use", - Description: "Implements a Yutori n1.5 computer use agent", + Name: "Yutori n2 Computer Use", + Description: "Implements a Yutori n2 computer use agent", Languages: []string{LanguageTypeScript, LanguagePython}, }, TemplateTzafonComputerUse: { diff --git a/pkg/templates/python/yutori/README.md b/pkg/templates/python/yutori/README.md index afca2702..16fe66b6 100644 --- a/pkg/templates/python/yutori/README.md +++ b/pkg/templates/python/yutori/README.md @@ -1,10 +1,8 @@ -# Kernel Python Sample App - Yutori n1.5 Computer Use +# Kernel Python Sample App - Yutori n2 Computer Use -This Kernel app implements a prompt loop using Yutori's Navigator n1.5 with Kernel's Computer Controls API. +This Kernel app implements a prompt loop using Yutori's Navigator n2 with Kernel's Computer Controls API. -[Navigator n1.5](https://yutori.com/blog/introducing-n1-5) is Yutori's pixels-to-actions LLM that predicts browser actions from screenshots. - -This template runs n1.5 in **computer-use-only mode**. n1.5 also supports a hybrid vision + DOM/JavaScript path (page-state extraction, custom JS, structured JSON output) for multi-field forms and bulk data extraction, but those tools are intentionally disabled here — see [Disabled tools](#disabled-tools). +[Navigator n2](https://docs.yutori.com/reference/n2) is Yutori's computer-use model. It reads a screenshot of the whole screen and answers with a batch of mouse and keyboard actions, and it can run shell commands and edit files on the machine it is driving. ## Setup @@ -28,7 +26,7 @@ kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https Optional payload fields: - `record_replay` (bool) — capture a video of the session (paid plans only). -- `kiosk` (bool) — launch the browser without address bar / tabs ([see below](#kiosk-mode)). +- `reasoning_effort` (`"none"`, `"low"`, `"medium"`, `"xhigh"`) — n2 reasons at `medium` by default. `xhigh` gives the longest traces and does best on hard multi-step tasks; `none` turns reasoning off. - `user_timezone` (IANA, e.g. `"America/New_York"`) and `user_location` (free text, e.g. `"New York, NY, US"`) — appended to the task message so the model has accurate temporal/locational grounding. More involved example (Kanban drag-and-drop): @@ -49,66 +47,62 @@ kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https When enabled, the response will include a `replay_url` field with a link to view the recorded session. -## Kiosk mode - -Prefer **non-kiosk mode** by default and when the agent is expected to switch domains via URL. Use **kiosk (`"kiosk": true`)** when: (1) you're recording sessions and want a cleaner UI in the replay, or (2) you're automating on a single website and the combination of the complex site layout and browser chrome (address bar, tabs) may confuse the agent. - -Note: In kiosk mode the agent may still try to use the address bar to enter URLs; it's not available, so it will eventually use `goto_url`, but those attempts may result in slowdown of the overall session. +## Screen Configuration -Default (non-kiosk): - -```bash -kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com, then navigate to ign.com and describe the page"}' -``` +n2 is a desktop model: it expects a screenshot of the **whole screen**, browser chrome included, and it navigates by clicking the address bar rather than through a dedicated navigation action. A Kernel `viewport` sets the browser window size, and screenshots from Computer Controls capture that window with its tabs and toolbar — which is exactly what n2 wants, so this template does not use kiosk mode. -With kiosk (single-site or recording): +This template runs at **1280x800**. Yutori also lists 1920x1080 and 1280x720 as resolutions in regular use; grounding may degrade at extreme aspect ratios. -```bash -kernel invoke python-yutori-cua cua-task --payload '{"query": "Enter https://example.com in the search box and then describe the page.", "kiosk": true}' -``` +> **Note:** n2 outputs coordinates in a 1000x1000 relative space, which are scaled to the actual screen dimensions per action. -## Viewport Configuration +See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. -Yutori n1.5 recommends a **1280×800 (WXGA, 16:10)** viewport for best grounding accuracy. +## Screenshots -> **Note:** n1.5 outputs coordinates in a 1000×1000 relative space, which are automatically scaled to the actual viewport dimensions. +Screenshots are converted to WebP before they are sent. Yutori caps requests at 10 MB, and a full-screen PNG trajectory blows past that on its own. -See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. +The loop also drops screenshots the model will not read: n2 keeps images from only the last 2 image-bearing messages, so older ones are stripped from each request while every message's text is kept. -## Screenshots +## Tools -Screenshots are automatically converted to WebP format for better compression across multi-step trajectories, as recommended by Yutori. +This template pins the `computer_use_tools-20260825` tool set. n2 rejects `disable_tools`, so all five tools are always served and the loop answers all of them. -## n1.5-latest Supported Actions +### `computer_batch` -This template uses the `browser_tools_core-20260403` tool set — coordinate-based browser actions that operate on screenshots only. +n2 returns a whole action list in one call. The loop runs them in order, stops at the first error, and replies with a single tool result carrying one screenshot taken after the last action that ran. Every coordinate in a batch refers to the screenshot from *before* the batch started. | Action | Description | |--------|-------------| | `left_click` | Left mouse click at coordinates (supports `modifier`) | | `double_click` | Double-click at coordinates (supports `modifier`) | | `triple_click` | Triple-click at coordinates (supports `modifier`) | -| `middle_click` | Middle mouse click at coordinates | -| `right_click` | Right mouse click at coordinates | -| `mouse_move` | Move mouse to coordinates without clicking | -| `mouse_down` | Press the left mouse button at coordinates | -| `mouse_up` | Release the left mouse button at coordinates | -| `scroll` | Scroll page in a direction | -| `type` | Type text into focused element | -| `key_press` | Send a single key or key combination | -| `hold_key` | Hold a key for a duration | -| `drag` | Click-and-drag operation | -| `wait` | Pause for UI to update | -| `refresh` | Reload current page | -| `go_back` | Navigate back in history | -| `go_forward` | Navigate forward in history | -| `goto_url` | Navigate to a URL | - -### Disabled tools - -The DOM/Playwright-based "expanded" tools (`extract_elements`, `find`, `set_element_value`, `execute_js`) are intentionally disabled via the `disable_tools` request parameter — this template runs computer-use only and does not expose a Playwright page to the model. +| `middle_click` | Middle mouse click at coordinates (supports `modifier`) | +| `right_click` | Right mouse click at coordinates (supports `modifier`) | +| `scroll` | Scroll up or down at coordinates, in wheel notches | +| `type` | Type text into the focused element | +| `key_press` | Press a key, combination, or sequence | +| `drag` | Drag from `start_coordinates` to `coordinates` | +| `mouse_move` | Move the mouse to coordinates without clicking | +| `mouse_down` | Press and hold the left button (coordinates optional) | +| `mouse_up` | Release the left button (coordinates optional) | +| `hold_key` | Hold a key down for a duration | +| `wait` | Pause without interacting | +| `screenshot` | Look without acting | + +Horizontal scrolling is not available, and there are no `goto_url` / `go_back` / `go_forward` / `refresh` actions — n2 drives the address bar and browser buttons like a person would. + +### `bash`, `read`, `write`, `edit` + +n2 can also work on the browser VM directly. These map onto Kernel's process and filesystem APIs: + +| Tool | Backed by | +|------|-----------| +| `bash` | `browsers.process.exec` — each call is a separate process; the working directory carries over between calls, environment variables do not. `run_in_background` detaches the command and reports its pid and log path. | +| `read` | `browsers.fs.read_file` — returns `cat -n` output, with `offset` / `limit` paging | +| `write` | `browsers.fs.write_file` | +| `edit` | read, exact-string replace, write. Refuses to edit a file that has not been read in this session. | ## Resources -- [Yutori n1.5 API Documentation](https://docs.yutori.com/reference/n1-5) +- [Yutori n2 API Documentation](https://docs.yutori.com/reference/n2) - [Kernel Documentation](https://www.kernel.sh/docs/quickstart) diff --git a/pkg/templates/python/yutori/loop.py b/pkg/templates/python/yutori/loop.py index 1a81c491..ead901b4 100644 --- a/pkg/templates/python/yutori/loop.py +++ b/pkg/templates/python/yutori/loop.py @@ -1,14 +1,15 @@ """ -Yutori n1.5 Sampling Loop +Yutori n2 Sampling Loop -Implements the agent loop for Yutori's n1.5-latest computer use model. -n1.5-latest uses an OpenAI-compatible API with tool_calls: -- Actions are returned via tool_calls in the assistant message -- Tool results use role: "tool" with matching tool_call_id +Implements the agent loop for Yutori's n2 computer use model. +n2 uses an OpenAI-compatible API with tool_calls: +- GUI actions arrive as a single `computer_batch` call holding an action list +- `bash`, `read`, `write`, and `edit` arrive as their own calls +- Every call needs one tool result with a matching tool_call_id - The model stops by returning content without tool_calls - Coordinates are returned in 1000x1000 space and need scaling -@see https://docs.yutori.com/reference/n1-5 +@see https://docs.yutori.com/reference/n2 """ from __future__ import annotations @@ -23,107 +24,96 @@ from kernel import Kernel from openai import OpenAI -from tools import ComputerTool, N15Action, ToolResult +from tools import ComputerTool, SystemTools -# Tools that require a Playwright page / DOM access. The default core tool set -# already excludes them, but we also list them in `disable_tools` so the -# exclusion is explicit and survives if the default ever changes. -DISABLED_TOOLS = ["extract_elements", "find", "set_element_value", "execute_js"] -TOOL_SET = "browser_tools_core-20260403" +# Pin the tool set so a change to the server default can't change which tools +# this run exposes. n2 rejects `disable_tools`, so the set is served whole. +TOOL_SET = "computer_use_tools-20260825" -NAVIGATOR_COORDINATE_SCALE = 1000 - -# Screenshot-trimming defaults mirror Yutori's reference loop: -# https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/payload.py -# Trimming is size-triggered — we only drop old screenshots when the payload -# exceeds MAX_REQUEST_BYTES, and we always keep at least KEEP_RECENT_SCREENSHOTS. +# Requests are capped at 10 MB. n2 also only reads images from the last 2 +# image-bearing messages, so anything older is dropped before sending rather +# than paying to upload screenshots the model will discard server-side. MAX_REQUEST_BYTES = 9_500_000 -KEEP_RECENT_SCREENSHOTS = 6 +KEEP_IMAGE_MESSAGES = 2 async def sampling_loop( *, - model: str = "n1.5-latest", + model: str = "n2", task: str, api_key: str, kernel: Kernel, session_id: str, - max_completion_tokens: int = 4096, + # Shared by the reasoning trace and the tool call, so a low value truncates + # the call. + max_completion_tokens: int = 16384, max_iterations: int = 100, - viewport_width: int = 1280, - viewport_height: int = 800, - kiosk_mode: bool = False, + screen_width: int = 1280, + screen_height: int = 800, + reasoning_effort: Optional[str] = None, user_timezone: str = "America/Los_Angeles", user_location: str = "San Francisco, CA, US", ) -> dict[str, Any]: - """Run the n1.5 sampling loop until the model stops calling tools or max iterations.""" - client = OpenAI( - api_key=api_key, - base_url="https://api.yutori.com/v1", - ) - - computer_tool = ComputerTool(kernel, session_id, viewport_width, viewport_height, kiosk_mode=kiosk_mode) + """Run the n2 sampling loop until the model stops calling tools or max iterations.""" + client = OpenAI(api_key=api_key, base_url="https://api.yutori.com/v1") - initial_screenshot = await computer_tool.screenshot() + computer_tool = ComputerTool(kernel, session_id, screen_width, screen_height) + system_tools = SystemTools(kernel, session_id) # Append location/timezone/current-date context to the task — mirrors Yutori's # format_task_with_context helper and helps the model with date-sensitive # judgments. https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/context.py - task_with_context = _format_task_with_context(task, user_timezone, user_location) - - user_content: list[dict[str, Any]] = [{"type": "text", "text": task_with_context}] - if initial_screenshot.get("base64_image"): - user_content.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/webp;base64,{initial_screenshot['base64_image']}" - }, - }) - conversation_messages: list[dict[str, Any]] = [ - {"role": "user", "content": user_content} + { + "role": "user", + "content": [ + {"type": "text", "text": _format_task_with_context(task, user_timezone, user_location)}, + _image_part(await computer_tool.screenshot()), + ], + } ] iteration = 0 final_answer: Optional[str] = None + prev_request_id: Optional[str] = None while iteration < max_iterations: iteration += 1 print(f"\n=== Iteration {iteration} ===") - request_messages, dropped = _trimmed_for_request(conversation_messages) - if dropped: - print(f"Trimmed {dropped} old screenshot(s) to fit request size limit") + request_messages = _trimmed_for_request(conversation_messages) + + # n2-specific knobs go in extra_body. + extra_body: dict[str, Any] = {"tool_set": TOOL_SET} + if reasoning_effort: + extra_body["reasoning_effort"] = reasoning_effort + if prev_request_id: + # Groups the trajectory into one conversation for usage reporting. + extra_body["prev_request_id"] = prev_request_id try: response = client.chat.completions.create( model=model, messages=request_messages, max_completion_tokens=max_completion_tokens, - temperature=0.3, - # n1.5-specific knobs go in extra_body. - # tool_set selects the core (coordinate-based) tools. - # disable_tools is a defense-in-depth exclusion of DOM/Playwright tools. - extra_body={ - "tool_set": TOOL_SET, - "disable_tools": DISABLED_TOOLS, - }, + temperature=0.6, + extra_body=extra_body, ) except Exception as api_error: print(f"API call failed: {api_error}") raise - if not response.choices or len(response.choices) == 0: - print(f"No choices in response: {response}") - raise ValueError("No choices in API response") + prev_request_id = getattr(response, "_request_id", None) or prev_request_id - choice = response.choices[0] - assistant_message = choice.message - if not assistant_message: + if not response.choices: + print(f"No choices in response: {response}") raise ValueError("No response from model") + assistant_message = response.choices[0].message print("Assistant content:", assistant_message.content or "(none)") + # Push the assistant message unchanged — `reasoning_content` rides along + # on it, and n2 needs it echoed back to keep its reasoning across turns. conversation_messages.append(assistant_message.model_dump(exclude_none=True)) tool_calls = assistant_message.tool_calls @@ -134,56 +124,25 @@ async def sampling_loop( print(f"No tool_calls, model is done. Final answer: {final_answer}") break - for tc in tool_calls: - action_name = tc.function.name - try: - args = json.loads(tc.function.arguments) - except json.JSONDecodeError: - print(f"Failed to parse tool_call arguments: {tc.function.arguments}") + for index, tool_call in enumerate(tool_calls): + # n2 usually answers with one call, but `parallel_tool_calls` is + # pinned on server-side so a turn can carry several. Only the first + # one was planned against the screenshot the model actually saw, so + # run it and make the rest re-plan. + if index > 0: conversation_messages.append({ "role": "tool", - "tool_call_id": tc.id, - "content": "Error: failed to parse arguments", + "tool_call_id": tool_call.id, + "content": ( + "Not executed: planned against a screenshot the previous call already " + "changed. Re-plan from the latest screenshot." + ), }) continue - action: N15Action = {"action_type": action_name, **args} - print(f"Executing action: {action_name}", args) - - scaled_action = _scale_coordinates(action, viewport_width, viewport_height) - - result: ToolResult - try: - result = await computer_tool.execute(scaled_action) - except Exception as e: - print(f"Action failed: {e}") - result = {"error": str(e)} - - if result.get("base64_image"): - conversation_messages.append({ - "role": "tool", - "tool_call_id": tc.id, - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/webp;base64,{result['base64_image']}" - }, - } - ], - }) - elif result.get("error"): - conversation_messages.append({ - "role": "tool", - "tool_call_id": tc.id, - "content": f"Action failed: {result['error']}", - }) - else: - conversation_messages.append({ - "role": "tool", - "tool_call_id": tc.id, - "content": result.get("output", "OK"), - }) + conversation_messages.append( + await _run_tool_call(tool_call, computer_tool, system_tools) + ) # If the loop exhausted iterations, prompt the model for a final summary so # the caller gets a usable answer instead of empty content. Mirrors Yutori's @@ -191,29 +150,23 @@ async def sampling_loop( if iteration >= max_iterations and not final_answer: print("Max iterations reached — requesting summary") try: - final_screenshot = await computer_tool.screenshot() - stop_content: list[dict[str, Any]] = [ - {"type": "text", "text": _format_stop_and_summarize(task)} - ] - if final_screenshot.get("base64_image"): - stop_content.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/webp;base64,{final_screenshot['base64_image']}" - }, - }) - conversation_messages.append({"role": "user", "content": stop_content}) + conversation_messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": _format_stop_and_summarize(task)}, + _image_part(await computer_tool.screenshot()), + ], + }) - summary_messages, _ = _trimmed_for_request(conversation_messages) summary_response = client.chat.completions.create( model=model, - messages=summary_messages, + messages=_trimmed_for_request(conversation_messages), max_completion_tokens=max_completion_tokens, - temperature=0.3, - extra_body={"tool_set": TOOL_SET, "disable_tools": DISABLED_TOOLS}, + temperature=0.6, + extra_body={"tool_set": TOOL_SET}, ) - summary = summary_response.choices[0].message if summary_response.choices else None - if summary: + if summary_response.choices: + summary = summary_response.choices[0].message conversation_messages.append(summary.model_dump(exclude_none=True)) final_answer = summary.content or None except Exception as summary_error: @@ -225,6 +178,52 @@ async def sampling_loop( } +async def _run_tool_call( + tool_call: Any, + computer_tool: ComputerTool, + system_tools: SystemTools, +) -> dict[str, Any]: + name = tool_call.function.name + + try: + args = json.loads(tool_call.function.arguments) + except json.JSONDecodeError: + print(f"Failed to parse tool_call arguments: {tool_call.function.arguments}") + return { + "role": "tool", + "tool_call_id": tool_call.id, + "content": "Error: failed to parse arguments", + } + + print(f"Executing tool: {name}", json.dumps(args)[:500]) + + if name == "computer_batch": + outcome = await computer_tool.run_batch(args.get("actions") or []) + return { + "role": "tool", + "tool_call_id": tool_call.id, + "content": [ + {"type": "text", "text": outcome.describe()}, + _image_part(await computer_tool.screenshot()), + ], + } + + try: + content = system_tools.execute(name, args) + except Exception as error: + print(f"{name} failed: {error}") + content = f"{name} failed: {error}" + + return {"role": "tool", "tool_call_id": tool_call.id, "content": content} + + +def _image_part(base64_image: str) -> dict[str, Any]: + return { + "type": "image_url", + "image_url": {"url": f"data:image/webp;base64,{base64_image}"}, + } + + def _format_task_with_context(task: str, user_timezone: str, user_location: str) -> str: """Append location, timezone, and current date/time to the task message.""" for timezone_name in [user_timezone, "America/Los_Angeles", "UTC"]: @@ -260,48 +259,28 @@ def _format_stop_and_summarize(task: str) -> str: ) -def _trimmed_for_request( - messages: list[dict[str, Any]], -) -> tuple[list[dict[str, Any]], int]: - """Return a deep-copied messages list with old screenshots stripped to fit MAX_REQUEST_BYTES. +def _trimmed_for_request(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Drop the screenshots n2 will not look at, keeping the text of every turn. - The most recent KEEP_RECENT_SCREENSHOTS screenshots are protected. The full - `messages` list is preserved unchanged for the caller's return value. + The model reads images from the last KEEP_IMAGE_MESSAGES image-bearing + messages and ignores the rest, so those are stripped every request. If the + payload is still over the cap after that, the previous screenshot goes too — + the latest one is never dropped, since a request without one badly degrades + grounding. """ trimmed = copy.deepcopy(messages) - size = _estimate_size(trimmed) - if size <= MAX_REQUEST_BYTES: - return trimmed, 0 - image_indices = [i for i, m in enumerate(trimmed) if _message_has_image(m)] - if not image_indices: - return trimmed, 0 - protected = set(image_indices[-max(1, KEEP_RECENT_SCREENSHOTS):]) - removed = 0 + for idx in image_indices[:-KEEP_IMAGE_MESSAGES]: + _strip_images(trimmed[idx]) - for idx in image_indices: - if size <= MAX_REQUEST_BYTES: + for idx in image_indices[-KEEP_IMAGE_MESSAGES:-1]: + if _estimate_size(trimmed) <= MAX_REQUEST_BYTES: break - if idx in protected: - continue - if _strip_one_image(trimmed[idx]): - removed += 1 - size = _estimate_size(trimmed) - - # If still over, strip from the protected window too — but always keep the latest. - if size > MAX_REQUEST_BYTES: - last_idx = image_indices[-1] - for idx in image_indices: - if size <= MAX_REQUEST_BYTES: - break - if idx == last_idx: - continue - if _strip_one_image(trimmed[idx]): - removed += 1 - size = _estimate_size(trimmed) + print("Payload still over the 10 MB cap — dropping the previous screenshot too") + _strip_images(trimmed[idx]) - return trimmed, removed + return trimmed def _estimate_size(messages: list[dict[str, Any]]) -> int: @@ -315,50 +294,18 @@ def _message_has_image(msg: dict[str, Any]) -> bool: return any(isinstance(p, dict) and p.get("type") == "image_url" for p in content) -def _strip_one_image(msg: dict[str, Any]) -> bool: +def _strip_images(msg: dict[str, Any]) -> None: content = msg.get("content") if not isinstance(content, list): - return False - - removed = False - new_content: list[dict[str, Any]] = [] - for part in content: - if not removed and isinstance(part, dict) and part.get("type") == "image_url": - removed = True - continue - new_content.append(part) + return - if not removed: - return False + new_content = [ + part for part in content + if not (isinstance(part, dict) and part.get("type") == "image_url") + ] has_text = any(isinstance(p, dict) and p.get("type") == "text" for p in new_content) if not has_text: - new_content.append({"type": "text", "text": "Screenshot omitted to stay under request size limit."}) + new_content.append({"type": "text", "text": "Screenshot omitted — superseded by a newer one."}) msg["content"] = new_content - return True - - -def _scale_coordinates(action: N15Action, viewport_width: int, viewport_height: int) -> N15Action: - scaled = dict(action) - - if "coordinates" in scaled and scaled["coordinates"]: - scaled["coordinates"] = _denormalize(scaled["coordinates"], viewport_width, viewport_height) - - if "start_coordinates" in scaled and scaled["start_coordinates"]: - scaled["start_coordinates"] = _denormalize(scaled["start_coordinates"], viewport_width, viewport_height) - - return scaled - - -def _denormalize(coords: list[int] | tuple[int, int], width: int, height: int) -> list[int]: - """Map [0, 1000] coordinates to viewport pixels and clamp to [0, dim-1]. - - Clamping prevents a boundary value like 1000 from landing one pixel outside - the viewport on a 1280x800 display. - """ - raw_x = round((coords[0] / NAVIGATOR_COORDINATE_SCALE) * width) - raw_y = round((coords[1] / NAVIGATOR_COORDINATE_SCALE) * height) - x = max(0, min(width - 1, raw_x)) - y = max(0, min(height - 1, raw_y)) - return [x, y] diff --git a/pkg/templates/python/yutori/main.py b/pkg/templates/python/yutori/main.py index 94d13e10..cacf3bd8 100644 --- a/pkg/templates/python/yutori/main.py +++ b/pkg/templates/python/yutori/main.py @@ -8,7 +8,7 @@ class _QueryInputOptional(TypedDict, total=False): record_replay: Optional[bool] - kiosk: Optional[bool] + reasoning_effort: Optional[str] user_timezone: Optional[str] user_location: Optional[str] @@ -35,14 +35,14 @@ async def cua_task( payload: QueryInput, ) -> QueryOutput: """ - Process a user query using Yutori n1.5 Computer Use with Kernel's browser automation. + Process a user query using Yutori n2 Computer Use with Kernel's browser automation. Args: ctx: Kernel context containing invocation information payload: An object containing: - query: The task/query string to process - record_replay: Optional boolean to enable video replay recording - - kiosk: Optional boolean to launch in kiosk mode + - reasoning_effort: Optional "none" | "low" | "medium" | "xhigh" - user_timezone: Optional IANA tz (e.g. "America/New_York") - user_location: Optional free-text location for model context @@ -54,27 +54,26 @@ async def cua_task( if not payload or not payload.get("query"): raise ValueError("Query is required") - record_replay = payload.get("record_replay", False) - kiosk_mode = payload.get("kiosk", False) - async with KernelBrowserSession( invocation_id=ctx.invocation_id, stealth=True, - record_replay=record_replay, - kiosk_mode=kiosk_mode, + record_replay=payload.get("record_replay", False), ) as session: print("Kernel browser live view url:", session.live_view_url) loop_kwargs: dict = { - "model": "n1.5-latest", + "model": "n2", "task": payload["query"], "api_key": str(api_key), "kernel": session.kernel, "session_id": str(session.session_id), - "viewport_width": session.viewport_width, - "viewport_height": session.viewport_height, - "kiosk_mode": kiosk_mode, + # n2 sees the whole screen, browser chrome included — the Kernel + # viewport is the window size, so it is the screen size here. + "screen_width": session.viewport_width, + "screen_height": session.viewport_height, } + if payload.get("reasoning_effort"): + loop_kwargs["reasoning_effort"] = payload["reasoning_effort"] if payload.get("user_timezone"): loop_kwargs["user_timezone"] = payload["user_timezone"] if payload.get("user_location"): diff --git a/pkg/templates/python/yutori/pyproject.toml b/pkg/templates/python/yutori/pyproject.toml index 87c17c70..7b82ec2e 100644 --- a/pkg/templates/python/yutori/pyproject.toml +++ b/pkg/templates/python/yutori/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "python-yutori-cua" version = "0.1.0" -description = "Kernel reference app for Yutori n1.5 Computer Use" +description = "Kernel reference app for Yutori n2 Computer Use" requires-python = ">=3.9" dependencies = [ "openai>=1.58.0", diff --git a/pkg/templates/python/yutori/session.py b/pkg/templates/python/yutori/session.py index 9bf020f2..93de7c43 100644 --- a/pkg/templates/python/yutori/session.py +++ b/pkg/templates/python/yutori/session.py @@ -39,9 +39,6 @@ class KernelBrowserSession: record_replay: bool = False replay_grace_period: float = 5.0 # Seconds to wait before stopping replay - # Kiosk mode (hides address bar and tabs in live view) - kiosk_mode: bool = False - # Invocation ID to link browser session to the action invocation invocation_id: Optional[str] = None @@ -64,7 +61,6 @@ async def __aenter__(self) -> "KernelBrowserSession": "width": self.viewport_width, "height": self.viewport_height, }, - kiosk_mode=self.kiosk_mode, ) self.session_id = browser.session_id diff --git a/pkg/templates/python/yutori/tools/__init__.py b/pkg/templates/python/yutori/tools/__init__.py index 5a1a4285..8a8394f3 100644 --- a/pkg/templates/python/yutori/tools/__init__.py +++ b/pkg/templates/python/yutori/tools/__init__.py @@ -1,11 +1,13 @@ -"""Yutori n1.5 Computer Tools.""" +"""Yutori n2 Computer Use Tools.""" -from .base import ToolError, ToolResult -from .computer import ComputerTool, N15Action +from .base import ToolError +from .computer import BatchOutcome, ComputerTool, N2Action +from .system import SystemTools __all__ = [ "ToolError", - "ToolResult", + "BatchOutcome", "ComputerTool", - "N15Action", + "N2Action", + "SystemTools", ] diff --git a/pkg/templates/python/yutori/tools/base.py b/pkg/templates/python/yutori/tools/base.py index d12a38e6..1a730994 100644 --- a/pkg/templates/python/yutori/tools/base.py +++ b/pkg/templates/python/yutori/tools/base.py @@ -1,6 +1,4 @@ -"""Base tool types for Yutori n1.""" - -from typing import TypedDict +"""Base tool types for Yutori n2.""" class ToolError(Exception): @@ -9,9 +7,3 @@ class ToolError(Exception): def __init__(self, message: str): self.message = message super().__init__(message) - - -class ToolResult(TypedDict, total=False): - base64_image: str - output: str - error: str diff --git a/pkg/templates/python/yutori/tools/computer.py b/pkg/templates/python/yutori/tools/computer.py index d4debded..e6085cd5 100644 --- a/pkg/templates/python/yutori/tools/computer.py +++ b/pkg/templates/python/yutori/tools/computer.py @@ -1,80 +1,96 @@ """ -Yutori n1.5 Computer Tool +Yutori n2 Computer Tool -Maps n1.5-latest action format to Kernel's Computer Controls API. -Screenshots are converted to WebP for better compression across multi-step trajectories. +Maps n2's `computer_batch` action list onto Kernel's Computer Controls API. +n2 plans every action in a batch against the screenshot taken *before* the batch +ran, so the batch executes sequentially, stops at the first error, and answers +with a single screenshot taken after the last action that ran. -@see https://docs.yutori.com/reference/n1-5 +@see https://docs.yutori.com/reference/n2 """ from __future__ import annotations import asyncio import base64 -import json +from dataclasses import dataclass from io import BytesIO -from typing import Any, Literal, TypedDict +from typing import Any, Literal, Optional, TypedDict from kernel import Kernel from PIL import Image -from .base import ToolError, ToolResult +from .base import ToolError TYPING_DELAY_MS = 12 -SCREENSHOT_DELAY_S = 0.15 -ACTION_DELAY_S = 0.3 +# Let the UI settle between actions in a batch, and again before the screenshot +# that the model will plan its next batch against. +INTER_ACTION_DELAY_S = 0.08 +SETTLE_DELAY_S = 0.4 -# n1.5 scroll `amount` is in "wheel units" where 1 unit ≈ 10% of the viewport -# height (~80px at 800px tall). Kernel's delta_y is a wheel-event repeat count -# where each tick is much smaller in practice, so we multiply. -SCROLL_NOTCHES_PER_AMOUNT = 4 +NAVIGATOR_COORDINATE_SCALE = 1000 + +# n2's scroll `amount` is a wheel notch count (1-50), which is exactly what +# Kernel's delta_x/delta_y take ("xdotool wheel units"), so it passes through. +DEFAULT_SCROLL_AMOUNT = 3 # WebP quality for screenshots. Kernel returns PNGs, which are crisp and # tolerate aggressive WebP compression with no visible degradation — matches # Yutori SDK's DEFAULT_WEBP_QUALITY_FOR_PNG=30 (yutori-sdk-python/yutori/ -# navigator/images.py). Lower values cut payload size substantially on long -# multi-step trajectories. +# navigator/images.py). Requests are capped at 10 MB and a trajectory carries +# several full-screen captures, so the compression matters. WEBP_QUALITY = 30 -N15ActionType = Literal[ +N2ActionName = Literal[ "left_click", "double_click", "triple_click", "middle_click", "right_click", - "mouse_move", - "mouse_down", - "mouse_up", "scroll", "type", "key_press", - "hold_key", "drag", + "mouse_move", + "mouse_down", + "mouse_up", + "hold_key", "wait", - "refresh", - "go_back", - "go_forward", - "goto_url", + "screenshot", ] -class N15Action(TypedDict, total=False): - action_type: N15ActionType - coordinates: tuple[int, int] | list[int] - start_coordinates: tuple[int, int] | list[int] - direction: Literal["up", "down", "left", "right"] - amount: int - text: str - key: str - modifier: str - duration: int - url: str +class N2Action(TypedDict, total=False): + """One `{name, arguments}` member of a `computer_batch` call.""" + + name: N2ActionName + arguments: dict[str, Any] + + +@dataclass +class BatchOutcome: + executed: int + total: int + # Set when an action raised; the remaining actions were skipped. + failed_index: Optional[int] = None + failed_name: Optional[str] = None + failed_message: Optional[str] = None + + def describe(self) -> str: + if self.failed_index is None: + return f"Executed {self.executed} of {self.total} actions." + return ( + f"Executed {self.executed} of {self.total} actions. " + f"Action {self.failed_index + 1} ({self.failed_name}) failed: {self.failed_message}. " + f"The remaining actions were skipped." + ) -# n1.5 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). +# n2 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). # Kernel's press_key expects XKeysym names (e.g. `Return`, `Ctrl`, `Page_Up`). -# Keys not in the map pass through unchanged (printable characters like `a`, -# `1`, `,` are already XKeysym). +# This map covers every key Yutori documents at +# https://docs.yutori.com/reference/n2#key-space — keys not in the map pass +# through unchanged (printable characters like `a` and `1` are already XKeysym). # # Sister implementation (Playwright target instead of XKeysym): # https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/keys.py @@ -115,6 +131,19 @@ class N15Action(TypedDict, total=False): "pagedown": "Page_Down", # Function keys **{f"f{i}": f"F{i}" for i in range(1, 13)}, + # Punctuation — n2 sends word forms, most of which are already XKeysym names + "minus": "minus", + "plus": "plus", + "equal": "equal", + "comma": "comma", + "period": "period", + "slash": "slash", + "backslash": "backslash", + "semicolon": "semicolon", + "quote": "apostrophe", + "backquote": "grave", + "bracketleft": "bracketleft", + "bracketright": "bracketright", # Locks / special "capslock": "Caps_Lock", "numlock": "Num_Lock", @@ -126,19 +155,11 @@ class N15Action(TypedDict, total=False): def _map_token(token: str) -> str: - lower = token.strip().lower() - return KEY_MAP.get(lower, token.strip()) - - -def _normalize_url(url: str) -> str: - trimmed = url.strip() - if "://" in trimmed: - return trimmed - return f"https://{trimmed}" + return KEY_MAP.get(token.strip().lower(), token.strip()) def _parse_key_expression(expr: str) -> list[str]: - """Parse an n1.5 key expression into one Kernel combo per sequential press. + """Parse an n2 key expression into one Kernel combo per sequential press. Spaces separate sequential presses; '+' separates simultaneous tokens within a press. Examples: @@ -154,285 +175,198 @@ def _parse_key_expression(expr: str) -> list[str]: ] +def _duration_ms(duration: Any, fallback_ms: int) -> int: + """n2 emits `duration` in seconds; Kernel takes milliseconds.""" + if isinstance(duration, (int, float)) and duration > 0: + return int(duration * 1000) + return fallback_ms + + class ComputerTool: - def __init__(self, kernel: Kernel, session_id: str, width: int = 1280, height: int = 800, kiosk_mode: bool = False): + def __init__(self, kernel: Kernel, session_id: str, screen_width: int = 1280, screen_height: int = 800): self.kernel = kernel self.session_id = session_id - self.width = width - self.height = height - self.kiosk_mode = kiosk_mode - - async def execute(self, action: N15Action) -> ToolResult: - action_type = action.get("action_type") - - handlers = { - "left_click": lambda a: self._handle_click(a, "left", 1), - "double_click": lambda a: self._handle_click(a, "left", 2), - "triple_click": lambda a: self._handle_click(a, "left", 3), - "middle_click": lambda a: self._handle_click(a, "middle", 1), - "right_click": lambda a: self._handle_click(a, "right", 1), - "mouse_move": self._handle_mouse_move, - "mouse_down": lambda a: self._handle_mouse_button(a, "down"), - "mouse_up": lambda a: self._handle_mouse_button(a, "up"), - "scroll": self._handle_scroll, - "type": self._handle_type, - "key_press": self._handle_key_press, - "hold_key": self._handle_hold_key, - "drag": self._handle_drag, - "wait": self._handle_wait, - "refresh": self._handle_refresh, - "go_back": self._handle_go_back, - "go_forward": self._handle_go_forward, - "goto_url": self._handle_goto_url, - } - - handler = handlers.get(action_type) - if not handler: - raise ToolError(f"Unknown action type: {action_type}") - - return await handler(action) - - async def _handle_click(self, action: N15Action, button: str, num_clicks: int) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) - modifier = action.get("modifier") + self.screen_width = screen_width + self.screen_height = screen_height + + async def run_batch(self, actions: list[N2Action]) -> BatchOutcome: + """Run a `computer_batch` action list in order, stopping at the first failure. + + The caller reports the outcome and a single post-batch screenshot back to n2. + """ + for index, action in enumerate(actions): + try: + await self._run_action(action) + except Exception as error: + return BatchOutcome( + executed=index, + total=len(actions), + failed_index=index, + failed_name=str(action.get("name")), + failed_message=str(error), + ) + if index < len(actions) - 1: + await asyncio.sleep(INTER_ACTION_DELAY_S) + + return BatchOutcome(executed=len(actions), total=len(actions)) + + async def _run_action(self, action: N2Action) -> None: + name = action.get("name") + args = action.get("arguments") or {} + + if name in ("left_click", "double_click", "triple_click"): + num_clicks = {"left_click": 1, "double_click": 2, "triple_click": 3}[name] + return self._click(args, "left", num_clicks) + if name == "middle_click": + return self._click(args, "middle", 1) + if name == "right_click": + return self._click(args, "right", 1) + if name == "scroll": + return self._scroll(args) + if name == "type": + return self._type(args) + if name == "key_press": + return self._key_press(args) + if name == "drag": + return self._drag(args) + if name == "mouse_move": + return self._mouse_move(args) + if name in ("mouse_down", "mouse_up"): + return self._mouse_button(args, name.split("_")[1]) + if name == "hold_key": + return self._hold_key(args) + if name == "wait": + return await asyncio.sleep(_duration_ms(args.get("duration"), 2000) / 1000) + if name == "screenshot": + # A batch already answers with a screenshot taken after its last + # action, so an explicit `screenshot` member has nothing to do. + return None + + raise ToolError(f"Unknown action: {name}") + + def _click(self, args: dict[str, Any], button: str, num_clicks: int) -> None: + x, y = self._require_coordinates(args.get("coordinates")) kwargs: dict[str, Any] = { - "x": coords["x"], - "y": coords["y"], + "x": x, + "y": y, "button": button, "click_type": "click", "num_clicks": num_clicks, } - if modifier: - kwargs["hold_keys"] = [_map_token(modifier)] + if args.get("modifier"): + kwargs["hold_keys"] = [_map_token(args["modifier"])] self.kernel.browsers.computer.click_mouse(self.session_id, **kwargs) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_mouse_move(self, action: N15Action) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) + def _mouse_move(self, args: dict[str, Any]) -> None: + x, y = self._require_coordinates(args.get("coordinates")) + self.kernel.browsers.computer.move_mouse(self.session_id, x=x, y=y) - self.kernel.browsers.computer.move_mouse( - self.session_id, - x=coords["x"], - y=coords["y"], - ) - - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_mouse_button(self, action: N15Action, click_type: str) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) + def _mouse_button(self, args: dict[str, Any], click_type: str) -> None: + # Coordinates are optional here — without them the button is pressed or + # released wherever the cursor already is, which is what a manual + # mouse_move -> mouse_down -> mouse_move -> mouse_up drag relies on. + if args.get("coordinates"): + x, y = self._require_coordinates(args["coordinates"]) + else: + position = self.kernel.browsers.computer.get_mouse_position(self.session_id) + x, y = position.x, position.y self.kernel.browsers.computer.click_mouse( self.session_id, - x=coords["x"], - y=coords["y"], + x=x, + y=y, button="left", click_type=click_type, ) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_scroll(self, action: N15Action) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) - direction = action.get("direction") - amount = max(action.get("amount", 3), 1) + def _scroll(self, args: dict[str, Any]) -> None: + x, y = self._require_coordinates(args.get("coordinates")) + direction = args.get("direction") - if direction not in ("up", "down", "left", "right"): + # n2 only scrolls vertically. + if direction not in ("up", "down"): raise ToolError(f"Invalid scroll direction: {direction}") - # Yutori 1 unit ≈ 10% of viewport height; scale into Kernel wheel-event ticks. - ticks = amount * SCROLL_NOTCHES_PER_AMOUNT - - delta_x = 0 - delta_y = 0 - - if direction == "up": - delta_y = -ticks - elif direction == "down": - delta_y = ticks - elif direction == "left": - delta_x = -ticks - elif direction == "right": - delta_x = ticks - - modifier = action.get("modifier") - scroll_kwargs: dict[str, Any] = { - "x": coords["x"], - "y": coords["y"], - "delta_x": delta_x, - "delta_y": delta_y, + notches = max(1, round(args.get("amount") or DEFAULT_SCROLL_AMOUNT)) + kwargs: dict[str, Any] = { + "x": x, + "y": y, + "delta_x": 0, + "delta_y": -notches if direction == "up" else notches, } - if modifier: - scroll_kwargs["hold_keys"] = [_map_token(modifier)] - - self.kernel.browsers.computer.scroll(self.session_id, **scroll_kwargs) + if args.get("modifier"): + kwargs["hold_keys"] = [_map_token(args["modifier"])] - await asyncio.sleep(SCREENSHOT_DELAY_S) - screenshot_result = await self.screenshot() - screenshot_result["output"] = f"Scrolled {amount} unit(s) {direction}." - return screenshot_result + self.kernel.browsers.computer.scroll(self.session_id, **kwargs) - async def _handle_type(self, action: N15Action) -> ToolResult: - text = action.get("text") + def _type(self, args: dict[str, Any]) -> None: + text = args.get("text") if not text: - raise ToolError("text is required for type action") + raise ToolError("text is required for type") - self.kernel.browsers.computer.type_text( - self.session_id, - text=text, - delay=TYPING_DELAY_MS, - ) + self.kernel.browsers.computer.type_text(self.session_id, text=text, delay=TYPING_DELAY_MS) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_key_press(self, action: N15Action) -> ToolResult: - key = action.get("key") + def _key_press(self, args: dict[str, Any]) -> None: + key = args.get("key") if not key: - raise ToolError("key is required for key_press action") + raise ToolError("key is required for key_press") - # n1.5 supports sequential presses ("down down down enter") — issue each + # n2 supports sequential presses ("down down down enter") — issue each # combo as its own press_key so they're seen as separate keystrokes. - combos = _parse_key_expression(key) - for combo in combos: + for combo in _parse_key_expression(key): self.kernel.browsers.computer.press_key(self.session_id, keys=[combo]) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_hold_key(self, action: N15Action) -> ToolResult: - key = action.get("key") + def _hold_key(self, args: dict[str, Any]) -> None: + key = args.get("key") if not key: - raise ToolError("key is required for hold_key action") - - # Yutori emits `duration` in seconds; Kernel SDK's press_key takes ms. - duration_s = action.get("duration") - duration_ms = int(duration_s * 1000) if duration_s and duration_s > 0 else 1000 + raise ToolError("key is required for hold_key") - combos = _parse_key_expression(key) - for combo in combos: + for combo in _parse_key_expression(key): self.kernel.browsers.computer.press_key( self.session_id, keys=[combo], - duration=duration_ms, + duration=_duration_ms(args.get("duration"), 1000), ) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_drag(self, action: N15Action) -> ToolResult: - start_coords = self._get_coordinates(action.get("start_coordinates")) - end_coords = self._get_coordinates(action.get("coordinates")) + def _drag(self, args: dict[str, Any]) -> None: + start_x, start_y = self._require_coordinates(args.get("start_coordinates")) + end_x, end_y = self._require_coordinates(args.get("coordinates")) self.kernel.browsers.computer.drag_mouse( self.session_id, - path=[[start_coords["x"], start_coords["y"]], [end_coords["x"], end_coords["y"]]], + path=[[start_x, start_y], [end_x, end_y]], button="left", ) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_wait(self, action: N15Action) -> ToolResult: - # Yutori emits `duration` in seconds (matches reference impl). - duration = action.get("duration") - seconds = duration if duration and duration > 0 else 2 - await asyncio.sleep(seconds) - return await self.screenshot() - - async def _handle_refresh(self, action: N15Action) -> ToolResult: - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["F5"], - ) - await asyncio.sleep(2) - return await self.screenshot() - - async def _handle_go_back(self, action: N15Action) -> ToolResult: - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Alt+Left"], - ) - await asyncio.sleep(1.5) - return await self.screenshot() + async def screenshot(self) -> str: + """Capture the whole screen — browser chrome included — as base64 WebP.""" + await asyncio.sleep(SETTLE_DELAY_S) - async def _handle_go_forward(self, action: N15Action) -> ToolResult: - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Alt+Right"], - ) - await asyncio.sleep(1.5) - return await self.screenshot() + response = self.kernel.browsers.computer.capture_screenshot(self.session_id) + image = Image.open(BytesIO(response.read())) + webp_buffer = BytesIO() + image.save(webp_buffer, "WEBP", quality=WEBP_QUALITY) - async def _handle_goto_url(self, action: N15Action) -> ToolResult: - url = action.get("url") - if not url: - raise ToolError("url is required for goto_url action") - target_url = _normalize_url(url) + return base64.b64encode(webp_buffer.getvalue()).decode("utf-8") - if self.kiosk_mode: - response = self.kernel.browsers.playwright.execute( - self.session_id, - code=f"await page.goto({json.dumps(target_url)});", - timeout_sec=60, - ) - if not response.success: - raise ToolError(response.error or "Playwright goto failed") - await asyncio.sleep(ACTION_DELAY_S) - return await self.screenshot() + def _require_coordinates(self, coords: Any) -> tuple[int, int]: + """Map [0, 1000] coordinates to screen pixels, clamped to [0, dim-1]. - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Ctrl+l"], - ) - await asyncio.sleep(ACTION_DELAY_S) + Clamping prevents a boundary value like 1000 from landing one pixel + outside the screen on a 1280x800 display. + """ + if not isinstance(coords, (list, tuple)) or len(coords) != 2: + raise ToolError(f"coordinates are required, got {coords!r}") - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Ctrl+a"], - ) - await asyncio.sleep(0.1) + nx, ny = coords + if not isinstance(nx, (int, float)) or not isinstance(ny, (int, float)): + raise ToolError(f"Invalid coordinates: {coords!r}") - self.kernel.browsers.computer.type_text( - self.session_id, - text=target_url, - delay=TYPING_DELAY_MS, - ) - await asyncio.sleep(ACTION_DELAY_S) + x = round(nx / NAVIGATOR_COORDINATE_SCALE * self.screen_width) + y = round(ny / NAVIGATOR_COORDINATE_SCALE * self.screen_height) - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Return"], + return ( + max(0, min(self.screen_width - 1, x)), + max(0, min(self.screen_height - 1, y)), ) - await asyncio.sleep(2) - return await self.screenshot() - - async def screenshot(self) -> ToolResult: - try: - response = self.kernel.browsers.computer.capture_screenshot( - self.session_id - ) - png_bytes = response.read() - img = Image.open(BytesIO(png_bytes)) - webp_buf = BytesIO() - img.save(webp_buf, "WEBP", quality=WEBP_QUALITY) - base64_image = base64.b64encode(webp_buf.getvalue()).decode("utf-8") - return {"base64_image": base64_image} - except Exception as e: - raise ToolError(f"Failed to take screenshot: {e}") - - def _get_coordinates( - self, coords: tuple[int, int] | list[int] | None - ) -> dict[str, int]: - if coords is None or len(coords) != 2: - return {"x": self.width // 2, "y": self.height // 2} - - x, y = coords - if not isinstance(x, (int, float)) or not isinstance(y, (int, float)) or x < 0 or y < 0: - raise ToolError(f"Invalid coordinates: {coords}") - - return {"x": int(x), "y": int(y)} diff --git a/pkg/templates/python/yutori/tools/system.py b/pkg/templates/python/yutori/tools/system.py new file mode 100644 index 00000000..a29e2d81 --- /dev/null +++ b/pkg/templates/python/yutori/tools/system.py @@ -0,0 +1,184 @@ +""" +Yutori n2 Shell and File Tools + +n2's tool set always ships `bash`, `read`, `write`, and `edit` alongside +`computer_batch` — `disable_tools` is rejected by the API, so a loop has to +answer these too. They run against the Kernel browser VM via the process and +filesystem APIs. + +@see https://docs.yutori.com/reference/n2 +""" + +from __future__ import annotations + +import base64 +import shlex +import time +from typing import Any, Optional + +from kernel import Kernel + +from .base import ToolError + +DEFAULT_TIMEOUT_SEC = 120 +MAX_TIMEOUT_SEC = 600 +DEFAULT_READ_LIMIT = 2000 +MAX_WRITE_CHARS = 256_000 + +# bash: "The working directory persists across calls; environment variables and +# shell functions do not." Each exec is its own process, so the command reports +# its final directory on a sentinel line that we strip before returning stdout. +CWD_SENTINEL = "__n2_cwd__" + + +def _decode(value: Optional[str]) -> str: + return base64.b64decode(value).decode("utf-8", errors="replace") if value else "" + + +def _format_command_result(stdout: str, stderr: str, exit_code: Optional[int]) -> str: + parts = [] + if stdout.strip(): + parts.append(stdout.rstrip()) + if stderr.strip(): + parts.append(f"stderr:\n{stderr.rstrip()}") + if exit_code: + parts.append(f"Exited with code {exit_code}.") + + return "\n".join(parts) if parts else "Command produced no output." + + +class SystemTools: + def __init__(self, kernel: Kernel, session_id: str): + self.kernel = kernel + self.session_id = session_id + self.cwd: Optional[str] = None + self.seen_paths: set[str] = set() + + def execute(self, name: str, args: dict[str, Any]) -> str: + handlers = { + "bash": self._bash, + "read": self._read, + "write": self._write, + "edit": self._edit, + } + handler = handlers.get(name) + if not handler: + raise ToolError(f"Unknown tool: {name}") + + return handler(args) + + def _bash(self, args: dict[str, Any]) -> str: + command = args.get("command") + if not command: + raise ToolError("command is required for bash") + + if args.get("run_in_background"): + return self._bash_background(command) + + script = "\n".join([ + command, + "n2_status=$?", + f"printf '\\n{CWD_SENTINEL}%s' \"$(pwd)\"", + "exit $n2_status", + ]) + + result = self.kernel.browsers.process.exec( + self.session_id, + command="bash", + args=["-lc", script], + cwd=self.cwd, + timeout_sec=min(args.get("timeout") or DEFAULT_TIMEOUT_SEC, MAX_TIMEOUT_SEC), + ) + + stdout = self._take_cwd(_decode(result.stdout_b64)) + + return _format_command_result(stdout, _decode(result.stderr_b64), result.exit_code) + + def _read(self, args: dict[str, Any]) -> str: + file_path = args.get("file_path") + if not file_path: + raise ToolError("file_path is required for read") + + response = self.kernel.browsers.fs.read_file(self.session_id, path=file_path) + lines = response.read().decode("utf-8", errors="replace").split("\n") + + offset = max(0, args.get("offset") or 0) + limit = max(1, args.get("limit") or DEFAULT_READ_LIMIT) + page = lines[offset : offset + limit] + + self.seen_paths.add(file_path) + + if not page: + return f"{file_path} has {len(lines)} line(s); offset {offset} is past the end." + + # cat -n format, so line numbers survive into the model's next edit. + return "\n".join(f"{offset + i + 1:>6}\t{line}" for i, line in enumerate(page)) + + def _write(self, args: dict[str, Any]) -> str: + file_path = args.get("file_path") + content = args.get("content") + if not file_path or content is None: + raise ToolError("file_path and content are required for write") + if len(content) > MAX_WRITE_CHARS: + raise ToolError(f"content exceeds the {MAX_WRITE_CHARS} character cap") + + self.kernel.browsers.fs.write_file(self.session_id, content.encode("utf-8"), path=file_path) + self.seen_paths.add(file_path) + + return f"Wrote {len(content)} character(s) to {file_path}." + + def _edit(self, args: dict[str, Any]) -> str: + file_path = args.get("file_path") + old_string = args.get("old_string") + new_string = args.get("new_string") + if not file_path or old_string is None or new_string is None: + raise ToolError("file_path, old_string, and new_string are required for edit") + # n2 is expected to know the current bytes before changing them. + if file_path not in self.seen_paths: + raise ToolError(f"{file_path} has not been read in this session — read it before editing.") + + response = self.kernel.browsers.fs.read_file(self.session_id, path=file_path) + original = response.read().decode("utf-8", errors="replace") + + occurrences = original.count(old_string) + if occurrences == 0: + raise ToolError(f"old_string not found in {file_path}") + + replace_all = bool(args.get("replace_all")) + if occurrences > 1 and not replace_all: + raise ToolError( + f"old_string matches {occurrences} times in {file_path} — " + f"pass replace_all or include more context." + ) + + updated = original.replace(old_string, new_string, -1 if replace_all else 1) + self.kernel.browsers.fs.write_file(self.session_id, updated.encode("utf-8"), path=file_path) + + return f"Replaced {occurrences if replace_all else 1} occurrence(s) in {file_path}." + + def _bash_background(self, command: str) -> str: + log_path = f"/tmp/n2-bg-{int(time.time() * 1000)}.log" + script = f"nohup bash -c {shlex.quote(command)} > {log_path} 2>&1 &\necho $!" + + result = self.kernel.browsers.process.exec( + self.session_id, + command="bash", + args=["-lc", script], + cwd=self.cwd, + ) + pid = _decode(result.stdout_b64).strip() + + return "\n".join([ + f"Started in the background with pid {pid}.", + f"Output is being written to {log_path} — use the read tool to check on it.", + f"Cancel it with: kill {pid}", + ]) + + def _take_cwd(self, stdout: str) -> str: + """Strip the trailing sentinel line and remember the directory it reported.""" + marker = stdout.rfind(f"\n{CWD_SENTINEL}") + if marker == -1: + return stdout + + self.cwd = stdout[marker + len(CWD_SENTINEL) + 1 :].strip() or self.cwd + return stdout[:marker] diff --git a/pkg/templates/typescript/yutori/README.md b/pkg/templates/typescript/yutori/README.md index 04be089f..477f1127 100644 --- a/pkg/templates/typescript/yutori/README.md +++ b/pkg/templates/typescript/yutori/README.md @@ -1,10 +1,8 @@ -# Kernel TypeScript Sample App - Yutori n1.5 Computer Use +# Kernel TypeScript Sample App - Yutori n2 Computer Use -This Kernel app implements a prompt loop using Yutori's Navigator n1.5 with Kernel's Computer Controls API. +This Kernel app implements a prompt loop using Yutori's Navigator n2 with Kernel's Computer Controls API. -[Navigator n1.5](https://yutori.com/blog/introducing-n1-5) is Yutori's pixels-to-actions LLM that predicts browser actions from screenshots. - -This template runs n1.5 in **computer-use-only mode**. n1.5 also supports a hybrid vision + DOM/JavaScript path (page-state extraction, custom JS, structured JSON output) for multi-field forms and bulk data extraction, but those tools are intentionally disabled here — see [Disabled tools](#disabled-tools). +[Navigator n2](https://docs.yutori.com/reference/n2) is Yutori's computer-use model. It reads a screenshot of the whole screen and answers with a batch of mouse and keyboard actions, and it can run shell commands and edit files on the machine it is driving. ## Setup @@ -28,7 +26,7 @@ kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://w Optional payload fields: - `record_replay` (bool) — capture a video of the session (paid plans only). -- `kiosk` (bool) — launch the browser without address bar / tabs ([see below](#kiosk-mode)). +- `reasoning_effort` (`"none"`, `"low"`, `"medium"`, `"xhigh"`) — n2 reasons at `medium` by default. `xhigh` gives the longest traces and does best on hard multi-step tasks; `none` turns reasoning off. - `user_timezone` (IANA, e.g. `"America/New_York"`) and `user_location` (free text, e.g. `"New York, NY, US"`) — appended to the task message so the model has accurate temporal/locational grounding. More involved example (Kanban drag-and-drop): @@ -49,66 +47,62 @@ kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://e When enabled, the response will include a `replay_url` field with a link to view the recorded session. -## Kiosk mode - -Prefer **non-kiosk mode** by default and when the agent is expected to switch domains via URL. Use **kiosk (`"kiosk": true`)** when: (1) you're recording sessions and want a cleaner UI in the replay, or (2) you're automating on a single website and the combination of the complex site layout and browser chrome (address bar, tabs) may confuse the agent. - -Note: In kiosk mode the agent may still try to use the address bar to enter URLs; it's not available, so it will eventually use `goto_url`, but those attempts may result in slowdown of the overall session. +## Screen Configuration -Default (non-kiosk): - -```bash -kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com, then navigate to ign.com and describe the page"}' -``` +n2 is a desktop model: it expects a screenshot of the **whole screen**, browser chrome included, and it navigates by clicking the address bar rather than through a dedicated navigation action. A Kernel `viewport` sets the browser window size, and screenshots from Computer Controls capture that window with its tabs and toolbar — which is exactly what n2 wants, so this template does not use kiosk mode. -With kiosk (single-site or recording): +This template runs at **1280x800**. Yutori also lists 1920x1080 and 1280x720 as resolutions in regular use; grounding may degrade at extreme aspect ratios. -```bash -kernel invoke ts-yutori-cua cua-task --payload '{"query": "Enter https://example.com in the search box and then describe the page.", "kiosk": true}' -``` +> **Note:** n2 outputs coordinates in a 1000x1000 relative space, which are scaled to the actual screen dimensions per action. -## Viewport Configuration +See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. -Yutori n1.5 recommends a **1280×800 (WXGA, 16:10)** viewport for best grounding accuracy. +## Screenshots -> **Note:** n1.5 outputs coordinates in a 1000×1000 relative space, which are automatically scaled to the actual viewport dimensions. +Screenshots are converted to WebP before they are sent. Yutori caps requests at 10 MB, and a full-screen PNG trajectory blows past that on its own. -See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. +The loop also drops screenshots the model will not read: n2 keeps images from only the last 2 image-bearing messages, so older ones are stripped from each request while every message's text is kept. -## Screenshots +## Tools -Screenshots are automatically converted to WebP format for better compression across multi-step trajectories, as recommended by Yutori. +This template pins the `computer_use_tools-20260825` tool set. n2 rejects `disable_tools`, so all five tools are always served and the loop answers all of them. -## n1.5-latest Supported Actions +### `computer_batch` -This template uses the `browser_tools_core-20260403` tool set — coordinate-based browser actions that operate on screenshots only. +n2 returns a whole action list in one call. The loop runs them in order, stops at the first error, and replies with a single tool result carrying one screenshot taken after the last action that ran. Every coordinate in a batch refers to the screenshot from *before* the batch started. | Action | Description | |--------|-------------| | `left_click` | Left mouse click at coordinates (supports `modifier`) | | `double_click` | Double-click at coordinates (supports `modifier`) | | `triple_click` | Triple-click at coordinates (supports `modifier`) | -| `middle_click` | Middle mouse click at coordinates | -| `right_click` | Right mouse click at coordinates | -| `mouse_move` | Move mouse to coordinates without clicking | -| `mouse_down` | Press the left mouse button at coordinates | -| `mouse_up` | Release the left mouse button at coordinates | -| `scroll` | Scroll page in a direction | -| `type` | Type text into focused element | -| `key_press` | Send a single key or key combination | -| `hold_key` | Hold a key for a duration | -| `drag` | Click-and-drag operation | -| `wait` | Pause for UI to update | -| `refresh` | Reload current page | -| `go_back` | Navigate back in history | -| `go_forward` | Navigate forward in history | -| `goto_url` | Navigate to a URL | - -### Disabled tools - -The DOM/Playwright-based "expanded" tools (`extract_elements`, `find`, `set_element_value`, `execute_js`) are intentionally disabled via the `disable_tools` request parameter — this template runs computer-use only and does not expose a Playwright page to the model. +| `middle_click` | Middle mouse click at coordinates (supports `modifier`) | +| `right_click` | Right mouse click at coordinates (supports `modifier`) | +| `scroll` | Scroll up or down at coordinates, in wheel notches | +| `type` | Type text into the focused element | +| `key_press` | Press a key, combination, or sequence | +| `drag` | Drag from `start_coordinates` to `coordinates` | +| `mouse_move` | Move the mouse to coordinates without clicking | +| `mouse_down` | Press and hold the left button (coordinates optional) | +| `mouse_up` | Release the left button (coordinates optional) | +| `hold_key` | Hold a key down for a duration | +| `wait` | Pause without interacting | +| `screenshot` | Look without acting | + +Horizontal scrolling is not available, and there are no `goto_url` / `go_back` / `go_forward` / `refresh` actions — n2 drives the address bar and browser buttons like a person would. + +### `bash`, `read`, `write`, `edit` + +n2 can also work on the browser VM directly. These map onto Kernel's process and filesystem APIs: + +| Tool | Backed by | +|------|-----------| +| `bash` | `browsers.process.exec` — each call is a separate process; the working directory carries over between calls, environment variables do not. `run_in_background` detaches the command and reports its pid and log path. | +| `read` | `browsers.fs.readFile` — returns `cat -n` output, with `offset` / `limit` paging | +| `write` | `browsers.fs.writeFile` | +| `edit` | read, exact-string replace, write. Refuses to edit a file that has not been read in this session. | ## Resources -- [Yutori n1.5 API Documentation](https://docs.yutori.com/reference/n1-5) +- [Yutori n2 API Documentation](https://docs.yutori.com/reference/n2) - [Kernel Documentation](https://www.kernel.sh/docs/quickstart) diff --git a/pkg/templates/typescript/yutori/index.ts b/pkg/templates/typescript/yutori/index.ts index 215a930b..f52fc45e 100644 --- a/pkg/templates/typescript/yutori/index.ts +++ b/pkg/templates/typescript/yutori/index.ts @@ -1,6 +1,6 @@ import { Kernel, type KernelContext } from '@onkernel/sdk'; import type OpenAI from 'openai'; -import { samplingLoop } from './loop'; +import { samplingLoop, type ReasoningEffort } from './loop'; import { KernelBrowserSession } from './session'; const kernel = new Kernel(); @@ -10,7 +10,7 @@ const app = kernel.app('ts-yutori-cua'); interface QueryInput { query: string; record_replay?: boolean; - kiosk?: boolean; + reasoning_effort?: ReasoningEffort; user_timezone?: string; user_location?: string; } @@ -35,13 +35,11 @@ app.action( throw new Error('Query is required'); } - // Create browser session with optional replay recording and kiosk mode - const kioskMode = payload.kiosk ?? false; + // Create browser session with optional replay recording const session = new KernelBrowserSession(kernel, { invocationId: ctx.invocation_id, stealth: true, recordReplay: payload.record_replay ?? false, - kioskMode, }); await session.start(); @@ -50,14 +48,16 @@ app.action( try { // Run the sampling loop const { finalAnswer, messages } = await samplingLoop({ - model: 'n1.5-latest', + model: 'n2', task: payload.query, apiKey: YUTORI_API_KEY, kernel, sessionId: session.sessionId, - viewportWidth: session.viewportWidth, - viewportHeight: session.viewportHeight, - kioskMode, + // n2 sees the whole screen, browser chrome included — the Kernel + // viewport is the window size, so it is the screen size here. + screenWidth: session.viewportWidth, + screenHeight: session.viewportHeight, + reasoningEffort: payload.reasoning_effort, userTimezone: payload.user_timezone, userLocation: payload.user_location, }); diff --git a/pkg/templates/typescript/yutori/loop.ts b/pkg/templates/typescript/yutori/loop.ts index 2b2c8c7a..b2c5c580 100644 --- a/pkg/templates/typescript/yutori/loop.ts +++ b/pkg/templates/typescript/yutori/loop.ts @@ -1,38 +1,38 @@ /** - * Yutori n1.5 Sampling Loop + * Yutori n2 Sampling Loop * - * Implements the agent loop for Yutori's n1.5-latest computer use model. - * n1.5-latest uses an OpenAI-compatible API with tool_calls: - * - Actions are returned via tool_calls in the assistant message - * - Tool results use role: "tool" with matching tool_call_id + * Implements the agent loop for Yutori's n2 computer use model. + * n2 uses an OpenAI-compatible API with tool_calls: + * - GUI actions arrive as a single `computer_batch` call holding an action list + * - `bash`, `read`, `write`, and `edit` arrive as their own calls + * - Every call needs one tool result with a matching tool_call_id * - The model stops by returning content without tool_calls * - Coordinates are returned in 1000x1000 space and need scaling * - * @see https://docs.yutori.com/reference/n1-5 + * @see https://docs.yutori.com/reference/n2 */ import OpenAI from 'openai'; import type { Kernel } from '@onkernel/sdk'; -import { ComputerTool, type N15Action, type ToolResult } from './tools/computer'; +import { ComputerTool, type BatchOutcome, type N2Action } from './tools/computer'; +import { SystemTools } from './tools/system'; -// Tools that require a Playwright page / DOM access. The default core tool set -// already excludes them, but we also list them in `disable_tools` so the -// exclusion is explicit and survives if the default ever changes. -const DISABLED_TOOLS = ['extract_elements', 'find', 'set_element_value', 'execute_js']; -const TOOL_SET = 'browser_tools_core-20260403'; +// Pin the tool set so a change to the server default can't change which tools +// this run exposes. n2 rejects `disable_tools`, so the set is served whole. +const TOOL_SET = 'computer_use_tools-20260825'; -const NAVIGATOR_COORDINATE_SCALE = 1000; +export type ReasoningEffort = 'none' | 'low' | 'medium' | 'xhigh'; -// Screenshot-trimming defaults mirror Yutori's reference loop: -// https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/payload.py -// Trimming is size-triggered — we only drop old screenshots when the payload -// exceeds MAX_REQUEST_BYTES, and we always keep at least KEEP_RECENT_SCREENSHOTS. +// Requests are capped at 10 MB. n2 also only reads images from the last 2 +// image-bearing messages, so anything older is dropped before sending rather +// than paying to upload screenshots the model will discard server-side. const MAX_REQUEST_BYTES = 9_500_000; -const KEEP_RECENT_SCREENSHOTS = 6; +const KEEP_IMAGE_MESSAGES = 2; interface YutoriExtras { tool_set: string; - disable_tools: string[]; + reasoning_effort?: ReasoningEffort; + prev_request_id?: string; } interface SamplingLoopOptions { @@ -43,9 +43,9 @@ interface SamplingLoopOptions { sessionId: string; maxCompletionTokens?: number; maxIterations?: number; - viewportWidth?: number; - viewportHeight?: number; - kioskMode?: boolean; + screenWidth?: number; + screenHeight?: number; + reasoningEffort?: ReasoningEffort; userTimezone?: string; userLocation?: string; } @@ -56,16 +56,18 @@ export interface SamplingLoopResult { } export async function samplingLoop({ - model = 'n1.5-latest', + model = 'n2', task, apiKey, kernel, sessionId, - maxCompletionTokens = 4096, + // Shared by the reasoning trace and the tool call, so a low value truncates + // the call. + maxCompletionTokens = 16384, maxIterations = 100, - viewportWidth = 1280, - viewportHeight = 800, - kioskMode = false, + screenWidth = 1280, + screenHeight = 800, + reasoningEffort, userTimezone = 'America/Los_Angeles', userLocation = 'San Francisco, CA, US', }: SamplingLoopOptions): Promise { @@ -74,80 +76,64 @@ export async function samplingLoop({ baseURL: 'https://api.yutori.com/v1', }); - const computerTool = new ComputerTool(kernel, sessionId, viewportWidth, viewportHeight, kioskMode); - - const initialScreenshot = await computerTool.screenshot(); + const computerTool = new ComputerTool(kernel, sessionId, screenWidth, screenHeight); + const systemTools = new SystemTools(kernel, sessionId); // Append location/timezone/current-date context to the task — mirrors Yutori's // format_task_with_context helper and helps the model with date-sensitive // judgments. https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/context.py - const taskWithContext = formatTaskWithContext(task, userTimezone, userLocation); - const conversationMessages: OpenAI.ChatCompletionMessageParam[] = [ { role: 'user', content: [ - { type: 'text', text: taskWithContext }, - ...(initialScreenshot.base64Image - ? [{ - type: 'image_url' as const, - image_url: { - url: `data:image/webp;base64,${initialScreenshot.base64Image}`, - }, - }] - : []), + { type: 'text', text: formatTaskWithContext(task, userTimezone, userLocation) }, + imagePart(await computerTool.screenshot()), ], }, ]; let iteration = 0; let finalAnswer: string | undefined; + let prevRequestId: string | undefined; while (iteration < maxIterations) { iteration++; console.log(`\n=== Iteration ${iteration} ===`); - const { messages: requestMessages, removed } = trimmedForRequest(conversationMessages); - if (removed > 0) { - console.log(`Trimmed ${removed} old screenshot(s) to fit request size limit`); - } + const requestMessages = trimmedForRequest(conversationMessages); + + // n2-specific knobs (not in OpenAI SDK types). The openai-node SDK + // serializes the body as-is, so these go at the top level via a spread — + // unlike the Python SDK, there is no `extra_body` kwarg here. + const yutoriExtras: YutoriExtras = { + tool_set: TOOL_SET, + ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), + // Groups the trajectory into one conversation for usage reporting. + ...(prevRequestId ? { prev_request_id: prevRequestId } : {}), + }; let response; try { - // n1.5-specific knobs (not in OpenAI SDK types). The openai-node SDK - // serializes the body as-is, so these go at the top level via a spread — - // unlike the Python SDK, there is no `extra_body` kwarg here. - // tool_set selects the core (coordinate-based) tools. - // disable_tools is a defense-in-depth exclusion of DOM/Playwright tools. - const yutoriExtras: YutoriExtras = { - tool_set: TOOL_SET, - disable_tools: DISABLED_TOOLS, - }; - response = await client.chat.completions.create({ - model, - messages: requestMessages, - max_completion_tokens: maxCompletionTokens, - temperature: 0.3, - ...yutoriExtras, - }); + response = await client.chat.completions.create( + requestBody(model, requestMessages, maxCompletionTokens, yutoriExtras), + ); } catch (apiError) { console.error('API call failed:', apiError); throw apiError; } - if (!response.choices || response.choices.length === 0) { - console.error('No choices in response:', JSON.stringify(response, null, 2)); - throw new Error('No choices in API response'); - } + prevRequestId = (response as { _request_id?: string })._request_id ?? prevRequestId; - const assistantMessage = response.choices[0]?.message; + const assistantMessage = response.choices?.[0]?.message; if (!assistantMessage) { + console.error('No choices in response:', JSON.stringify(response, null, 2)); throw new Error('No response from model'); } console.log('Assistant content:', assistantMessage.content || '(none)'); - // Preserve full assistant message (including tool_calls) in history + // Push the assistant message unchanged — `reasoning_content` rides along on + // it, and n2 needs it echoed back to keep its reasoning across turns. conversationMessages.push(assistantMessage); const toolCalls = assistantMessage.tool_calls; @@ -159,67 +145,21 @@ export async function samplingLoop({ break; } - for (const toolCall of toolCalls) { - const actionName = toolCall.function.name; - let args: Record; - try { - args = JSON.parse(toolCall.function.arguments); - } catch { - console.error('Failed to parse tool_call arguments:', toolCall.function.arguments); + for (const [index, toolCall] of toolCalls.entries()) { + // n2 usually answers with one call, but `parallel_tool_calls` is pinned on + // server-side so a turn can carry several. Only the first one was planned + // against the screenshot the model actually saw, so run it and make the + // rest re-plan. + if (index > 0) { conversationMessages.push({ role: 'tool', tool_call_id: toolCall.id, - content: 'Error: failed to parse arguments', + content: 'Not executed: planned against a screenshot the previous call already changed. Re-plan from the latest screenshot.', }); continue; } - const action: N15Action = { - action_type: actionName as N15Action['action_type'], - ...args, - }; - - console.log('Executing action:', actionName, args); - - const scaledAction = scaleCoordinates(action, viewportWidth, viewportHeight); - - let result: ToolResult; - try { - result = await computerTool.execute(scaledAction); - } catch (error) { - console.error('Action failed:', error); - result = { - error: error instanceof Error ? error.message : String(error), - }; - } - - if (result.base64Image) { - conversationMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - // Yutori n1 accepts image content arrays in tool messages (not yet in OpenAI SDK types) - content: [ - { - type: 'image_url', - image_url: { - url: `data:image/webp;base64,${result.base64Image}`, - }, - }, - ] as unknown as string, - }); - } else if (result.error) { - conversationMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: `Action failed: ${result.error}`, - }); - } else { - conversationMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: result.output || 'OK', - }); - } + conversationMessages.push(await runToolCall(toolCall, computerTool, systemTools)); } } @@ -229,27 +169,20 @@ export async function samplingLoop({ if (iteration >= maxIterations && !finalAnswer) { console.log('Max iterations reached — requesting summary'); try { - const finalScreenshot = await computerTool.screenshot(); conversationMessages.push({ role: 'user', content: [ { type: 'text', text: formatStopAndSummarize(task) }, - ...(finalScreenshot.base64Image - ? [{ - type: 'image_url' as const, - image_url: { url: `data:image/webp;base64,${finalScreenshot.base64Image}` }, - }] - : []), + imagePart(await computerTool.screenshot()), ], }); - const { messages: summaryMessages } = trimmedForRequest(conversationMessages); - const summaryResponse = await client.chat.completions.create({ - model, - messages: summaryMessages, - max_completion_tokens: maxCompletionTokens, - temperature: 0.3, - ...({ tool_set: TOOL_SET, disable_tools: DISABLED_TOOLS } satisfies YutoriExtras), - }); + + const summaryResponse = await client.chat.completions.create( + requestBody(model, trimmedForRequest(conversationMessages), maxCompletionTokens, { + tool_set: TOOL_SET, + }), + ); + const summary = summaryResponse.choices[0]?.message; if (summary) { conversationMessages.push(summary); @@ -266,6 +199,80 @@ export async function samplingLoop({ }; } +// n2's knobs are not in the OpenAI SDK types — the openai-node SDK serializes +// the body as-is, so they ride at the top level. `reasoning_effort` collides +// with OpenAI's own narrower enum, hence the cast on the way out. +function requestBody( + model: string, + messages: OpenAI.ChatCompletionMessageParam[], + maxCompletionTokens: number, + extras: YutoriExtras, +): OpenAI.ChatCompletionCreateParamsNonStreaming { + return { + model, + messages, + max_completion_tokens: maxCompletionTokens, + temperature: 0.6, + ...extras, + } as OpenAI.ChatCompletionCreateParamsNonStreaming; +} + +async function runToolCall( + toolCall: OpenAI.ChatCompletionMessageToolCall, + computerTool: ComputerTool, + systemTools: SystemTools, +): Promise { + const name = toolCall.function.name; + + let args: Record; + try { + args = JSON.parse(toolCall.function.arguments); + } catch { + console.error('Failed to parse tool_call arguments:', toolCall.function.arguments); + return { role: 'tool', tool_call_id: toolCall.id, content: 'Error: failed to parse arguments' }; + } + + console.log('Executing tool:', name, JSON.stringify(args)); + + if (name === 'computer_batch') { + const outcome = await computerTool.runBatch((args.actions ?? []) as N2Action[]); + + return { + role: 'tool', + tool_call_id: toolCall.id, + // n2 accepts image content arrays in tool messages (not yet in OpenAI SDK types) + content: [ + { type: 'text', text: describeOutcome(outcome) }, + imagePart(await computerTool.screenshot()), + ] as unknown as string, + }; + } + + try { + return { role: 'tool', tool_call_id: toolCall.id, content: await systemTools.execute(name, args) }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`${name} failed:`, message); + return { role: 'tool', tool_call_id: toolCall.id, content: `${name} failed: ${message}` }; + } +} + +function describeOutcome({ executed, total, failure }: BatchOutcome): string { + if (!failure) { + return `Executed ${executed} of ${total} actions.`; + } + + return ( + `Executed ${executed} of ${total} actions. ` + + `Action ${failure.index + 1} (${failure.name}) failed: ${failure.message}. ` + + `The remaining actions were skipped.` + ); +} + +function imagePart(base64Image: string): OpenAI.ChatCompletionContentPartImage { + return { type: 'image_url', image_url: { url: `data:image/webp;base64,${base64Image}` } }; +} + function formatTaskWithContext(task: string, userTimezone: string, userLocation: string): string { const now = new Date(); const tzLabel = resolveTimezone(userTimezone); @@ -319,30 +326,6 @@ function formatStopAndSummarize(task: string): string { ); } -function scaleCoordinates(action: N15Action, viewportWidth: number, viewportHeight: number): N15Action { - const scaled = { ...action }; - - if (scaled.coordinates) { - scaled.coordinates = denormalize(scaled.coordinates, viewportWidth, viewportHeight); - } - - if (scaled.start_coordinates) { - scaled.start_coordinates = denormalize(scaled.start_coordinates, viewportWidth, viewportHeight); - } - - return scaled; -} - -// Map [0, 1000] coordinates into viewport pixels and clamp to [0, dim-1] so a -// boundary value like 1000 doesn't land one pixel outside the viewport. -function denormalize(coords: [number, number], width: number, height: number): [number, number] { - const rawX = Math.round((coords[0] / NAVIGATOR_COORDINATE_SCALE) * width); - const rawY = Math.round((coords[1] / NAVIGATOR_COORDINATE_SCALE) * height); - const x = Math.max(0, Math.min(width - 1, rawX)); - const y = Math.max(0, Math.min(height - 1, rawY)); - return [x, y]; -} - interface ImagePart { type: 'image_url'; image_url: { url: string }; @@ -365,70 +348,54 @@ function messageHasImage(msg: OpenAI.ChatCompletionMessageParam): boolean { return content.some((p) => typeof p === 'object' && p !== null && (p as { type?: unknown }).type === 'image_url'); } -function stripOneImage(msg: OpenAI.ChatCompletionMessageParam): boolean { +function stripImages(msg: OpenAI.ChatCompletionMessageParam): void { const content = (msg as { content?: unknown }).content; - if (!Array.isArray(content)) return false; + if (!Array.isArray(content)) return; - let removed = false; - const next: ContentPart[] = []; - for (const part of content as ContentPart[]) { - if (!removed && typeof part === 'object' && part !== null && (part as { type?: unknown }).type === 'image_url') { - removed = true; - continue; - } - next.push(part); - } - if (!removed) return false; + const next = (content as ContentPart[]).filter( + (p) => !(typeof p === 'object' && p !== null && (p as { type?: unknown }).type === 'image_url'), + ); const hasText = next.some((p) => typeof p === 'object' && p !== null && (p as { type?: unknown }).type === 'text'); if (!hasText) { - next.push({ type: 'text', text: 'Screenshot omitted to stay under request size limit.' }); + next.push({ type: 'text', text: 'Screenshot omitted — superseded by a newer one.' }); } (msg as { content: unknown }).content = next; - return true; } +/** + * Drop the screenshots n2 will not look at, keeping the text of every turn. + * + * The model reads images from the last KEEP_IMAGE_MESSAGES image-bearing + * messages and ignores the rest, so those are stripped every request. If the + * payload is still over the cap after that, the older of the kept screenshots + * goes too — the latest one is always kept, since a request with no screenshot + * badly degrades grounding. + */ function trimmedForRequest( messages: OpenAI.ChatCompletionMessageParam[], -): { messages: OpenAI.ChatCompletionMessageParam[]; removed: number } { +): OpenAI.ChatCompletionMessageParam[] { // Deep-copy so the caller's full history is preserved unchanged. const trimmed = JSON.parse(JSON.stringify(messages)) as OpenAI.ChatCompletionMessageParam[]; - let size = estimateSize(trimmed); - if (size <= MAX_REQUEST_BYTES) return { messages: trimmed, removed: 0 }; - const imageIndices: number[] = []; for (let i = 0; i < trimmed.length; i++) { if (messageHasImage(trimmed[i]!)) imageIndices.push(i); } - if (imageIndices.length === 0) return { messages: trimmed, removed: 0 }; - - const keep = Math.max(1, KEEP_RECENT_SCREENSHOTS); - const protectedIdx = new Set(imageIndices.slice(-keep)); - let removed = 0; - - for (const idx of imageIndices) { - if (size <= MAX_REQUEST_BYTES) break; - if (protectedIdx.has(idx)) continue; - if (stripOneImage(trimmed[idx]!)) { - removed++; - size = estimateSize(trimmed); - } + + for (const idx of imageIndices.slice(0, -KEEP_IMAGE_MESSAGES)) { + stripImages(trimmed[idx]!); } - // If still over, strip from the protected window too — but always keep the latest. - if (size > MAX_REQUEST_BYTES) { - const lastIdx = imageIndices[imageIndices.length - 1]!; - for (const idx of imageIndices) { - if (size <= MAX_REQUEST_BYTES) break; - if (idx === lastIdx) continue; - if (stripOneImage(trimmed[idx]!)) { - removed++; - size = estimateSize(trimmed); - } - } + // The latest screenshot is never dropped — a request without one badly + // degrades grounding — so the previous one is the only thing left to give up. + const kept = imageIndices.slice(-KEEP_IMAGE_MESSAGES); + for (const idx of kept.slice(0, -1)) { + if (estimateSize(trimmed) <= MAX_REQUEST_BYTES) break; + console.warn('Payload still over the 10 MB cap — dropping the previous screenshot too'); + stripImages(trimmed[idx]!); } - return { messages: trimmed, removed }; + return trimmed; } diff --git a/pkg/templates/typescript/yutori/session.ts b/pkg/templates/typescript/yutori/session.ts index b0edcc3d..47185f2f 100644 --- a/pkg/templates/typescript/yutori/session.ts +++ b/pkg/templates/typescript/yutori/session.ts @@ -22,8 +22,6 @@ export interface SessionOptions { viewportWidth?: number; /** Viewport height */ viewportHeight?: number; - /** Launch browser in kiosk mode (hides address bar and tabs) */ - kioskMode?: boolean; } export interface SessionInfo { @@ -45,7 +43,6 @@ const DEFAULT_OPTIONS: Required> = { replayGracePeriod: 5.0, viewportWidth: 1280, viewportHeight: 800, - kioskMode: false, }; /** @@ -105,10 +102,6 @@ export class KernelBrowserSession { return this.options.viewportHeight; } - get kioskMode(): boolean { - return this.options.kioskMode; - } - get info(): SessionInfo { return { sessionId: this.sessionId, @@ -130,7 +123,6 @@ export class KernelBrowserSession { width: this.options.viewportWidth, height: this.options.viewportHeight, }, - kiosk_mode: this.options.kioskMode, }); this._sessionId = browser.session_id ?? null; diff --git a/pkg/templates/typescript/yutori/tools/computer.ts b/pkg/templates/typescript/yutori/tools/computer.ts index a201e009..431402da 100644 --- a/pkg/templates/typescript/yutori/tools/computer.ts +++ b/pkg/templates/typescript/yutori/tools/computer.ts @@ -1,10 +1,12 @@ /** - * Yutori n1.5 Computer Tool + * Yutori n2 Computer Tool * - * Maps n1.5-latest action format to Kernel's Computer Controls API. - * Screenshots are converted to WebP for better compression across multi-step trajectories. + * Maps n2's `computer_batch` action list onto Kernel's Computer Controls API. + * n2 plans every action in a batch against the screenshot taken *before* the + * batch ran, so the batch executes sequentially, stops at the first error, and + * answers with a single screenshot taken after the last action that ran. * - * @see https://docs.yutori.com/reference/n1-5 + * @see https://docs.yutori.com/reference/n2 */ import { Buffer } from 'buffer'; @@ -12,27 +14,24 @@ import type { Kernel } from '@onkernel/sdk'; import sharp from 'sharp'; const TYPING_DELAY_MS = 12; -const SCREENSHOT_DELAY_MS = 150; -const ACTION_DELAY_MS = 300; +// Let the UI settle between actions in a batch, and again before the screenshot +// that the model will plan its next batch against. +const INTER_ACTION_DELAY_MS = 80; +const SETTLE_DELAY_MS = 400; -// n1.5 scroll `amount` is in "wheel units" where 1 unit ≈ 10% of the viewport -// height (~80px at 800px tall). Kernel's `delta_y` is a wheel-event repeat -// count where each tick is much smaller in practice, so we multiply. -const SCROLL_NOTCHES_PER_AMOUNT = 4; +const NAVIGATOR_COORDINATE_SCALE = 1000; + +// n2's scroll `amount` is a wheel notch count (1-50), which is exactly what +// Kernel's delta_x/delta_y take ("xdotool wheel units"), so it passes through. +const DEFAULT_SCROLL_AMOUNT = 3; // WebP quality for screenshots. Kernel returns PNGs, which are crisp and // tolerate aggressive WebP compression with no visible degradation — matches // Yutori SDK's DEFAULT_WEBP_QUALITY_FOR_PNG=30 (yutori-sdk-python/yutori/ -// navigator/images.py). Lower values cut payload size substantially on long -// multi-step trajectories. +// navigator/images.py). Requests are capped at 10 MB and a trajectory carries +// several full-screen captures, so the compression matters. const WEBP_QUALITY = 30; -export interface ToolResult { - base64Image?: string; - output?: string; - error?: string; -} - export class ToolError extends Error { constructor(message: string) { super(message); @@ -40,44 +39,52 @@ export class ToolError extends Error { } } -export type N15ActionType = +export type N2ActionName = | 'left_click' | 'double_click' | 'triple_click' | 'middle_click' | 'right_click' - | 'mouse_move' - | 'mouse_down' - | 'mouse_up' | 'scroll' | 'type' | 'key_press' - | 'hold_key' | 'drag' + | 'mouse_move' + | 'mouse_down' + | 'mouse_up' + | 'hold_key' | 'wait' - | 'refresh' - | 'go_back' - | 'go_forward' - | 'goto_url'; + | 'screenshot'; -export interface N15Action { - action_type: N15ActionType; +export interface N2ActionArgs { coordinates?: [number, number]; start_coordinates?: [number, number]; - direction?: 'up' | 'down' | 'left' | 'right'; + direction?: 'up' | 'down'; amount?: number; text?: string; key?: string; modifier?: string; duration?: number; - url?: string; } -// n1.5 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). +/** One `{name, arguments}` member of a `computer_batch` call. */ +export interface N2Action { + name: N2ActionName; + arguments?: N2ActionArgs; +} + +export interface BatchOutcome { + executed: number; + total: number; + /** Set when an action threw; the remaining actions were skipped. */ + failure?: { index: number; name: string; message: string }; +} + +// n2 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). // Kernel's press_key expects XKeysym names (e.g. `Return`, `Ctrl`, `Page_Up`). // This map covers every key Yutori documents at -// https://docs.yutori.com/reference/n1-5#key-space — keys not in the map pass -// through unchanged (printable characters like `a`, `1`, `,` are already XKeysym). +// https://docs.yutori.com/reference/n2#key-space — keys not in the map pass +// through unchanged (printable characters like `a` and `1` are already XKeysym). // // Sister implementation (Playwright target instead of XKeysym): // https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/keys.py @@ -119,6 +126,19 @@ const KEY_MAP: Record = { // Function keys f1: 'F1', f2: 'F2', f3: 'F3', f4: 'F4', f5: 'F5', f6: 'F6', f7: 'F7', f8: 'F8', f9: 'F9', f10: 'F10', f11: 'F11', f12: 'F12', + // Punctuation — n2 sends word forms, most of which are already XKeysym names + minus: 'minus', + plus: 'plus', + equal: 'equal', + comma: 'comma', + period: 'period', + slash: 'slash', + backslash: 'backslash', + semicolon: 'semicolon', + quote: 'apostrophe', + backquote: 'grave', + bracketleft: 'bracketleft', + bracketright: 'bracketright', // Locks / special capslock: 'Caps_Lock', numlock: 'Num_Lock', @@ -133,14 +153,9 @@ function mapToken(token: string): string { return KEY_MAP[lower] ?? token.trim(); } -function normalizeUrl(url: string): string { - const trimmed = url.trim(); - return trimmed.includes('://') ? trimmed : `https://${trimmed}`; -} - -// Parse an n1.5 key expression into one Kernel combo string per sequential -// press. Spaces separate sequential presses; `+` separates simultaneous tokens -// within a press. Examples: +// Parse an n2 key expression into one Kernel combo string per sequential press. +// Spaces separate sequential presses; `+` separates simultaneous tokens within a +// press. Examples: // "enter" -> ["Return"] // "ctrl+c" -> ["Ctrl+c"] // "down down enter" -> ["Down", "Down", "Return"] @@ -156,327 +171,226 @@ function parseKeyExpression(expr: string): string[] { export class ComputerTool { private kernel: Kernel; private sessionId: string; - private width: number; - private height: number; - private kioskMode: boolean; + private screenWidth: number; + private screenHeight: number; - constructor(kernel: Kernel, sessionId: string, width = 1280, height = 800, kioskMode = false) { + constructor(kernel: Kernel, sessionId: string, screenWidth = 1280, screenHeight = 800) { this.kernel = kernel; this.sessionId = sessionId; - this.width = width; - this.height = height; - this.kioskMode = kioskMode; + this.screenWidth = screenWidth; + this.screenHeight = screenHeight; + } + + /** + * Run a `computer_batch` action list in order, stopping at the first failure. + * The caller reports the outcome and a single post-batch screenshot back to n2. + */ + async runBatch(actions: N2Action[]): Promise { + for (let i = 0; i < actions.length; i++) { + const action = actions[i]!; + try { + await this.runAction(action); + } catch (error) { + return { + executed: i, + total: actions.length, + failure: { + index: i, + name: action.name, + message: error instanceof Error ? error.message : String(error), + }, + }; + } + if (i < actions.length - 1) { + await this.sleep(INTER_ACTION_DELAY_MS); + } + } + + return { executed: actions.length, total: actions.length }; } - async execute(action: N15Action): Promise { - const { action_type } = action; + private async runAction(action: N2Action): Promise { + const args = action.arguments ?? {}; - switch (action_type) { + switch (action.name) { case 'left_click': - return this.handleClick(action, 'left', 1); + return this.click(args, 'left', 1); case 'double_click': - return this.handleClick(action, 'left', 2); + return this.click(args, 'left', 2); case 'triple_click': - return this.handleClick(action, 'left', 3); + return this.click(args, 'left', 3); case 'middle_click': - return this.handleClick(action, 'middle', 1); + return this.click(args, 'middle', 1); case 'right_click': - return this.handleClick(action, 'right', 1); - case 'mouse_move': - return this.handleMouseMove(action); - case 'mouse_down': - return this.handleMouseButton(action, 'down'); - case 'mouse_up': - return this.handleMouseButton(action, 'up'); + return this.click(args, 'right', 1); case 'scroll': - return this.handleScroll(action); + return this.scroll(args); case 'type': - return this.handleType(action); + return this.type(args); case 'key_press': - return this.handleKeyPress(action); - case 'hold_key': - return this.handleHoldKey(action); + return this.keyPress(args); case 'drag': - return this.handleDrag(action); + return this.drag(args); + case 'mouse_move': + return this.mouseMove(args); + case 'mouse_down': + return this.mouseButton(args, 'down'); + case 'mouse_up': + return this.mouseButton(args, 'up'); + case 'hold_key': + return this.holdKey(args); case 'wait': - return this.handleWait(action); - case 'refresh': - return this.handleRefresh(); - case 'go_back': - return this.handleGoBack(); - case 'go_forward': - return this.handleGoForward(); - case 'goto_url': - return this.handleGotoUrl(action); + return this.sleep(durationMs(args.duration, 2000)); + case 'screenshot': + // A batch already answers with a screenshot taken after its last action, + // so an explicit `screenshot` member has nothing to do. + return; default: - throw new ToolError(`Unknown action type: ${action_type}`); + throw new ToolError(`Unknown action: ${action.name}`); } } - private async handleClick(action: N15Action, button: 'left' | 'right' | 'middle', numClicks: number): Promise { - const coords = this.getCoordinates(action.coordinates); - const holdKeys = action.modifier ? [mapToken(action.modifier)] : undefined; + private async click(args: N2ActionArgs, button: 'left' | 'right' | 'middle', numClicks: number): Promise { + const { x, y } = this.requireCoordinates(args.coordinates); + const holdKeys = args.modifier ? [mapToken(args.modifier)] : undefined; await this.kernel.browsers.computer.clickMouse(this.sessionId, { - x: coords.x, - y: coords.y, + x, + y, button, click_type: 'click', num_clicks: numClicks, ...(holdKeys ? { hold_keys: holdKeys } : {}), }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleMouseMove(action: N15Action): Promise { - const coords = this.getCoordinates(action.coordinates); - - await this.kernel.browsers.computer.moveMouse(this.sessionId, { - x: coords.x, - y: coords.y, - }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); + private async mouseMove(args: N2ActionArgs): Promise { + const { x, y } = this.requireCoordinates(args.coordinates); + await this.kernel.browsers.computer.moveMouse(this.sessionId, { x, y }); } - private async handleMouseButton(action: N15Action, clickType: 'down' | 'up'): Promise { - const coords = this.getCoordinates(action.coordinates); + private async mouseButton(args: N2ActionArgs, clickType: 'down' | 'up'): Promise { + // Coordinates are optional here — without them the button is pressed or + // released wherever the cursor already is, which is what a manual + // mouse_move -> mouse_down -> mouse_move -> mouse_up drag relies on. + const { x, y } = args.coordinates + ? this.requireCoordinates(args.coordinates) + : await this.kernel.browsers.computer.getMousePosition(this.sessionId); await this.kernel.browsers.computer.clickMouse(this.sessionId, { - x: coords.x, - y: coords.y, + x, + y, button: 'left', click_type: clickType, }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleScroll(action: N15Action): Promise { - const coords = this.getCoordinates(action.coordinates); - const direction = action.direction; - const amount = Math.max(action.amount ?? 3, 1); + private async scroll(args: N2ActionArgs): Promise { + const { x, y } = this.requireCoordinates(args.coordinates); + const direction = args.direction; - if (!direction || !['up', 'down', 'left', 'right'].includes(direction)) { + // n2 only scrolls vertically. + if (direction !== 'up' && direction !== 'down') { throw new ToolError(`Invalid scroll direction: ${direction}`); } - // Yutori 1 unit ≈ 10% of viewport height; scale into Kernel wheel-event ticks. - const ticks = amount * SCROLL_NOTCHES_PER_AMOUNT; - - let delta_x = 0; - let delta_y = 0; - - switch (direction) { - case 'up': - delta_y = -ticks; - break; - case 'down': - delta_y = ticks; - break; - case 'left': - delta_x = -ticks; - break; - case 'right': - delta_x = ticks; - break; - } - - const holdKeys = action.modifier ? [mapToken(action.modifier)] : undefined; + const notches = Math.max(1, Math.round(args.amount ?? DEFAULT_SCROLL_AMOUNT)); + const holdKeys = args.modifier ? [mapToken(args.modifier)] : undefined; await this.kernel.browsers.computer.scroll(this.sessionId, { - x: coords.x, - y: coords.y, - delta_x, - delta_y, + x, + y, + delta_x: 0, + delta_y: direction === 'up' ? -notches : notches, ...(holdKeys ? { hold_keys: holdKeys } : {}), }); - - await this.sleep(SCREENSHOT_DELAY_MS); - const screenshotResult = await this.screenshot(); - return { - ...screenshotResult, - output: `Scrolled ${amount} unit(s) ${direction}.`, - }; } - private async handleType(action: N15Action): Promise { - const text = action.text; - if (!text) { - throw new ToolError('text is required for type action'); + private async type(args: N2ActionArgs): Promise { + if (!args.text) { + throw new ToolError('text is required for type'); } await this.kernel.browsers.computer.typeText(this.sessionId, { - text, + text: args.text, delay: TYPING_DELAY_MS, }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleKeyPress(action: N15Action): Promise { - const key = action.key; - if (!key) { - throw new ToolError('key is required for key_press action'); + private async keyPress(args: N2ActionArgs): Promise { + if (!args.key) { + throw new ToolError('key is required for key_press'); } - // n1.5 supports sequential presses ("down down down enter") — issue each - // combo as its own pressKey so they're seen as separate keystrokes. - const combos = parseKeyExpression(key); - for (const combo of combos) { + // n2 supports sequential presses ("down down down enter") — issue each combo + // as its own pressKey so they're seen as separate keystrokes. + for (const combo of parseKeyExpression(args.key)) { await this.kernel.browsers.computer.pressKey(this.sessionId, { keys: [combo] }); } - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleHoldKey(action: N15Action): Promise { - const key = action.key; - if (!key) { - throw new ToolError('key is required for hold_key action'); + private async holdKey(args: N2ActionArgs): Promise { + if (!args.key) { + throw new ToolError('key is required for hold_key'); } - // Yutori emits `duration` in seconds; Kernel SDK's pressKey takes ms. - const durationMs = action.duration && action.duration > 0 ? Math.round(action.duration * 1000) : 1000; - - const combos = parseKeyExpression(key); - for (const combo of combos) { + for (const combo of parseKeyExpression(args.key)) { await this.kernel.browsers.computer.pressKey(this.sessionId, { keys: [combo], - duration: durationMs, + duration: durationMs(args.duration, 1000), }); } - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleDrag(action: N15Action): Promise { - const startCoords = this.getCoordinates(action.start_coordinates); - const endCoords = this.getCoordinates(action.coordinates); + private async drag(args: N2ActionArgs): Promise { + const start = this.requireCoordinates(args.start_coordinates); + const end = this.requireCoordinates(args.coordinates); await this.kernel.browsers.computer.dragMouse(this.sessionId, { - path: [[startCoords.x, startCoords.y], [endCoords.x, endCoords.y]], + path: [[start.x, start.y], [end.x, end.y]], button: 'left', }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleWait(action: N15Action): Promise { - // Yutori emits `duration` in seconds (matches reference impl). - const durationMs = action.duration && action.duration > 0 ? Math.round(action.duration * 1000) : 2000; - await this.sleep(durationMs); - return this.screenshot(); - } + /** Capture the whole screen — browser chrome included — as base64 WebP. */ + async screenshot(): Promise { + await this.sleep(SETTLE_DELAY_MS); - private async handleRefresh(): Promise { - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['F5'], - }); + const response = await this.kernel.browsers.computer.captureScreenshot(this.sessionId); + const pngBuffer = Buffer.from(await (await response.blob()).arrayBuffer()); + const webpBuffer = await sharp(pngBuffer).webp({ quality: WEBP_QUALITY }).toBuffer(); - await this.sleep(2000); - return this.screenshot(); + return webpBuffer.toString('base64'); } - private async handleGoBack(): Promise { - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Alt+Left'], - }); - - await this.sleep(1500); - return this.screenshot(); - } - - private async handleGoForward(): Promise { - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Alt+Right'], - }); - - await this.sleep(1500); - return this.screenshot(); - } - - private async handleGotoUrl(action: N15Action): Promise { - const url = action.url; - if (!url) { - throw new ToolError('url is required for goto_url action'); - } - const targetUrl = normalizeUrl(url); - - if (this.kioskMode) { - const response = await this.kernel.browsers.playwright.execute(this.sessionId, { - code: `await page.goto(${JSON.stringify(targetUrl)});`, - timeout_sec: 60, - }); - if (!response.success) { - throw new ToolError(response.error ?? 'Playwright goto failed'); - } - await this.sleep(ACTION_DELAY_MS); - return this.screenshot(); - } - - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Ctrl+l'], - }); - await this.sleep(ACTION_DELAY_MS); - - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Ctrl+a'], - }); - await this.sleep(100); - - await this.kernel.browsers.computer.typeText(this.sessionId, { - text: targetUrl, - delay: TYPING_DELAY_MS, - }); - await this.sleep(ACTION_DELAY_MS); - - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Return'], - }); - - await this.sleep(2000); - return this.screenshot(); - } - - async screenshot(): Promise { - try { - const response = await this.kernel.browsers.computer.captureScreenshot(this.sessionId); - const blob = await response.blob(); - const arrayBuffer = await blob.arrayBuffer(); - const pngBuffer = Buffer.from(arrayBuffer); - const webpBuffer = await sharp(pngBuffer).webp({ quality: WEBP_QUALITY }).toBuffer(); - - return { - base64Image: webpBuffer.toString('base64'), - }; - } catch (error) { - throw new ToolError(`Failed to take screenshot: ${error}`); - } - } - - private getCoordinates(coords?: [number, number]): { x: number; y: number } { + // Map [0, 1000] coordinates into screen pixels and clamp to [0, dim-1] so a + // boundary value like 1000 doesn't land one pixel outside the screen. + private requireCoordinates(coords?: [number, number]): { x: number; y: number } { if (!coords || coords.length !== 2) { - return { x: Math.floor(this.width / 2), y: Math.floor(this.height / 2) }; + throw new ToolError(`coordinates are required, got ${JSON.stringify(coords)}`); } - const [x, y] = coords; - if (typeof x !== 'number' || typeof y !== 'number' || x < 0 || y < 0) { + const [nx, ny] = coords; + if (typeof nx !== 'number' || typeof ny !== 'number') { throw new ToolError(`Invalid coordinates: ${JSON.stringify(coords)}`); } - return { x, y }; + return { + x: clamp(Math.round((nx / NAVIGATOR_COORDINATE_SCALE) * this.screenWidth), this.screenWidth), + y: clamp(Math.round((ny / NAVIGATOR_COORDINATE_SCALE) * this.screenHeight), this.screenHeight), + }; } private sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } } + +function clamp(value: number, dimension: number): number { + return Math.max(0, Math.min(dimension - 1, value)); +} + +// n2 emits `duration` in seconds; Kernel and setTimeout take milliseconds. +function durationMs(duration: number | undefined, fallbackMs: number): number { + return duration && duration > 0 ? Math.round(duration * 1000) : fallbackMs; +} diff --git a/pkg/templates/typescript/yutori/tools/system.ts b/pkg/templates/typescript/yutori/tools/system.ts new file mode 100644 index 00000000..e53ac6fb --- /dev/null +++ b/pkg/templates/typescript/yutori/tools/system.ts @@ -0,0 +1,227 @@ +/** + * Yutori n2 Shell and File Tools + * + * n2's tool set always ships `bash`, `read`, `write`, and `edit` alongside + * `computer_batch` — `disable_tools` is rejected by the API, so a loop has to + * answer these too. They run against the Kernel browser VM via the process and + * filesystem APIs. + * + * @see https://docs.yutori.com/reference/n2 + */ + +import { Buffer } from 'buffer'; +import type { Kernel } from '@onkernel/sdk'; +import { ToolError } from './computer'; + +const DEFAULT_TIMEOUT_SEC = 120; +const MAX_TIMEOUT_SEC = 600; +const DEFAULT_READ_LIMIT = 2000; +const MAX_WRITE_CHARS = 256_000; + +// bash: "The working directory persists across calls; environment variables and +// shell functions do not." Each exec is its own process, so the command reports +// its final directory on a sentinel line that we strip before returning stdout. +const CWD_SENTINEL = '__n2_cwd__'; + +interface BashArgs { + command?: string; + timeout?: number; + run_in_background?: boolean; +} + +interface ReadArgs { + file_path?: string; + offset?: number; + limit?: number; +} + +interface WriteArgs { + file_path?: string; + content?: string; +} + +interface EditArgs { + file_path?: string; + old_string?: string; + new_string?: string; + replace_all?: boolean; +} + +export class SystemTools { + private kernel: Kernel; + private sessionId: string; + private cwd: string | undefined; + private seenPaths = new Set(); + + constructor(kernel: Kernel, sessionId: string) { + this.kernel = kernel; + this.sessionId = sessionId; + } + + async execute(name: string, args: Record): Promise { + switch (name) { + case 'bash': + return this.bash(args as BashArgs); + case 'read': + return this.read(args as ReadArgs); + case 'write': + return this.write(args as WriteArgs); + case 'edit': + return this.edit(args as EditArgs); + default: + throw new ToolError(`Unknown tool: ${name}`); + } + } + + private async bash(args: BashArgs): Promise { + const command = args.command; + if (!command) { + throw new ToolError('command is required for bash'); + } + + const timeoutSec = Math.min(args.timeout ?? DEFAULT_TIMEOUT_SEC, MAX_TIMEOUT_SEC); + + if (args.run_in_background) { + return this.bashBackground(command); + } + + const script = [ + command, + 'n2_status=$?', + `printf '\\n${CWD_SENTINEL}%s' "$(pwd)"`, + 'exit $n2_status', + ].join('\n'); + + const result = await this.kernel.browsers.process.exec(this.sessionId, { + command: 'bash', + args: ['-lc', script], + cwd: this.cwd ?? null, + timeout_sec: timeoutSec, + }); + + const stdout = this.takeCwd(decode(result.stdout_b64)); + const stderr = decode(result.stderr_b64); + + return formatCommandResult(stdout, stderr, result.exit_code); + } + + private async read(args: ReadArgs): Promise { + const filePath = args.file_path; + if (!filePath) { + throw new ToolError('file_path is required for read'); + } + + const response = await this.kernel.browsers.fs.readFile(this.sessionId, { path: filePath }); + const lines = (await response.text()).split('\n'); + + const offset = Math.max(0, args.offset ?? 0); + const limit = Math.max(1, args.limit ?? DEFAULT_READ_LIMIT); + const page = lines.slice(offset, offset + limit); + + this.seenPaths.add(filePath); + + if (page.length === 0) { + return `${filePath} has ${lines.length} line(s); offset ${offset} is past the end.`; + } + + // cat -n format, so line numbers survive into the model's next edit. + return page.map((line, i) => `${String(offset + i + 1).padStart(6)}\t${line}`).join('\n'); + } + + private async write(args: WriteArgs): Promise { + const filePath = args.file_path; + const content = args.content; + if (!filePath || content === undefined) { + throw new ToolError('file_path and content are required for write'); + } + if (content.length > MAX_WRITE_CHARS) { + throw new ToolError(`content exceeds the ${MAX_WRITE_CHARS} character cap`); + } + + await this.kernel.browsers.fs.writeFile(this.sessionId, content, { path: filePath }); + this.seenPaths.add(filePath); + + return `Wrote ${content.length} character(s) to ${filePath}.`; + } + + private async edit(args: EditArgs): Promise { + const { file_path: filePath, old_string: oldString, new_string: newString } = args; + if (!filePath || oldString === undefined || newString === undefined) { + throw new ToolError('file_path, old_string, and new_string are required for edit'); + } + // n2 is expected to know the current bytes before changing them. + if (!this.seenPaths.has(filePath)) { + throw new ToolError(`${filePath} has not been read in this session — read it before editing.`); + } + + const response = await this.kernel.browsers.fs.readFile(this.sessionId, { path: filePath }); + const original = await response.text(); + + const occurrences = original.split(oldString).length - 1; + if (occurrences === 0) { + throw new ToolError(`old_string not found in ${filePath}`); + } + if (occurrences > 1 && !args.replace_all) { + throw new ToolError( + `old_string matches ${occurrences} times in ${filePath} — pass replace_all or include more context.`, + ); + } + + // The replacer function form is required — a string replacement would + // expand `$$`, `$&`, and friends inside n2's new_string. + const updated = args.replace_all + ? original.split(oldString).join(newString) + : original.replace(oldString, () => newString); + + await this.kernel.browsers.fs.writeFile(this.sessionId, updated, { path: filePath }); + + return `Replaced ${args.replace_all ? occurrences : 1} occurrence(s) in ${filePath}.`; + } + + private async bashBackground(command: string): Promise { + const logPath = `/tmp/n2-bg-${Date.now()}.log`; + const script = `nohup bash -c ${shellQuote(command)} > ${logPath} 2>&1 &\necho $!`; + + const result = await this.kernel.browsers.process.exec(this.sessionId, { + command: 'bash', + args: ['-lc', script], + cwd: this.cwd ?? null, + }); + + const pid = decode(result.stdout_b64).trim(); + + return [ + `Started in the background with pid ${pid}.`, + `Output is being written to ${logPath} — use the read tool to check on it.`, + `Cancel it with: kill ${pid}`, + ].join('\n'); + } + + /** Strip the trailing sentinel line and remember the directory it reported. */ + private takeCwd(stdout: string): string { + const marker = stdout.lastIndexOf(`\n${CWD_SENTINEL}`); + if (marker === -1) { + return stdout; + } + + this.cwd = stdout.slice(marker + CWD_SENTINEL.length + 1).trim() || this.cwd; + return stdout.slice(0, marker); + } +} + +function decode(base64?: string): string { + return base64 ? Buffer.from(base64, 'base64').toString('utf-8') : ''; +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +function formatCommandResult(stdout: string, stderr: string, exitCode?: number): string { + const parts: string[] = []; + if (stdout.trim()) parts.push(stdout.trimEnd()); + if (stderr.trim()) parts.push(`stderr:\n${stderr.trimEnd()}`); + if (exitCode) parts.push(`Exited with code ${exitCode}.`); + + return parts.length > 0 ? parts.join('\n') : 'Command produced no output.'; +}