fix(agent): adapt image tool outputs for Chat Completions

openkb/agent/chat.py

@@ -24,6 +24,7 @@
 from prompt_toolkit.styles import Style
 
 from openkb.agent.chat_session import ChatSession
+from openkb.agent.model_compat import with_chat_completions_compat
 from openkb.agent.query import (
     MAX_TURNS,
     build_chat_agent,
@@ -354,7 +355,12 @@ async def _run_turn(
 
     new_input = session.history + [{"role": "user", "content": user_input}]
 
-    result = Runner.run_streamed(agent, new_input, max_turns=MAX_TURNS)
+    result = Runner.run_streamed(
+        agent,
+        new_input,
+        max_turns=MAX_TURNS,
+        run_config=with_chat_completions_compat(None),
+    )
 
     print()
     collected: list[str] = []

openkb/agent/model_compat.py

@@ -0,0 +1,118 @@
+"""Model-protocol compatibility helpers for agent runs."""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Awaitable, Callable
+from dataclasses import replace
+from typing import Any
+
+from agents import RunConfig
+from agents.run_config import CallModelData, ModelInputData
+
+_IMAGE_OUTPUT_ACK = "Image returned successfully; inspect the following user image."
+_IMAGE_INPUT_NOTE = "Images returned by the preceding tool call(s):"
+
+
+def adapt_image_tool_outputs_for_chat_completions(
+    data: CallModelData[Any],
+) -> ModelInputData:
+    """Move image tool outputs into a following user message.
+
+    Chat Completions accepts images in user content but restricts tool content
+    to text. The Agents SDK uses Responses-style ``input_image`` parts for
+    ``ToolOutputImage``, so rewrite those parts immediately before the model
+    call while preserving a text result for every tool call.
+    """
+    adapted: list[Any] = []
+    pending_images: list[dict[str, Any]] = []
+
+    def flush_images() -> None:
+        if not pending_images:
+            return
+        adapted.append(
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": _IMAGE_INPUT_NOTE},
+                    *pending_images,
+                ],
+            }
+        )
+        pending_images.clear()
+
+    for item in data.model_data.input:
+        if not (isinstance(item, dict) and item.get("type") == "function_call_output"):
+            flush_images()
+            adapted.append(item)
+            continue
+
+        output = item.get("output")
+        if not isinstance(output, list):
+            adapted.append(item)
+            continue
+
+        images = [
+            part for part in output if isinstance(part, dict) and part.get("type") == "input_image"
+        ]
+        if not images:
+            adapted.append(item)
+            continue
+
+        text_parts = [
+            part for part in output if isinstance(part, dict) and part.get("type") == "input_text"
+        ]
+        adapted.append({**item, "output": text_parts or _IMAGE_OUTPUT_ACK})
+        pending_images.extend(images)
+
+    flush_images()
+    return ModelInputData(input=adapted, instructions=data.model_data.instructions)
+
+
+class _ChatCompletionsCompatFilter:
+    """Compose an existing model-input filter with the image compatibility pass."""
+
+    def __init__(
+        self,
+        existing_filter: Callable[[CallModelData[Any]], Awaitable[ModelInputData] | ModelInputData],
+    ) -> None:
+        self._existing_filter = existing_filter
+
+    async def __call__(self, data: CallModelData[Any]) -> ModelInputData:
+        filtered = self._existing_filter(data)
+        if inspect.isawaitable(filtered):
+            filtered = await filtered
+        return adapt_image_tool_outputs_for_chat_completions(
+            CallModelData(
+                model_data=filtered,
+                agent=data.agent,
+                context=data.context,
+            )
+        )
+
+
+CHAT_COMPLETIONS_RUN_CONFIG = RunConfig(
+    call_model_input_filter=adapt_image_tool_outputs_for_chat_completions
+)
+
+
+def with_chat_completions_compat(run_config: RunConfig | None) -> RunConfig:
+    """Return a run config that preserves existing settings and adapts image outputs."""
+    if run_config is None:
+        return CHAT_COMPLETIONS_RUN_CONFIG
+
+    existing_filter = run_config.call_model_input_filter
+    if existing_filter is None:
+        return replace(
+            run_config,
+            call_model_input_filter=adapt_image_tool_outputs_for_chat_completions,
+        )
+    if existing_filter is adapt_image_tool_outputs_for_chat_completions or isinstance(
+        existing_filter, _ChatCompletionsCompatFilter
+    ):
+        return run_config
+
+    return replace(
+        run_config,
+        call_model_input_filter=_ChatCompletionsCompatFilter(existing_filter),
+    )

openkb/agent/query.py

@@ -7,6 +7,7 @@
 
 from agents import Agent, Runner, ToolOutputImage, ToolOutputText, function_tool
 
+from openkb.agent.model_compat import with_chat_completions_compat
 from openkb.agent.tools import (
     artifact_event_from_write,
     get_wiki_page_content,
@@ -159,10 +160,11 @@ async def iter_agent_response_events(
     from agents import RawResponsesStreamEvent, RunItemStreamEvent
     from openai.types.responses import ResponseTextDeltaEvent
 
-    result = (
-        Runner.run_streamed(agent, input_data, max_turns=max_turns, run_config=run_config)
-        if run_config
-        else Runner.run_streamed(agent, input_data, max_turns=max_turns)
+    result = Runner.run_streamed(
+        agent,
+        input_data,
+        max_turns=max_turns,
+        run_config=with_chat_completions_compat(run_config),
     )
     collected: list[str] = []
     pending_calls: dict[str, tuple[str, str]] = {}
@@ -387,10 +389,11 @@ async def run_query(
     agent = build_query_agent(wiki_root, model, language=language, bundle=bundle)
 
     if not stream:
-        result = (
-            await Runner.run(agent, question, max_turns=MAX_TURNS, run_config=run_config)
-            if run_config
-            else await Runner.run(agent, question, max_turns=MAX_TURNS)
+        result = await Runner.run(
+            agent,
+            question,
+            max_turns=MAX_TURNS,
+            run_config=with_chat_completions_compat(run_config),
         )
         return result.final_output or ""
 
@@ -425,10 +428,11 @@ def _start_live() -> Live | None:
     live: Live | None = None
     last_was_text = False
     need_blank_before_text = False
-    result = (
-        Runner.run_streamed(agent, question, max_turns=MAX_TURNS, run_config=run_config)
-        if run_config
-        else Runner.run_streamed(agent, question, max_turns=MAX_TURNS)
+    result = Runner.run_streamed(
+        agent,
+        question,
+        max_turns=MAX_TURNS,
+        run_config=with_chat_completions_compat(run_config),
     )
     collected: list[str] = []
     segment: list[str] = []

openkb/agent/skill_runner.py

@@ -32,6 +32,7 @@
 
 from agents import Runner, function_tool
 
+from openkb.agent.model_compat import with_chat_completions_compat
 from openkb.agent.query import build_query_agent, build_run_config_from_bundle
 from openkb.agent.skills import _parse_frontmatter, scan_local_skills
 from openkb.agent.tools import read_kb_file, write_kb_file
@@ -190,15 +191,17 @@ def read_output_or_skill_file(path: str) -> str:
 
     # Per-KB credential isolation: when a bundle is supplied (REST path) the
     # RunConfig carries a dedicated LitellmModel with this KB's api_key/base_url,
-    # overriding the process-global provider for this run only. When bundle is
-    # None (CLI path) build_run_config_from_bundle returns None and the call is
-    # byte-identical to the pre-bundle behavior (no run_config kwarg passed).
+    # overriding the process-global provider for this run only. The compatibility
+    # helper preserves that config and adapts image tool outputs for Chat
+    # Completions; without a bundle it creates the minimal compatible config.
     run_config = build_run_config_from_bundle(model, bundle)
     try:
-        if run_config:
-            await Runner.run(agent, user_seed, max_turns=max_turns, run_config=run_config)
-        else:
-            await Runner.run(agent, user_seed, max_turns=max_turns)
+        await Runner.run(
+            agent,
+            user_seed,
+            max_turns=max_turns,
+            run_config=with_chat_completions_compat(run_config),
+        )
     except MaxTurnsExceeded as exc:
         raise RuntimeError(
             f"Skill {skill_name!r} hit the {max_turns}-step cap before "

openkb/skill/creator.py

@@ -20,6 +20,7 @@
 from agents import Agent, Runner, ToolOutputImage, ToolOutputText, function_tool
 from agents.model_settings import ModelSettings
 
+from openkb.agent.model_compat import with_chat_completions_compat
 from openkb.config import LlmCredentialBundle, resolve_model_settings
 from openkb.prompts import load_prompt
 from openkb.schema import get_agents_md
@@ -228,17 +229,19 @@ async def run_skill_create(
     # Per-KB credential isolation: when a bundle is supplied (REST path) the
     # RunConfig carries a dedicated LitellmModel with this KB's api_key/base_url,
     # overriding the process-global provider for this run only. When bundle is
-    # None (CLI path) build_run_config_from_bundle returns None and the call is
-    # byte-identical to the pre-bundle behavior (no run_config kwarg passed).
+    # None (CLI path) build_run_config_from_bundle returns None, so the shared
+    # Chat Completions image compatibility config is used instead.
     # Lazy import mirrors this module's convention of keeping query imports local.
     from openkb.agent.query import build_run_config_from_bundle
 
     run_config = build_run_config_from_bundle(model, bundle)
     try:
-        if run_config:
-            await Runner.run(agent, seed, max_turns=MAX_TURNS, run_config=run_config)
-        else:
-            await Runner.run(agent, seed, max_turns=MAX_TURNS)
+        await Runner.run(
+            agent,
+            seed,
+            max_turns=MAX_TURNS,
+            run_config=with_chat_completions_compat(run_config),
+        )
     except MaxTurnsExceeded as exc:
         raise RuntimeError(
             f"Skill compilation hit the {MAX_TURNS}-step cap before finishing. "