diff --git a/openkb/agent/chat.py b/openkb/agent/chat.py index 898831fc0..fca4b47ad 100644 --- a/openkb/agent/chat.py +++ b/openkb/agent/chat.py @@ -24,6 +24,7 @@ from prompt_toolkit.styles import Style from openkb.agent.chat_session import ChatSession +from openkb.agent.model_compat import with_chat_completions_compat from openkb.agent.query import ( MAX_TURNS, build_chat_agent, @@ -354,7 +355,12 @@ async def _run_turn( new_input = session.history + [{"role": "user", "content": user_input}] - result = Runner.run_streamed(agent, new_input, max_turns=MAX_TURNS) + result = Runner.run_streamed( + agent, + new_input, + max_turns=MAX_TURNS, + run_config=with_chat_completions_compat(None), + ) print() collected: list[str] = [] diff --git a/openkb/agent/model_compat.py b/openkb/agent/model_compat.py new file mode 100644 index 000000000..e1cec1a47 --- /dev/null +++ b/openkb/agent/model_compat.py @@ -0,0 +1,118 @@ +"""Model-protocol compatibility helpers for agent runs.""" + +from __future__ import annotations + +import inspect +from collections.abc import Awaitable, Callable +from dataclasses import replace +from typing import Any + +from agents import RunConfig +from agents.run_config import CallModelData, ModelInputData + +_IMAGE_OUTPUT_ACK = "Image returned successfully; inspect the following user image." +_IMAGE_INPUT_NOTE = "Images returned by the preceding tool call(s):" + + +def adapt_image_tool_outputs_for_chat_completions( + data: CallModelData[Any], +) -> ModelInputData: + """Move image tool outputs into a following user message. + + Chat Completions accepts images in user content but restricts tool content + to text. The Agents SDK uses Responses-style ``input_image`` parts for + ``ToolOutputImage``, so rewrite those parts immediately before the model + call while preserving a text result for every tool call. + """ + adapted: list[Any] = [] + pending_images: list[dict[str, Any]] = [] + + def flush_images() -> None: + if not pending_images: + return + adapted.append( + { + "role": "user", + "content": [ + {"type": "input_text", "text": _IMAGE_INPUT_NOTE}, + *pending_images, + ], + } + ) + pending_images.clear() + + for item in data.model_data.input: + if not (isinstance(item, dict) and item.get("type") == "function_call_output"): + flush_images() + adapted.append(item) + continue + + output = item.get("output") + if not isinstance(output, list): + adapted.append(item) + continue + + images = [ + part for part in output if isinstance(part, dict) and part.get("type") == "input_image" + ] + if not images: + adapted.append(item) + continue + + text_parts = [ + part for part in output if isinstance(part, dict) and part.get("type") == "input_text" + ] + adapted.append({**item, "output": text_parts or _IMAGE_OUTPUT_ACK}) + pending_images.extend(images) + + flush_images() + return ModelInputData(input=adapted, instructions=data.model_data.instructions) + + +class _ChatCompletionsCompatFilter: + """Compose an existing model-input filter with the image compatibility pass.""" + + def __init__( + self, + existing_filter: Callable[[CallModelData[Any]], Awaitable[ModelInputData] | ModelInputData], + ) -> None: + self._existing_filter = existing_filter + + async def __call__(self, data: CallModelData[Any]) -> ModelInputData: + filtered = self._existing_filter(data) + if inspect.isawaitable(filtered): + filtered = await filtered + return adapt_image_tool_outputs_for_chat_completions( + CallModelData( + model_data=filtered, + agent=data.agent, + context=data.context, + ) + ) + + +CHAT_COMPLETIONS_RUN_CONFIG = RunConfig( + call_model_input_filter=adapt_image_tool_outputs_for_chat_completions +) + + +def with_chat_completions_compat(run_config: RunConfig | None) -> RunConfig: + """Return a run config that preserves existing settings and adapts image outputs.""" + if run_config is None: + return CHAT_COMPLETIONS_RUN_CONFIG + + existing_filter = run_config.call_model_input_filter + if existing_filter is None: + return replace( + run_config, + call_model_input_filter=adapt_image_tool_outputs_for_chat_completions, + ) + if existing_filter is adapt_image_tool_outputs_for_chat_completions or isinstance( + existing_filter, _ChatCompletionsCompatFilter + ): + return run_config + + return replace( + run_config, + call_model_input_filter=_ChatCompletionsCompatFilter(existing_filter), + ) diff --git a/openkb/agent/query.py b/openkb/agent/query.py index da1a939ef..6860b08b8 100644 --- a/openkb/agent/query.py +++ b/openkb/agent/query.py @@ -7,6 +7,7 @@ from agents import Agent, Runner, ToolOutputImage, ToolOutputText, function_tool +from openkb.agent.model_compat import with_chat_completions_compat from openkb.agent.tools import ( artifact_event_from_write, get_wiki_page_content, @@ -159,10 +160,11 @@ async def iter_agent_response_events( from agents import RawResponsesStreamEvent, RunItemStreamEvent from openai.types.responses import ResponseTextDeltaEvent - result = ( - Runner.run_streamed(agent, input_data, max_turns=max_turns, run_config=run_config) - if run_config - else Runner.run_streamed(agent, input_data, max_turns=max_turns) + result = Runner.run_streamed( + agent, + input_data, + max_turns=max_turns, + run_config=with_chat_completions_compat(run_config), ) collected: list[str] = [] pending_calls: dict[str, tuple[str, str]] = {} @@ -387,10 +389,11 @@ async def run_query( agent = build_query_agent(wiki_root, model, language=language, bundle=bundle) if not stream: - result = ( - await Runner.run(agent, question, max_turns=MAX_TURNS, run_config=run_config) - if run_config - else await Runner.run(agent, question, max_turns=MAX_TURNS) + result = await Runner.run( + agent, + question, + max_turns=MAX_TURNS, + run_config=with_chat_completions_compat(run_config), ) return result.final_output or "" @@ -425,10 +428,11 @@ def _start_live() -> Live | None: live: Live | None = None last_was_text = False need_blank_before_text = False - result = ( - Runner.run_streamed(agent, question, max_turns=MAX_TURNS, run_config=run_config) - if run_config - else Runner.run_streamed(agent, question, max_turns=MAX_TURNS) + result = Runner.run_streamed( + agent, + question, + max_turns=MAX_TURNS, + run_config=with_chat_completions_compat(run_config), ) collected: list[str] = [] segment: list[str] = [] diff --git a/openkb/agent/skill_runner.py b/openkb/agent/skill_runner.py index 0e8a4222d..bb09ab1b0 100644 --- a/openkb/agent/skill_runner.py +++ b/openkb/agent/skill_runner.py @@ -32,6 +32,7 @@ from agents import Runner, function_tool +from openkb.agent.model_compat import with_chat_completions_compat from openkb.agent.query import build_query_agent, build_run_config_from_bundle from openkb.agent.skills import _parse_frontmatter, scan_local_skills from openkb.agent.tools import read_kb_file, write_kb_file @@ -190,15 +191,17 @@ def read_output_or_skill_file(path: str) -> str: # Per-KB credential isolation: when a bundle is supplied (REST path) the # RunConfig carries a dedicated LitellmModel with this KB's api_key/base_url, - # overriding the process-global provider for this run only. When bundle is - # None (CLI path) build_run_config_from_bundle returns None and the call is - # byte-identical to the pre-bundle behavior (no run_config kwarg passed). + # overriding the process-global provider for this run only. The compatibility + # helper preserves that config and adapts image tool outputs for Chat + # Completions; without a bundle it creates the minimal compatible config. run_config = build_run_config_from_bundle(model, bundle) try: - if run_config: - await Runner.run(agent, user_seed, max_turns=max_turns, run_config=run_config) - else: - await Runner.run(agent, user_seed, max_turns=max_turns) + await Runner.run( + agent, + user_seed, + max_turns=max_turns, + run_config=with_chat_completions_compat(run_config), + ) except MaxTurnsExceeded as exc: raise RuntimeError( f"Skill {skill_name!r} hit the {max_turns}-step cap before " diff --git a/openkb/skill/creator.py b/openkb/skill/creator.py index f90024174..aab0b3b5d 100644 --- a/openkb/skill/creator.py +++ b/openkb/skill/creator.py @@ -20,6 +20,7 @@ from agents import Agent, Runner, ToolOutputImage, ToolOutputText, function_tool from agents.model_settings import ModelSettings +from openkb.agent.model_compat import with_chat_completions_compat from openkb.config import LlmCredentialBundle, resolve_model_settings from openkb.prompts import load_prompt from openkb.schema import get_agents_md @@ -228,17 +229,19 @@ async def run_skill_create( # Per-KB credential isolation: when a bundle is supplied (REST path) the # RunConfig carries a dedicated LitellmModel with this KB's api_key/base_url, # overriding the process-global provider for this run only. When bundle is - # None (CLI path) build_run_config_from_bundle returns None and the call is - # byte-identical to the pre-bundle behavior (no run_config kwarg passed). + # None (CLI path) build_run_config_from_bundle returns None, so the shared + # Chat Completions image compatibility config is used instead. # Lazy import mirrors this module's convention of keeping query imports local. from openkb.agent.query import build_run_config_from_bundle run_config = build_run_config_from_bundle(model, bundle) try: - if run_config: - await Runner.run(agent, seed, max_turns=MAX_TURNS, run_config=run_config) - else: - await Runner.run(agent, seed, max_turns=MAX_TURNS) + await Runner.run( + agent, + seed, + max_turns=MAX_TURNS, + run_config=with_chat_completions_compat(run_config), + ) except MaxTurnsExceeded as exc: raise RuntimeError( f"Skill compilation hit the {MAX_TURNS}-step cap before finishing. " diff --git a/tests/test_model_compat.py b/tests/test_model_compat.py new file mode 100644 index 000000000..ed0ad8f29 --- /dev/null +++ b/tests/test_model_compat.py @@ -0,0 +1,177 @@ +"""Tests for model-protocol compatibility adapters.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from agents.models.chatcmpl_converter import Converter +from agents.run_config import CallModelData, ModelInputData + +from openkb.agent.model_compat import ( + CHAT_COMPLETIONS_RUN_CONFIG, + adapt_image_tool_outputs_for_chat_completions, + with_chat_completions_compat, +) + + +def _adapt(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + data = CallModelData( + model_data=ModelInputData(input=items, instructions="system"), + agent=None, # type: ignore[arg-type] + context=None, + ) + return adapt_image_tool_outputs_for_chat_completions(data).input + + +def test_moves_image_tool_output_to_following_user_message(): + items = [ + {"role": "user", "content": "Describe the image."}, + { + "type": "function_call", + "call_id": "call_1", + "name": "get_image", + "arguments": '{"image_path":"figure.png"}', + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA", + } + ], + }, + ] + + messages = Converter.items_to_messages( + _adapt(items), + model="openai/qwen3.6-plus", + preserve_tool_output_all_content=True, + ) + + assert messages[2]["role"] == "tool" + assert isinstance(messages[2]["content"], str) + assert messages[3] == { + "role": "user", + "content": [ + { + "type": "text", + "text": "Images returned by the preceding tool call(s):", + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AAAA", + "detail": "auto", + }, + }, + ], + } + + +def test_keeps_parallel_tool_outputs_before_synthetic_user_message(): + items = [ + {"role": "user", "content": "Compare the files."}, + { + "type": "function_call", + "call_id": "call_1", + "name": "get_image", + "arguments": "{}", + }, + { + "type": "function_call", + "call_id": "call_2", + "name": "read_file", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA", + } + ], + }, + { + "type": "function_call_output", + "call_id": "call_2", + "output": "file contents", + }, + ] + + adapted = _adapt(items) + + assert [item.get("type") for item in adapted[3:5]] == [ + "function_call_output", + "function_call_output", + ] + assert adapted[5]["role"] == "user" + + +def test_uses_default_compat_config_when_run_config_is_missing(): + assert with_chat_completions_compat(None) is CHAT_COMPLETIONS_RUN_CONFIG + + +def test_adds_compat_filter_without_replacing_existing_settings(): + from agents import RunConfig + + original = RunConfig(model="litellm/openai/qwen3.6-plus") + + merged = with_chat_completions_compat(original) + + assert merged is not original + assert merged.model == original.model + assert merged.call_model_input_filter is adapt_image_tool_outputs_for_chat_completions + + +@pytest.mark.asyncio +async def test_composes_existing_filter_before_image_compat(): + from agents import RunConfig + + def existing_filter(data: CallModelData[Any]) -> ModelInputData: + return ModelInputData( + input=[{"role": "user", "content": "prefixed"}, *data.model_data.input], + instructions=data.model_data.instructions, + ) + + merged = with_chat_completions_compat( + RunConfig( + model="litellm/openai/qwen3.6-plus", + call_model_input_filter=existing_filter, + ) + ) + data = CallModelData( + model_data=ModelInputData( + input=[ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA", + } + ], + } + ], + instructions="system", + ), + agent=None, # type: ignore[arg-type] + context=None, + ) + + result = merged.call_model_input_filter(data) + assert result is not None + if hasattr(result, "__await__"): + result = await result + + assert result.input[0] == {"role": "user", "content": "prefixed"} + assert result.input[1]["output"] == ( + "Image returned successfully; inspect the following user image." + ) + assert result.input[2]["role"] == "user" + assert with_chat_completions_compat(merged) is merged diff --git a/tests/test_skill_runner.py b/tests/test_skill_runner.py index 205daac6c..b19e91244 100644 --- a/tests/test_skill_runner.py +++ b/tests/test_skill_runner.py @@ -25,6 +25,7 @@ import pytest +from openkb.agent.model_compat import adapt_image_tool_outputs_for_chat_completions from openkb.agent.skill_runner import ( MAX_TURNS, SkillNotFoundError, @@ -103,6 +104,7 @@ async def test_run_skill_loads_body_into_instructions(tmp_path: Path): async def fake_runner_run(agent, seed, **kw): captured["instructions"] = agent.instructions captured["tools"] = [getattr(t, "name", "?") for t in agent.tools] + captured["run_config"] = kw["run_config"] return MagicMock() with patch("openkb.agent.skill_runner.Runner.run", new=fake_runner_run): @@ -120,6 +122,10 @@ async def fake_runner_run(agent, seed, **kw): # The skill-runner's two distinguishing tools are wired in. assert "write_file" in captured["tools"] assert "read_output_or_skill_file" in captured["tools"] + assert ( + captured["run_config"].call_model_input_filter + is adapt_image_tool_outputs_for_chat_completions + ) # Return shape assert isinstance(result, SkillRunResult) assert result.skill_name == "marker-skill"