From 30bf008cb9ad7325041f804c6b9935535fec0e4a Mon Sep 17 00:00:00 2001 From: CocoRoF Date: Sun, 11 Oct 2026 13:56:38 +0900 Subject: [PATCH] 2.79.1: OpenAI gpt-6 and later are reasoning families gpt-6-luna matched none of the reasoning prefixes (o1, o3, o4, gpt-5), so its agent turns went to Chat Completions with tools and every one failed: "Function tools with reasoning_effort are not supported ... use /v1/responses or set reasoning_effort to 'none'". Found on Odysseus prod. - Match GPT generations 5 and later by number, so tool turns take the Responses wire and requests get max_completion_tokens and default sampling up front. - A heal teaches the client: the max_tokens rename or the tools+reasoning refusal remembers the model for the process. - The tools+reasoning 400 is retried once with reasoning_effort "none". Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 21 +++++++ pyproject.toml | 2 +- src/geny_executor/llm_client/openai.py | 57 ++++++++++++++++++- tests/llm_client/unit/test_openai_client.py | 62 ++++++++++++++++++++- tests/unit/test_openai_responses_routing.py | 6 ++ 5 files changed, 142 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3747835..355885d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [2.79.1] - 2026-10-11 + +### Fixed + +- **OpenAI: every GPT generation from 5 on is a reasoning family.** The + static prefix table named `o1`, `o3`, `o4` and `gpt-5`, so `gpt-6-luna` + matched nothing. Its agent turns went to Chat Completions with tools, + where the vendor refuses tools on a reasoning model (`Function tools with + reasoning_effort are not supported ... use /v1/responses or set + reasoning_effort to 'none'`), and every turn failed on a correctly + configured account. Generations are now matched by number (`gpt-5`, + `gpt-6.1-sol`, `gpt-7`, `gpt-10-mini`), so their tool turns take the + Responses wire and they get `max_completion_tokens` and default sampling + up front. +- **A heal teaches the client.** When the vendor names the + `max_tokens` rename, or refuses tools with reasoning on Chat Completions, + the model is remembered for the process: the next call is built right + and the next tool turn goes to Responses. The tools-with-reasoning 400 + itself is retried once with `reasoning_effort: "none"`, the vendor's own + documented way out, so the turn that discovers the model still answers. + ## [2.79.0] — 2026-09-29 Phase 4 of the 2026-09-29 harness audit: quality. diff --git a/pyproject.toml b/pyproject.toml index a8a2b63..d3d9174 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "geny-executor" -version = "2.79.0" +version = "2.79.1" description = "Harness-engineered agent pipeline library — one 21-stage architecture, many model accounts behind one conversation" readme = "README.md" license = "Apache-2.0" diff --git a/src/geny_executor/llm_client/openai.py b/src/geny_executor/llm_client/openai.py index 0a9440f..7654999 100644 --- a/src/geny_executor/llm_client/openai.py +++ b/src/geny_executor/llm_client/openai.py @@ -10,6 +10,7 @@ import json import logging +import re from typing import Any, AsyncIterator, Dict, List, Optional from geny_executor.core.errors import APIError, ErrorCategory @@ -55,11 +56,45 @@ "gpt-5", ) +# The prefix table went stale exactly the way the note above predicts: +# ``gpt-6-luna`` shipped, matched none of the prefixes, and on an agent turn +# went to Chat Completions with tools — where the vendor refuses tools on a +# reasoning model. Every agent turn 400'd on a correctly configured account +# (Odysseus prod, 2026-10-11). Every GPT generation from 5 on is a reasoning +# family, so name the generations by number instead of listing them. +_REASONING_GENERATION = re.compile(r"^gpt-(?:[5-9]|[1-9]\d)(?:\D|$)") + +# Models the vendor has told us, at runtime, are reasoning models (a heal +# below fired for them). Process-local: it only spares the retry on the +# next call and the routing mistake on the next tool turn. +_LEARNED_REASONING_MODELS: set[str] = set() + + +def _is_reasoning_family(model: str) -> bool: + name = (model or "").strip().lower() + return ( + any(name.startswith(prefix) for prefix in _MAX_COMPLETION_TOKENS_PREFIXES) + or bool(_REASONING_GENERATION.match(name)) + or name in _LEARNED_REASONING_MODELS + ) + + +def _learn_reasoning_model(model: str) -> None: + name = (model or "").strip().lower() + if name and name not in _LEARNED_REASONING_MODELS: + _LEARNED_REASONING_MODELS.add(name) + logger.warning( + "openai: %r behaves as a reasoning model — remembered for this process " + "(max_completion_tokens, default sampling, tool turns on Responses). " + "Teach _is_reasoning_family about this family.", + model, + ) + def _model_requires_max_completion_tokens(model: str) -> bool: """True iff ``model`` belongs to a family known to reject ``max_tokens`` in favour of ``max_completion_tokens``.""" - return any(model.startswith(prefix) for prefix in _MAX_COMPLETION_TOKENS_PREFIXES) + return _is_reasoning_family(model) def _model_rejects_sampling_controls(model: str) -> bool: @@ -80,7 +115,7 @@ def _model_rejects_sampling_controls(model: str) -> bool: is one the model will not honour either way. The alternative is a 400 on every call. """ - return any(model.startswith(prefix) for prefix in _MAX_COMPLETION_TOKENS_PREFIXES) + return _is_reasoning_family(model) class OpenAIResponsesClient(ResponsesClient): @@ -215,13 +250,29 @@ def _heal_request_kwargs( Covers reasoning families the static ``_MAX_COMPLETION_TOKENS_PREFIXES`` table doesn't know yet. """ + msg = str(getattr(exc, "message", "") or exc).lower() + if kwargs.get("tools") and "function tools with reasoning_effort" in msg: + # The model reasons and this wire will not carry tools with it. + # Remember the model so its next tool turn goes to Responses + # (``_prefers_responses``); answer this one with reasoning off, + # which is the vendor's own documented way out. + _learn_reasoning_model(kwargs.get("model", "")) + if kwargs.get("reasoning_effort") == "none": + return None + retry = dict(kwargs) + retry["reasoning_effort"] = "none" + for key in ("temperature", "top_p"): + retry.pop(key, None) + if "max_tokens" in retry: + retry["max_completion_tokens"] = retry.pop("max_tokens") + return retry if "max_tokens" not in kwargs: return None - msg = str(getattr(exc, "message", "") or exc).lower() if "max_tokens" not in msg: return None if "max_completion_tokens" not in msg and "not supported" not in msg: return None + _learn_reasoning_model(kwargs.get("model", "")) retry = dict(kwargs) retry["max_completion_tokens"] = retry.pop("max_tokens") return retry diff --git a/tests/llm_client/unit/test_openai_client.py b/tests/llm_client/unit/test_openai_client.py index de020c1..83fc6ee 100644 --- a/tests/llm_client/unit/test_openai_client.py +++ b/tests/llm_client/unit/test_openai_client.py @@ -39,6 +39,16 @@ # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _forget_learned_models(): + """Heals teach the module which models reason; keep that out of the next test.""" + from geny_executor.llm_client import openai as mod + + mod._LEARNED_REASONING_MODELS.clear() + yield + mod._LEARNED_REASONING_MODELS.clear() + + def _req(**overrides: Any) -> APIRequest: base: Dict[str, Any] = { "model": "gpt-4o", @@ -193,7 +203,8 @@ def test_parse_usage_tolerates_missing_details() -> None: @pytest.mark.parametrize( "model", - ["o1", "o1-preview", "o3", "o3-mini", "o4-mini", "gpt-5", "gpt-5.2-codex"], + ["o1", "o1-preview", "o3", "o3-mini", "o4-mini", "gpt-5", "gpt-5.2-codex", + "gpt-6-luna", "gpt-6.1-sol", "gpt-7", "gpt-10-mini"], ) def test_reasoning_families_require_max_completion_tokens(model: str) -> None: assert _model_requires_max_completion_tokens(model) is True @@ -201,7 +212,7 @@ def test_reasoning_families_require_max_completion_tokens(model: str) -> None: @pytest.mark.parametrize( "model", - ["gpt-4o", "gpt-4.1-mini", "gpt-3.5-turbo", "open-mistral", "llama-3"], + ["gpt-4o", "gpt-4.1-mini", "gpt-3.5-turbo", "open-mistral", "llama-3", "gpt-4.5-preview"], ) def test_classic_families_keep_max_tokens(model: str) -> None: assert _model_requires_max_completion_tokens(model) is False @@ -500,3 +511,50 @@ def test_a_classic_model_is_never_sent_an_effort() -> None: )) assert kwargs["tools"] assert "reasoning_effort" not in kwargs + + +# --------------------------------------------------------------------------- +# tools + reasoning on Chat Completions: reactive (2026-10-11, gpt-6-luna) +# --------------------------------------------------------------------------- + +_TOOLS_REASONING_400 = ( + "Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not " + "supported for gpt-9-test in /v1/chat/completions. To use function tools, use /v1/responses " + "or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort'}}" +) + + +def test_tools_reasoning_400_retries_with_effort_none_and_learns_the_model() -> None: + from geny_executor.llm_client import openai as mod + + client = OpenAIClient(api_key="sk-mock") + model = "acme-reasoner-x" # a name no static table knows + mod._LEARNED_REASONING_MODELS.discard(model) + kwargs = {"model": model, "messages": [], "tools": [{"type": "function"}], "max_tokens": 512, "temperature": 0.2} + healed = client._heal_request_kwargs(kwargs, RuntimeError(_TOOLS_REASONING_400)) + assert healed is not None + assert healed["reasoning_effort"] == "none" + assert healed["max_completion_tokens"] == 512 and "max_tokens" not in healed + assert "temperature" not in healed + assert kwargs["max_tokens"] == 512 # pure: the original is untouched + # remembered — the next build uses the reasoning-family kwargs + assert _model_requires_max_completion_tokens(model) is True + mod._LEARNED_REASONING_MODELS.discard(model) + + +def test_tools_reasoning_400_does_not_loop_once_effort_is_none() -> None: + client = OpenAIClient(api_key="sk-mock") + kwargs = {"model": "gpt-6-luna", "messages": [], "tools": [{"type": "function"}], "reasoning_effort": "none"} + assert client._heal_request_kwargs(kwargs, RuntimeError(_TOOLS_REASONING_400)) is None + + +def test_max_tokens_heal_teaches_the_model() -> None: + from geny_executor.llm_client import openai as mod + + client = OpenAIClient(api_key="sk-mock") + model = "acme-reasoner-y" + mod._LEARNED_REASONING_MODELS.discard(model) + healed = client._heal_request_kwargs({"model": model, "max_tokens": 64}, RuntimeError(_RENAME_400)) + assert healed == {"model": model, "max_completion_tokens": 64} + assert _model_requires_max_completion_tokens(model) is True + mod._LEARNED_REASONING_MODELS.discard(model) diff --git a/tests/unit/test_openai_responses_routing.py b/tests/unit/test_openai_responses_routing.py index 6bf81a7..2a7f217 100644 --- a/tests/unit/test_openai_responses_routing.py +++ b/tests/unit/test_openai_responses_routing.py @@ -65,6 +65,12 @@ def test_reasoning_model_with_tools_goes_to_responses(self) -> None: client = OpenAIClient(api_key="sk-test") assert client._prefers_responses(_request(client)) is True + @pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "gpt-7"]) + def test_newer_generations_with_tools_go_to_responses(self, model: str) -> None: + """gpt-6-luna fell through the prefix table and 400'd every agent turn.""" + client = OpenAIClient(api_key="sk-test") + assert client._prefers_responses(_request(client, model=model)) is True + def test_ordinary_model_stays_on_chat_completions(self) -> None: client = OpenAIClient(api_key="sk-test") assert client._prefers_responses(_request(client, model="gpt-4o")) is False