From c3dee3035103b9809b70524de7f90f78dd694f93 Mon Sep 17 00:00:00 2001 From: Gideon Balogun Date: Thu, 5 Nov 2026 13:52:32 +0100 Subject: [PATCH] feat(models): add native Gemini reference adapter Add native streaming, function calls, thought-signature replay, token usage and error mapping. Integrate the recorder, redact Google API key headers, and cover the adapter with offline conformance and failure tests. --- CHANGELOG.md | 11 + README.md | 9 +- examples/providers/fixtures/README.md | 16 +- .../providers/fixtures/gemini_native.jsonl | 3 + examples/providers/gemini_native.py | 354 +++++++++++++ examples/providers/record.py | 7 +- src/agentling/testing.py | 4 +- tests/test_examples.py | 7 + tests/test_providers_gemini.py | 479 ++++++++++++++++++ tests/test_testing.py | 5 +- 10 files changed, 886 insertions(+), 9 deletions(-) create mode 100644 examples/providers/fixtures/gemini_native.jsonl create mode 100644 examples/providers/gemini_native.py create mode 100644 tests/test_providers_gemini.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bc443cc..83e9d5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to Agentling are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project aims to follow [semantic versioning](https://semver.org/). +## [Unreleased] + +### Added + +- A copyable native Gemini streaming adapter, recorder integration and an offline + conformance fixture, with tool calling, thought-signature replay and token usage. + +### Fixed + +- Redact Google's `x-goog-api-key` header when recording HTTP fixtures. + ## [0.2.0] - 2026-11-03 **Breaking: Agentling no longer ships a model provider.** Every removed or diff --git a/README.md b/README.md index b2e4b7a..5db81c0 100644 --- a/README.md +++ b/README.md @@ -240,10 +240,11 @@ class Model(Protocol): ) -> AsyncIterator[Delta]: ... ``` -[`examples/providers/`](examples/providers/) has three adapters to copy: +[`examples/providers/`](examples/providers/) has four adapters to copy: `openai_chat.py` for OpenAI and compatible servers (LM Studio, vLLM, Ollama's `/v1`, OpenRouter, Gemini's compatibility endpoint), `anthropic_messages.py` -for Claude, and `ollama_native.py` for Ollama's own API. Point `base_url` at +for Claude, `ollama_native.py` for Ollama's own API, and `gemini_native.py` +for Gemini's native API. Point `base_url` at your endpoint: ```python @@ -274,6 +275,10 @@ the reference adapter closest to your backend: - [`ollama_native.py`](examples/providers/ollama_native.py): Ollama's `/api/chat`. Reads newline-delimited JSON, whole tool calls, and usage from the final chunk. +- [`gemini_native.py`](examples/providers/gemini_native.py): Gemini's native + `streamGenerateContent` API. Maps function calls, cached and thinking tokens, + and replays model parts with their thought signatures through `provider_data`. + Reads `GEMINI_API_KEY` (or `GOOGLE_API_KEY`); `base_url` includes `/v1beta`. **What you yield.** Each `Delta` carries any of these, all optional: diff --git a/examples/providers/fixtures/README.md b/examples/providers/fixtures/README.md index c844dd4..5af429b 100644 --- a/examples/providers/fixtures/README.md +++ b/examples/providers/fixtures/README.md @@ -6,7 +6,7 @@ recorded=path)` replays them in order: line 1 answers the text turn, line 2 the tool turn, and line 3, when present, must make the adapter raise a `ModelError`. -Request headers that carry a key (`authorization`, `x-api-key`, `api-key`) +Request headers that carry a key (`authorization`, `x-api-key`, `api-key`, `x-goog-api-key`) are stored as `‹redacted›`. ## Where each line came from @@ -24,6 +24,20 @@ live recording by running `record.py`, then update this table. | `openai_chat_lmstudio.jsonl` | hand-written | hand-written, tool call id `""` | none | LM Studio's OpenAI-compatible server, which sends empty tool call ids | | `anthropic_messages.jsonl` | hand-written | hand-written, a signed thinking block, a redacted thinking block and a tool call | hand-written, 529 `overloaded_error` | Anthropic Messages streaming and errors docs | | `ollama_native.jsonl` | recorded 3 Nov 2026, Ollama 0.40.14 with `qwen3:8b` | recorded 3 Nov 2026, same server; the call carries `id` and `function.index` | hand-written, 404 model not found | Ollama `/api/chat` docs and `server/routes.go` for line 3 | +| `gemini_native.jsonl` | hand-written | hand-written, thought and function-call signatures, cached and thinking tokens | hand-written, 429 `RESOURCE_EXHAUSTED` with `retry-after` | [Gemini generateContent REST reference](https://ai.google.dev/api/generate-content), including `Part`, `FunctionCall` and `UsageMetadata` | + +The Gemini fixture was captured through `record.py` over `httpx.MockTransport`; +it is not a live Gemini recording. Replace it with a free-tier recording when a +key is available: + +```bash +uv run python -m examples.providers.record gemini_native \\ + --model gemini-2.5-flash --api-key-env GEMINI_API_KEY \\ + --out examples/providers/fixtures/gemini_native.jsonl +``` + +The recorder writes the two successful turns; keep or replace the third error +exchange separately, as described above. Read the request bodies before committing. Request bodies are recorded as sent. The recorder redacts keys in headers, not in bodies, so read a new fixture before committing it; the test suite diff --git a/examples/providers/fixtures/gemini_native.jsonl b/examples/providers/fixtures/gemini_native.jsonl new file mode 100644 index 0000000..1416bf9 --- /dev/null +++ b/examples/providers/fixtures/gemini_native.jsonl @@ -0,0 +1,3 @@ +{"request": {"method": "POST", "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", "headers": {"host": "generativelanguage.googleapis.com", "accept": "*/*", "accept-encoding": "identity", "connection": "keep-alive", "user-agent": "python-httpx/0.28.1", "x-goog-api-key": "‹redacted›", "content-length": "181", "content-type": "application/json"}, "body": {"contents": [{"role": "user", "parts": [{"text": "Reply with exactly the word: pong"}]}], "systemInstruction": {"parts": [{"text": "You are a test subject. Follow instructions exactly."}]}}}, "status": 200, "headers": {"content-type": "text/event-stream", "content-length": "191"}, "body_lines": ["data: {\"candidates\": [{\"index\": 0, \"content\": {\"role\": \"model\", \"parts\": [{\"text\": \"pong\"}]}, \"finishReason\": \"STOP\"}], \"usageMetadata\": {\"promptTokenCount\": 16, \"candidatesTokenCount\": 1}}", "", ""]} +{"request": {"method": "POST", "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", "headers": {"host": "generativelanguage.googleapis.com", "accept": "*/*", "accept-encoding": "identity", "connection": "keep-alive", "user-agent": "python-httpx/0.28.1", "x-goog-api-key": "‹redacted›", "content-length": "472", "content-type": "application/json"}, "body": {"contents": [{"role": "user", "parts": [{"text": "Call the echo tool with text set to ping. Do not answer in prose."}]}], "systemInstruction": {"parts": [{"text": "You are a test subject. Follow instructions exactly."}]}, "tools": [{"functionDeclarations": [{"name": "echo", "description": "Echo the given text back.", "parametersJsonSchema": {"type": "object", "properties": {"text": {"type": "string", "description": "The text to echo."}}, "required": ["text"], "additionalProperties": false}}]}]}}, "status": 200, "headers": {"content-type": "text/event-stream", "content-length": "500"}, "body_lines": ["data: {\"candidates\": [{\"index\": 0, \"content\": {\"role\": \"model\", \"parts\": [{\"text\": \"Use echo.\", \"thought\": true, \"thoughtSignature\": \"fixture-signature\"}]}}]}", "", "data: {\"candidates\": [{\"index\": 0, \"content\": {\"role\": \"model\", \"parts\": [{\"functionCall\": {\"name\": \"echo\", \"args\": {\"text\": \"ping\"}}, \"thoughtSignature\": \"fixture-function-signature\"}]}, \"finishReason\": \"STOP\"}], \"usageMetadata\": {\"promptTokenCount\": 28, \"candidatesTokenCount\": 6, \"thoughtsTokenCount\": 4, \"cachedContentTokenCount\": 8}}", "", ""]} +{"request": {"method": "POST", "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", "headers": {"host": "generativelanguage.googleapis.com", "accept": "*/*", "accept-encoding": "identity", "connection": "keep-alive", "user-agent": "python-httpx/0.28.1", "x-goog-api-key": "‹redacted›", "content-length": "181", "content-type": "application/json"}, "body": {"contents": [{"role": "user", "parts": [{"text": "Reply with exactly the word: pong"}]}], "systemInstruction": {"parts": [{"text": "You are a test subject. Follow instructions exactly."}]}}}, "status": 429, "headers": {"content-type": "application/json", "retry-after": "3"}, "body_lines": ["{\"error\": {\"code\": 429, \"status\": \"RESOURCE_EXHAUSTED\", \"message\": \"Quota exceeded.\"}}"]} diff --git a/examples/providers/gemini_native.py b/examples/providers/gemini_native.py new file mode 100644 index 0000000..2a9ebdb --- /dev/null +++ b/examples/providers/gemini_native.py @@ -0,0 +1,354 @@ +"""A copyable adapter for Gemini's native streaming generateContent API. + +Uses httpx, not Google's SDK, and never retries; the agent loop owns retries. +Raw model parts round-trip under provider_data["gemini"]["parts"], preserving +thought signatures on text and function calls without merging or reordering them. + + model = GeminiModel("gemini-2.5-flash") # reads GEMINI_API_KEY + agent = Agent(model=model, tools=[...]) +""" + +from __future__ import annotations + +import copy +import json +import os +from collections.abc import AsyncIterator, Sequence +from typing import Any +from urllib.parse import quote + +import httpx + +from agentling import ( + ChatMessage, + Delta, + FinishReason, + ModelAuthError, + ModelBadRequestError, + ModelConnectionError, + ModelContextLengthError, + ModelError, + ModelRateLimitError, + ModelServerError, + ModelTimeoutError, + ToolCallDelta, + ToolDefinition, + Usage, +) + +_FINISH_REASONS: dict[str, FinishReason] = { + "STOP": "stop", + "MAX_TOKENS": "length", + "SAFETY": "content_filter", + "RECITATION": "content_filter", + "BLOCKLIST": "content_filter", + "PROHIBITED_CONTENT": "content_filter", + "SPII": "content_filter", + "IMAGE_SAFETY": "content_filter", + "IMAGE_PROHIBITED_CONTENT": "content_filter", + "MALFORMED_FUNCTION_CALL": "error", + "UNEXPECTED_TOOL_CALL": "error", + "TOO_MANY_TOOL_CALLS": "error", + "MISSING_THOUGHT_SIGNATURE": "error", + "MALFORMED_RESPONSE": "error", +} +_NAMESPACE = "gemini" + + +class GeminiModel: + """Stream one candidate from Gemini's native REST API.""" + + def __init__( + self, + model: str, + *, + api_key: str | None = None, + base_url: str = "https://generativelanguage.googleapis.com/v1beta", + timeout: float = 60.0, + transport: httpx.AsyncBaseTransport | None = None, + extra_body: dict[str, Any] | None = None, + ) -> None: + """Configure the endpoint, optional request overrides and test transport. + + `model` is a model name, optionally prefixed with `models/`. The key + defaults to GEMINI_API_KEY, then GOOGLE_API_KEY. `extra_body` is merged + last, for settings such as generationConfig and safetySettings. + """ + self.model = model.removeprefix("models/") + self.api_key = ( + api_key + if api_key is not None + else os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") + ) + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.transport = transport + self.extra_body = copy.deepcopy(extra_body or {}) + + async def stream( + self, + messages: Sequence[ChatMessage], + tools: Sequence[ToolDefinition] = (), + ) -> AsyncIterator[Delta]: + """Yield text, visible thoughts and complete function-call parts as Deltas.""" + body = {**request_body(messages, tools), **self.extra_body} + headers = {"x-goog-api-key": self.api_key} if self.api_key else {} + url = ( + f"{self.base_url}/models/{quote(self.model, safe='')}:streamGenerateContent" + ) + decoder = _TurnDecoder() + try: + async with httpx.AsyncClient( + transport=self.transport, timeout=self.timeout + ) as client: + async with client.stream( + "POST", url, params={"alt": "sse"}, json=body, headers=headers + ) as response: + if response.status_code != 200: + await response.aread() + raise error_for_response(response) + async for line in response.aiter_lines(): + for delta in decoder.decode(line): + yield delta + yield decoder.finish() + except httpx.TimeoutException as exc: + raise ModelTimeoutError(f"Request timed out: {exc}") from exc + except httpx.TransportError as exc: + raise ModelConnectionError( + f"Could not reach {self.base_url}: {exc}" + ) from exc + except httpx.HTTPError as exc: + raise ModelServerError(f"Bad response from {self.base_url}: {exc}") from exc + + +def request_body( + messages: Sequence[ChatMessage], tools: Sequence[ToolDefinition] +) -> dict[str, Any]: + """Map system instructions, history and JSON Schema tool declarations.""" + body: dict[str, Any] = {"contents": _wire_messages(messages)} + system = "\n\n".join(m.content for m in messages if m.role == "system") + if system: + body["systemInstruction"] = {"parts": [{"text": system}]} + if tools: + body["tools"] = [ + { + "functionDeclarations": [ + { + "name": tool.name, + "description": tool.description, + "parametersJsonSchema": copy.deepcopy(tool.parameters), + } + for tool in tools + ] + } + ] + return body + + +def _model_parts(message: ChatMessage) -> list[dict[str, Any]]: + data = (message.provider_data or {}).get(_NAMESPACE) + if isinstance(data, dict) and isinstance(data.get("parts"), list): + return copy.deepcopy(data["parts"]) + parts: list[dict[str, Any]] = [] + if message.content: + parts.append({"text": message.content}) + parts.extend( + { + "functionCall": { + "id": c.id, + "name": c.name, + "args": copy.deepcopy(c.arguments), + } + } + for c in message.tool_calls + ) + return parts + + +def _wire_messages(messages: Sequence[ChatMessage]) -> list[dict[str, Any]]: + wire: list[dict[str, Any]] = [] + # Agentling synthesises absent IDs; don't send those back as native IDs. + calls: dict[str, tuple[str, str | None]] = {} + for message in messages: + if message.role == "system": + continue + role = "model" if message.role == "assistant" else "user" + if message.role == "assistant": + parts = _model_parts(message) + native_calls = [p["functionCall"] for p in parts if "functionCall" in p] + for call, native in zip(message.tool_calls, native_calls): + calls[call.id] = (call.name, native.get("id")) + elif message.role == "tool": + if message.tool_call_id not in calls: + raise ModelBadRequestError("Tool result has no matching function call.") + name, native_id = calls[message.tool_call_id] + response: dict[str, Any] = { + "name": name, + "response": {"result": message.content}, + } + if native_id: + response["id"] = native_id + parts = [{"functionResponse": response}] + else: + parts = [{"text": message.content}] if message.content else [] + if not parts: + continue + if wire and wire[-1]["role"] == role: + wire[-1]["parts"].extend(parts) + else: + wire.append({"role": role, "parts": parts}) + return wire + + +class _TurnDecoder: + """Keep cumulative metadata for the final Delta; stream useful parts now.""" + + def __init__(self) -> None: + self._parts: list[dict[str, Any]] = [] + self._tool_count = 0 + self._usage: dict[str, int] = {} + self._finish_reason: str | None = None + self._saw_event = False + + def decode(self, line: str) -> list[Delta]: + if not line.startswith("data:"): + return [] + self._saw_event = True + try: + event = json.loads(line[5:].strip()) + if not isinstance(event, dict): + raise TypeError("event must be an object") + if "error" in event: + error = event["error"] + raise _classify(int(error.get("code", 500)), error) + usage = event.get("usageMetadata") + if isinstance(usage, dict): + for key in ( + "promptTokenCount", + "toolUsePromptTokenCount", + "candidatesTokenCount", + "thoughtsTokenCount", + "cachedContentTokenCount", + ): + if key in usage: + value = usage[key] + if type(value) is not int or value < 0: + raise TypeError( + "token counts must be non-negative integers" + ) + self._usage[key] = value + if (event.get("promptFeedback") or {}).get("blockReason"): + self._finish_reason = "SAFETY" + candidates = event.get("candidates") or [] + # Agentling represents one answer, even if extra_body requests several. + candidate: dict[str, Any] = next( + (c for c in candidates if c.get("index", 0) == 0), {} + ) + if candidate.get("finishReason"): + if not isinstance(candidate["finishReason"], str): + raise TypeError("finish reason must be a string") + self._finish_reason = candidate["finishReason"] + result: list[Delta] = [] + for part in (candidate.get("content") or {}).get("parts", []): + if not isinstance(part, dict): + raise TypeError("part must be an object") + self._parts.append(copy.deepcopy(part)) + if part.get("text"): + if not isinstance(part["text"], str): + raise TypeError("text must be a string") + if part.get("thought"): + result.append(Delta(reasoning=part["text"])) + else: + result.append(Delta(content=part["text"])) + if "functionCall" in part: + call = part["functionCall"] + if not isinstance(call.get("name"), str) or not call["name"]: + raise TypeError("function call must have a name") + if call.get("id") is not None and not isinstance(call["id"], str): + raise TypeError("function call id must be a string") + args = call.get("args", {}) + if not isinstance(args, dict): + raise TypeError("function arguments must be an object") + result.append( + Delta( + tool_calls=[ + ToolCallDelta( + index=self._tool_count, + id=call.get("id"), + name=call["name"], + arguments=json.dumps(args), + ) + ] + ) + ) + self._tool_count += 1 + return result + except (ValueError, TypeError, AttributeError) as exc: + raise ModelServerError( + "Malformed Gemini stream event.", retryable=False + ) from exc + + def finish(self) -> Delta: + if not self._saw_event or self._finish_reason is None: + raise ModelServerError( + "Gemini stream ended without a finish reason.", retryable=False + ) + reason = _FINISH_REASONS.get(self._finish_reason, "unknown") + if reason == "stop" and self._tool_count: + reason = "tool_calls" + usage = None + if self._usage: + # Gemini reports thoughts separately from candidate output tokens. + usage = Usage( + input_tokens=self._usage.get("promptTokenCount", 0) + + self._usage.get("toolUsePromptTokenCount", 0), + output_tokens=self._usage.get("candidatesTokenCount", 0) + + self._usage.get("thoughtsTokenCount", 0), + cached_input_tokens=self._usage.get("cachedContentTokenCount", 0), + reasoning_tokens=self._usage.get("thoughtsTokenCount", 0), + ) + return Delta( + usage=usage, + finish_reason=reason, + provider_data={_NAMESPACE: {"parts": self._parts}} if self._parts else None, + ) + + +def error_for_response(response: httpx.Response) -> ModelError: + """Classify HTTP errors without leaking request headers or whole bodies.""" + try: + payload = response.json() + except ValueError: + payload = {} + error = payload.get("error", {}) if isinstance(payload, dict) else {} + return _classify(response.status_code, error, response.headers.get("retry-after")) + + +def _classify(status: int, error: Any, retry_after: str | None = None) -> ModelError: + detail = error if isinstance(error, dict) else {} + message = str(detail.get("message") or "The model endpoint rejected the request.")[ + :200 + ] + text = f"HTTP {status} from the model endpoint: {message}" + if status in (401, 403) or "API_KEY_INVALID" in str(detail.get("details", "")): + return ModelAuthError(text, status=status) + if status == 429: + try: + delay = max(0.0, float(retry_after)) if retry_after is not None else None + except ValueError: + delay = None + return ModelRateLimitError(text, status=status, retry_after=delay) + if status == 408: + return ModelTimeoutError(text, status=status) + if status >= 500: + return ModelServerError(text, status=status) + if status == 400 and any( + marker in message.casefold() + for marker in ( + "input token count", + "maximum number of tokens", + "context length", + ) + ): + return ModelContextLengthError(text, status=status) + return ModelBadRequestError(text, status=status) diff --git a/examples/providers/record.py b/examples/providers/record.py index ad7b1d4..87b44e8 100644 --- a/examples/providers/record.py +++ b/examples/providers/record.py @@ -35,10 +35,11 @@ write_fixture, ) from examples.providers.anthropic_messages import AnthropicMessagesModel +from examples.providers.gemini_native import GeminiModel from examples.providers.ollama_native import OllamaModel from examples.providers.openai_chat import OpenAIChatModel -ADAPTERS = ("openai_chat", "anthropic_messages", "ollama_native") +ADAPTERS = ("openai_chat", "anthropic_messages", "ollama_native", "gemini_native") def build( @@ -51,13 +52,15 @@ def build( ) -> Model: """Construct one of the reference adapters around `transport`.""" - built: OpenAIChatModel | AnthropicMessagesModel | OllamaModel + built: OpenAIChatModel | AnthropicMessagesModel | OllamaModel | GeminiModel if adapter == "openai_chat": built = OpenAIChatModel(model, api_key=api_key, transport=transport) elif adapter == "anthropic_messages": built = AnthropicMessagesModel(model, api_key=api_key, transport=transport) elif adapter == "ollama_native": built = OllamaModel(model, api_key=api_key, transport=transport) + elif adapter == "gemini_native": + built = GeminiModel(model, api_key=api_key, transport=transport) else: raise ValueError(f"Unknown adapter {adapter!r}. Choose one of {ADAPTERS}.") if base_url: diff --git a/src/agentling/testing.py b/src/agentling/testing.py index b9566cc..b224036 100644 --- a/src/agentling/testing.py +++ b/src/agentling/testing.py @@ -46,7 +46,7 @@ class _TransportBase: # type: ignore[no-redef] ScriptedTurn = ChatMessage | Sequence[Delta] | Exception _REDACTED = "‹redacted›" -_SECRET_HEADERS = frozenset({"authorization", "x-api-key", "api-key"}) +_SECRET_HEADERS = frozenset({"authorization", "x-api-key", "api-key", "x-goog-api-key"}) # Replayed bodies are plain text, so headers about the original encoding # would make httpx decode them a second time. _DROPPED_ON_REPLAY = frozenset( @@ -512,7 +512,7 @@ class RecordingTransport(_TransportBase): Wraps `inner` (a real network transport by default). Each request asks for an unencoded body so the recording holds the bytes the adapter - parsed. API keys in the authorization, x-api-key and api-key headers are + parsed. API keys in authorization, x-api-key, api-key and x-goog-api-key are replaced before anything is stored. Request bodies are recorded as sent, so a key a backend takes in the body is kept: check a fixture before committing it. diff --git a/tests/test_examples.py b/tests/test_examples.py index 801f5d0..f2c3d47 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -33,6 +33,7 @@ read_fixture, ) from examples.providers.anthropic_messages import AnthropicMessagesModel +from examples.providers.gemini_native import GeminiModel from examples.providers.ollama_native import OllamaModel from examples.providers.openai_chat import OpenAIChatModel @@ -237,11 +238,16 @@ def _ollama(transport: httpx.AsyncBaseTransport | None) -> OllamaModel: return OllamaModel("qwen3:8b", transport=transport) +def _gemini(transport: httpx.AsyncBaseTransport | None) -> GeminiModel: + return GeminiModel("gemini-2.5-flash", api_key="k", transport=transport) + + @pytest.mark.parametrize( ("factory", "fixture", "error_status"), [ (_anthropic, "anthropic_messages.jsonl", 529), (_ollama, "ollama_native.jsonl", 404), + (_gemini, "gemini_native.jsonl", 429), ], ) async def test_reference_adapter_fixture_passes_check_model( @@ -315,6 +321,7 @@ def test_record_help_lists_the_adapters() -> None: "openai_chat", "anthropic_messages", "ollama_native", + "gemini_native", "--base-url", "--model", "--api-key-env", diff --git a/tests/test_providers_gemini.py b/tests/test_providers_gemini.py new file mode 100644 index 0000000..0dbaf11 --- /dev/null +++ b/tests/test_providers_gemini.py @@ -0,0 +1,479 @@ +"""Gemini's native reference adapter, exercised offline over httpx.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from agentling import ( + ChatMessage, + Delta, + ModelAuthError, + ModelBadRequestError, + ModelConnectionError, + ModelContextLengthError, + ModelError, + ModelRateLimitError, + ModelServerError, + ModelTimeoutError, + ToolCall, + ToolDefinition, + Usage, + agglomerate_deltas, +) +from agentling.testing import check_model, read_fixture +from examples.providers.gemini_native import GeminiModel +from examples.providers.record import record + +_USER = [ChatMessage("user", "hi")] +_FIXTURE = Path("examples/providers/fixtures/gemini_native.jsonl") + + +def _chunk(*parts: dict[str, Any], finish: str | None = None) -> dict[str, Any]: + candidate: dict[str, Any] = { + "index": 0, + "content": {"role": "model", "parts": list(parts)}, + } + if finish: + candidate["finishReason"] = finish + return {"candidates": [candidate]} + + +def _transport( + events: list[Any], + *, + requests: list[httpx.Request] | None = None, + status: int = 200, + body: str | None = None, + headers: dict[str, str] | None = None, +) -> httpx.MockTransport: + def handler(request: httpx.Request) -> httpx.Response: + if requests is not None: + requests.append(request) + content = ( + body + if body is not None + else "".join(f"data: {json.dumps(e)}\n\n" for e in events) + ) + return httpx.Response(status, content=content, headers=headers) + + return httpx.MockTransport(handler) + + +def _model(transport: httpx.AsyncBaseTransport, **kwargs: Any) -> GeminiModel: + return GeminiModel("gemini-test", api_key="test-key", transport=transport, **kwargs) + + +async def _collect( + model: GeminiModel, + messages: list[ChatMessage] | None = None, + tools: list[ToolDefinition] | None = None, +) -> list[Delta]: + return [d async for d in model.stream(messages or _USER, tools or [])] + + +async def test_text_thoughts_usage_and_metadata_stream() -> None: + events = [ + {"modelVersion": "gemini-test"}, + _chunk({"thought": True, "text": "Thinking", "thoughtSignature": "signature"}), + _chunk({"text": "Hel"}), + _chunk({"text": "lo"}, finish="STOP"), + { + "usageMetadata": { + "promptTokenCount": 20, + "cachedContentTokenCount": 5, + "candidatesTokenCount": 6, + "thoughtsTokenCount": 8, + "toolUsePromptTokenCount": 3, + } + }, + ] + deltas = await _collect(_model(_transport(events))) + result = agglomerate_deltas(deltas) + assert result.content == "Hello" + assert result.reasoning == "Thinking" + assert result.usage == Usage(23, 14, cached_input_tokens=5, reasoning_tokens=8) + assert result.finish_reason == "stop" + assert sum(d.finish_reason is not None for d in deltas) == 1 + assert all( + d.content or d.reasoning or d.tool_calls or d.usage or d.finish_reason + for d in deltas + ) + assert deltas[-1].provider_data == { + "gemini": { + "parts": [ + events[1]["candidates"][0]["content"]["parts"][0], + {"text": "Hel"}, + {"text": "lo"}, + ] + } + } + + +async def test_parallel_calls_and_signatures_round_trip_without_native_ids() -> None: + parts: list[dict[str, Any]] = [ + {"text": "Calling tools.", "thoughtSignature": "text-signature"}, + { + "functionCall": {"name": "echo", "args": {"text": "a"}}, + "thoughtSignature": "call-signature", + }, + {"functionCall": {"name": "echo", "args": {"text": "b"}}}, + ] + requests: list[httpx.Request] = [] + model = _model(_transport([_chunk(*parts, finish="STOP")], requests=requests)) + deltas = await _collect(model) + result = agglomerate_deltas(deltas) + assert result.finish_reason == "tool_calls" + assert [c.index for d in deltas for c in d.tool_calls] == [0, 1] + assert [(c.name, c.arguments) for c in result.tool_calls] == [ + ("echo", {"text": "a"}), + ("echo", {"text": "b"}), + ] + history = [ + ChatMessage("user", "call"), + result, + *[ + ChatMessage("tool", f"result-{i}", tool_call_id=c.id) + for i, c in enumerate(result.tool_calls) + ], + ChatMessage("user", "continue"), + ] + before = copy.deepcopy(history) + await _collect(model, history) + contents = json.loads(requests[-1].content)["contents"] + assert contents[1] == {"role": "model", "parts": parts} + assert contents[2] == { + "role": "user", + "parts": [ + {"functionResponse": {"name": "echo", "response": {"result": "result-0"}}}, + {"functionResponse": {"name": "echo", "response": {"result": "result-1"}}}, + {"text": "continue"}, + ], + } + assert history == before + + +async def test_native_call_ids_are_returned_in_function_responses() -> None: + requests: list[httpx.Request] = [] + model = _model( + _transport( + [ + _chunk( + {"functionCall": {"id": "native-1", "name": "echo", "args": {}}}, + finish="STOP", + ) + ], + requests=requests, + ) + ) + result = agglomerate_deltas(await _collect(model)) + assert result.tool_calls == [ToolCall("native-1", "echo", {})] + await _collect( + model, [*_USER, result, ChatMessage("tool", "done", tool_call_id="native-1")] + ) + assert ( + json.loads(requests[-1].content)["contents"][-1]["parts"][0][ + "functionResponse" + ]["id"] + == "native-1" + ) + + +async def test_request_schema_system_extra_body_and_other_provider_data() -> None: + requests: list[httpx.Request] = [] + model = GeminiModel( + "models/gemini-test", + api_key="key", + base_url="https://test/v1beta/", + transport=_transport( + [_chunk({"text": "ok"}, finish="STOP")], requests=requests + ), + extra_body={"generationConfig": {"maxOutputTokens": 100}, "safetySettings": []}, + ) + messages = [ + ChatMessage("system", "sys1"), + ChatMessage("system", "sys2"), + *_USER, + ChatMessage( + "assistant", + content="calling", + tool_calls=[ToolCall("c1", "echo", {"text": "x"})], + provider_data={"anthropic": {"blocks": [{"secret": "foreign"}]}}, + ), + ChatMessage("tool", "x", tool_call_id="c1"), + ] + tools = [ + ToolDefinition( + "echo", + "Echo.", + { + "type": "object", + "properties": {"text": {"type": "string"}}, + "additionalProperties": False, + }, + ) + ] + before = copy.deepcopy((messages, tools)) + await _collect(model, messages, tools) + (request,) = requests + assert ( + str(request.url) + == "https://test/v1beta/models/gemini-test:streamGenerateContent?alt=sse" + ) + assert request.headers["x-goog-api-key"] == "key" + assert "key=" not in str(request.url) + body = json.loads(request.content) + assert body["systemInstruction"] == {"parts": [{"text": "sys1\n\nsys2"}]} + assert body["tools"] == [ + { + "functionDeclarations": [ + { + "name": "echo", + "description": "Echo.", + "parametersJsonSchema": tools[0].parameters, + } + ] + } + ] + assert body["generationConfig"] == {"maxOutputTokens": 100} + assert body["safetySettings"] == [] + assert body["contents"][1] == { + "role": "model", + "parts": [ + {"text": "calling"}, + {"functionCall": {"id": "c1", "name": "echo", "args": {"text": "x"}}}, + ], + } + assert "foreign" not in request.content.decode() + assert (messages, tools) == before + + +@pytest.mark.parametrize( + "reason,expected", + [ + ("STOP", "stop"), + ("MAX_TOKENS", "length"), + ("SAFETY", "content_filter"), + ("RECITATION", "content_filter"), + ("BLOCKLIST", "content_filter"), + ("PROHIBITED_CONTENT", "content_filter"), + ("SPII", "content_filter"), + ("MALFORMED_FUNCTION_CALL", "error"), + ("UNEXPECTED_TOOL_CALL", "error"), + ("OTHER", "unknown"), + ("FUTURE_REASON", "unknown"), + ], +) +async def test_finish_reason_mapping(reason: str, expected: str) -> None: + result = agglomerate_deltas( + await _collect(_model(_transport([_chunk({"text": "x"}, finish=reason)]))) + ) + assert result.finish_reason == expected + assert result.usage is None + + +async def test_prompt_block_is_a_content_filter_finish() -> None: + result = agglomerate_deltas( + await _collect( + _model(_transport([{"promptFeedback": {"blockReason": "SAFETY"}}])) + ) + ) + assert result.finish_reason == "content_filter" + assert result.content == "" + + +async def test_usage_is_cumulative_and_second_candidate_is_ignored() -> None: + events = [ + {"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1}}, + { + "candidates": [ + {"index": 1, "content": {"parts": [{"text": "ignore"}]}}, + { + "index": 0, + "content": {"parts": [{"text": "answer"}]}, + "finishReason": "STOP", + }, + ], + "usageMetadata": {"candidatesTokenCount": 4}, + }, + ] + result = agglomerate_deltas(await _collect(_model(_transport(events)))) + assert result.content == "answer" + assert result.usage == Usage(7, 4) + + +@pytest.mark.parametrize( + "status,error_type,message", + [ + (400, ModelBadRequestError, "bad field"), + (401, ModelAuthError, "bad key"), + (403, ModelAuthError, "denied"), + (404, ModelBadRequestError, "not found"), + (408, ModelTimeoutError, "timeout"), + (429, ModelRateLimitError, "slow down"), + (500, ModelServerError, "internal"), + (503, ModelServerError, "unavailable"), + (504, ModelServerError, "deadline"), + ( + 400, + ModelContextLengthError, + "The input token count exceeds the maximum number of tokens allowed", + ), + ], +) +async def test_http_errors_are_classified_without_retries( + status: int, error_type: type[ModelError], message: str +) -> None: + requests: list[httpx.Request] = [] + transport = _transport( + [], + status=status, + body=json.dumps( + {"error": {"code": status, "message": message}, "request": "private body"} + ), + requests=requests, + headers={"retry-after": "3"}, + ) + with pytest.raises(error_type) as caught: + await _collect(_model(transport)) + assert type(caught.value) is error_type + assert caught.value.status == status + assert caught.value.retry_after == (3.0 if status == 429 else None) + assert len(requests) == 1 + assert "private body" not in str(caught.value) + + +async def test_invalid_api_key_reported_as_400_is_auth_error() -> None: + error = { + "error": { + "code": 400, + "message": "API key not valid", + "details": [{"reason": "API_KEY_INVALID"}], + } + } + with pytest.raises(ModelAuthError): + await _collect(_model(_transport([], status=400, body=json.dumps(error)))) + + +@pytest.mark.parametrize("retry_after,expected", [("-2", 0.0), ("bad", None)]) +async def test_retry_after_is_parsed(retry_after: str, expected: float | None) -> None: + with pytest.raises(ModelRateLimitError) as caught: + await _collect( + _model(_transport([], status=429, headers={"retry-after": retry_after})) + ) + assert caught.value.retry_after == expected + + +@pytest.mark.parametrize( + "exception,error_type", + [ + (httpx.ConnectError, ModelConnectionError), + (httpx.ReadTimeout, ModelTimeoutError), + (httpx.RemoteProtocolError, ModelConnectionError), + ], +) +async def test_transport_errors_are_classified( + exception: type[httpx.TransportError], error_type: type[ModelError] +) -> None: + def fail(request: httpx.Request) -> httpx.Response: + raise exception("failure", request=request) + + with pytest.raises(error_type): + await _collect(_model(httpx.MockTransport(fail))) + + +async def test_midstream_error_is_classified() -> None: + events = [ + _chunk({"text": "partial"}), + {"error": {"code": 503, "message": "unavailable"}}, + ] + with pytest.raises(ModelServerError, match="unavailable"): + await _collect(_model(_transport(events))) + + +@pytest.mark.parametrize( + "events", + [ + [[]], + [{"candidates": [None]}], + [_chunk({"functionCall": {"args": {}}})], + [_chunk({"functionCall": {"name": "echo", "args": []}})], + [_chunk({"text": 1})], + [_chunk({"functionCall": {"id": 1, "name": "echo"}})], + [{"candidates": [{"finishReason": ["STOP"]}]}], + ], +) +async def test_malformed_stream_events_fail_as_model_errors(events: list[Any]) -> None: + with pytest.raises(ModelServerError, match="Malformed") as caught: + await _collect(_model(_transport(events))) + assert caught.value.retryable is False + + +@pytest.mark.parametrize( + "body", + ["data: invalid\n\n", "data: null\n\n", '{"candidates": []}', "data: {}\n\n"], +) +async def test_invalid_or_incomplete_streams_fail(body: str) -> None: + with pytest.raises(ModelServerError) as caught: + await _collect(_model(_transport([], body=body))) + assert caught.value.retryable is False + + +async def test_orphan_tool_response_is_bad_request() -> None: + with pytest.raises(ModelBadRequestError, match="matching function call"): + await _collect( + _model(_transport([])), + [ChatMessage("tool", "result", tool_call_id="missing")], + ) + + +@pytest.mark.parametrize("count", [True, -1, "3"]) +async def test_malformed_usage_does_not_escape_as_invalid_usage(count: Any) -> None: + events = [ + { + **_chunk({"text": "answer"}, finish="STOP"), + "usageMetadata": {"cachedContentTokenCount": count}, + } + ] + with pytest.raises(ModelServerError, match="Malformed") as caught: + await _collect(_model(_transport(events))) + assert caught.value.retryable is False + + +def test_key_environment_and_explicit_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GOOGLE_API_KEY", "google-key") + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + assert GeminiModel("m").api_key == "google-key" + monkeypatch.setenv("GEMINI_API_KEY", "gemini-key") + assert GeminiModel("m").api_key == "gemini-key" + assert GeminiModel("m", api_key="override").api_key == "override" + assert GeminiModel("m", api_key="").api_key == "" + + +async def test_recorder_redacts_google_key_and_replays_contract(tmp_path: Path) -> None: + responses = read_fixture(_FIXTURE)[:2] + + def handler(request: httpx.Request) -> httpx.Response: + exchange = responses.pop(0) + return httpx.Response(200, content="\n".join(exchange.body_lines)) + + out = tmp_path / "gemini.jsonl" + report = await record( + "gemini_native", + model="gemini-2.5-flash", + api_key="private-google-key", + out=out, + inner=httpx.MockTransport(handler), + ) + assert report.ok, str(report) + assert "private-google-key" not in out.read_text() + assert {e.request["headers"]["x-goog-api-key"] for e in read_fixture(out)} == { + "‹redacted›" + } + replay = await check_model(lambda t: GeminiModel("m", transport=t), recorded=out) + assert replay.ok, str(replay) diff --git a/tests/test_testing.py b/tests/test_testing.py index 8849c67..3946ddf 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -457,6 +457,7 @@ async def test_recording_transport_redacts_keys() -> None: "Authorization": "Bearer sk-test-1", "X-Api-Key": "sk-ant-test-2", "api-key": "k3", + "X-Goog-Api-Key": "google-test-key", "X-Trace": "t", } @@ -465,10 +466,10 @@ async def test_recording_transport_redacts_keys() -> None: assert response.text == "hello" recorded = recorder.exchanges[0].request["headers"] - for name in ("authorization", "x-api-key", "api-key"): + for name in ("authorization", "x-api-key", "api-key", "x-goog-api-key"): assert recorded[name] == "‹redacted›" assert recorded["x-trace"] == "t" - for secret in ("sk-test-1", "sk-ant-test-2", "k3"): + for secret in ("sk-test-1", "sk-ant-test-2", "k3", "google-test-key"): assert secret not in repr(recorder.exchanges) assert recorder.exchanges[0].request["body"] == {"a": 1}