From 87fc64e5a151748551448766c0475408fe56b8dd Mon Sep 17 00:00:00 2001 From: epanonymous <93846049+epanonymous@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:40:49 -0400 Subject: [PATCH 1/3] docs: describe live resource rail and hosted MCP contracts --- README.md | 1 + docs/api-reference.md | 218 ++++++++++++++++++++++++++++++++++++++++-- docs/mcp-server.md | 46 ++++++++- docs/tools.md | 66 +++++++++++++ 4 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 docs/tools.md diff --git a/README.md b/README.md index 2cf7c63..e24b4cf 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ print(agent["name"], agent["credits"]) - [5-Minute Quickstart](docs/quickstart.md) - [API Reference](docs/api-reference.md) +- [Tool discovery and execution](docs/tools.md) - [Pricing](docs/pricing.md) - [MCP Server Guide](docs/mcp-server.md) - [x402 USDC Payments](docs/x402-payments.md) diff --git a/docs/api-reference.md b/docs/api-reference.md index ad891c7..88993de 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -104,17 +104,219 @@ Requires `Authorization: Bearer `. Response: ## Resource discovery -| Method | Endpoint | Description | +Use the [tool loop guide](tools.md) for discover → inspect → estimate → run → poll. +Tool discovery is separate from agent discovery at `/v1/agents/discover`. + +| Method | Endpoint | Authentication / cost | |---|---|---| -| `GET` | `/v1/agents/resources/catalog` | Browse available resources | -| `POST` | `/v1/agents/resources/estimate` | Estimate karma for a planned call | +| `GET` | `/v1/agents/resources/discover?q=...` | Public, free | +| `GET` | `/v1/agents/resources/inspect/{slug}` | Public, free | +| `POST` | `/v1/agents/resources/estimate/{slug}` | Public, free | +| `POST` | `/v1/agents/resources/estimate` | Public, free; legacy body-slug alias | +| `POST` | `/v1/agents/resources/run/{slug}` | Bearer key; executes and can spend credits | +| `GET` | `/v1/agents/resources/runs/{run_id}` | Bearer key; polls and settles the existing run | +| `GET` | `/v1/agents/resources/catalog` | Legacy discovery alias; still returned HTTP 200 on 2026-10-30 | -`/v1/agents/resources/catalog` query parameters: +The live run route requires the slug in the path: bare `/v1/agents/resources/run` returned HTTP 404. Treat slugs as opaque identifiers; preserve embedded `/` segments. -| Param | Type | Notes | -|---|---|---| -| `category` | string | `ai`, `communication`, `utility`, `storage` | -| `status` | string | `available`, `coming_soon`, `unavailable` | +### Live discovery and inspection + +Captured with curl against `api.deva.me` on 2026-10-30, without authentication. These are complete JSON responses, reformatted for readability; catalog contents and prices can change. + +```bash +curl -sS 'https://api.deva.me/v1/agents/resources/discover?q=market%20intel%20leaderboard&limit=1' +``` + +HTTP 200: + +```json +{ + "items": [ + { + "provider": "DEVA", + "slug": "deva/market-intel-leaderboard", + "name": "market intel leaderboard", + "description": "Whale smart money trades on Solana, Base and Ethereum plus Hyperliquid positions and quality scores. Filter by chain; spot assets use mint/contract addresses.", + "categories": [ + "base", + "ethereum", + "finance", + "hyperliquid", + "smart money", + "solana" + ], + "pricing_summary": "$0.0010 / call; included on paid Agent Key plans", + "unit_price_usd": 0.001, + "payable_with": [ + "credits" + ], + "run_kind": "SYNC", + "provider_tool_id": "market-intel-leaderboard", + "provider_native_id": "market-intel-leaderboard", + "popularity": 0, + "pricing": { + "unit_name": "call", + "unit_price_usd": "0.001", + "paid_plan_price_usd": "0" + }, + "pricing_model": "PER_EVENT", + "min_cap_usd": null, + "relevance_score": 1.0, + "burn_rate": null + } + ], + "total": 20, + "next_cursor": "MQ==", + "next": { + "inspect_url": "/v1/agents/resources/inspect/{slug}" + } +} +``` + +Discovery accepts `q`, `category`, `provider`, `status`, `cursor`, `limit` (1–100, default 50), and `min_score` (0–1). Pass `next_cursor` as `cursor` for another page. Select by capability and schema, without branching on provider identity. + +Inspect the exact slug returned above: + +```bash +curl -sS 'https://api.deva.me/v1/agents/resources/inspect/deva/market-intel-leaderboard' +``` + +HTTP 200: + +```json +{ + "slug": "deva/market-intel-leaderboard", + "provider": "DEVA", + "input_schema": { + "type": "object", + "title": "LeaderboardQuery", + "properties": { + "chain": { + "enum": [ + "hl", + "solana", + "base", + "eth" + ], + "type": "string", + "title": "Chain", + "default": "hl" + }, + "limit": { + "type": "integer", + "title": "Limit", + "default": 50, + "maximum": 50, + "minimum": 1 + }, + "window": { + "enum": [ + "day", + "week", + "30d", + "90d", + "allTime" + ], + "type": "string", + "title": "Window", + "default": "30d" + } + }, + "additionalProperties": false + }, + "output_hint": "Same data envelope as /v1/market-intel", + "pricing": { + "unit_name": "call", + "unit_price_usd": "0.001", + "paid_plan_price_usd": "0" + }, + "defaults": {}, + "caps": { + "max_total_charge_usd_required": false + }, + "build": null, + "min_cap_usd": null, + "next": { + "run_url": "/v1/agents/resources/run/deva/market-intel-leaderboard" + } +} +``` + +### Estimate before spending + +`params` must follow the inspected `input_schema`. The body-slug alias was also checked live: + +```bash +curl -sS 'https://api.deva.me/v1/agents/resources/estimate' \ + -H 'Content-Type: application/json' \ + -d '{"slug":"deva/market-intel-leaderboard","params":{"chain":"hl","limit":1,"window":"30d"},"maxTotalChargeUsd":0.01}' +``` + +HTTP 200: + +```json +{ + "amount_usd": 0.001, + "amount_credits": 0.001, + "deterministic": true, + "cap_usd": 0.01, + "breakdown": {}, + "basis": "per call; included on paid Agent Key plans", + "sample_size": null, + "cap_credits": null, + "min_charge_usd": null, + "min_cap_usd": null, + "next": { + "run_url": "/v1/agents/resources/run/deva/market-intel-leaderboard" + } +} +``` + +For the preferred `/estimate/{slug}` route, send the same body without `slug`. Estimates are free and do not reserve credits. `deterministic: false` means the amount is not a guaranteed bill. Use the returned `min_cap_usd` and inspect's `caps.max_total_charge_usd_required` to choose a valid cap. REST names the cap `maxTotalChargeUsd`; hosted MCP names it `max_total_charge_credits`. One credit equals one USD. + +### Run and poll + +The following authenticated commands are templates, **not captured successful runs**. This documentation session had no API key. Executing the first command can spend credits. + +```bash +curl -sS 'https://api.deva.me/v1/agents/resources/run/deva/market-intel-leaderboard' \ + -H "Authorization: Bearer $DEVA_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"params":{"chain":"hl","limit":1,"window":"30d"},"maxTotalChargeUsd":0.01}' + +# Only for an asynchronous response: set RUN_ID to its run_id. +curl -sS "https://api.deva.me/v1/agents/resources/runs/$RUN_ID" \ + -H "Authorization: Bearer $DEVA_API_KEY" +``` + +Verified response contracts: + +- **HTTP 200, synchronous:** `status`, `charged`, `credits_charged`, `actual_cost_usd`, `run_id` (nullable), `output`, `message`. Read `output` directly; do not poll a completed synchronous result. +- **HTTP 202, asynchronous:** `run_id`, `status: "running"`, `poll`, `max_total_charge_credits`, `next`. Poll the same run until `status` is `succeeded` or `failed`. +- **Poll HTTP 200:** `status`, `run_id`, and nullable `credits_charged`, `actual_cost_usd`, `items`, `dataset_id`, `result_count`, `usage_breakdown`, `max_total_charge_credits`, `cap_hit`, `message`, `next`. Check the body status even when HTTP is 200. `cap_hit: true` means output may be partial. + +These unauthenticated probes were captured live (the poll identifier is deliberately a placeholder): + +```bash +curl -sS 'https://api.deva.me/v1/agents/resources/run/deva/market-intel-leaderboard' \ + -H 'Content-Type: application/json' \ + -d '{"params":{"chain":"hl","limit":1,"window":"30d"},"maxTotalChargeUsd":0.01}' +curl -sS 'https://api.deva.me/v1/agents/resources/runs/documentation-example' +``` + +Both returned HTTP 401: + +```json +{ + "detail": { + "success": false, + "error": "Missing API key", + "hint": "Include 'Authorization: Bearer deva_xxx' header" + } +} +``` + +See [errors and budget guards](tools.md#errors-and-budget-guards) before executing paid calls. Never automatically retry a run POST. ## Key-value storage diff --git a/docs/mcp-server.md b/docs/mcp-server.md index dc8a9b1..0926e97 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -2,6 +2,50 @@ Use `@deva-me/mcp-server` to expose Deva API capabilities as MCP tools in Claude Code, Cursor, OpenClaw, and other MCP-compatible clients. +## Hosted MCP (available now) + +Send JSON-RPC requests to **`https://api.deva.me/v1/mcp`** with `Authorization: Bearer $DEVA_API_KEY`. This endpoint authenticates with a Deva API key; it does not use OAuth. Configure your remote MCP client with this URL and Bearer header, or inspect the tool list directly: + +```bash +curl -sS https://api.deva.me/v1/mcp \ + -H "Authorization: Bearer $DEVA_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' +``` + +The current hosted tool registry defines these names (without a `deva_` prefix): + +| Tool | Purpose | +|---|---| +| `list_resources` | Discover tools with `q` and optional filters | +| `inspect_resource` | Read a `slug`'s input schema and cap requirements | +| `estimate_resource` | Estimate `slug` + `params`, optionally with `max_total_charge_credits` | +| `run_resource` | Execute `slug` + `params` with `max_total_charge_credits`; can spend | +| `get_run` | Poll `run_id`; optionally trim displayed results | +| `get_balance` | Read spendable credits | +| `introspect_key` | Read key identity, claim state, and balance | +| `create_payment_request` | Create a funding checkout link | +| `get_claim_status` | Read claim state | +| `get_payout_status` | Read reward payout state | + +Use `list_resources` → `inspect_resource` → `estimate_resource` → `run_resource` → `get_run`. Discovery, inspection, and estimation are free. Poll an asynchronous run until `succeeded` or `failed`; read synchronous output directly. Check MCP `result.isError` as well as HTTP status. Never automatically retry `run_resource`. + +Hosted MCP uses `max_total_charge_credits` for the cap and `min_cap_credits` in inspection; REST uses `maxTotalChargeUsd` and `min_cap_usd`. One credit equals one USD. Read the [tool loop guide](tools.md) for budget guards and failure handling. + +## npm stdio tools (next release) + +The generic tool loop arrives in the **next `@deva-me/mcp-server` release**. These are stdio names, separate from the hosted names above: + +| Stdio tool | Operation | +|---|---| +| `deva_tools_discover` | Discover tools | +| `deva_tools_inspect` | Inspect a discovered slug | +| `deva_tools_estimate` | Estimate a run without spending | +| `deva_tools_run` | Start a capped run | +| `deva_tools_get_run` | Poll an existing run | + +Check the installed server's `tools/list` for availability and argument schemas. Do not assume these tools exist in the current npm release. The existing `deva_resources_catalog` tool is legacy discovery. + ## Install and run ```bash @@ -78,7 +122,7 @@ DEVA_MCP_PROFILE = "default" | Social | `deva_social_post_create`, `deva_social_feed_get`, `deva_social_agents_search` | | AI resources | `deva_ai_chat_completions`, `deva_ai_tts`, `deva_ai_embeddings` | | Storage | `deva_storage_kv_set`, `deva_storage_file_upload` | -| Balance and pricing | `deva_balance_get`, `deva_cost_estimate`, `deva_resources_catalog` | +| Balance and pricing | `deva_balance_get`, `deva_cost_estimate` | | Messaging | `deva_messaging_send`, `deva_messaging_inbox_get`, `deva_messaging_reply` | ## Environment variables diff --git a/docs/tools.md b/docs/tools.md new file mode 100644 index 0000000..bbd5a71 --- /dev/null +++ b/docs/tools.md @@ -0,0 +1,66 @@ +# Discover and run tools + +Use one loop for Deva and third-party tools: **discover → inspect → estimate → run → poll**. The REST base URL is `https://api.deva.me/v1/agents/resources`. See the [API reference](api-reference.md#resource-discovery) for curl requests and captured live responses, or the [MCP guide](mcp-server.md) for tool names. + +## The loop + +1. **Discover:** `GET /discover?q=...` returns `items`, `total`, and `next_cursor`. Choose a relevant `items[].slug`; do not construct a slug or select based on its provider prefix. Discover is public and free. +2. **Inspect:** `GET /inspect/{slug}` returns `input_schema`, `defaults`, `pricing`, and `caps`. Build `params` from that schema. Inspect is public and free. +3. **Estimate:** `POST /estimate/{slug}` with `{"params": {...}, "maxTotalChargeUsd": 0.01}` returns `amount_credits`, `deterministic`, `basis`, and cap information. Estimate is public and free; it neither runs the tool nor reserves funds. The numeric cap here is only an example: your chosen tool may require more. +4. **Run:** `POST /run/{slug}` with the same parameters and cap, plus `Authorization: Bearer $DEVA_API_KEY`. This is the spending step. The key must permit resource execution and have sufficient spendable credits. Run can reserve a budget before actual usage is settled. Some tools are free or included in your plan; check the estimate and final charge. +5. **Poll, when needed:** `GET /runs/{run_id}` with the same key until `status` is `succeeded` or `failed`. Polling does not start another paid execution; it can settle the charge for the existing run. The key needs resource read access. Save the run ID before polling so you can resume after a timeout. + +For REST, keep `/` characters inside a slug as path segments. The older `POST /estimate` accepts `slug` in its body. Bare `POST /run` is not a working route. + +## Synchronous and asynchronous results + +Discovery advertises `run_kind` (`SYNC` or `ASYNC`). Handle the actual run response as authoritative: + +- HTTP **200** returns a synchronous result. Check `status`, read `output`, and record `credits_charged`. A nullable `run_id` is not an instruction to poll. +- HTTP **202** returns `run_id` and `poll`. Wait between GETs (for example, two seconds) and stop at a terminal status. Terminal output is in `items`, with `result_count` and `credits_charged`. A successful HTTP response can still contain `status: "failed"`. + +A poll timeout does not cancel execution. Keep the ID and resume polling; do not submit another run. `cap_hit: true` means the result was truncated at the spending cap. Poll options `fields`, `max_items`, and `summary` only reduce the response displayed; they do not reduce the bill. Save large results to a file instead of copying them into model context. + +## Errors and budget guards + +| HTTP status | Meaning | Action | +|---|---|---| +| 401 | Missing or invalid API key | Supply a valid Bearer key for run/poll. | +| 402 | Insufficient credits or a cap that exceeds spendable balance | Read the error reason. `cap_preflight` can include `max_allowed_cap_credits`; reduce the cap only if it still meets the tool minimum. `cap_floor_above_balance` requires funding before that tool can run. | +| 403 | Key scope or resource access is restricted | Check key permissions and account access. | +| 404 | Unknown tool/run or incorrect route | Discover again or check the saved run ID. | +| 422 | Invalid parameters or cap | Follow the inspected schema and error details. | +| 503 | Provider unavailable | Preserve any known run ID and check it later. Do not automatically repeat the run POST. | + +**Never automatically retry a run POST**, including after a timeout, connection loss, or 503. The request may already have started an execution. The REST run route supports an `Idempotency-Key` header for intentional duplicate protection; keep the same key for the same intended request, but do not assume an ambiguous response means nothing ran. + +> **Budget guard:** Keep both a per-task credit budget and a maximum run count. Before each POST, require a positive finite cap no larger than your remaining task budget and no smaller than `min_cap_usd`. Compare `amount_credits` to your cap, but do not mistake an estimate for enforcement. Send `maxTotalChargeUsd` on the run itself. Reserve the full cap locally before submitting, and keep that reservation after ambiguous failures until reconciled. One credit equals one USD. Never raise a budget automatically to satisfy a tool minimum. + +## Runnable examples and tests + +The examples use HTTP directly, accept an exact discovered slug plus schema-valid JSON, and submit at most one run. They fail closed on invalid caps, over-budget estimates, and unknown statuses. No run POST is retried. + +```bash +export DEVA_API_KEY=deva_your_api_key +# This command can spend up to the supplied cap. +python3 examples/python/tools_loop.py \ + --query 'market intel leaderboard' --slug 'deva/market-intel-leaderboard' \ + --params '{"chain":"hl","limit":1,"window":"30d"}' --budget 0.01 + +# Node.js 22.18+ (native TypeScript type stripping); same spending behavior. +node examples/typescript/tools_loop.ts \ + --query 'market intel leaderboard' --slug 'deva/market-intel-leaderboard' \ + --params '{"chain":"hl","limit":1,"window":"30d"}' --budget 0.01 +``` + +- [Python example](../examples/python/tools_loop.py) uses the standard library. +- [TypeScript example](../examples/typescript/tools_loop.ts) uses built-in `fetch`. + +Run mocked HTTP unit tests from the repository root: + +```bash +python3 -m unittest discover -s tests -p 'test_*.py' +node --test tests/tools_loop.test.ts +``` + +Both suites include a live smoke test skipped unless `DEVA_API_KEY` is set. The smoke tests discover, inspect, estimate, and verify authenticated run lookup; they never start a paid run. Successful paid run/poll responses were not captured for this docs update because no key was available. From e1802305e88fdbe2b0f0eab6023605e99cb92c06 Mon Sep 17 00:00:00 2001 From: epanonymous <93846049+epanonymous@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:43:21 -0400 Subject: [PATCH 2/3] docs: add capped Python and TypeScript tool loops with HTTP tests --- .github/workflows/tool-examples.yml | 20 +++++++ docs/tools.md | 2 +- examples/python/tools_loop.py | 76 ++++++++++++++++++++++++++ examples/typescript/tools_loop.ts | 68 +++++++++++++++++++++++ tests/test_tools_loop.py | 85 +++++++++++++++++++++++++++++ tests/tools_loop.test.ts | 78 ++++++++++++++++++++++++++ 6 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tool-examples.yml create mode 100644 examples/python/tools_loop.py create mode 100644 examples/typescript/tools_loop.ts create mode 100644 tests/test_tools_loop.py create mode 100644 tests/tools_loop.test.ts diff --git a/.github/workflows/tool-examples.yml b/.github/workflows/tool-examples.yml new file mode 100644 index 0000000..b1e27d9 --- /dev/null +++ b/.github/workflows/tool-examples.yml @@ -0,0 +1,20 @@ +name: Tool loop examples +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: actions/setup-node@v4 + with: + node-version: '24' + - run: python3 -m unittest discover -s tests -p 'test_*.py' + - run: node --test tests/tools_loop.test.ts diff --git a/docs/tools.md b/docs/tools.md index bbd5a71..f402a82 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -29,7 +29,7 @@ A poll timeout does not cancel execution. Keep the ID and resume polling; do not | 402 | Insufficient credits or a cap that exceeds spendable balance | Read the error reason. `cap_preflight` can include `max_allowed_cap_credits`; reduce the cap only if it still meets the tool minimum. `cap_floor_above_balance` requires funding before that tool can run. | | 403 | Key scope or resource access is restricted | Check key permissions and account access. | | 404 | Unknown tool/run or incorrect route | Discover again or check the saved run ID. | -| 422 | Invalid parameters or cap | Follow the inspected schema and error details. | +| 400 / 422 | Invalid parameters or cap | Follow the inspected schema and error details. | | 503 | Provider unavailable | Preserve any known run ID and check it later. Do not automatically repeat the run POST. | **Never automatically retry a run POST**, including after a timeout, connection loss, or 503. The request may already have started an execution. The REST run route supports an `Idempotency-Key` header for intentional duplicate protection; keep the same key for the same intended request, but do not assume an ambiguous response means nothing ran. diff --git a/examples/python/tools_loop.py b/examples/python/tools_loop.py new file mode 100644 index 0000000..92f6aa4 --- /dev/null +++ b/examples/python/tools_loop.py @@ -0,0 +1,76 @@ +"""One capped tool execution. No automatic retries; Python standard library only.""" +import argparse +import json +import math +import os +import sys +import time +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + +BASE = 'https://api.deva.me/v1/agents/resources' + + +def request(method, path, body=None, key=None): + headers = {'Content-Type': 'application/json'} + if key: + headers['Authorization'] = f'Bearer {key}' + req = Request(BASE + path, data=None if body is None else json.dumps(body).encode(), + headers=headers, method=method) + # urlopen does not retry POSTs. HTTP errors propagate to the caller. + with urlopen(req, timeout=60) as response: + return response.status, json.load(response) + + +def run_loop(query, slug, params, budget, key, *, http=request, sleep=time.sleep, + max_polls=60, on_run=lambda run_id: print(f'Run ID: {run_id}', file=sys.stderr)): + if not key: + raise ValueError('Set DEVA_API_KEY') + if not math.isfinite(budget) or budget <= 0 or not isinstance(params, dict): + raise ValueError('Require a positive finite budget and an object for params') + if max_polls < 1: + raise ValueError('max_polls must be positive') + _, page = http('GET', '/discover?' + urlencode({'q': query, 'limit': 100})) + if not any(item['slug'] == slug for item in page['items']): + raise ValueError('Slug not in this discovery page; refine your query') + path_slug = quote(slug, safe='/') + _, spec = http('GET', '/inspect/' + path_slug) + # Read input_schema before preparing params; server validates the supplied object. + body = {'params': params, 'maxTotalChargeUsd': budget} + _, estimate = http('POST', '/estimate/' + path_slug, body) + for floor in (spec.get('min_cap_usd'), estimate.get('min_cap_usd')): + if floor is not None and (not math.isfinite(float(floor)) or float(floor) > budget): + raise ValueError('Tool minimum exceeds budget') + amount = estimate.get('amount_credits') + if amount is not None and (not math.isfinite(float(amount)) or float(amount) < 0 or float(amount) > budget): + raise ValueError('Estimate is invalid or exceeds budget') + # One-call task budget: reserve the entire cap before this single POST. + # Do not release that reservation or submit again on an ambiguous failure. + code, result = http('POST', '/run/' + path_slug, body, key) + if code == 200: + if result.get('status') != 'succeeded': + raise RuntimeError('Synchronous run failed: ' + str(result.get('message'))) + return result + if code != 202 or not result.get('run_id'): + raise RuntimeError('Unexpected run response; do not resubmit') + run_id = result['run_id'] + on_run(run_id) + for _ in range(max_polls): + sleep(2) + _, result = http('GET', '/runs/' + quote(run_id, safe=''), key=key) + if result['status'] == 'succeeded': + return result + if result['status'] != 'running': + raise RuntimeError(f'Run {run_id}: {result["status"]}; do not resubmit') + raise TimeoutError(f'Poll limit reached; resume GET /runs/{run_id}; do not resubmit') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--query', required=True) + parser.add_argument('--slug', required=True) + parser.add_argument('--params', required=True, type=json.loads) + parser.add_argument('--budget', required=True, type=float) + args = parser.parse_args() + print(json.dumps(run_loop(args.query, args.slug, args.params, args.budget, + os.environ.get('DEVA_API_KEY')), indent=2)) diff --git a/examples/typescript/tools_loop.ts b/examples/typescript/tools_loop.ts new file mode 100644 index 0000000..4305077 --- /dev/null +++ b/examples/typescript/tools_loop.ts @@ -0,0 +1,68 @@ +/** One capped execution using built-in fetch. Never automatically retry a run. */ +import { parseArgs } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +export const BASE = 'https://api.deva.me/v1/agents/resources'; +type Json = Record; +type Http = (method: string, path: string, body?: Json, key?: string) => Promise<[number, Json]>; + +export const request: Http = async (method, path, body, key) => { + const response = await fetch(BASE + path, { + method, + headers: { 'Content-Type': 'application/json', ...(key ? { Authorization: `Bearer ${key}` } : {}) }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(60_000), + redirect: 'error', + }); + if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`); + return [response.status, await response.json() as Json]; +}; + +export async function runLoop(query: string, slug: string, params: Json, budget: number, key: string, + { http = request, sleep = (ms: number) => new Promise(r => setTimeout(r, ms)), + maxPolls = 60, onRun = (id: string) => console.error(`Run ID: ${id}`) } = {}) { + if (!key) throw new Error('Set DEVA_API_KEY'); + if (!Number.isFinite(budget) || budget <= 0 || !params || typeof params !== 'object' || Array.isArray(params)) + throw new Error('Require a positive finite budget and an object for params'); + if (!Number.isInteger(maxPolls) || maxPolls < 1) throw new Error('maxPolls must be positive'); + const [, page] = await http('GET', '/discover?' + new URLSearchParams({ q: query, limit: '100' })); + if (!page.items.some((item: Json) => item.slug === slug)) throw new Error('Slug not in this discovery page; refine your query'); + const pathSlug = slug.split('/').map(encodeURIComponent).join('/'); + const [, spec] = await http('GET', '/inspect/' + pathSlug); + // Read input_schema before preparing params; the server validates the supplied object. + const body = { params, maxTotalChargeUsd: budget }; + const [, estimate] = await http('POST', '/estimate/' + pathSlug, body); + for (const floor of [spec.min_cap_usd, estimate.min_cap_usd]) { + if (floor != null && (!Number.isFinite(Number(floor)) || Number(floor) > budget)) + throw new Error('Tool minimum exceeds budget'); + } + const amount = estimate.amount_credits; + if (amount != null && (!Number.isFinite(Number(amount)) || Number(amount) < 0 || Number(amount) > budget)) + throw new Error('Estimate is invalid or exceeds budget'); + // One-call task budget: reserve the whole cap before submitting this single POST. + // An ambiguous failure does not release the reservation or authorize another run. + const [code, initial] = await http('POST', '/run/' + pathSlug, body, key); + if (code === 200) { + if (initial.status !== 'succeeded') throw new Error(`Synchronous run failed: ${initial.message}`); + return initial; + } + if (code !== 202 || !initial.run_id) throw new Error('Unexpected run response; do not resubmit'); + const id = initial.run_id; + onRun(id); + for (let i = 0; i < maxPolls; i++) { + await sleep(2000); + const [, result] = await http('GET', '/runs/' + encodeURIComponent(id), undefined, key); + if (result.status === 'succeeded') return result; + if (result.status !== 'running') throw new Error(`Run ${id}: ${result.status}; do not resubmit`); + } + throw new Error(`Poll limit reached; resume GET /runs/${id}; do not resubmit`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const { values } = parseArgs({ options: { + query: { type: 'string' }, slug: { type: 'string' }, params: { type: 'string' }, budget: { type: 'string' }, + } }); + if (!values.query || !values.slug || !values.params || !values.budget) throw new Error('Supply --query --slug --params --budget'); + const result = await runLoop(values.query, values.slug, JSON.parse(values.params), Number(values.budget), process.env.DEVA_API_KEY ?? ''); + console.log(JSON.stringify(result, null, 2)); +} diff --git a/tests/test_tools_loop.py b/tests/test_tools_loop.py new file mode 100644 index 0000000..9c4334b --- /dev/null +++ b/tests/test_tools_loop.py @@ -0,0 +1,85 @@ +import importlib.util +import io +import json +import os +from pathlib import Path +import unittest +from unittest.mock import patch +from urllib.error import HTTPError + +spec = importlib.util.spec_from_file_location('tools_loop', Path(__file__).parents[1] / 'examples/python/tools_loop.py') +loop = importlib.util.module_from_spec(spec) +spec.loader.exec_module(loop) +SLUG = 'deva/example' + + +class Response(io.BytesIO): + def __init__(self, status, body): + super().__init__(json.dumps(body).encode()) + self.status = status + + +class LoopTests(unittest.TestCase): + def execute(self, tail, *, estimate=None, budget=0.01, max_polls=3): + replies = [Response(200, {'items': [{'slug': SLUG}]}), + Response(200, {'input_schema': {}, 'min_cap_usd': None}), + Response(200, estimate or {'amount_credits': 0.001})] + tail + with patch.object(loop, 'urlopen', side_effect=replies) as http: + self.http = http + return loop.run_loop('example', SLUG, {}, budget, 'test-key', + sleep=lambda _: None, on_run=lambda _: None, max_polls=max_polls) + + def test_sync_and_cap_and_auth(self): + result = self.execute([Response(200, {'status': 'succeeded', 'output': {'ok': True}})]) + self.assertTrue(result['output']['ok']) + requests = [call.args[0] for call in self.http.call_args_list] + self.assertEqual(len(requests), 4) + self.assertFalse(requests[0].has_header('Authorization')) + self.assertEqual(requests[-1].get_header('Authorization'), 'Bearer test-key') + self.assertEqual(json.loads(requests[-1].data)['maxTotalChargeUsd'], 0.01) + self.assertTrue(requests[-1].full_url.endswith('/run/deva/example')) + + def test_async_polls_same_id(self): + result = self.execute([Response(202, {'run_id': 'run-1', 'status': 'running'}), + Response(200, {'status': 'running'}), + Response(200, {'status': 'succeeded', 'items': [1]})]) + self.assertEqual(result['items'], [1]) + self.assertEqual([c.args[0].method for c in self.http.call_args_list], ['GET', 'GET', 'POST', 'POST', 'GET', 'GET']) + self.assertTrue(self.http.call_args.args[0].full_url.endswith('/runs/run-1')) + + def test_failures_never_retry_post(self): + for error in [TimeoutError('timeout'), HTTPError('url', 402, 'credits', {}, None), HTTPError('url', 503, 'unavailable', {}, None)]: + with self.subTest(error=error), self.assertRaises(type(error)): + self.execute([error]) + self.assertEqual(self.http.call_count, 4) + + def test_rejects_estimate_above_cap_and_minimum(self): + for estimate in [{'amount_credits': 1}, {'amount_credits': None, 'min_cap_usd': 1}, {'amount_credits': 'NaN'}]: + with self.subTest(estimate=estimate), self.assertRaises(ValueError): + self.execute([], estimate=estimate) + self.assertEqual(self.http.call_count, 3) + + def test_invalid_budgets_do_not_call_http(self): + for budget in [0, -1, float('nan'), float('inf')]: + with self.assertRaises(ValueError): + self.execute([], budget=budget) + self.assertEqual(self.http.call_count, 0) + + def test_terminal_failure_and_timeout(self): + with self.assertRaisesRegex(RuntimeError, 'failed'): + self.execute([Response(202, {'run_id': 'run-1'}), Response(200, {'status': 'failed'})]) + with self.assertRaisesRegex(TimeoutError, 'resume GET /runs/run-1'): + self.execute([Response(202, {'run_id': 'run-1'}), Response(200, {'status': 'running'})], max_polls=1) + + @unittest.skipUnless(os.getenv('DEVA_API_KEY'), 'DEVA_API_KEY not set; no paid execution') + def test_live_smoke(self): + _, page = loop.request('GET', '/discover?q=market%20intel%20leaderboard&limit=1') + slug = page['items'][0]['slug'] + _, spec = loop.request('GET', '/inspect/' + loop.quote(slug, safe='/')) + self.assertIn('input_schema', spec) + _, estimate = loop.request('POST', '/estimate/' + loop.quote(slug, safe='/'), {'params': {}}) + self.assertIn('amount_credits', estimate) + # A missing run must reach authenticated lookup (404), not authentication (401). + with self.assertRaises(HTTPError) as error: + loop.request('GET', '/runs/00000000-0000-0000-0000-000000000000', key=os.environ['DEVA_API_KEY']) + self.assertEqual(error.exception.code, 404) diff --git a/tests/tools_loop.test.ts b/tests/tools_loop.test.ts new file mode 100644 index 0000000..ae544fb --- /dev/null +++ b/tests/tools_loop.test.ts @@ -0,0 +1,78 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { runLoop, request } from '../examples/typescript/tools_loop.ts'; + +const slug = 'deva/example'; +async function execute(t: any, tail: any[], estimate: any = { amount_credits: 0.001 }, budget = 0.01, maxPolls = 3) { + const replies: any[] = [ [200, { items: [{ slug }] }], [200, { input_schema: {} }], [200, estimate], ...tail ]; + const calls: any[] = []; + t.mock.method(globalThis, 'fetch', async (url: string, init: any) => { + calls.push({ url, ...init }); + const next = replies.shift(); + if (next instanceof Error) throw next; + assert.ok(next, 'Unexpected HTTP call (possible retry)'); + return new Response(JSON.stringify(next[1]), { status: next[0] }); + }); + try { + return { result: await runLoop('example', slug, {}, budget, 'test-key', { + sleep: async () => {}, onRun: () => {}, maxPolls, + }), calls }; + } catch (error) { return { error, calls }; } +} + +test('sync output, cap, slug path and auth', async t => { + const { result, error, calls } = await execute(t, [[200, { status: 'succeeded', output: { ok: true } }]]); + assert.ifError(error); + assert.equal(result?.output.ok, true); + assert.equal(calls.length, 4); + assert.equal(calls[0].headers.Authorization, undefined); + assert.equal(calls[3].headers.Authorization, 'Bearer test-key'); + assert.ok(calls[3].url.endsWith('/run/deva/example')); + assert.equal(JSON.parse(calls[3].body).maxTotalChargeUsd, 0.01); +}); +test('async polls same run without another POST', async t => { + const { result, error, calls } = await execute(t, [[202, { run_id: 'r1' }], [200, { status: 'running' }], [200, { status: 'succeeded', items: [1] }]]); + assert.ifError(error); + assert.deepEqual(result?.items, [1]); + assert.deepEqual(calls.map(c => c.method), ['GET', 'GET', 'POST', 'POST', 'GET', 'GET']); + assert.ok(calls[5].url.endsWith('/runs/r1')); +}); +for (const failure of [new Error('timeout'), [402, { error: 'insufficient credits' }], [503, { error: 'unavailable' }]]) { + test(`no retry after ${JSON.stringify(failure)}`, async t => { + const { error, calls } = await execute(t, [failure]); + assert.ok(error); + assert.equal(calls.length, 4); + }); +} +for (const estimate of [{ amount_credits: 1 }, { min_cap_usd: 1 }, { amount_credits: 'NaN' }]) { + test(`reject estimate ${JSON.stringify(estimate)}`, async t => { + const { error, calls } = await execute(t, [], estimate); + assert.ok(error); + assert.equal(calls.length, 3); + }); +} +for (const budget of [0, -1, NaN, Infinity]) { + test(`reject budget ${budget}`, async t => { + const { error, calls } = await execute(t, [], undefined, budget); + assert.ok(error); + assert.equal(calls.length, 0); + }); +} +test('failed terminal state', async t => { + const { error } = await execute(t, [[202, { run_id: 'r1' }], [200, { status: 'failed' }]]); + assert.match(String(error), /failed/); +}); +test('poll timeout preserves ID', async t => { + const { error, calls } = await execute(t, [[202, { run_id: 'r1' }], [200, { status: 'running' }]], undefined, 0.01, 1); + assert.match(String(error), /resume GET \/runs\/r1/); + assert.equal(calls.length, 5); +}); +test('live smoke without paid execution', { skip: !process.env.DEVA_API_KEY }, async () => { + const [, page] = await request('GET', '/discover?q=market%20intel%20leaderboard&limit=1'); + const slugPath = page.items[0].slug.split('/').map(encodeURIComponent).join('/'); + const [, spec] = await request('GET', '/inspect/' + slugPath); + assert.ok(spec.input_schema); + const [, estimate] = await request('POST', '/estimate/' + slugPath, { params: {} }); + assert.ok('amount_credits' in estimate); + await assert.rejects(request('GET', '/runs/00000000-0000-0000-0000-000000000000', undefined, process.env.DEVA_API_KEY), /HTTP 404/); +}); From 033e01a5304b2fb71b41de4fc760b7e4890329ac Mon Sep 17 00:00:00 2001 From: epanonymous <93846049+epanonymous@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:29:38 -0400 Subject: [PATCH 3/3] fix: harden tool loop redirects and execution recovery --- docs/api-reference.md | 2 +- docs/tools.md | 6 +-- examples/python/tools_loop.py | 26 ++++++++--- examples/typescript/tools_loop.ts | 9 +++- tests/test_tools_loop.py | 74 +++++++++++++++++++++++++++++-- tests/tools_loop.test.ts | 27 ++++++++++- 6 files changed, 127 insertions(+), 17 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 88993de..b7c68ed 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -289,7 +289,7 @@ curl -sS "https://api.deva.me/v1/agents/resources/runs/$RUN_ID" \ -H "Authorization: Bearer $DEVA_API_KEY" ``` -Verified response contracts: +Expected response contracts (successful authenticated run/poll responses were not captured for this guide): - **HTTP 200, synchronous:** `status`, `charged`, `credits_charged`, `actual_cost_usd`, `run_id` (nullable), `output`, `message`. Read `output` directly; do not poll a completed synchronous result. - **HTTP 202, asynchronous:** `run_id`, `status: "running"`, `poll`, `max_total_charge_credits`, `next`. Poll the same run until `status` is `succeeded` or `failed`. diff --git a/docs/tools.md b/docs/tools.md index f402a82..aa53ec2 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -19,7 +19,7 @@ Discovery advertises `run_kind` (`SYNC` or `ASYNC`). Handle the actual run respo - HTTP **200** returns a synchronous result. Check `status`, read `output`, and record `credits_charged`. A nullable `run_id` is not an instruction to poll. - HTTP **202** returns `run_id` and `poll`. Wait between GETs (for example, two seconds) and stop at a terminal status. Terminal output is in `items`, with `result_count` and `credits_charged`. A successful HTTP response can still contain `status: "failed"`. -A poll timeout does not cancel execution. Keep the ID and resume polling; do not submit another run. `cap_hit: true` means the result was truncated at the spending cap. Poll options `fields`, `max_items`, and `summary` only reduce the response displayed; they do not reduce the bill. Save large results to a file instead of copying them into model context. +A poll timeout or transport error does not cancel execution. The examples include the run ID and resume path in poll errors. Keep the ID and resume polling; do not submit another run. `cap_hit: true` means output may be partial at the spending cap. The examples return the complete response, including this flag. Poll options `fields`, `max_items`, and `summary` only reduce the response displayed; they do not reduce the bill. Save large results to a file instead of copying them into model context. ## Errors and budget guards @@ -32,13 +32,13 @@ A poll timeout does not cancel execution. Keep the ID and resume polling; do not | 400 / 422 | Invalid parameters or cap | Follow the inspected schema and error details. | | 503 | Provider unavailable | Preserve any known run ID and check it later. Do not automatically repeat the run POST. | -**Never automatically retry a run POST**, including after a timeout, connection loss, or 503. The request may already have started an execution. The REST run route supports an `Idempotency-Key` header for intentional duplicate protection; keep the same key for the same intended request, but do not assume an ambiguous response means nothing ran. +**Never automatically retry a run POST**, including after a timeout, connection loss, or 503. The request may already have started an execution. Support for the REST run route’s `Idempotency-Key` header was not verified by an authenticated capture for this guide. Do not rely on it to make resubmission safe after an ambiguous response. > **Budget guard:** Keep both a per-task credit budget and a maximum run count. Before each POST, require a positive finite cap no larger than your remaining task budget and no smaller than `min_cap_usd`. Compare `amount_credits` to your cap, but do not mistake an estimate for enforcement. Send `maxTotalChargeUsd` on the run itself. Reserve the full cap locally before submitting, and keep that reservation after ambiguous failures until reconciled. One credit equals one USD. Never raise a budget automatically to satisfy a tool minimum. ## Runnable examples and tests -The examples use HTTP directly, accept an exact discovered slug plus schema-valid JSON, and submit at most one run. They fail closed on invalid caps, over-budget estimates, and unknown statuses. No run POST is retried. +The examples use HTTP directly, accept an exact discovered slug plus schema-valid JSON, and submit at most one run. They fail closed on invalid caps, over-budget estimates, and unknown statuses. No run POST is retried. A missing amount or a non-deterministic estimate is not a guaranteed price; the validated cap sent to the server remains the spending limit. ```bash export DEVA_API_KEY=deva_your_api_key diff --git a/examples/python/tools_loop.py b/examples/python/tools_loop.py index 92f6aa4..a23956c 100644 --- a/examples/python/tools_loop.py +++ b/examples/python/tools_loop.py @@ -5,20 +5,29 @@ import os import sys import time +from urllib.error import HTTPError from urllib.parse import quote, urlencode -from urllib.request import Request, urlopen +from urllib.request import HTTPRedirectHandler, Request, build_opener BASE = 'https://api.deva.me/v1/agents/resources' +class RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise HTTPError(req.full_url, code, 'Redirect refused', headers, fp) + + +_opener = build_opener(RejectRedirects()) + + def request(method, path, body=None, key=None): headers = {'Content-Type': 'application/json'} if key: headers['Authorization'] = f'Bearer {key}' req = Request(BASE + path, data=None if body is None else json.dumps(body).encode(), headers=headers, method=method) - # urlopen does not retry POSTs. HTTP errors propagate to the caller. - with urlopen(req, timeout=60) as response: + # Never follow redirects or retry POSTs; credentials stay on the original request. + with _opener.open(req, timeout=60) as response: return response.status, json.load(response) @@ -28,8 +37,8 @@ def run_loop(query, slug, params, budget, key, *, http=request, sleep=time.sleep raise ValueError('Set DEVA_API_KEY') if not math.isfinite(budget) or budget <= 0 or not isinstance(params, dict): raise ValueError('Require a positive finite budget and an object for params') - if max_polls < 1: - raise ValueError('max_polls must be positive') + if type(max_polls) is not int or max_polls < 1: + raise ValueError('max_polls must be a positive integer') _, page = http('GET', '/discover?' + urlencode({'q': query, 'limit': 100})) if not any(item['slug'] == slug for item in page['items']): raise ValueError('Slug not in this discovery page; refine your query') @@ -49,7 +58,7 @@ def run_loop(query, slug, params, budget, key, *, http=request, sleep=time.sleep code, result = http('POST', '/run/' + path_slug, body, key) if code == 200: if result.get('status') != 'succeeded': - raise RuntimeError('Synchronous run failed: ' + str(result.get('message'))) + raise RuntimeError('Synchronous run failed: ' + str(result.get('message')) + '; do not resubmit') return result if code != 202 or not result.get('run_id'): raise RuntimeError('Unexpected run response; do not resubmit') @@ -57,7 +66,10 @@ def run_loop(query, slug, params, budget, key, *, http=request, sleep=time.sleep on_run(run_id) for _ in range(max_polls): sleep(2) - _, result = http('GET', '/runs/' + quote(run_id, safe=''), key=key) + try: + _, result = http('GET', '/runs/' + quote(run_id, safe=''), key=key) + except Exception as error: + raise RuntimeError(f'Poll failed; resume GET /runs/{quote(run_id, safe="")}; do not resubmit') from error if result['status'] == 'succeeded': return result if result['status'] != 'running': diff --git a/examples/typescript/tools_loop.ts b/examples/typescript/tools_loop.ts index 4305077..a8c2acf 100644 --- a/examples/typescript/tools_loop.ts +++ b/examples/typescript/tools_loop.ts @@ -43,7 +43,7 @@ export async function runLoop(query: string, slug: string, params: Json, budget: // An ambiguous failure does not release the reservation or authorize another run. const [code, initial] = await http('POST', '/run/' + pathSlug, body, key); if (code === 200) { - if (initial.status !== 'succeeded') throw new Error(`Synchronous run failed: ${initial.message}`); + if (initial.status !== 'succeeded') throw new Error(`Synchronous run failed: ${initial.message}; do not resubmit`); return initial; } if (code !== 202 || !initial.run_id) throw new Error('Unexpected run response; do not resubmit'); @@ -51,7 +51,12 @@ export async function runLoop(query: string, slug: string, params: Json, budget: onRun(id); for (let i = 0; i < maxPolls; i++) { await sleep(2000); - const [, result] = await http('GET', '/runs/' + encodeURIComponent(id), undefined, key); + let result: Json; + try { + [, result] = await http('GET', '/runs/' + encodeURIComponent(id), undefined, key); + } catch (cause) { + throw new Error(`Poll failed; resume GET /runs/${encodeURIComponent(id)}; do not resubmit`, { cause }); + } if (result.status === 'succeeded') return result; if (result.status !== 'running') throw new Error(`Run ${id}: ${result.status}; do not resubmit`); } diff --git a/tests/test_tools_loop.py b/tests/test_tools_loop.py index 9c4334b..39de2c6 100644 --- a/tests/test_tools_loop.py +++ b/tests/test_tools_loop.py @@ -2,8 +2,11 @@ import io import json import os -from pathlib import Path import unittest +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread from unittest.mock import patch from urllib.error import HTTPError @@ -19,19 +22,48 @@ def __init__(self, status, body): self.status = status +@contextmanager +def redirect_server(code): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + calls.append((self.command, self.path, self.headers.get('Authorization'))) + self.send_response(code if self.path != '/destination' else 200) + self.send_header('Location', f'http://localhost:{self.server.server_port}/destination') + self.end_headers() + self.wfile.write(b'{}') + + do_POST = do_GET + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = Thread(target=lambda: server.serve_forever(poll_interval=0.01)) + thread.start() + try: + yield f'http://127.0.0.1:{server.server_port}', calls + finally: + server.shutdown() + server.server_close() + thread.join() + + class LoopTests(unittest.TestCase): def execute(self, tail, *, estimate=None, budget=0.01, max_polls=3): replies = [Response(200, {'items': [{'slug': SLUG}]}), Response(200, {'input_schema': {}, 'min_cap_usd': None}), Response(200, estimate or {'amount_credits': 0.001})] + tail - with patch.object(loop, 'urlopen', side_effect=replies) as http: + with patch.object(loop._opener, 'open', side_effect=replies) as http: self.http = http return loop.run_loop('example', SLUG, {}, budget, 'test-key', sleep=lambda _: None, on_run=lambda _: None, max_polls=max_polls) def test_sync_and_cap_and_auth(self): - result = self.execute([Response(200, {'status': 'succeeded', 'output': {'ok': True}})]) + result = self.execute([Response(200, {'status': 'succeeded', 'output': {'ok': True}, 'cap_hit': True})]) self.assertTrue(result['output']['ok']) + self.assertTrue(result['cap_hit']) requests = [call.args[0] for call in self.http.call_args_list] self.assertEqual(len(requests), 4) self.assertFalse(requests[0].has_header('Authorization')) @@ -71,6 +103,42 @@ def test_terminal_failure_and_timeout(self): with self.assertRaisesRegex(TimeoutError, 'resume GET /runs/run-1'): self.execute([Response(202, {'run_id': 'run-1'}), Response(200, {'status': 'running'})], max_polls=1) + def test_rejects_redirects_without_forwarding_credentials(self): + for code in [301, 302, 303, 307, 308]: + for method, path, body in [('POST', '/run/example', {}), ('GET', '/runs/r1', None)]: + with self.subTest(code=code, method=method), redirect_server(code) as (base, calls): + with patch.object(loop, 'BASE', base), self.assertRaises(HTTPError) as error: + loop.request(method, path, body, 'test-key') + self.assertEqual(error.exception.code, code) + error.exception.close() + self.assertEqual(calls, [(method, path, 'Bearer test-key')]) + + def test_invalid_poll_limits_do_not_call_http(self): + for limit in [1.5, 1.0, True, False, '2', None, 0, -1, float('nan'), float('inf')]: + with self.subTest(limit=limit), self.assertRaisesRegex(ValueError, 'positive integer'): + self.execute([], max_polls=limit) + self.assertEqual(self.http.call_count, 0) + + def test_sync_non_success_never_polls_or_resubmits(self): + for status in ['failed', 'running', 'unknown', None]: + with self.subTest(status=status), self.assertRaisesRegex(RuntimeError, 'do not resubmit'): + self.execute([Response(200, {'status': status})]) + self.assertEqual(self.http.call_count, 4) + + def test_unexpected_success_response_never_polls_or_resubmits(self): + for code, body in [(201, {'status': 'succeeded'}), (206, {'run_id': 'r1'}), + (202, {}), (202, {'run_id': None}), (202, {'run_id': ''})]: + with self.subTest(code=code, body=body), self.assertRaisesRegex(RuntimeError, 'do not resubmit'): + self.execute([Response(code, body)]) + self.assertEqual(self.http.call_count, 4) + + def test_poll_errors_preserve_resume_guidance_and_cause(self): + for error in [TimeoutError('timeout'), HTTPError('url', 503, 'unavailable', {}, None)]: + with self.subTest(error=error), self.assertRaisesRegex(RuntimeError, 'resume GET /runs/run-1; do not resubmit') as raised: + self.execute([Response(202, {'run_id': 'run-1'}), error]) + self.assertIs(raised.exception.__cause__, error) + self.assertEqual(self.http.call_count, 5) + @unittest.skipUnless(os.getenv('DEVA_API_KEY'), 'DEVA_API_KEY not set; no paid execution') def test_live_smoke(self): _, page = loop.request('GET', '/discover?q=market%20intel%20leaderboard&limit=1') diff --git a/tests/tools_loop.test.ts b/tests/tools_loop.test.ts index ae544fb..521f88f 100644 --- a/tests/tools_loop.test.ts +++ b/tests/tools_loop.test.ts @@ -21,9 +21,10 @@ async function execute(t: any, tail: any[], estimate: any = { amount_credits: 0. } test('sync output, cap, slug path and auth', async t => { - const { result, error, calls } = await execute(t, [[200, { status: 'succeeded', output: { ok: true } }]]); + const { result, error, calls } = await execute(t, [[200, { status: 'succeeded', output: { ok: true }, cap_hit: true }]]); assert.ifError(error); assert.equal(result?.output.ok, true); + assert.equal(result?.cap_hit, true); assert.equal(calls.length, 4); assert.equal(calls[0].headers.Authorization, undefined); assert.equal(calls[3].headers.Authorization, 'Bearer test-key'); @@ -76,3 +77,27 @@ test('live smoke without paid execution', { skip: !process.env.DEVA_API_KEY }, a assert.ok('amount_credits' in estimate); await assert.rejects(request('GET', '/runs/00000000-0000-0000-0000-000000000000', undefined, process.env.DEVA_API_KEY), /HTTP 404/); }); + +for (const status of ['failed', 'running', 'unknown', undefined]) { + test(`sync non-success ${status} never polls or resubmits`, async t => { + const { error, calls } = await execute(t, [[200, { status }]]); + assert.match(String(error), /do not resubmit/); + assert.equal(calls.length, 4); + }); +} +for (const response of [[201, { status: 'succeeded' }], [206, { run_id: 'r1' }], + [202, {}], [202, { run_id: null }], [202, { run_id: '' }]]) { + test(`unexpected success ${JSON.stringify(response)} never polls or resubmits`, async t => { + const { error, calls } = await execute(t, [response]); + assert.match(String(error), /do not resubmit/); + assert.equal(calls.length, 4); + }); +} +for (const failure of [new Error('timeout'), [503, { error: 'unavailable' }]]) { + test(`poll error preserves resume guidance ${JSON.stringify(failure)}`, async t => { + const { error, calls } = await execute(t, [[202, { run_id: 'r1' }], failure]); + assert.match(String(error), /resume GET \/runs\/r1; do not resubmit/); + assert.ok(error instanceof Error && error.cause instanceof Error); + assert.equal(calls.length, 5); + }); +}