Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions src/llm/src/bedrock/client.lua
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ local thinking = require("thinking")
local sigv4 = require("sigv4")
local credentials = require("bedrock_credentials")

local stream_redactors = setmetatable({}, { __mode = "k" })

local bedrock_client = {}

bedrock_client._http_client = http_client
Expand Down Expand Up @@ -108,7 +110,7 @@ local function parse_error_response(http_response: transport.HttpResponse): tran
end

-- Make a signed POST request to a bedrock-runtime path
local function signed_request(path, payload, options)
local function signed_request(path, payload, options): (any, transport.RequestError?)
options = options or {}
local config = resolve_config()
local timeout, timeout_err = transport.resolve_timeout(options.timeout, "options.timeout", config.timeout, config.timeout_error)
Expand Down Expand Up @@ -181,25 +183,29 @@ local function signed_request(path, payload, options)
local retry = transport.request_retry(options.retry, config.retry)
local response, request_error = transport.send(send_once, parse_error_response, retry, config.deadline_at)
if not response then
return nil, request_error
return nil, transport.redact_error(request_error, transport.request_secrets(signed_headers, config.headers))
end

if options.stream and response.stream then
return {
local stream_response = {
stream = response.stream,
status_code = response.status_code,
headers = response.headers,
metadata = extract_response_metadata(response)
}
stream_redactors[stream_response] = function(value)
return transport.redact_error(value, transport.request_secrets(signed_headers, config.headers))
end
return stream_response
end

local parsed, parse_err = json.decode(tostring(response.body))
if parse_err then
return nil, {
return nil, transport.redact_error({
status_code = response.status_code,
message = "Failed to parse Bedrock response: " .. parse_err,
metadata = extract_response_metadata(response)
}
}, transport.request_secrets(signed_headers, config.headers))
end

parsed.metadata = extract_response_metadata(response)
Expand Down Expand Up @@ -405,6 +411,7 @@ end

-- Process ConverseStream eventstream, dispatching to callbacks as each event arrives
function bedrock_client.process_converse_stream(stream_response, callbacks)
local redact_error = stream_redactors[stream_response]
if not stream_response or not stream_response.stream then
return nil, "Invalid stream response"
end
Expand All @@ -431,6 +438,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks)
-- Converse stream events use top-level keys instead of a "type" field
if event._exception then
local exc = event._exception
if redact_error then exc = redact_error(exc) end
error_reported = true
on_error({ message = exc.message, type = exc.type })
return tostring(exc.message)
Expand Down Expand Up @@ -534,6 +542,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks)

local stream_err = read_eventstream(stream_response.stream, handle_event)
if stream_err then
if redact_error then stream_err = redact_error(stream_err) end
if not error_reported then
on_error({ message = stream_err })
end
Expand Down
18 changes: 13 additions & 5 deletions src/llm/src/claude/client.lua
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ local ctx = require("ctx")
local transport = require("transport")
local thinking = require("thinking")

local stream_redactors = setmetatable({}, { __mode = "k" })

type ClaudeConfig = {
api_key: string?,
base_url: string,
Expand Down Expand Up @@ -157,7 +159,6 @@ local function prepare_headers(api_key, api_version, beta_features, method, addi
return headers
end


function claude_client.request(endpoint_path, payload, options): (any, transport.RequestError?)
options = options or {}
local method = options.method or "POST"
Expand Down Expand Up @@ -207,25 +208,29 @@ function claude_client.request(endpoint_path, payload, options): (any, transport
local retry = transport.request_retry(options.retry, config.retry)
local response, request_error = transport.send(send_once, parse_error_response, retry, config.deadline_at)
if not response then
return nil, request_error
return nil, transport.redact_error(request_error, transport.request_secrets(headers, config.headers))
end

if options.stream and response.stream then
return {
local stream_response = {
stream = response.stream,
status_code = response.status_code,
headers = response.headers,
metadata = extract_response_metadata(response)
}
stream_redactors[stream_response] = function(value)
return transport.redact_error(value, transport.request_secrets(headers, config.headers))
end
return stream_response
end

local parsed, parse_err = json.decode(response.body or "")
if parse_err then
return nil, {
return nil, transport.redact_error({
status_code = response.status_code,
message = "Failed to parse Claude response: " .. parse_err,
metadata = extract_response_metadata(response)
}
}, transport.request_secrets(headers, config.headers))
end

parsed.metadata = extract_response_metadata(response)
Expand Down Expand Up @@ -253,6 +258,7 @@ local function stream_result(full_content, tool_calls, thinking_blocks, layout,
end

function claude_client.process_stream(stream_response, callbacks)
local redact_error = stream_redactors[stream_response]
if not stream_response or not stream_response.stream then
return nil, "Invalid stream response"
end
Expand All @@ -278,6 +284,7 @@ function claude_client.process_stream(stream_response, callbacks)
local chunk, err = stream_response.stream:read()

if err then
if redact_error then err = redact_error(err) end
on_error({ message = err })
return nil, err
end
Expand Down Expand Up @@ -419,6 +426,7 @@ function claude_client.process_stream(stream_response, callbacks)
return full_content, nil, result
elseif event_type == "error" then
if data and data.error then
if redact_error then data.error = redact_error(data.error) end
on_error(data.error)
return nil, data.error.message
end
Expand Down
39 changes: 29 additions & 10 deletions src/llm/src/google/client.lua
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ local output = require("output")
local transport = require("transport")
local mapper = require("google_mapper")

local stream_redactors = setmetatable({}, { __mode = "k" })

local function redact_request_error(value, http_options, method)
local configured_headers = {}
for name in pairs(http_options.headers) do
if name ~= "Accept" and (name ~= "Content-Type" or method == "GET") then
configured_headers[name] = true
end
end
return transport.redact_error(value, transport.request_secrets(http_options.headers, configured_headers))
end

type StreamInput = {
stream: any,
metadata: table?,
Expand Down Expand Up @@ -71,6 +83,7 @@ local function parse_error_response(http_response: transport.HttpResponse): tran
end

function client.process_stream(stream_response: StreamInput, callbacks: StreamCallbacks?): (string?, string?, StreamResult?)
local redact_error = stream_redactors[stream_response]
if not stream_response or not stream_response.stream then
return nil, "Invalid stream response"
end
Expand Down Expand Up @@ -101,7 +114,8 @@ function client.process_stream(stream_response: StreamInput, callbacks: StreamCa
if data_line == "[DONE]" then done = true; return nil end
local parsed, parse_err = json.decode(data_line)
if parse_err or type(parsed) ~= "table" then
return "Invalid Google stream event: " .. tostring(parse_err or "expected object")
local message = "Invalid Google stream event: " .. tostring(parse_err or "expected object")
return redact_error and redact_error(message) or message
end

if parsed.error then
Expand All @@ -110,6 +124,7 @@ function client.process_stream(stream_response: StreamInput, callbacks: StreamCa
code = parsed.error.code,
status = parsed.error.status
}
if redact_error then stream_error_info = redact_error(stream_error_info) end
return tostring(stream_error_info.message)
end

Expand Down Expand Up @@ -158,7 +173,10 @@ function client.process_stream(stream_response: StreamInput, callbacks: StreamCa

while not done do
local chunk, err = stream_response.stream:read()
if err then on_error({ message = err }); return nil, err end
if err then
if redact_error then err = redact_error(err) end
on_error({ message = err }); return nil, err
end
if not chunk then
-- Some gateways omit the final blank line. Do not discard a final event,
-- but reject a torn/malformed JSON payload rather than silently losing it.
Expand Down Expand Up @@ -201,7 +219,7 @@ end

--- Process a streaming response and send chunks via output.streamer.
--- Returns an aggregated Google-like response compatible with map_success_response().
local function handle_stream_response(response, http_options)
local function handle_stream_response(response, http_options, method)
local reply_to = http_options.stream_reply_to :: string?
local topic = http_options.stream_topic :: string?
local streamer = output.streamer(reply_to, topic, http_options.stream_buffer_size or 10)
Expand Down Expand Up @@ -252,10 +270,11 @@ local function handle_stream_response(response, http_options)
end,
}

local _, stream_err = client.process_stream(
{ stream = response.stream, metadata = {} },
callbacks :: StreamCallbacks
)
local stream_response = { stream = response.stream, metadata = {} }
stream_redactors[stream_response] = function(value)
return redact_request_error(value, http_options, method)
end
local _, stream_err = client.process_stream(stream_response, callbacks :: StreamCallbacks)

if stream_err then
return nil, {
Expand Down Expand Up @@ -314,12 +333,12 @@ function client.request(method, url, http_options, retry: transport.Retry?, dead
local failure: any = request_error
failure.stream_started = false
end
return nil, request_error
return nil, redact_request_error(request_error, http_options, method)
end

-- Streaming: process stream, send chunks via streamer, return aggregated response
if http_options.stream and response.stream then
return handle_stream_response(response, http_options)
return handle_stream_response(response, http_options, method)
end

local parsed, parse_err = json.decode(response.body or "")
Expand All @@ -329,7 +348,7 @@ function client.request(method, url, http_options, retry: transport.Retry?, dead
message = "Failed to parse Google response: " .. parse_err,
metadata = {}
}
return nil, parse_error
return nil, redact_request_error(parse_error, http_options, method)
end

parsed.metadata = extract_response_metadata(parsed)
Expand Down
21 changes: 15 additions & 6 deletions src/llm/src/openai/client.lua
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ local env = require("env")
local ctx = require("ctx")
local transport = require("transport")

local stream_redactors = setmetatable({}, { __mode = "k" })

type OpenAIConfig = {
api_key: string?,
base_url: string,
Expand Down Expand Up @@ -172,33 +174,37 @@ function openai_client.request(endpoint_path, payload, options)
local retry = transport.request_retry(options.retry, config.retry)
local response, request_error = transport.send(send_once, parse_error_response, retry, config.deadline_at)
if not response then
return nil, request_error
return nil, transport.redact_error(request_error, transport.request_secrets(headers, config.headers))
end

if options.stream and response.stream then
return {
local stream_response = {
stream = response.stream,
status_code = response.status_code,
headers = response.headers,
metadata = extract_response_metadata(response)
}
stream_redactors[stream_response] = function(value)
return transport.redact_error(value, transport.request_secrets(headers, config.headers))
end
return stream_response
end

local parsed, parse_err = json.decode(response.body or "")
if parse_err then
return nil, {
return nil, transport.redact_error({
status_code = response.status_code,
message = "Failed to parse OpenAI response: " .. parse_err,
metadata = extract_response_metadata(response)
}
}, transport.request_secrets(headers, config.headers))
end

if type(parsed) ~= "table" then
return nil, {
return nil, transport.redact_error({
status_code = response.status_code,
message = "Invalid OpenAI response: expected a JSON object",
metadata = extract_response_metadata(response)
}
}, transport.request_secrets(headers, config.headers))
end

parsed.metadata = extract_response_metadata(response)
Expand All @@ -210,6 +216,7 @@ end
-- names the event (e.g. response.output_text.delta). Multiple events are
-- separated by a blank line (\n\n).
function openai_client.process_stream(stream_response, callbacks): (string?, any, any)
local redact_error = stream_redactors[stream_response]
if not stream_response or not stream_response.stream then
return nil, "Invalid stream response"
end
Expand Down Expand Up @@ -314,6 +321,7 @@ function openai_client.process_stream(stream_response, callbacks): (string?, any
local chunk, err = stream_response.stream:read()

if err then
if redact_error then err = redact_error(err) end
on_error(err)
return nil, err
end
Expand Down Expand Up @@ -465,6 +473,7 @@ function openai_client.process_stream(stream_response, callbacks): (string?, any
type = err_payload and err_payload.type,
param = err_payload and err_payload.param
}
if redact_error then error_info = redact_error(error_info) end
on_error(error_info)
close_stream()
return nil, error_info.message, { error = error_info }
Expand Down
Loading
Loading