Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/llm/src/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,16 @@ Embeddings move only between the routes of the resolved card, because vectors of

Per-call options, never sent to the driver:

`generate` accepts `prepare_request(context)` to shape a fresh request for each
attempted model or route. The context contains `model` (the resolved card, or
`nil` for a direct provider call), `provider_id`, and `provider_model`. Return
`{ messages = prompt_input, max_tokens = optional_limit }`, or `nil, err`.
Messages use the same input formats as `generate`; build fresh messages and do
not mutate the supplied model card. Preparation runs before output limits and
context admission, so each candidate's limits still apply. Errors use the
existing preflight failure behavior. Without this callback, request construction
is unchanged. The callback runs locally and is never sent to a driver.

```lua
llm.generate(builder, {
model = "claude",
Expand Down
29 changes: 27 additions & 2 deletions src/llm/src/llm.lua
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,8 @@ type GenerateOptions = {
fallback: any?,
route: table?,
deadline_ms: number?,
cache_ttl: string?
cache_ttl: string?,
prepare_request: ((table) -> (table?, error?))?
}

type EmbedOptions = {
Expand Down Expand Up @@ -203,7 +204,7 @@ local llm = {}
-- Contract constants
local USAGE_TRACKER_CONTRACT = "wippy.llm:usage_tracker"
local MODEL_RESOLVER_CONTRACT = "wippy.llm:model_resolver"
local CALL_OPTIONS = fallback.set_of({ "fallback", "route", "deadline_ms", "checkpoint" })
local CALL_OPTIONS = fallback.set_of({ "fallback", "route", "deadline_ms", "checkpoint", "prepare_request" })

-- Dependency injection fields
llm._models = nil
Expand Down Expand Up @@ -1362,6 +1363,9 @@ local function call(spec, options)
end

function llm.generate(prompt_input, options)
if options and options.prepare_request ~= nil and type(options.prepare_request) ~= "function" then
return nil, errors.new({ kind = errors.INVALID, retryable = false, message = "prepare_request must be a function" })
end
if not options or type(options.model) ~= "string" or options.model == "" then
return nil, errors.new({ kind = errors.INVALID, retryable = false, message = "Model is required in options" })
end
Expand Down Expand Up @@ -1393,6 +1397,27 @@ function llm.generate(prompt_input, options)
return { messages = request.messages, model = wire_model, options = {} }
end,
prepare = function(contract_args, provider_info, call_options, providers_module, card)
if call_options.prepare_request then
local prepared, err = call_options.prepare_request({
model = card, provider_id = provider_info.id, provider_model = contract_args.model,
})
if err then return nil, native_error(err, errors.INVALID, false) end
if type(prepared) ~= "table" then
return nil, errors.new({ kind = errors.INVALID, retryable = false,
message = "prepare_request must return a request object" })
end
local messages, message_err = prepare_messages(prepared.messages)
if not messages then return nil, message_err end
contract_args.messages = messages
if prepared.max_tokens ~= nil then
if type(prepared.max_tokens) ~= "number" or prepared.max_tokens < 1
or prepared.max_tokens == math.huge or prepared.max_tokens ~= math.floor(prepared.max_tokens) then
return nil, errors.new({ kind = errors.INVALID, retryable = false,
message = "prepare_request max_tokens must be a positive integer" })
end
contract_args.options.max_tokens = prepared.max_tokens
end
end
local adjusted, route_err = prepare_route(contract_args, provider_info, call_options, providers_module, card)
if not adjusted then
return nil, route_err
Expand Down
9 changes: 9 additions & 0 deletions src/llm/test/_index.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1604,3 +1604,12 @@ entries:
fallback: wippy.llm.util:fallback
test: wippy.test:test
method: run_tests

- name: request_preparation_test
kind: function.lua
meta: {type: test, suite: request_preparation}
source: file://request_preparation_test.lua
method: run
imports:
test: wippy.test:test
llm: wippy.llm:llm
126 changes: 126 additions & 0 deletions src/llm/test/request_preparation_test.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
local test = require("test")
local llm = require("llm")

return test.run_cases(function()
describe("Per-candidate generation request preparation", function()
local saved, cards, sent
before_each(function()
saved = {llm._models, llm._providers, llm._usage_tracker, llm._model_resolver}
cards = {
first = {name = "first", output_tokens = 128, max_tokens = 2048,
capabilities = {"generate"}, providers = {{id = "provider", provider_model = "first-wire"}},
fallback = {"second"}},
second = {name = "second", output_tokens = 64, max_tokens = 2048,
capabilities = {"generate"}, providers = {{id = "provider", provider_model = "second-wire"}}},
}
sent = {}
llm._model_resolver = false
llm._usage_tracker = false
llm._models = {
get_by_name = function(name) return cards[name] end,
get_by_class = function() return {cards.first, cards.second} end,
}
llm._providers = {
driver_declares_legacy_reasoning_flag = function() return false end,
driver_output_policy = function() return nil end,
open = function() return {generate = function(_, args)
sent[#sent + 1] = args
if args.model == "first-wire" then
return nil, errors.new({kind = errors.INTERNAL, retryable = true,
message = "unavailable", details = {error_type = "server_error"}})
end
return {result = "done", tokens = {}}, nil
end} end,
}
end)
after_each(function()
llm._models, llm._providers, llm._usage_tracker, llm._model_resolver =
saved[1], saved[2], saved[3], saved[4]
end)

for _, selector in ipairs({"first", "class:vision"}) do
it("prepares and caps each attempted model for " .. selector, function()
local prepared = {}
local options = {model = selector, max_tokens = 16,
prepare_request = function(context)
prepared[#prepared + 1] = context.model.name
test.eq(context.provider_id, "provider")
return {messages = {{role = "user", content = {{type = "text", text = context.model.name}}}},
max_tokens = 512}
end}
local result, err = llm.generate("original", options)
test.is_nil(err, tostring(err))
test.eq(result.result, "done")
test.eq(#prepared, 2)
test.eq(#sent, 2)
for i, name in ipairs({"first", "second"}) do
test.eq(prepared[i], name)
test.eq(sent[i].messages[1].content[1].text, name)
test.eq(sent[i].options.max_tokens, cards[name].output_tokens)
test.is_nil(sent[i].options.prepare_request)
end
test.eq(options.max_tokens, 16)
end)
end

it("keeps the existing request when no preparation is supplied", function()
local result, err = llm.generate("original", {model = "first", max_tokens = 32})
test.is_nil(err, tostring(err))
test.eq(result.result, "done")
test.eq(#sent, 2)
for _, args in ipairs(sent) do
test.eq(args.messages[1].content[1].text, "original")
test.eq(args.options.max_tokens, 32)
end
end)

it("preserves a native preparation error before a provider request", function()
local cause = errors.new({kind = errors.INVALID, retryable = false, message = "invalid profile"})
local result, err = llm.generate("original", {model = "first",
prepare_request = function() return nil, cause end})
test.is_nil(result)
test.eq(err, cause)
test.eq(#sent, 0)
end)

it("admits the prepared messages against the candidate context window", function()
local result, err = llm.generate("original", {model = "first", fallback = false,
prepare_request = function()
return {messages = {{role = "user", content = {{type = "text", text = string.rep("x", 20000)}}}},
max_tokens = 16}
end})
test.is_nil(result)
test.eq(err:details().error_type, "context_window")
test.eq(#sent, 0)
end)

it("rejects invalid preparation results before calling a provider", function()
for _, prepare in ipairs({
false,
function() return nil end,
function() return {} end,
function() return {messages = "question", max_tokens = 0} end,
function() return {messages = "question", max_tokens = math.huge} end,
function() return {messages = "question", max_tokens = 1.5} end,
}) do
local result, err = llm.generate("original", {model = "first", prepare_request = prepare})
test.is_nil(result)
test.eq(err:kind(), errors.INVALID)
test.is_false(err:retryable())
test.eq(#sent, 0)
end
end)

it("supplies no model card for a direct provider call", function()
local result, err = llm.generate("original", {model = "second-wire", provider_id = "provider",
prepare_request = function(context)
test.is_nil(context.model)
test.eq(context.provider_model, "second-wire")
return {messages = {{role = "user", content = {{type = "text", text = "direct"}}}}}
end})
test.is_nil(err, tostring(err))
test.eq(result.result, "done")
test.eq(sent[1].messages[1].content[1].text, "direct")
end)
end)
end)
Loading