diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 2b15065..81079d0 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -874,6 +874,16 @@ Embeddings move only between the routes of the resolved card, because vectors of Per-call options, never sent to the driver: +`generate` accepts `prepare_request(context)` to shape a fresh request for each +attempted model or route. The context contains `model` (the resolved card, or +`nil` for a direct provider call), `provider_id`, and `provider_model`. Return +`{ messages = prompt_input, max_tokens = optional_limit }`, or `nil, err`. +Messages use the same input formats as `generate`; build fresh messages and do +not mutate the supplied model card. Preparation runs before output limits and +context admission, so each candidate's limits still apply. Errors use the +existing preflight failure behavior. Without this callback, request construction +is unchanged. The callback runs locally and is never sent to a driver. + ```lua llm.generate(builder, { model = "claude", diff --git a/src/llm/src/llm.lua b/src/llm/src/llm.lua index f74b041..82ba097 100644 --- a/src/llm/src/llm.lua +++ b/src/llm/src/llm.lua @@ -157,7 +157,8 @@ type GenerateOptions = { fallback: any?, route: table?, deadline_ms: number?, - cache_ttl: string? + cache_ttl: string?, + prepare_request: ((table) -> (table?, error?))? } type EmbedOptions = { @@ -203,7 +204,7 @@ local llm = {} -- Contract constants local USAGE_TRACKER_CONTRACT = "wippy.llm:usage_tracker" local MODEL_RESOLVER_CONTRACT = "wippy.llm:model_resolver" -local CALL_OPTIONS = fallback.set_of({ "fallback", "route", "deadline_ms", "checkpoint" }) +local CALL_OPTIONS = fallback.set_of({ "fallback", "route", "deadline_ms", "checkpoint", "prepare_request" }) -- Dependency injection fields llm._models = nil @@ -1362,6 +1363,9 @@ local function call(spec, options) end function llm.generate(prompt_input, options) + if options and options.prepare_request ~= nil and type(options.prepare_request) ~= "function" then + return nil, errors.new({ kind = errors.INVALID, retryable = false, message = "prepare_request must be a function" }) + end if not options or type(options.model) ~= "string" or options.model == "" then return nil, errors.new({ kind = errors.INVALID, retryable = false, message = "Model is required in options" }) end @@ -1393,6 +1397,27 @@ function llm.generate(prompt_input, options) return { messages = request.messages, model = wire_model, options = {} } end, prepare = function(contract_args, provider_info, call_options, providers_module, card) + if call_options.prepare_request then + local prepared, err = call_options.prepare_request({ + model = card, provider_id = provider_info.id, provider_model = contract_args.model, + }) + if err then return nil, native_error(err, errors.INVALID, false) end + if type(prepared) ~= "table" then + return nil, errors.new({ kind = errors.INVALID, retryable = false, + message = "prepare_request must return a request object" }) + end + local messages, message_err = prepare_messages(prepared.messages) + if not messages then return nil, message_err end + contract_args.messages = messages + if prepared.max_tokens ~= nil then + if type(prepared.max_tokens) ~= "number" or prepared.max_tokens < 1 + or prepared.max_tokens == math.huge or prepared.max_tokens ~= math.floor(prepared.max_tokens) then + return nil, errors.new({ kind = errors.INVALID, retryable = false, + message = "prepare_request max_tokens must be a positive integer" }) + end + contract_args.options.max_tokens = prepared.max_tokens + end + end local adjusted, route_err = prepare_route(contract_args, provider_info, call_options, providers_module, card) if not adjusted then return nil, route_err diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index 9e58ae3..7ce706c 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -1604,3 +1604,12 @@ entries: fallback: wippy.llm.util:fallback test: wippy.test:test method: run_tests + + - name: request_preparation_test + kind: function.lua + meta: {type: test, suite: request_preparation} + source: file://request_preparation_test.lua + method: run + imports: + test: wippy.test:test + llm: wippy.llm:llm diff --git a/src/llm/test/request_preparation_test.lua b/src/llm/test/request_preparation_test.lua new file mode 100644 index 0000000..2694ddc --- /dev/null +++ b/src/llm/test/request_preparation_test.lua @@ -0,0 +1,126 @@ +local test = require("test") +local llm = require("llm") + +return test.run_cases(function() + describe("Per-candidate generation request preparation", function() + local saved, cards, sent + before_each(function() + saved = {llm._models, llm._providers, llm._usage_tracker, llm._model_resolver} + cards = { + first = {name = "first", output_tokens = 128, max_tokens = 2048, + capabilities = {"generate"}, providers = {{id = "provider", provider_model = "first-wire"}}, + fallback = {"second"}}, + second = {name = "second", output_tokens = 64, max_tokens = 2048, + capabilities = {"generate"}, providers = {{id = "provider", provider_model = "second-wire"}}}, + } + sent = {} + llm._model_resolver = false + llm._usage_tracker = false + llm._models = { + get_by_name = function(name) return cards[name] end, + get_by_class = function() return {cards.first, cards.second} end, + } + llm._providers = { + driver_declares_legacy_reasoning_flag = function() return false end, + driver_output_policy = function() return nil end, + open = function() return {generate = function(_, args) + sent[#sent + 1] = args + if args.model == "first-wire" then + return nil, errors.new({kind = errors.INTERNAL, retryable = true, + message = "unavailable", details = {error_type = "server_error"}}) + end + return {result = "done", tokens = {}}, nil + end} end, + } + end) + after_each(function() + llm._models, llm._providers, llm._usage_tracker, llm._model_resolver = + saved[1], saved[2], saved[3], saved[4] + end) + + for _, selector in ipairs({"first", "class:vision"}) do + it("prepares and caps each attempted model for " .. selector, function() + local prepared = {} + local options = {model = selector, max_tokens = 16, + prepare_request = function(context) + prepared[#prepared + 1] = context.model.name + test.eq(context.provider_id, "provider") + return {messages = {{role = "user", content = {{type = "text", text = context.model.name}}}}, + max_tokens = 512} + end} + local result, err = llm.generate("original", options) + test.is_nil(err, tostring(err)) + test.eq(result.result, "done") + test.eq(#prepared, 2) + test.eq(#sent, 2) + for i, name in ipairs({"first", "second"}) do + test.eq(prepared[i], name) + test.eq(sent[i].messages[1].content[1].text, name) + test.eq(sent[i].options.max_tokens, cards[name].output_tokens) + test.is_nil(sent[i].options.prepare_request) + end + test.eq(options.max_tokens, 16) + end) + end + + it("keeps the existing request when no preparation is supplied", function() + local result, err = llm.generate("original", {model = "first", max_tokens = 32}) + test.is_nil(err, tostring(err)) + test.eq(result.result, "done") + test.eq(#sent, 2) + for _, args in ipairs(sent) do + test.eq(args.messages[1].content[1].text, "original") + test.eq(args.options.max_tokens, 32) + end + end) + + it("preserves a native preparation error before a provider request", function() + local cause = errors.new({kind = errors.INVALID, retryable = false, message = "invalid profile"}) + local result, err = llm.generate("original", {model = "first", + prepare_request = function() return nil, cause end}) + test.is_nil(result) + test.eq(err, cause) + test.eq(#sent, 0) + end) + + it("admits the prepared messages against the candidate context window", function() + local result, err = llm.generate("original", {model = "first", fallback = false, + prepare_request = function() + return {messages = {{role = "user", content = {{type = "text", text = string.rep("x", 20000)}}}}, + max_tokens = 16} + end}) + test.is_nil(result) + test.eq(err:details().error_type, "context_window") + test.eq(#sent, 0) + end) + + it("rejects invalid preparation results before calling a provider", function() + for _, prepare in ipairs({ + false, + function() return nil end, + function() return {} end, + function() return {messages = "question", max_tokens = 0} end, + function() return {messages = "question", max_tokens = math.huge} end, + function() return {messages = "question", max_tokens = 1.5} end, + }) do + local result, err = llm.generate("original", {model = "first", prepare_request = prepare}) + test.is_nil(result) + test.eq(err:kind(), errors.INVALID) + test.is_false(err:retryable()) + test.eq(#sent, 0) + end + end) + + it("supplies no model card for a direct provider call", function() + local result, err = llm.generate("original", {model = "second-wire", provider_id = "provider", + prepare_request = function(context) + test.is_nil(context.model) + test.eq(context.provider_model, "second-wire") + return {messages = {{role = "user", content = {{type = "text", text = "direct"}}}}} + end}) + test.is_nil(err, tostring(err)) + test.eq(result.result, "done") + test.eq(sent[1].messages[1].content[1].text, "direct") + end) + end) +end)