package router import ( "errors" "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" ) // budgetFallback finds a model this caller can still afford. // // A spent budget was a dead end: the check refused the request and that was // the end of it. The accounting layer has always exempted a model that costs // nothing, with a comment saying it is "what lets a key with no budget left // still be served a cheap fallback" — describing a caller that was never // written, so the exemption could not fire. // // It matters most for the audience this is built for. A team trying models // will exhaust a quota, and being dropped to a smaller model is a far better // answer than a refusal: it makes a budget a guardrail rather than a wall. // // Silent, with the served model in a header, which is what capacity fallback // already does. Two rules for the same situation would be worse than one // imperfect rule, and a client that cares can read the header. func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accounting.Reservation, bool) { _, entry := r.resolve(req.Model) for _, name := range entry.Fallbacks { // The allowlist still applies. A key that may not use a model does not // gain access to it by running out of money, which would make a budget // a way around an allowlist rather than a limit on top of one. if !r.permits(p, name) { continue } // And it has to be able to hold the prompt. The fit was checked against // the model the caller asked for, before anyone knew a different one // would serve — so without this a 20k prompt sent to a 32k model lands // on a 2k one, and the engine truncates it or refuses. Either way the // caller sees a failure naming a model they never chose. // // The history is not trimmed to make it fit. Dropping a caller's // messages to stay under a limit they did not ask about is a worse // answer than saying the prompt is too large, and it would be done // silently, to a request that was already being quietly moved. canonical, cand := r.resolve(name) if why := checkContextFits(cand, req.contextTokens(), req.MaxTokens); why != "" { continue } res, err := r.acct.Check(p, canonical, req.estimatedTokens()) if err != nil { continue } req.Model = name return res, true } return accounting.Reservation{}, false } // isBudgetSpent reports whether a limit error is the one a fallback can answer. // // Only a spent budget. A rate limit is answered by waiting a second, and moving // the request to another model would spend someone else's capacity to avoid a // delay they did not ask to avoid. func isBudgetSpent(err error) bool { var le *accounting.LimitError if !errors.As(err, &le) { return false } return errors.Is(le.Err, accounting.ErrBudgetSpent) }