package router import ( "fmt" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // checkContextFits reports why a request cannot fit in a model's context, or // "" when it can. // // Refused here rather than at the engine. An over-long prompt used to be // admitted, scheduled, given a replica and a slot, and only then rejected โ€” // paying the whole cost of the request to learn something knowable from the // request itself. The engine's error also arrives in whatever shape that engine // chose, which is not something a client can act on. // // The prompt estimate is bytes over four, the same crude stand-in used for // scoring, so the comparison is approximate in one direction: it can let // through a request the engine will refuse. That is the right way round. A // tokenizer would make it exact, and needing one is why plan ยง7.5 tier 2 is v2. // To keep an estimate that reads slightly low from refusing valid requests, a // margin is allowed before anything is rejected. func checkContextFits(m catalog.Model, promptTokens, maxTokens int) string { if m.ContextLength <= 0 { // Unknown, which is every model that has not declared one. Saying // nothing beats guessing a limit and refusing work over it. return "" } if m.MaxOutput > 0 && maxTokens > m.MaxOutput { return fmt.Sprintf("max_tokens is %d, but %q allows at most %d", maxTokens, m.Name, m.MaxOutput) } // Only the prompt is estimated; max_tokens is exact when given. A request // that does not say how much it wants back is judged on its prompt alone, // because assuming the whole remaining window would refuse most requests. need := promptTokens + maxTokens allowed := m.ContextLength + contextEstimateMargin(m.ContextLength) if need <= allowed { return "" } if maxTokens > 0 { return fmt.Sprintf( "about %d prompt tokens plus max_tokens %d exceeds the %d-token context of %q", promptTokens, maxTokens, m.ContextLength, m.Name) } return fmt.Sprintf("the prompt is about %d tokens, over the %d-token context of %q", promptTokens, m.ContextLength, m.Name) } // contextEstimateMargin is the slack allowed for the bytes-over-four estimate. // // That estimate is wrong in both directions depending on the text โ€” code and // non-Latin scripts tokenize very differently from English prose โ€” and being // wrong by a few percent must not turn a valid request into a refusal. Ten // percent is enough to absorb that while still catching a prompt that is // genuinely too big, which is the case worth catching. func contextEstimateMargin(context int) int { return context / 10 }