Files
llmgw/internal/router/chat.go
T
Ilker ManapandClaude Opus 5 f80eef9fa6 Stop the chat page asking for more tokens than the model allows
The page imposes its own ceiling on a turn, so a playground request cannot
occupy a replica for ten minutes. It sent that ceiling regardless of the
model, and a model declaring a smaller max_output then answered every turn
with "max_tokens is 2048, but this model allows at most 1024" — the gateway
refusing its own page, over a number the person never chose and has no way
to change.

The ceiling is the page's to impose on itself and the model's limit is not
negotiable, so it sends the smaller of the two.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-28 00:48:07 +02:00

193 lines
7.2 KiB
Go

package router
import (
"bytes"
"encoding/json"
"errors"
"io"
"net/http"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// registerChat mounts the in-browser chat for signed-in people.
//
// It exists so someone can try a model without setting up a client, and — more
// often — so they can tell whether a problem is the gateway or their code.
//
// The browser never sees an API key. The page carries a session, the server
// turns that into the same identity an API key would produce, and the request
// goes down the ordinary path: admission, model allowlist, rate limits, budget,
// accounting. Anything else would make this page a hole in every limit the rest
// of the gateway enforces.
func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) {
mux.HandleFunc("GET /chat", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
web.Chat(user, r.modelsFor(req, user), version.Version)(w, req)
}))
mux.HandleFunc("POST /chat/send", pages.Guard(r.chatSend))
// Behind the same guard as the rest of the page: it takes a POST body and
// returns a file, so it wants a session and the CSRF check for the same
// reasons /chat/send does.
mux.HandleFunc("POST /chat/download", pages.Guard(
func(w http.ResponseWriter, req *http.Request, _ store.User) {
web.CodeArchive(r.cfg.MaxBodyBytes)(w, req)
}))
}
// chatSend dispatches one turn on behalf of a signed-in person.
func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.User) {
principal, err := r.principalFor(req, user)
if err != nil {
http.Error(w, "your account has no project to bill this to", http.StatusForbidden)
return
}
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
http.Error(w, "could not read the request", http.StatusBadRequest)
return
}
var in struct {
Model string `json:"model"`
Messages []json.RawMessage `json:"messages"`
}
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" || len(in.Messages) == 0 {
http.Error(w, "a model and at least one message are required", http.StatusBadRequest)
return
}
// Rebuilt rather than forwarded, so the page cannot set fields it has no
// business setting — a stream:false that would hang the reader, a
// max_tokens large enough to occupy a replica for an hour.
_, model := r.resolve(in.Model)
upstream, err := json.Marshal(map[string]any{
"model": in.Model,
"messages": in.Messages,
"stream": true,
"max_tokens": chatTokenCap(model),
})
if err != nil {
http.Error(w, "could not build the request", http.StatusInternalServerError)
return
}
next := req.Clone(auth.WithPrincipal(req.Context(), principal))
next.Body = io.NopCloser(bytes.NewReader(upstream))
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
// The same handler the API serves, so there is exactly one request path and
// no second place for its rules to drift.
r.handleInference("/v1/chat/completions")(w, next)
}
// chatMaxTokens bounds one turn from the page.
//
// A playground should not be the thing that ties up a replica for ten minutes,
// and someone who wants a longer answer has an API key for that.
//
// 512 was chosen when every model here was a 0.5B answering questions in a
// paragraph. It is about a screenful of code, so a coding model cut off
// mid-function on the first real request — and the page said nothing, because
// it never read finish_reason. The cap being too low was a nuisance; the reply
// ending silently was the actual defect, and both are fixed.
const chatMaxTokens = 2048
// chatTokenCap is that bound, lowered to whatever the model allows.
//
// Sending the constant regardless made the gateway refuse its own page: a model
// declaring max_output 1024 answered every turn from /chat with "max_tokens is
// 2048, but this model allows at most 1024", which is a true sentence about a
// number the person never chose and cannot change. The ceiling is the page's to
// impose on itself, and the model's limit is not negotiable, so the smaller of
// the two is the only answer that is right in both directions.
func chatTokenCap(m catalog.Model) int {
if m.MaxOutput > 0 && m.MaxOutput < chatMaxTokens {
return m.MaxOutput
}
return chatMaxTokens
}
// principalFor builds the identity a signed-in person acts under.
//
// The key id is synthetic and marked as a session, because there is no key: it
// keeps usage rows attributable and rate limits per person, while making it
// obvious in the usage table that these requests came from the page rather than
// from a credential somebody holds.
func (r *Router) principalFor(req *http.Request, user store.User) (auth.Principal, error) {
if r.store == nil || user.ProjectID == "" {
return auth.Principal{}, errors.New("no project")
}
eff, err := r.store.ResolveProject(req.Context(), user.ProjectID)
if err != nil {
return auth.Principal{}, err
}
return auth.Principal{
KeyID: "session:" + user.ID,
KeyPrefix: "session",
ProjectID: eff.ProjectID,
TeamID: eff.TeamID,
OrgID: eff.OrgID,
Band: eff.Band,
RoutingProfile: eff.RoutingProfile,
Limits: eff,
}, nil
}
// modelsFor lists what this person may actually ask for.
//
// Served rather than catalogued: offering a model with no replica produces a
// 503 the reader cannot interpret, and the allowlist is applied so the picker
// never shows a model the request would refuse.
//
// One entry per model, under its catalog name. An endpoint advertises every
// name a client may address it by — the model's own and all of its aliases,
// because that is what the router matches against — so taking those strings as
// they come listed the same model once for each name it has. A picker offering
// "coder", "coder-7b" and "qwen2.5-coder-7b-instruct" as three choices invites
// somebody to compare them.
//
// Real models are ordered ahead of simulated ones so the browser's default
// selection — the first option — is always a model that answers. Sorted purely
// by name, "big-model" came first and every visitor who pressed Send without
// touching the picker got placeholder text.
func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
principal, err := r.principalFor(req, user)
if err != nil {
return nil
}
seen := map[string]struct{}{}
var out []web.ChatModel
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
// Resolved rather than taken as sent: an alias and the name it
// stands for are one model, and the catalog is what says so.
name, entry := r.resolve(m)
if _, dup := seen[name]; dup {
continue
}
seen[name] = struct{}{}
if r.permits(principal, name) {
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()})
}
}
}
slices.SortFunc(out, func(a, b web.ChatModel) int {
if a.Simulated != b.Simulated {
if b.Simulated {
return -1
}
return 1
}
return strings.Compare(a.Name, b.Name)
})
return out
}