diff --git a/docs/onboarding.md b/docs/onboarding.md index 8c36aeb..f0d4e65 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -236,6 +236,27 @@ Tell them out of band and have them change it from their profile. Their sessions are closed — if they lost the password, the reason might be that someone else has it. Their API keys keep working; those are separate credentials. +## Trying a model in the browser + +`/chat` is a playground for anyone signed in. It exists so somebody can see +whether a model works without setting up a client, and — more often — so they +can tell whether a problem is the gateway or their code. + +**The browser never holds a key.** The page carries a session; the server turns +that into the same identity an API key would produce, resolved from the person's +project, and the request goes down the ordinary path: admission, model +allowlist, rate limits, budget, accounting. Anything else would make the page a +hole in every limit the rest of the gateway enforces. + +Usage from the page is attributed to `session:` rather than a key, so +it is billed to the right project and visibly did not come from a credential +somebody holds. One turn is capped at 512 tokens: a playground should not be the +thing that ties up a replica for ten minutes, and anyone who wants a longer +answer has an API key for that. + +Nothing is stored. The conversation lives in the page and is gone when it is +closed. + ## Seeing the cluster The dashboard — nodes, replicas, models, why something did not get placed, usage diff --git a/internal/router/chat.go b/internal/router/chat.go new file mode 100644 index 0000000..a99d1bc --- /dev/null +++ b/internal/router/chat.go @@ -0,0 +1,138 @@ +package router + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "slices" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" + "gitea.mynodes.xyz/ilker/llmgw/internal/web" +) + +// registerChat mounts the in-browser chat for signed-in people. +// +// It exists so someone can try a model without setting up a client, and — more +// often — so they can tell whether a problem is the gateway or their code. +// +// The browser never sees an API key. The page carries a session, the server +// turns that into the same identity an API key would produce, and the request +// goes down the ordinary path: admission, model allowlist, rate limits, budget, +// accounting. Anything else would make this page a hole in every limit the rest +// of the gateway enforces. +func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) { + mux.HandleFunc("GET /chat", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) { + web.Chat(user, r.modelsFor(req, user), version.Version)(w, req) + })) + mux.HandleFunc("POST /chat/send", pages.Guard(r.chatSend)) +} + +// chatSend dispatches one turn on behalf of a signed-in person. +func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.User) { + principal, err := r.principalFor(req, user) + if err != nil { + http.Error(w, "your account has no project to bill this to", http.StatusForbidden) + return + } + + body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)) + if err != nil { + http.Error(w, "could not read the request", http.StatusBadRequest) + return + } + + var in struct { + Model string `json:"model"` + Messages []json.RawMessage `json:"messages"` + } + if err := json.Unmarshal(body, &in); err != nil || in.Model == "" || len(in.Messages) == 0 { + http.Error(w, "a model and at least one message are required", http.StatusBadRequest) + return + } + + // Rebuilt rather than forwarded, so the page cannot set fields it has no + // business setting — a stream:false that would hang the reader, a + // max_tokens large enough to occupy a replica for an hour. + upstream, err := json.Marshal(map[string]any{ + "model": in.Model, + "messages": in.Messages, + "stream": true, + "max_tokens": chatMaxTokens, + }) + if err != nil { + http.Error(w, "could not build the request", http.StatusInternalServerError) + return + } + + next := req.Clone(auth.WithPrincipal(req.Context(), principal)) + next.Body = io.NopCloser(bytes.NewReader(upstream)) + next.ContentLength = int64(len(upstream)) + next.Header.Set("Content-Type", "application/json") + + // The same handler the API serves, so there is exactly one request path and + // no second place for its rules to drift. + r.handleInference("/v1/chat/completions")(w, next) +} + +// chatMaxTokens bounds one turn from the page. +// +// A playground should not be the thing that ties up a replica for ten minutes, +// and someone who wants a longer answer has an API key for that. +const chatMaxTokens = 512 + +// principalFor builds the identity a signed-in person acts under. +// +// The key id is synthetic and marked as a session, because there is no key: it +// keeps usage rows attributable and rate limits per person, while making it +// obvious in the usage table that these requests came from the page rather than +// from a credential somebody holds. +func (r *Router) principalFor(req *http.Request, user store.User) (auth.Principal, error) { + if r.store == nil || user.ProjectID == "" { + return auth.Principal{}, errors.New("no project") + } + eff, err := r.store.ResolveProject(req.Context(), user.ProjectID) + if err != nil { + return auth.Principal{}, err + } + return auth.Principal{ + KeyID: "session:" + user.ID, + KeyPrefix: "session", + ProjectID: eff.ProjectID, + TeamID: eff.TeamID, + OrgID: eff.OrgID, + Band: eff.Band, + RoutingProfile: eff.RoutingProfile, + Limits: eff, + }, nil +} + +// modelsFor lists what this person may actually ask for. +// +// Served rather than catalogued: offering a model with no replica produces a +// 503 the reader cannot interpret, and the allowlist is applied so the picker +// never shows a model the request would refuse. +func (r *Router) modelsFor(req *http.Request, user store.User) []string { + principal, err := r.principalFor(req, user) + if err != nil { + return nil + } + seen := map[string]struct{}{} + var out []string + for _, s := range r.reg.Snapshots("") { + for _, m := range s.Models { + if _, dup := seen[m]; dup { + continue + } + seen[m] = struct{}{} + if r.permits(principal, m) { + out = append(out, m) + } + } + } + slices.Sort(out) + return out +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index d011b87..d70f765 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -108,6 +108,7 @@ func (r *Router) Handler() http.Handler { // they are exempt below. pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log) pages.Register(mux) + r.registerChat(mux, pages) if r.clusterView != nil { // The cluster dashboard, reachable from a browser without exposing @@ -127,7 +128,8 @@ func (r *Router) Handler() http.Handler { // stays open because a load balancer has no key. return auth.Middleware(r.auth, r.log, "/health", "/", "/_static/", "/claim/", - "/signup", "/login", "/logout", "/profile", "/profile/", "/admin", "/admin/")(mux) + "/signup", "/login", "/logout", "/profile", "/profile/", + "/admin", "/admin/", "/chat", "/chat/")(mux) } // parsedRequest is what the router needs from a request. Everything else in the diff --git a/internal/store/users.go b/internal/store/users.go index 83be434..0fc139a 100644 --- a/internal/store/users.go +++ b/internal/store/users.go @@ -256,3 +256,41 @@ func (s *SQLite) AllProjects(ctx context.Context) ([]ProjectPath, error) { type ProjectPath struct { ID, Path string } + +// ResolveProject returns the limits an account inherits from its project. +// +// The same walk as Resolve, minus the key. A person signed in to the chat page +// has no key in their hand — the key was shown once and is theirs to keep — so +// their identity there is their project, and it must be resolved the same way +// or the page would be a hole in the limits everything else obeys. +func (s *SQLite) ResolveProject(ctx context.Context, projectID string) (Effective, error) { + var e Effective + row := s.db.QueryRowContext(ctx, + `SELECT p.id, t.id, o.id + FROM projects p + JOIN teams t ON t.id = p.team_id + JOIN organizations o ON o.id = t.org_id + WHERE p.id = ?`, projectID) + if err := row.Scan(&e.ProjectID, &e.TeamID, &e.OrgID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Effective{}, ErrNotFound + } + return Effective{}, err + } + + // Widest first, so a narrower scope can only tighten what it inherits. + for _, step := range []struct { + scope Scope + id string + }{{ScopeOrg, e.OrgID}, {ScopeTeam, e.TeamID}, {ScopeProject, e.ProjectID}} { + l, err := s.LimitsFor(ctx, step.scope, step.id) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return Effective{}, err + } + e.tighten(l) + } + return e, nil +} diff --git a/internal/web/chat.go b/internal/web/chat.go new file mode 100644 index 0000000..eda240f --- /dev/null +++ b/internal/web/chat.go @@ -0,0 +1,65 @@ +package web + +import ( + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type chatData struct { + User store.User + Models []string + CSRF string + Version string + Now string +} + +// Chat renders the in-browser chat for one signed-in person. +func Chat(user store.User, models []string, version string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + noStore(w) + render(w, "chat", chatData{ + User: user, Models: models, CSRF: csrfFor(r), + Version: version, Now: time.Now().UTC().Format(time.RFC3339), + }) + } +} + +// Guard runs a handler only for a signed-in, approved account. +// +// Exported because the chat lives in the router — it needs the endpoint +// registry and the request path — while the session machinery lives here. The +// alternative was a second copy of the session check next to the one thing that +// must never have two copies. +// +// Its POSTs carry the CSRF token in a header rather than a form field, because +// they are JSON. A header is a stronger check than a field: a cross-site form +// post cannot set one at all. +func (p *AccountPages) Guard( + next func(w http.ResponseWriter, r *http.Request, user store.User), +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + noStore(w) + sess, user, ok := p.session(r) + if !ok { + if r.Method == http.MethodGet { + http.Redirect(w, r, "/login", http.StatusSeeOther) + } else { + http.Error(w, "not signed in", http.StatusUnauthorized) + } + return + } + if r.Method != http.MethodGet { + token := r.Header.Get("X-CSRF-Token") + if token == "" { + token = r.FormValue("csrf") + } + if !p.accounts.CheckCSRF(sess, token) { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + } + next(w, r, user) + } +} diff --git a/internal/web/templates/account.html b/internal/web/templates/account.html index 0de16f7..4b97118 100644 --- a/internal/web/templates/account.html +++ b/internal/web/templates/account.html @@ -53,6 +53,7 @@ Tell whoever you asked for access that you have signed up.

Your keys

{{.User.Name}} · {{.User.Email}} · project {{.User.ProjectID}}