diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 868fdc3..0ce4129 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -122,6 +122,7 @@ func newCtlCmd() *cobra.Command { cmd.AddCommand( newNodesCmd(client, &asJSON), newLogsCmd(client, &asJSON), + newRequestCmd(client, &asJSON), newModelsCmd(client, &asJSON), newReplicasCmd(client, &asJSON), newExplainCmd(client, &asJSON), diff --git a/internal/cli/ctl_request.go b/internal/cli/ctl_request.go new file mode 100644 index 0000000..b6af3b4 --- /dev/null +++ b/internal/cli/ctl_request.go @@ -0,0 +1,45 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "request ID", + Short: "What one request went through: replica, queue, first token, tokens", + Long: "Every response carries its id in X-Request-Id, and a client may send its\n" + + "own. Given one, this says which replica answered, how long admission held\n" + + "it, how long the first token took and what it cost — the answer to \"why\n" + + "was this slow\". The access log has the same fields under \"request\".", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var v adminapi.RequestView + if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(v) + } + model := v.Model + if v.ServedModel != "" && v.ServedModel != v.Model { + model += " (served by " + v.ServedModel + ")" + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "request %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z")) + fmt.Fprintf(out, "model %s\n", model) + fmt.Fprintf(out, "replica %s\n", or(v.Replica, "- (none answered)")) + fmt.Fprintf(out, "status %d\n", v.Status) + fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs) + fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens) + fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID)) + return nil + }, + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 3742e0b..355f506 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest) mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs) mux.HandleFunc("POST /api/v1/restart", a.restart) + mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) } @@ -1115,3 +1116,52 @@ func (a *API) retryModel(w http.ResponseWriter, r *http.Request) { was := a.ctl.Reconciler().Retry(name) writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was}) } + +// requestFinder looks a request up by the id it was given. Asked of the usage +// source by type assertion, so one without it answers 501 rather than every +// source having to grow the method. +type requestFinder interface { + UsageByRequest(ctx context.Context, requestID string) (store.Usage, bool, error) +} + +// RequestView is what one request went through. +type RequestView struct { + RequestID string `json:"request_id"` + At time.Time `json:"at"` + KeyID string `json:"key_id"` + UserID string `json:"user_id,omitempty"` + ProjectID string `json:"project_id"` + Model string `json:"model"` + ServedModel string `json:"served_model"` + Replica string `json:"replica,omitempty"` + Status int `json:"status"` + LatencyMs int64 `json:"latency_ms"` + QueueMs int64 `json:"queue_ms"` + TTFTMs int64 `json:"ttft_ms"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + CostMicros int64 `json:"cost_micros"` +} + +func (a *API) request(w http.ResponseWriter, r *http.Request) { + f, ok := a.usage.(requestFinder) + if !ok { + writeError(w, http.StatusNotImplemented, "this control plane keeps no usage records") + return + } + u, found, err := f.UsageByRequest(r.Context(), r.PathValue("id")) + switch { + case err != nil: + writeError(w, http.StatusInternalServerError, err.Error()) + return + case !found: + writeError(w, http.StatusNotFound, "no request with that id; records are written a few seconds after a request ends") + return + } + writeJSON(w, http.StatusOK, RequestView{ + RequestID: u.RequestID, At: u.At, KeyID: u.KeyID, UserID: u.UserID, ProjectID: u.ProjectID, + Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status, + LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs, + PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros, + }) +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 241356c..6596582 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -3,6 +3,8 @@ package router import ( "bytes" "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -68,6 +70,7 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err r.m.observeAdmissionRejected(req.Band, reason) writeError(w, status, "server_error", err.Error()) r.m.observeRequest(r.metricModel(req.Model), "", status, time.Since(start), nil, 0) + r.accessLog(req, dispatchResult{err: err}, status, time.Since(start)) } // prefixBytes is how much of the prompt is kept for prefix-aware routing. @@ -224,6 +227,12 @@ type parsedRequest struct { UpstreamBody []byte Header http.Header + // RequestID follows the request through the access log, the usage record + // and the engine, and goes back to the client in X-Request-Id. + RequestID string + // QueueMs is how long admission held it. + QueueMs int64 + SessionID string Band string // KeyID, ProjectID and Tenant are empty when authentication is off. @@ -275,8 +284,19 @@ type inferenceBody struct { func (r *Router) handleInference(path string) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { start := time.Now() + // First, so even a request refused as malformed can be named when + // somebody asks what happened to it. + rid := requestID(req.Header.Get("X-Request-Id")) + w.Header().Set("X-Request-Id", rid) parsed, err := r.parse(req, path) + if err == nil { + parsed.RequestID = rid + if parsed.Header == nil { + parsed.Header = http.Header{} + } + parsed.Header.Set("X-Request-Id", rid) + } if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0) @@ -339,6 +359,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { } defer lease.Release() queued := time.Since(start) + parsed.QueueMs = queued.Milliseconds() r.m.observeAdmission(lease.Band(), r.metricModel(parsed.Model), queued) if queued > time.Millisecond { w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10)) @@ -422,6 +443,7 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR r.m.observeRequest(r.metricModel(req.Model), res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) r.account(req, res, status, time.Since(start)) + r.accessLog(req, res, status, time.Since(start)) } // account settles a finished request against its key. @@ -437,6 +459,8 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, Model: req.Model, ServedModel: res.model, Status: status, LatencyMs: took.Milliseconds(), + RequestID: req.RequestID, Replica: res.endpointID, + QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(), } if u.ServedModel == "" { u.ServedModel = req.Model @@ -468,6 +492,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start)) } // parse reads the body and extracts the routing-relevant fields. @@ -854,3 +879,55 @@ func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) { } writeJSON(w, http.StatusOK, out) } + +// requestID is the client's X-Request-Id when it is one worth keeping, or a +// new one. Kept only when short and plain: it is written into logs and a +// header, and a client-chosen string with newlines or megabytes in it is not +// an id. +func requestID(sent string) string { + if n := len(sent); n > 0 && n <= 128 && strings.IndexFunc(sent, func(c rune) bool { + return !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || + c == '-' || c == '_' || c == '.' || c == ':') + }) < 0 { + return sent + } + var b [12]byte + _, _ = rand.Read(b[:]) + return "req_" + hex.EncodeToString(b[:]) +} + +// accessLog writes one line per finished request: enough to follow a slow or +// failed request from the id a client reports to the replica that served it, +// and where its time went. Before this the only per-request lines were for +// disconnects, early stream ends and upstream retries. +func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, took time.Duration) { + attrs := []any{ + "request_id", req.RequestID, "status", status, "model", req.Model, + "duration_ms", took.Milliseconds(), "queue_ms", req.QueueMs, "stream", req.Stream, + } + if req.RequestedModel != "" && req.RequestedModel != req.Model { + attrs = append(attrs, "requested_model", req.RequestedModel) + } + if res.model != "" && res.model != req.Model { + attrs = append(attrs, "served_model", res.model) + } + if res.endpointID != "" { + attrs = append(attrs, "replica", res.endpointID) + } + if res.outcome.TTFT > 0 { + attrs = append(attrs, "ttft_ms", res.outcome.TTFT.Milliseconds()) + } + if u := res.outcome.Usage; u != nil { + attrs = append(attrs, "prompt_tokens", u.PromptTokens, "completion_tokens", u.CompletionTokens) + } + if req.KeyID != "" { + attrs = append(attrs, "key", req.KeyID, "project", req.ProjectID) + } + if req.UserID != "" { + attrs = append(attrs, "user", req.UserID) + } + if res.err != nil { + attrs = append(attrs, "err", res.err.Error()) + } + r.log.Info("request", attrs...) +} diff --git a/internal/router/pricing_test.go b/internal/router/pricing_test.go index 9135cf8..40d06ae 100644 --- a/internal/router/pricing_test.go +++ b/internal/router/pricing_test.go @@ -140,3 +140,51 @@ func waitForUsage(t *testing.T, rt *Router) store.KeyUsage { t.Fatal("no usage was recorded") return store.KeyUsage{} } + +func TestARequestCanBeFollowedByItsID(t *testing.T) { + // "Why was my request slow?" had no answer: the queue time went out in a + // header and nowhere else, and nothing tied a request to its replica. + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + + rec := post(t, rt.Handler(), + map[string]any{"model": "q14", "messages": []any{ + map[string]any{"role": "user", "content": "hello"}}}, + map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "client-abc.1"}) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if got := rec.Header().Get("X-Request-Id"); got != "client-abc.1" { + t.Errorf("the client's id came back as %q", got) + } + + deadline := time.Now().Add(5 * time.Second) + for { + u, ok, err := rt.store.UsageByRequest(t.Context(), "client-abc.1") + if err != nil { + t.Fatal(err) + } + if ok { + if u.Replica == "" || u.Status != 200 { + t.Errorf("recorded %+v; expected the replica that answered", u) + } + break + } + if time.Now().After(deadline) { + t.Fatal("the request's usage cannot be found by its id") + } + time.Sleep(20 * time.Millisecond) + } +} + +func TestAnIDThatIsNotOneIsReplaced(t *testing.T) { + for _, bad := range []string{"", "has space", "new\nline", strings.Repeat("x", 200)} { + got := requestID(bad) + if got == bad || !strings.HasPrefix(got, "req_") { + t.Errorf("%q became %q", bad, got) + } + } + if requestID("req_1") != "req_1" { + t.Error("a plain id was not kept") + } +} diff --git a/internal/router/proxy/proxy.go b/internal/router/proxy/proxy.go index 693e13d..a7eb286 100644 --- a/internal/router/proxy/proxy.go +++ b/internal/router/proxy/proxy.go @@ -526,7 +526,7 @@ func copyHeaders(dst, src http.Header) { // llmgw, not llmgw to the engine. llmgw's own key for the engine is set by Do. func cloneForUpstream(h http.Header) http.Header { out := make(http.Header, 4) - for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} { + for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate", "X-Request-Id"} { if v := h.Get(k); v != "" { out.Set(k, v) } diff --git a/internal/store/migrations/postgres/0012_usage_diagnostics.sql b/internal/store/migrations/postgres/0012_usage_diagnostics.sql new file mode 100644 index 0000000..59c1929 --- /dev/null +++ b/internal/store/migrations/postgres/0012_usage_diagnostics.sql @@ -0,0 +1,14 @@ +-- Ported from the SQLite migration of the same number. +-- What a request went through, beside what it cost: which replica answered, +-- how long it queued in admission, and how long the first token took. Without +-- them "why was my request slow" had no answer — the queue time went out in a +-- response header and nowhere else. +-- +-- request_id is the id the client sent or the gateway made, echoed in the +-- X-Request-Id header and the access log, so one request can be followed from +-- the client's error to the replica that served it. +ALTER TABLE usage ADD COLUMN request_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE usage ADD COLUMN replica TEXT NOT NULL DEFAULT ''; +ALTER TABLE usage ADD COLUMN queue_ms BIGINT NOT NULL DEFAULT 0; +ALTER TABLE usage ADD COLUMN ttft_ms BIGINT NOT NULL DEFAULT 0; +CREATE INDEX usage_by_request ON usage(request_id) WHERE request_id <> ''; diff --git a/internal/store/migrations/sqlite/0012_usage_diagnostics.sql b/internal/store/migrations/sqlite/0012_usage_diagnostics.sql new file mode 100644 index 0000000..16f8f06 --- /dev/null +++ b/internal/store/migrations/sqlite/0012_usage_diagnostics.sql @@ -0,0 +1,13 @@ +-- What a request went through, beside what it cost: which replica answered, +-- how long it queued in admission, and how long the first token took. Without +-- them "why was my request slow" had no answer — the queue time went out in a +-- response header and nowhere else. +-- +-- request_id is the id the client sent or the gateway made, echoed in the +-- X-Request-Id header and the access log, so one request can be followed from +-- the client's error to the replica that served it. +ALTER TABLE usage ADD COLUMN request_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE usage ADD COLUMN replica TEXT NOT NULL DEFAULT ''; +ALTER TABLE usage ADD COLUMN queue_ms INTEGER NOT NULL DEFAULT 0; +ALTER TABLE usage ADD COLUMN ttft_ms INTEGER NOT NULL DEFAULT 0; +CREATE INDEX usage_by_request ON usage(request_id) WHERE request_id <> ''; diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index a9ed2ec..3cb4c30 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -424,8 +424,9 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error { return s.inTx(ctx, func(tx conn) error { stmt, err := tx.PrepareContext(ctx, `INSERT INTO usage (ts, key_id, project_id, model, served_model, - prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id, + request_id, replica, queue_ms, ttft_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) if err != nil { return err } @@ -433,7 +434,8 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error { for _, r := range records { if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID, r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens, - r.CostMicros, r.Status, r.LatencyMs, r.UserID); err != nil { + r.CostMicros, r.Status, r.LatencyMs, r.UserID, + r.RequestID, r.Replica, r.QueueMs, r.TTFTMs); err != nil { return err } } @@ -828,3 +830,25 @@ func (s *DB) SnapshotTo(ctx context.Context, path string) error { _, err := s.raw.ExecContext(ctx, `VACUUM INTO ?`, path) return err } + +// UsageByRequest finds the record of one request by the id it was given, or +// ok=false when none has that id. +func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool, error) { + var u Usage + var ts int64 + err := s.db.QueryRowContext(ctx, + `SELECT ts, key_id, project_id, COALESCE(user_id, ''), model, served_model, prompt_tokens, + completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms + FROM usage WHERE request_id = ? ORDER BY ts DESC LIMIT 1`, requestID). + Scan(&ts, &u.KeyID, &u.ProjectID, &u.UserID, &u.Model, &u.ServedModel, &u.PromptTokens, + &u.CompletionTokens, &u.CostMicros, &u.Status, &u.LatencyMs, &u.RequestID, &u.Replica, + &u.QueueMs, &u.TTFTMs) + if errors.Is(err, sql.ErrNoRows) { + return Usage{}, false, nil + } + if err != nil { + return Usage{}, false, err + } + u.At = time.UnixMilli(ts) + return u, true, nil +} diff --git a/internal/store/store.go b/internal/store/store.go index b0f4141..109bbd5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -244,6 +244,15 @@ type Usage struct { CostMicros int64 Status int LatencyMs int64 + + // RequestID is the id the client sent in X-Request-Id or the gateway made. + RequestID string + // Replica answered the request; empty when none did. + Replica string + // QueueMs is time spent waiting in admission; TTFTMs is time to the first + // token, 0 when none came. + QueueMs int64 + TTFTMs int64 } // Store is the persistent half of tenancy and accounting.