Follow one request from its id to the replica that served it
"Why was this request slow?" had no answer. Nothing tied a request to the replica that served it, the time admission held it went out in a response header and was never kept, and the only per-request log lines were for disconnects, early stream ends and upstream retries. Every request now has an id: the client's X-Request-Id when it is a plain string of reasonable length, otherwise one the gateway makes. It is set on the response before anything else, so a request refused as malformed can be named too, and forwarded to the engine. One access log line per request carries it with the model asked for and served, the replica, status, time in the queue, time to the first token, total time, tokens, and the key, project and user. Usage records gain the id, the replica, queue time and time to first token, tried against a restored copy of the live store before being shipped: 591 rows kept, columns and index in place. `llmgw ctl request ID` reads one back through the admin API. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
91fe24a105
commit
f8572fb5ee
10 files changed
+285
-4
No files matched your search
@@ -122,6 +122,7 @@ func newCtlCmd() *cobra.Command {
|
||||
cmd.AddCommand(
|
||||
newNodesCmd(client, &asJSON),
|
||||
newLogsCmd(client, &asJSON),
|
||||
newRequestCmd(client, &asJSON),
|
||||
newModelsCmd(client, &asJSON),
|
||||
newReplicasCmd(client, &asJSON),
|
||||
newExplainCmd(client, &asJSON),
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "request ID",
|
||||
Short: "What one request went through: replica, queue, first token, tokens",
|
||||
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
|
||||
"own. Given one, this says which replica answered, how long admission held\n" +
|
||||
"it, how long the first token took and what it cost — the answer to \"why\n" +
|
||||
"was this slow\". The access log has the same fields under \"request\".",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var v adminapi.RequestView
|
||||
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
model := v.Model
|
||||
if v.ServedModel != "" && v.ServedModel != v.Model {
|
||||
model += " (served by " + v.ServedModel + ")"
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "request %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z"))
|
||||
fmt.Fprintf(out, "model %s\n", model)
|
||||
fmt.Fprintf(out, "replica %s\n", or(v.Replica, "- (none answered)"))
|
||||
fmt.Fprintf(out, "status %d\n", v.Status)
|
||||
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
}
|
||||
|
||||
@@ -1115,3 +1116,52 @@ func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
|
||||
was := a.ctl.Reconciler().Retry(name)
|
||||
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
|
||||
}
|
||||
|
||||
// requestFinder looks a request up by the id it was given. Asked of the usage
|
||||
// source by type assertion, so one without it answers 501 rather than every
|
||||
// source having to grow the method.
|
||||
type requestFinder interface {
|
||||
UsageByRequest(ctx context.Context, requestID string) (store.Usage, bool, error)
|
||||
}
|
||||
|
||||
// RequestView is what one request went through.
|
||||
type RequestView struct {
|
||||
RequestID string `json:"request_id"`
|
||||
At time.Time `json:"at"`
|
||||
KeyID string `json:"key_id"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
ProjectID string `json:"project_id"`
|
||||
Model string `json:"model"`
|
||||
ServedModel string `json:"served_model"`
|
||||
Replica string `json:"replica,omitempty"`
|
||||
Status int `json:"status"`
|
||||
LatencyMs int64 `json:"latency_ms"`
|
||||
QueueMs int64 `json:"queue_ms"`
|
||||
TTFTMs int64 `json:"ttft_ms"`
|
||||
PromptTokens int64 `json:"prompt_tokens"`
|
||||
CompletionTokens int64 `json:"completion_tokens"`
|
||||
CostMicros int64 `json:"cost_micros"`
|
||||
}
|
||||
|
||||
func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := a.usage.(requestFinder)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane keeps no usage records")
|
||||
return
|
||||
}
|
||||
u, found, err := f.UsageByRequest(r.Context(), r.PathValue("id"))
|
||||
switch {
|
||||
case err != nil:
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
case !found:
|
||||
writeError(w, http.StatusNotFound, "no request with that id; records are written a few seconds after a request ends")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, RequestView{
|
||||
RequestID: u.RequestID, At: u.At, KeyID: u.KeyID, UserID: u.UserID, ProjectID: u.ProjectID,
|
||||
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
|
||||
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
})
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package router
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -68,6 +70,7 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err
|
||||
r.m.observeAdmissionRejected(req.Band, reason)
|
||||
writeError(w, status, "server_error", err.Error())
|
||||
r.m.observeRequest(r.metricModel(req.Model), "", status, time.Since(start), nil, 0)
|
||||
r.accessLog(req, dispatchResult{err: err}, status, time.Since(start))
|
||||
}
|
||||
|
||||
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
|
||||
@@ -224,6 +227,12 @@ type parsedRequest struct {
|
||||
UpstreamBody []byte
|
||||
Header http.Header
|
||||
|
||||
// RequestID follows the request through the access log, the usage record
|
||||
// and the engine, and goes back to the client in X-Request-Id.
|
||||
RequestID string
|
||||
// QueueMs is how long admission held it.
|
||||
QueueMs int64
|
||||
|
||||
SessionID string
|
||||
Band string
|
||||
// KeyID, ProjectID and Tenant are empty when authentication is off.
|
||||
@@ -275,8 +284,19 @@ type inferenceBody struct {
|
||||
func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
start := time.Now()
|
||||
// First, so even a request refused as malformed can be named when
|
||||
// somebody asks what happened to it.
|
||||
rid := requestID(req.Header.Get("X-Request-Id"))
|
||||
w.Header().Set("X-Request-Id", rid)
|
||||
|
||||
parsed, err := r.parse(req, path)
|
||||
if err == nil {
|
||||
parsed.RequestID = rid
|
||||
if parsed.Header == nil {
|
||||
parsed.Header = http.Header{}
|
||||
}
|
||||
parsed.Header.Set("X-Request-Id", rid)
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
|
||||
@@ -339,6 +359,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
}
|
||||
defer lease.Release()
|
||||
queued := time.Since(start)
|
||||
parsed.QueueMs = queued.Milliseconds()
|
||||
r.m.observeAdmission(lease.Band(), r.metricModel(parsed.Model), queued)
|
||||
if queued > time.Millisecond {
|
||||
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
|
||||
@@ -422,6 +443,7 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
|
||||
|
||||
r.m.observeRequest(r.metricModel(req.Model), res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
|
||||
r.account(req, res, status, time.Since(start))
|
||||
r.accessLog(req, res, status, time.Since(start))
|
||||
}
|
||||
|
||||
// account settles a finished request against its key.
|
||||
@@ -437,6 +459,8 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
|
||||
Model: req.Model, ServedModel: res.model, Status: status,
|
||||
LatencyMs: took.Milliseconds(),
|
||||
RequestID: req.RequestID, Replica: res.endpointID,
|
||||
QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(),
|
||||
}
|
||||
if u.ServedModel == "" {
|
||||
u.ServedModel = req.Model
|
||||
@@ -468,6 +492,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro
|
||||
// which is absent when waiting does not help.
|
||||
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
|
||||
r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0)
|
||||
r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start))
|
||||
}
|
||||
|
||||
// parse reads the body and extracts the routing-relevant fields.
|
||||
@@ -854,3 +879,55 @@ func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// requestID is the client's X-Request-Id when it is one worth keeping, or a
|
||||
// new one. Kept only when short and plain: it is written into logs and a
|
||||
// header, and a client-chosen string with newlines or megabytes in it is not
|
||||
// an id.
|
||||
func requestID(sent string) string {
|
||||
if n := len(sent); n > 0 && n <= 128 && strings.IndexFunc(sent, func(c rune) bool {
|
||||
return !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' ||
|
||||
c == '-' || c == '_' || c == '.' || c == ':')
|
||||
}) < 0 {
|
||||
return sent
|
||||
}
|
||||
var b [12]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "req_" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// accessLog writes one line per finished request: enough to follow a slow or
|
||||
// failed request from the id a client reports to the replica that served it,
|
||||
// and where its time went. Before this the only per-request lines were for
|
||||
// disconnects, early stream ends and upstream retries.
|
||||
func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
|
||||
attrs := []any{
|
||||
"request_id", req.RequestID, "status", status, "model", req.Model,
|
||||
"duration_ms", took.Milliseconds(), "queue_ms", req.QueueMs, "stream", req.Stream,
|
||||
}
|
||||
if req.RequestedModel != "" && req.RequestedModel != req.Model {
|
||||
attrs = append(attrs, "requested_model", req.RequestedModel)
|
||||
}
|
||||
if res.model != "" && res.model != req.Model {
|
||||
attrs = append(attrs, "served_model", res.model)
|
||||
}
|
||||
if res.endpointID != "" {
|
||||
attrs = append(attrs, "replica", res.endpointID)
|
||||
}
|
||||
if res.outcome.TTFT > 0 {
|
||||
attrs = append(attrs, "ttft_ms", res.outcome.TTFT.Milliseconds())
|
||||
}
|
||||
if u := res.outcome.Usage; u != nil {
|
||||
attrs = append(attrs, "prompt_tokens", u.PromptTokens, "completion_tokens", u.CompletionTokens)
|
||||
}
|
||||
if req.KeyID != "" {
|
||||
attrs = append(attrs, "key", req.KeyID, "project", req.ProjectID)
|
||||
}
|
||||
if req.UserID != "" {
|
||||
attrs = append(attrs, "user", req.UserID)
|
||||
}
|
||||
if res.err != nil {
|
||||
attrs = append(attrs, "err", res.err.Error())
|
||||
}
|
||||
r.log.Info("request", attrs...)
|
||||
}
|
||||
@@ -140,3 +140,51 @@ func waitForUsage(t *testing.T, rt *Router) store.KeyUsage {
|
||||
t.Fatal("no usage was recorded")
|
||||
return store.KeyUsage{}
|
||||
}
|
||||
|
||||
func TestARequestCanBeFollowedByItsID(t *testing.T) {
|
||||
// "Why was my request slow?" had no answer: the queue time went out in a
|
||||
// header and nowhere else, and nothing tied a request to its replica.
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
|
||||
rec := post(t, rt.Handler(),
|
||||
map[string]any{"model": "q14", "messages": []any{
|
||||
map[string]any{"role": "user", "content": "hello"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "client-abc.1"})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("X-Request-Id"); got != "client-abc.1" {
|
||||
t.Errorf("the client's id came back as %q", got)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
u, ok, err := rt.store.UsageByRequest(t.Context(), "client-abc.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
if u.Replica == "" || u.Status != 200 {
|
||||
t.Errorf("recorded %+v; expected the replica that answered", u)
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the request's usage cannot be found by its id")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIDThatIsNotOneIsReplaced(t *testing.T) {
|
||||
for _, bad := range []string{"", "has space", "new\nline", strings.Repeat("x", 200)} {
|
||||
got := requestID(bad)
|
||||
if got == bad || !strings.HasPrefix(got, "req_") {
|
||||
t.Errorf("%q became %q", bad, got)
|
||||
}
|
||||
}
|
||||
if requestID("req_1") != "req_1" {
|
||||
t.Error("a plain id was not kept")
|
||||
}
|
||||
}
|
||||
@@ -526,7 +526,7 @@ func copyHeaders(dst, src http.Header) {
|
||||
// llmgw, not llmgw to the engine. llmgw's own key for the engine is set by Do.
|
||||
func cloneForUpstream(h http.Header) http.Header {
|
||||
out := make(http.Header, 4)
|
||||
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} {
|
||||
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate", "X-Request-Id"} {
|
||||
if v := h.Get(k); v != "" {
|
||||
out.Set(k, v)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- What a request went through, beside what it cost: which replica answered,
|
||||
-- how long it queued in admission, and how long the first token took. Without
|
||||
-- them "why was my request slow" had no answer — the queue time went out in a
|
||||
-- response header and nowhere else.
|
||||
--
|
||||
-- request_id is the id the client sent or the gateway made, echoed in the
|
||||
-- X-Request-Id header and the access log, so one request can be followed from
|
||||
-- the client's error to the replica that served it.
|
||||
ALTER TABLE usage ADD COLUMN request_id TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN replica TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN queue_ms BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE usage ADD COLUMN ttft_ms BIGINT NOT NULL DEFAULT 0;
|
||||
CREATE INDEX usage_by_request ON usage(request_id) WHERE request_id <> '';
|
||||
@@ -0,0 +1,13 @@
|
||||
-- What a request went through, beside what it cost: which replica answered,
|
||||
-- how long it queued in admission, and how long the first token took. Without
|
||||
-- them "why was my request slow" had no answer — the queue time went out in a
|
||||
-- response header and nowhere else.
|
||||
--
|
||||
-- request_id is the id the client sent or the gateway made, echoed in the
|
||||
-- X-Request-Id header and the access log, so one request can be followed from
|
||||
-- the client's error to the replica that served it.
|
||||
ALTER TABLE usage ADD COLUMN request_id TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN replica TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN queue_ms INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE usage ADD COLUMN ttft_ms INTEGER NOT NULL DEFAULT 0;
|
||||
CREATE INDEX usage_by_request ON usage(request_id) WHERE request_id <> '';
|
||||
@@ -424,8 +424,9 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
return s.inTx(ctx, func(tx conn) error {
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -433,7 +434,8 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
for _, r := range records {
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID); err != nil {
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -828,3 +830,25 @@ func (s *DB) SnapshotTo(ctx context.Context, path string) error {
|
||||
_, err := s.raw.ExecContext(ctx, `VACUUM INTO ?`, path)
|
||||
return err
|
||||
}
|
||||
|
||||
// UsageByRequest finds the record of one request by the id it was given, or
|
||||
// ok=false when none has that id.
|
||||
func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool, error) {
|
||||
var u Usage
|
||||
var ts int64
|
||||
err := s.db.QueryRowContext(ctx,
|
||||
`SELECT ts, key_id, project_id, COALESCE(user_id, ''), model, served_model, prompt_tokens,
|
||||
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms
|
||||
FROM usage WHERE request_id = ? ORDER BY ts DESC LIMIT 1`, requestID).
|
||||
Scan(&ts, &u.KeyID, &u.ProjectID, &u.UserID, &u.Model, &u.ServedModel, &u.PromptTokens,
|
||||
&u.CompletionTokens, &u.CostMicros, &u.Status, &u.LatencyMs, &u.RequestID, &u.Replica,
|
||||
&u.QueueMs, &u.TTFTMs)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Usage{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Usage{}, false, err
|
||||
}
|
||||
u.At = time.UnixMilli(ts)
|
||||
return u, true, nil
|
||||
}
|
||||
@@ -244,6 +244,15 @@ type Usage struct {
|
||||
CostMicros int64
|
||||
Status int
|
||||
LatencyMs int64
|
||||
|
||||
// RequestID is the id the client sent in X-Request-Id or the gateway made.
|
||||
RequestID string
|
||||
// Replica answered the request; empty when none did.
|
||||
Replica string
|
||||
// QueueMs is time spent waiting in admission; TTFTMs is time to the first
|
||||
// token, 0 when none came.
|
||||
QueueMs int64
|
||||
TTFTMs int64
|
||||
}
|
||||
|
||||
// Store is the persistent half of tenancy and accounting.
|
||||
|
||||
Reference in new issue
Block a user