Follow one request from its id to the replica that served it

"Why was this request slow?" had no answer. Nothing tied a request to the
replica that served it, the time admission held it went out in a response
header and was never kept, and the only per-request log lines were for
disconnects, early stream ends and upstream retries.

Every request now has an id: the client's X-Request-Id when it is a plain
string of reasonable length, otherwise one the gateway makes. It is set on
the response before anything else, so a request refused as malformed can be
named too, and forwarded to the engine. One access log line per request
carries it with the model asked for and served, the replica, status, time in
the queue, time to the first token, total time, tokens, and the key,
project and user.

Usage records gain the id, the replica, queue time and time to first token,
tried against a restored copy of the live store before being shipped: 591
rows kept, columns and index in place. `llmgw ctl request ID` reads one back
through the admin API.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 02:36:45 +02:00
1 parent 91fe24a105
commit f8572fb5ee
10 files changed
+285 -4

No files matched your search

+50
View File
@@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
mux.HandleFunc("POST /api/v1/restart", a.restart)
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
}
@@ -1115,3 +1116,52 @@ func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
was := a.ctl.Reconciler().Retry(name)
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
}
// requestFinder looks a request up by the id it was given. Asked of the usage
// source by type assertion, so one without it answers 501 rather than every
// source having to grow the method.
type requestFinder interface {
UsageByRequest(ctx context.Context, requestID string) (store.Usage, bool, error)
}
// RequestView is what one request went through.
type RequestView struct {
RequestID string `json:"request_id"`
At time.Time `json:"at"`
KeyID string `json:"key_id"`
UserID string `json:"user_id,omitempty"`
ProjectID string `json:"project_id"`
Model string `json:"model"`
ServedModel string `json:"served_model"`
Replica string `json:"replica,omitempty"`
Status int `json:"status"`
LatencyMs int64 `json:"latency_ms"`
QueueMs int64 `json:"queue_ms"`
TTFTMs int64 `json:"ttft_ms"`
PromptTokens int64 `json:"prompt_tokens"`
CompletionTokens int64 `json:"completion_tokens"`
CostMicros int64 `json:"cost_micros"`
}
func (a *API) request(w http.ResponseWriter, r *http.Request) {
f, ok := a.usage.(requestFinder)
if !ok {
writeError(w, http.StatusNotImplemented, "this control plane keeps no usage records")
return
}
u, found, err := f.UsageByRequest(r.Context(), r.PathValue("id"))
switch {
case err != nil:
writeError(w, http.StatusInternalServerError, err.Error())
return
case !found:
writeError(w, http.StatusNotFound, "no request with that id; records are written a few seconds after a request ends")
return
}
writeJSON(w, http.StatusOK, RequestView{
RequestID: u.RequestID, At: u.At, KeyID: u.KeyID, UserID: u.UserID, ProjectID: u.ProjectID,
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
})
}