Ilker ManapandClaude Opus 5.5
f8572fb5ee
"Why was this request slow?" had no answer. Nothing tied a request to the replica that served it, the time admission held it went out in a response header and was never kept, and the only per-request log lines were for disconnects, early stream ends and upstream retries. Every request now has an id: the client's X-Request-Id when it is a plain string of reasonable length, otherwise one the gateway makes. It is set on the response before anything else, so a request refused as malformed can be named too, and forwarded to the engine. One access log line per request carries it with the model asked for and served, the replica, status, time in the queue, time to the first token, total time, tokens, and the key, project and user. Usage records gain the id, the replica, queue time and time to first token, tried against a restored copy of the live store before being shipped: 591 rows kept, columns and index in place. `llmgw ctl request ID` reads one back through the admin API. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
46 lines
1.7 KiB
Go
46 lines
1.7 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "request ID",
|
|
Short: "What one request went through: replica, queue, first token, tokens",
|
|
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
|
|
"own. Given one, this says which replica answered, how long admission held\n" +
|
|
"it, how long the first token took and what it cost — the answer to \"why\n" +
|
|
"was this slow\". The access log has the same fields under \"request\".",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
var v adminapi.RequestView
|
|
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
|
|
return err
|
|
}
|
|
if *asJSON {
|
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
|
}
|
|
model := v.Model
|
|
if v.ServedModel != "" && v.ServedModel != v.Model {
|
|
model += " (served by " + v.ServedModel + ")"
|
|
}
|
|
out := cmd.OutOrStdout()
|
|
fmt.Fprintf(out, "request %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z"))
|
|
fmt.Fprintf(out, "model %s\n", model)
|
|
fmt.Fprintf(out, "replica %s\n", or(v.Replica, "- (none answered)"))
|
|
fmt.Fprintf(out, "status %d\n", v.Status)
|
|
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
|
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
|
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
|
return nil
|
|
},
|
|
}
|
|
}
|