Requests kept and replayed: keys requests per project with retention and a share, ctl request --body and --replay

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 22:51:21 +02:00
1 parent 841ee21c12
commit 32ff1dbd6f
22 files changed
+757 -10

No files matched your search

+85 -3
View File
@@ -10,18 +10,34 @@ import (
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
var (
body bool
replay string
)
c := &cobra.Command{
Use: "request ID",
Short: "What one request went through: replica, queue, first token, tokens, guardrails",
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
"own. Given one, this says which replica answered, how long admission held\n" +
"it, how long the first token took and what it cost — the answer to \"why\n" +
"was this slow\". The access log has the same fields under \"request\".",
Args: cobra.ExactArgs(1),
"was this slow\". The access log has the same fields under \"request\".\n\n" +
"For a project that keeps its requests (`llmgw keys requests`), --body\n" +
"shows what the model was sent and what came back, and --replay MODEL\n" +
"sends the same request to another model, past the limits and the\n" +
"guardrails, and shows its answer beside the first.",
Example: " llmgw ctl request req_3f2a…\n llmgw ctl request req_3f2a… --body\n llmgw ctl request req_3f2a… --replay qwen2.5-14b-instruct",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if replay != "" {
return replayRequest(cmd, client, args[0], replay, *asJSON)
}
if body {
return showRequestBody(cmd, client, args[0], *asJSON)
}
var v adminapi.RequestView
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
return err
@@ -65,4 +81,70 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return nil
},
}
c.Flags().BoolVar(&body, "body", false, "show the request and its answer, for a project that keeps them")
c.Flags().StringVar(&replay, "replay", "", "send the kept request to this model and show both answers")
return c
}
func showRequestBody(cmd *cobra.Command, client func() *ctlClient, id string, asJSON bool) error {
var b store.RequestBody
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(id)+"/body", &b); err != nil {
return err
}
if asJSON {
return json.NewEncoder(cmd.OutOrStdout()).Encode(b)
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "request %s at %s, %s to %s (served by %s), status %d\n", b.RequestID,
b.At.UTC().Format("2006-01-02 15:04:05Z"), b.Path, b.Model, b.ServedModel, b.Status)
fmt.Fprintf(out, "kept until %s\n\n", b.ExpiresAt.UTC().Format("2006-01-02 15:04Z"))
fmt.Fprintln(out, "--- sent")
fmt.Fprintln(out, indentJSON(b.Request))
fmt.Fprintln(out, "--- answered")
fmt.Fprintln(out, or(b.Response, "(no text: an error, or not a completion)"))
if b.Truncated {
fmt.Fprintln(out, "(longer than what is kept; cut here)")
}
return nil
}
func replayRequest(cmd *cobra.Command, client func() *ctlClient, id, model string, asJSON bool) error {
var v adminapi.ReplayView
body, _ := json.Marshal(map[string]string{"model": model})
if err := client().post(cmd.Context(), "/api/v1/requests/"+url.PathEscape(id)+"/replay", body, &v); err != nil {
return err
}
if asJSON {
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "--- %s answered (status %d)\n%s\n\n", v.Original.ServedModel, v.Original.Status,
or(v.Original.Response, "(no text)"))
fmt.Fprintf(out, "--- %s answers (status %d, %d ms)\n%s\n", v.Model, v.Status, v.TookMs, or(replyText(v.Body), string(v.Body)))
return nil
}
// replyText is a completion's text, or "" when it is not one.
func replyText(body json.RawMessage) string {
var c struct {
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
Text string `json:"text"`
} `json:"choices"`
}
if json.Unmarshal(body, &c) != nil || len(c.Choices) == 0 {
return ""
}
return c.Choices[0].Message.Content + c.Choices[0].Text
}
func indentJSON(s string) string {
var v any
if json.Unmarshal([]byte(s), &v) != nil {
return s
}
b, _ := json.MarshalIndent(v, "", " ")
return string(b)
}
+1
View File
@@ -42,6 +42,7 @@ func newTenancyCmd() *cobra.Command {
newKeyTTLCmd(open),
newLimitsCmd(open),
newThreadsCmd(open),
newRequestLogCmd(open),
newMemoryCmd(open),
newMoveStoreCmd(),
)
+74
View File
@@ -0,0 +1,74 @@
package cli
import (
"fmt"
"github.com/spf13/cobra"
)
func newRequestLogCmd(open opener) *cobra.Command {
var (
project string
hours int
sample float64
off bool
)
cmd := &cobra.Command{
Use: "requests",
Short: "Keep a project's requests and answers, to follow one up or replay it",
Long: "With this on, what each request of the project sent the model — after the\n" +
"guardrails' redaction, with any stored conversation in it — and the answer\n" +
"the client got are kept for --retain-hours. `llmgw ctl request ID --body`\n" +
"shows one, and `--replay MODEL` sends it to another model.\n\n" +
"Off for every project until this is run, like conversations: llmgw\n" +
"otherwise keeps no text. --sample keeps a share of the requests rather\n" +
"than all of them. A project under a guardrails policy with no_store keeps\n" +
"nothing whatever this says.",
Example: " llmgw keys requests --project Acme/ml/train --retain-hours 72\n" +
" llmgw keys requests --project Acme/ml/train --retain-hours 168 --sample 0.05\n" +
" llmgw keys requests --project Acme/ml/train --off",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if !off && hours <= 0 {
return fmt.Errorf("--retain-hours is required: say how long requests are kept, or pass --off")
}
if sample <= 0 || sample > 1 {
return fmt.Errorf("--sample is a share: above 0, at most 1")
}
s, err := open(cmd.Context())
if err != nil {
return err
}
defer s.Close()
p, err := resolveProject(cmd.Context(), s, project)
if err != nil {
return err
}
if off {
hours = 0
}
if err := s.SetRequestLog(cmd.Context(), p.ID, hours, sample); err != nil {
return err
}
if hours == 0 {
fmt.Fprintf(cmd.OutOrStdout(), "%s keeps no requests. Those kept are removed at the next prune.\n", p.Path)
return nil
}
fmt.Fprintf(cmd.OutOrStdout(), "%s keeps %s of its requests and answers for %d hours.\n",
p.Path, share(sample), hours)
return nil
},
}
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
cmd.Flags().IntVar(&hours, "retain-hours", 0, "how long a request and its answer are kept")
cmd.Flags().Float64Var(&sample, "sample", 1, "the share of requests kept, above 0 and at most 1")
cmd.Flags().BoolVar(&off, "off", false, "stop keeping requests for this project")
return cmd
}
func share(f float64) string {
if f >= 1 {
return "every one"
}
return fmt.Sprintf("%g%%", f*100)
}
+1
View File
@@ -292,6 +292,7 @@ func newServerCmd() *cobra.Command {
adminAPI = api
api.SetReleases(cfg.Releases)
api.SetNotifier(notifier)
api.SetReplayer(rt.Replay)
if st := rt.Store(); st != nil && gate != nil {
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
}
+33 -2
View File
@@ -70,6 +70,8 @@ type peopleData struct {
limits map[string]store.Limits
threads map[string]int // project id → hours kept, conversations
notes map[string]int // project id → hours kept, agent notes
// requests is how each project keeps its requests and answers.
requests map[string]store.RequestLog
// guardPolicies are server.yaml's guardrail policies, guardDefault those
// applied everywhere, and guardAt which are attached where.
guardPolicies []string
@@ -122,7 +124,7 @@ func (m consoleModel) fetchPeople() tea.Cmd {
d.guardDefault = cfg.Guard.Default
d.guardAt, _ = s.GuardAttachments(ctx)
}
d.threads, d.notes = map[string]int{}, map[string]int{}
d.threads, d.notes, d.requests = map[string]int{}, map[string]int{}, map[string]store.RequestLog{}
for _, r := range d.orgs {
if r.level != "project" {
continue
@@ -133,6 +135,9 @@ func (m consoleModel) fetchPeople() tea.Cmd {
if t, err := s.MemoryRetention(ctx, r.id); err == nil {
d.notes[r.id] = int(t.Hours())
}
if l, err := s.RequestLogOf(ctx, r.id); err == nil {
d.requests[r.id] = l
}
}
return consolePeopleMsg(d)
}
@@ -502,7 +507,33 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
}
items = append(items,
retention("Keep conversations", "threads", "conversations", m.people.threads[r.id]),
retention("Keep agent notes", "memory", "notes", m.people.notes[r.id]))
retention("Keep agent notes", "memory", "notes", m.people.notes[r.id]),
menuItem{"Keep requests…", func(m *consoleModel) tea.Cmd {
l := m.people.requests[r.id]
sample := "1"
if l.Sample > 0 {
sample = strconv.FormatFloat(l.Sample, 'f', -1, 64)
}
m.openForm(&formDialog{title: "Keep requests · " + r.path,
note: "Off by default: llmgw keeps no requests unless a project asks for it.\n" +
"Kept, `llmgw ctl request ID --body` shows one and --replay sends it to another model.",
fields: []formField{
textF("hours", strconv.Itoa(int(l.Retain.Hours())), "how long they are kept; 0 keeps none"),
textF("share", sample, "of requests kept, above 0 and at most 1"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
h, err := strconv.Atoi(v[0])
if err != nil || h < 0 {
return nil, fmt.Errorf("hours is a whole number")
}
args := []string{"keys", "requests", "--project=" + r.id}
if h == 0 {
return m.runCLI("Keep requests", append(args, "--off")...), nil
}
return m.runCLI("Keep requests", append(args, "--retain-hours="+v[0], "--sample="+v[1])...), nil
}})
return nil
}})
}
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Remove " + r.path,